Mongo之ChangeStream详解

简介

Change Stream可以直译为"变更流"，也就是说会将数据库中的所有变更以流式的方式呈现出来。用户可以很方便地对数据库建立一个监听（订阅）进程，一旦数据库发生变更，使用change stream的客户端都可以收到相应的通知。使用场景可以包括但不限于以下几种：

• 多个MongoDB集群之间的增量数据同步；

• 高风险操作的审计（删库删表）；

• 将MongoDB的变更订阅到其他关联系统实现离线分析/计算等等；

特征

change stream的一些特性事项

• change stream对于副本集和分片集群都可用。副本集时，可以在副本集中任意一个成员上建立监听流；分片集群时则只能在mongos上建立监听流。

• 使用条件：1）WT引擎；2）副本集协议为 pv1 ；3）4.0及以前的版本，要求支持readConcern为 “majority” 。

• 粒度可调整，可选择配置在单个表、单个库或者整个集群上。但是无法配置为 admin/local/config 库或者 system.xxx 表。

• 4.0以后的版本可以指定 startAtOperationTime 来表示在某个特定的时间开始监听change Stream。但是要求给定的时间点必须在所选择节点的有效oplog时间范围中。

使用

1. 搭建集群

下面通过搭建一个单节点的副本集mongodb来试验下
修改配置并启动

bind_ip=0.0.0.0
port=27017
replSet=ts
fork=true # 以创建子进程的方式运行
dbpath=/data/mongodb/db #日志输出方式数据库路径
logappend=true #日志输出方式，日志append而不是overwrite
logpath=/data/mongodb/logs/mongo.log #日志路径
auth=false #开启安全验证（可以不开启）

配置集群

> cfg = {"_id" : "ts", "members" : [{"_id" : 0,"host" : "9.135.77.164:27017"}]}
{
        "_id" : "ts",
        "members" : [
                {
                        "_id" : 0,
                        "host" : "9.135.77.164:27017"
                }
        ]
}
> 
> 
> rs.initiate(cfg)
{ "ok" : 1 }

查看状态

ts:PRIMARY> rs.status()
{
        "set" : "ts",
        "date" : ISODate("2023-04-03T03:33:48.813Z"),
        "myState" : 1,
        "term" : NumberLong(1),
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 1,
        "writeMajorityCount" : 1,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1680492817, 8),
                        "t" : NumberLong(1)
                },
                "lastCommittedWallTime" : ISODate("2023-04-03T03:33:37.357Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1680492817, 8),
                        "t" : NumberLong(1)
                },
                "readConcernMajorityWallTime" : ISODate("2023-04-03T03:33:37.357Z"),
                "appliedOpTime" : {
                        "ts" : Timestamp(1680492817, 8),
                        "t" : NumberLong(1)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1680492817, 8),
                        "t" : NumberLong(1)
                },
                "lastAppliedWallTime" : ISODate("2023-04-03T03:33:37.357Z"),
                "lastDurableWallTime" : ISODate("2023-04-03T03:33:37.357Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1680492817, 7),
        "lastStableCheckpointTimestamp" : Timestamp(1680492817, 7),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "electionTimeout",
                "lastElectionDate" : ISODate("2023-04-03T03:33:37.288Z"),
                "electionTerm" : NumberLong(1),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(0, 0),
                        "t" : NumberLong(-1)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1680492817, 1),
                        "t" : NumberLong(-1)
                },
                "numVotesNeeded" : 1,
                "priorityAtElection" : 1,
                "electionTimeoutMillis" : NumberLong(10000),
                "newTermStartDate" : ISODate("2023-04-03T03:33:37.329Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2023-04-03T03:33:37.356Z")
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "9.135.77.164:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 218,
                        "optime" : {
                                "ts" : Timestamp(1680492817, 8),
                                "t" : NumberLong(1)
                        },
                        "optimeDate" : ISODate("2023-04-03T03:33:37Z"),
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1680492817, 2),
                        "electionDate" : ISODate("2023-04-03T03:33:37Z"),
                        "configVersion" : 1,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1680492817, 8),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1680492817, 8)
}

下面通过一个简单的示例来看看怎么使用它

2. 简要代码

package main
import (
"context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
"time"
)
func getDb() *mongo.Database {
var err error
   clientOptions := options.Client().ApplyURI(
"mongodb://9.135.77.164:27017").SetConnectTimeout(5 * time.Second)
// 连接到MongoDB
   client, err := mongo.Connect(context.TODO(), clientOptions)
if err != nil {
      log.Fatal(err)
   }
//选择数据库
return client.Database("audit")
}
type StreamObject struct {
   Id *WatchId `bson:"_id"`
   OperationType string
   FullDocument map[string]interface{}
   Ns NS
   UpdateDescription map[string]interface{}
   DocumentKey map[string]interface{}
}
type NS struct {
   Database string `bson:"db"`
   Collection string `bson:"coll"`
}
type WatchId struct {
   Data string `bson:"_data"`
}
const (
   OperationTypeInsert = "insert"
   OperationTypeDelete = "delete"
   OperationTypeUpdate = "update"
   OperationTypeReplace = "replace"
)
var resumeToken bson.Raw
func Sync() {
//go syncMaster()
//
//for {
// time.Sleep(2 * time.Second)
//}
for {
//获得主库数据连接
      client := getDb()
      log.Println("start watch")
      watch(client)
   }
}
func syncMaster() {
for {
//获得主库数据连接
      client := getDb()
      watch(client)
   }
}
func watch(client *mongo.Database) {
defer func() {
      err := recover()
if err != nil {
         log.Printf("同步出现异常: %+v \n", err)
      }
   }()
//设置过滤条件
   pipeline := mongo.Pipeline{
      bson.D{{"$match",
         bson.M{"operationType": bson.M{"$in": bson.A{"insert", "delete", "replace", "update"}}},
      }},
   }
//当前时间前一小时
   now := time.Now()
   m, _ := time.ParseDuration("-1h")
   now = now.Add(m)
   timestamp := &primitive.Timestamp{
      T: uint32(now.Unix()),
      I: 0,
   }
//设置监听option
   opt := options.ChangeStream().SetFullDocument(options.UpdateLookup).SetStartAtOperationTime(timestamp)
if resumeToken != nil {
      opt.SetResumeAfter(resumeToken)
      opt.SetStartAtOperationTime(nil)
   }
//获得watch监听
   watch, err := client.Watch(context.TODO(), pipeline, opt)
if err != nil {
      log.Fatal("watch监听失败：", err)
   }
   log.Println("watching")
for watch.Next(context.TODO()) {
var stream StreamObject
      err = watch.Decode(&stream)
if err != nil {
         log.Println("watch数据失败：", err)
      }
      log.Println("=============", stream.FullDocument["_id"])
//保存现在resumeToken
      resumeToken = watch.ResumeToken()
switch stream.OperationType {
case OperationTypeInsert:
         log.Println("insert record：", stream.FullDocument)
case OperationTypeDelete:
         log.Println("delete record：", stream.FullDocument)
case OperationTypeUpdate:
         log.Println("update record：", stream.FullDocument)
      }
   }
}

3. 启动

2023/04/03 20:50:40 start watch
2023/04/03 20:50:40 watching

4. 更新数据

ts:PRIMARY> db.operate_record.update({"_id":6}, {$set: {"operator": "ttttttttt"}})

5. 查看监听

2023/04/03 20:54:00 ============= 6
2023/04/03 20:54:00 update record： map[_id:6 action:创建资源 created_at:1676950105535 description: detail: message: object:1 operate_time:1676950105535 operator:ttttttttt project_id:111 result:true trace_85251e-c4a9-427e-9a38-a3c5264d497d]

https://mongoing.com/archives/75336
https://blog.csdn.net/it_freshman/article/details/115659883
https://cloud.tencent.com/developer/article/1792748