cdc
A Go library to assist in monitoring data changes and syncing data from MySQL/Mongo.
how to use
With Go module support, simply add the following import
import "github.com/Rayson9418/cdc"
demo
check the dir examples
for more detail
monitor mongo changed data
func DemoMonitor() error {
// Init position store
if err := cdcredis.InitClient(options.CdcOpt.Redis); err != nil {
Logger.Fatal("init redis client with opt err", zap.Error(err))
}
store := cdcstore.NewStreamRedisStore("mongo:oplog:pos")
// New handler for specific collection
handler := NewDemoHandler()
// New mongo monitor
m, err := cdcmongo.NewDefaultMonitor(options.CdcOpt.Mongo)
if err != nil {
return err
}
// Set position store for monitor
m.SetStore(store)
// Add handlers for monitor
if err = m.AddHandler(handler); err != nil {
return err
}
return m.StartMonitor()
}
monitor mysql changed data
func DemoMonitor() error {
// InitClient position store
if err := cdcredis.InitClient(options.CdcOpt.Redis); err != nil {
Logger.Fatal("init redis client with opt err", zap.Error(err))
}
store := cdcstore.NewBinlogRedisStore("mysql:binlog:pos")
// New handler for specific collection
handler := NewDemoHandler()
// New row event monitor
m, err := cdcmysql.NewRowEventMonitor(options.CdcOpt.Mysql)
if err != nil {
return err
}
// Set position store for monitor
m.SetStore(store)
// Add handlers for monitor
if err = m.AddHandler(handler); err != nil {
return err
}
return m.StartMonitor()
}
sync data from mongo/mysql
// Keep sync always
func DemoSyncAlways() error {
opt := cdcsyncer.NewDefaultOpt()
opt.StartHour = options.CdcOpt.Syncer.StartHour
opt.EndHour = options.CdcOpt.Syncer.EndHour
opt.BatchLimit = options.CdcOpt.Syncer.BatchLimit
opt.Interval = options.CdcOpt.Syncer.Interval
demo1Syncer := NewDemo1Syncer()
return cdcsyncer.StartSyncer(demo1Syncer)
}
// Sync once
func DemoSyncOnce() error {
opt := cdcsyncer.NewDefaultOpt()
opt.StartHour = options.CdcOpt.Syncer.StartHour
opt.EndHour = options.CdcOpt.Syncer.EndHour
opt.BatchLimit = options.CdcOpt.Syncer.BatchLimit
opt.Interval = options.CdcOpt.Syncer.Interval
demo1Syncer := NewDemo1Syncer()
return cdcsyncer.SyncOnce(demo1Syncer)
}
// Sync on time
func DemoSyncOnTime() error {
opt := cdcsyncer.NewDefaultOpt()
opt.StartHour = options.CdcOpt.Syncer.StartHour
opt.EndHour = options.CdcOpt.Syncer.EndHour
opt.BatchLimit = options.CdcOpt.Syncer.BatchLimit
opt.Interval = options.CdcOpt.Syncer.Interval
demo1Syncer := NewDemo1Syncer()
return cdcsyncer.StartSyncerOnTime(demo1Syncer)
}
run
If you have Docker and Docker Compose installed, you can run the code using the following steps.
-
Clone this repository to your machine, and switch to the directory examples
.
# clone this repository
git clone git@github.com:Rayson9418/cdc.git
# switch to the directory
cd cdc/examples
-
Run the script to build the container for compilation, and set up the middleware environment.
bash build_on_docker.sh -t
-
Compile the code and then run it.
make build && ./examples