binlog

package module
v0.0.0-...-dd9f466 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 21, 2022 License: Apache-2.0 Imports: 25 Imported by: 0

README

binlog

mysql binlog replication protocol in golang

documentation

Documentation

Overview

Package binlog implements mysql binlog replication protocol.

This library is mainly aimed to provide RBR event parsing.

to connect to mysql server: 

bl, err := binlog.Dial("tcp", "localhost:3306", 5*time.Second)
if err != nil {
	return err
}
if bl.IsSSLSupported() {
	if err = bl.UpgradeSSL(tlsConfig); err != nil {
		return err
	}
}
if err := bl.Authenticate("root", "secret"); err != nil {
	return err
}

to get binlog events from server: 

serverID := 0 // use non-zero to wait for more data after end of last log
if serverID !=0 {
	// send heartbeatEvent when there is no more data
	if err := bl.SetHeartbeatPeriod(30 * time.Second); err != nil {
		return err
	}
}
if err := bl.Seek(serverID, "binlog.000001", 4); err != nil {
	return err
}
for {
	e, err := bl.NextEvent()
	if err != nil {
		if err == io.EOF {
			break
		}
		return err
	}
	re, ok := e.Data.(binlog.RowsEvent)
	if !ok {
		continue
	}
	fmt.Sprintf("Table: %s.%s\n", re.TableMap.SchemaName, re.TableMap.TableName)
	if e.IsWriteRows() {
		fmt.Println("Action: insert")
	else if e.IsUpdateRows() {
		fmt.Println("Action: update")
	else if e.IsDeleteRows() {
		fmt.Println("Action: delete")
	}
	for {
		row, _, err := bl.NextRow()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		for i, v := range row {
			col := d.Columns()[i]
			fmt.Sprintf("col=%s ordinal=%d value=%v\n", col.Name, col.Ordinal, v)
		}
	}
}

this package also supports the following:

  • dump to local directory
  • resume dump from where it left
  • read binlog files from dump directory as if it is server

for example usage see cmd/binlog/main.go

Index

Constants

This section is empty.

Variables

View Source 
var ErrMalformedPacket = errors.New("binlog: malformed packet")

ErrMalformedPacket used to indicate malformed packet.

Functions

This section is empty.

Types

type Column 

type Column struct {
	Ordinal  int
	Type     ColumnType
	Nullable bool
	Unsigned bool
	Meta     uint16
	Charset  uint64 // value zero means unknown.

	// following are populated only if
	// system variable binlog_row_metadata==FULL
	Name   string
	Values []string // permitted values for Enum and Set type.
}

Column captures column info for TableMapEvent and RowsEvent.

type ColumnType 

type ColumnType uint8

ColumnType used in TableMapEvent and RowsEvent. 

const (
	TypeDecimal    ColumnType = 0x00
	TypeTiny       ColumnType = 0x01 // int8 or uint8. TINYINT
	TypeShort      ColumnType = 0x02 // int16 or uint16. SMALLINT
	TypeLong       ColumnType = 0x03 // int32 or uint32. INT
	TypeFloat      ColumnType = 0x04 // float32. FLOAT
	TypeDouble     ColumnType = 0x05 // float64. DOUBLE
	TypeNull       ColumnType = 0x06
	TypeTimestamp  ColumnType = 0x07
	TypeLongLong   ColumnType = 0x08 // int64 or uint64. BIGINT
	TypeInt24      ColumnType = 0x09 // int32 or uint32. MEDIUMINT
	TypeDate       ColumnType = 0x0a // time.Time(UTC). DATE
	TypeTime       ColumnType = 0x0b
	TypeDateTime   ColumnType = 0x0c
	TypeYear       ColumnType = 0x0d // int. YEAR
	TypeNewDate    ColumnType = 0x0e
	TypeVarchar    ColumnType = 0x0f // string. VARCHAR
	TypeBit        ColumnType = 0x10 // uint64. BIT
	TypeTimestamp2 ColumnType = 0x11 // time.Time(LOCAL). TIMESTAMP
	TypeDateTime2  ColumnType = 0x12 // time.Time(UTC). DATETIME
	TypeTime2      ColumnType = 0x13 // time.Duration. TIME
	TypeJSON       ColumnType = 0xf5 // JSON, JSON
	TypeNewDecimal ColumnType = 0xf6 // Decimal. DECIMAL NUMERIC
	TypeEnum       ColumnType = 0xf7 // Enum. ENUM
	TypeSet        ColumnType = 0xf8 // Set. SET
	TypeTinyBlob   ColumnType = 0xf9
	TypeMediumBlob ColumnType = 0xfa
	TypeLongBlob   ColumnType = 0xfb
	TypeBlob       ColumnType = 0xfc // []byte or string. TINYBLOB BLOB MEDIUMBLOB LONGBLOB TINYTEXT TEXT MEDIUMTEXT LONGTEXT
	TypeVarString  ColumnType = 0xfd
	TypeString     ColumnType = 0xfe // string. CHAR
	TypeGeometry   ColumnType = 0xff
)

ColumnType Constants

https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType

func (ColumnType) String 

func (t ColumnType) String() string

type Decimal 

type Decimal string

A Decimal represents a MySQL Decimal/Numeric literal.

https://dev.mysql.com/doc/refman/8.0/en/fixed-point-types.html

func (Decimal) BigFloat 

func (d Decimal) BigFloat() (*big.Float, error)

BigFloat returns the number as a *big.Float.

func (Decimal) Float64 

func (d Decimal) Float64() (float64, error)

Float64 returns the number as a float64.

func (Decimal) MarshalJSON 

func (d Decimal) MarshalJSON() ([]byte, error)

func (Decimal) String 

func (d Decimal) String() string

type Enum 

type Enum struct {
	// index value. refers to a position in list of permitted values.
	// begins with 1.
	// 0 means empty string invalid value.
	Val uint16

	// list of permitted values.
	// will be populated only if system
	// variable binlog_row_metadata==FULL
	Values []string
}

Enum represents value of TypeEnum.

https://dev.mysql.com/doc/refman/8.0/en/enum.html

func (Enum) MarshalJSON 

func (e Enum) MarshalJSON() ([]byte, error)

func (Enum) String 

func (e Enum) String() string

type Event 

type Event struct {
	Header EventHeader
	Data   interface{} // one of XXXEvent
}

Event represents Binlog Event.

type EventHeader 

type EventHeader struct {
	Timestamp uint32    // seconds since unix epoch
	EventType EventType // binlog event type
	ServerID  uint32    // server-id of the originating mysql-server
	EventSize uint32    // size of the event (header + post-header + body)
	LogFile   string    // logfile of the next event
	NextPos   uint32    // position of the next event
	Flags     uint16    // flags
}

EventHeader represents Binlog Event Header.

https://dev.mysql.com/doc/internals/en/binlog-event-header.html https://dev.mysql.com/doc/internals/en/event-header-fields.html

type EventType 

type EventType uint8

EventType represents Binlog Event Type. 

const (
	UNKNOWN_EVENT            EventType = 0x00 // should never occur. used when event cannot be recognized.
	START_EVENT_V3           EventType = 0x01 // descriptor event written to binlog beginning. deprecated.
	QUERY_EVENT              EventType = 0x02 // written when an updating statement is done.
	STOP_EVENT               EventType = 0x03 // written when mysqld stops.
	ROTATE_EVENT             EventType = 0x04 // written when mysqld switches to a new binary log file.
	INTVAR_EVENT             EventType = 0x05 // if stmt uses AUTO_INCREMENT col or LAST_INSERT_ID().
	LOAD_EVENT               EventType = 0x06 // used for LOAD DATA INFILE statements in MySQL 3.23.
	SLAVE_EVENT              EventType = 0x07 // not used.
	CREATE_FILE_EVENT        EventType = 0x08 // used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
	APPEND_BLOCK_EVENT       EventType = 0x09 // used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
	EXEC_LOAD_EVENT          EventType = 0x0a // used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
	DELETE_FILE_EVENT        EventType = 0x0b // used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
	NEW_LOAD_EVENT           EventType = 0x0c // used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
	RAND_EVENT               EventType = 0x0d // if stmt uses RAND().
	USER_VAR_EVENT           EventType = 0x0e // if stmt uses a user variable.
	FORMAT_DESCRIPTION_EVENT EventType = 0x0f // descriptor event written to binlog beginning.
	XID_EVENT                EventType = 0x10 // for XA commit transaction.
	BEGIN_LOAD_QUERY_EVENT   EventType = 0x11 // used for LOAD DATA INFILE statements in MySQL 5.0.
	EXECUTE_LOAD_QUERY_EVENT EventType = 0x12 // used for LOAD DATA INFILE statements in MySQL 5.0.
	TABLE_MAP_EVENT          EventType = 0x13 // precedes rbr event. contains table definition.
	WRITE_ROWS_EVENTv0       EventType = 0x14 // logs inserts of rows in a single table.
	UPDATE_ROWS_EVENTv0      EventType = 0x15 // logs updates of rows in a single table.
	DELETE_ROWS_EVENTv0      EventType = 0x16 // logs deletions of rows in a single table.
	WRITE_ROWS_EVENTv1       EventType = 0x17 // logs inserts of rows in a single table.
	UPDATE_ROWS_EVENTv1      EventType = 0x18 // logs updates of rows in a single table.
	DELETE_ROWS_EVENTv1      EventType = 0x19 // logs inserts of rows in a single table.
	INCIDENT_EVENT           EventType = 0x1a // used to log an out of the ordinary event that occurred on the master.
	HEARTBEAT_EVENT          EventType = 0x1b // to signal that master is still alive. not written to file.
	IGNORABLE_EVENT          EventType = 0x1c
	ROWS_QUERY_EVENT         EventType = 0x1d
	WRITE_ROWS_EVENTv2       EventType = 0x1e // logs inserts of rows in a single table.
	UPDATE_ROWS_EVENTv2      EventType = 0x1f // logs updates of rows in a single table.
	DELETE_ROWS_EVENTv2      EventType = 0x20 // logs inserts of rows in a single table.
	GTID_EVENT               EventType = 0x21
	ANONYMOUS_GTID_EVENT     EventType = 0x22
	PREVIOUS_GTIDS_EVENT     EventType = 0x23
)

Event Type Constants.

https://dev.mysql.com/doc/internals/en/binlog-event-type.html https://dev.mysql.com/doc/internals/en/event-meanings.html

func (EventType) IsDeleteRows 

func (t EventType) IsDeleteRows() bool

IsDeleteRows tells if this EventType DELETE_ROWS_EVENT. MySQL has multiple versions of DELETE_ROWS_EVENT.

func (EventType) IsUpdateRows 

func (t EventType) IsUpdateRows() bool

IsUpdateRows tells if this EventType UPDATE_ROWS_EVENT. MySQL has multiple versions of UPDATE_ROWS_EVENT.

func (EventType) IsWriteRows 

func (t EventType) IsWriteRows() bool

IsWriteRows tells if this EventType WRITE_ROWS_EVENT. MySQL has multiple versions of WRITE_ROWS_EVENT.

func (EventType) String 

func (t EventType) String() string

type FormatDescriptionEvent 

type FormatDescriptionEvent struct {
	BinlogVersion          uint16 // version of this binlog format
	ServerVersion          string // version of the MySQL Server that created the binlog
	CreateTimestamp        uint32 // seconds since Unix epoch when the binlog was created
	EventHeaderLength      uint8  // length of the Binlog Event Header of next events
	EventTypeHeaderLengths []byte // post-header lengths for different event-types
}

FormatDescriptionEvent is written to the beginning of the each binary log file. This event is used as of MySQL 5.0; it supersedes START_EVENT_V3.

https://dev.mysql.com/doc/internals/en/format-description-event.html

type HeartbeatEvent 

type HeartbeatEvent struct{}

HeartbeatEvent sent by a master to a slave to let the slave know that the master is still alive. Not written to log files.

https://dev.mysql.com/doc/internals/en/heartbeat-event.html

type IncidentEvent 

type IncidentEvent struct {
	Type    uint16
	Message string
}

IncidentEvent used to log an out of the ordinary event that occurred on the master. It notifies the slave that something happened on the master that might cause data to be in an inconsistent state.

https://dev.mysql.com/doc/internals/en/incident-event.html

type IntVarEvent 

type IntVarEvent struct {
	// Type indicates subtype.
	//
	// INSERT_ID_EVENT(0x02) indicates the value to use for an AUTO_INCREMENT column in the next statement.
	//
	// LAST_INSERT_ID_EVENT(0x01) indicates the value to use for the LAST_INSERT_ID() function in the next statement.
	Type  uint8
	Value uint64
}

IntVarEvent written every time a statement uses an AUTO_INCREMENT column or the LAST_INSERT_ID() function. It precedes other events for the statement. This is written only before a QUERY_EVENT and is not used with row-based logging.

https://dev.mysql.com/doc/internals/en/intvar-event.html

type JSON 

type JSON struct{ Val interface{} }

Json represents value of TypeJSON

https://dev.mysql.com/doc/refman/8.0/en/json.html

func (JSON) MarshalJSON 

func (j JSON) MarshalJSON() ([]byte, error)

type Local 

type Local struct {
	// contains filtered or unexported fields
}

Local represents connection to local dump directory.

func Open 

func Open(dir string) (*Local, error)

Open connects to dump directory specified.

func (*Local) ListFiles 

func (bl *Local) ListFiles() ([]string, error)

ListFiles lists the binary log files in dump directory.

func (*Local) MasterStatus 

func (bl *Local) MasterStatus() (file string, pos uint32, err error)

MasterStatus provides status information about the binary log files in dump directory.

func (*Local) NextEvent 

func (bl *Local) NextEvent() (Event, error)

NextEvent return next binlog event.

return io.EOF when there are no more Events

func (*Local) NextRow 

func (bl *Local) NextRow() (values []interface{}, valuesBeforeUpdate []interface{}, err error)

NextRow returns next row for RowsEvent. Returns io.EOF when there are no more rows. valuesBeforeUpdate should be used only for events UPDATE_ROWS_EVENTv1, UPDATE_ROWS_EVENTv2.

func (*Local) RemoveFirstFile 

func (bl *Local) RemoveFirstFile() error

RemoveFirstFile deletes the first binary log file from dump directory.

func (*Local) Seek 

func (bl *Local) Seek(serverID uint32, fileName string, position uint32) error

Seek requests binlog at fileName and position.

if serverID is zero, NextEvent return io.EOF when there are no more events. if serverID is non-zero, NextEvent waits for new events.

type QueryEvent 

type QueryEvent struct {
	SlaveProxyID  uint32
	ExecutionTIme uint32
	ErrorCode     uint16
	StatusVars    []byte
	Schema        string
	Query         string
}

QueryEvent is written when an updating statement is done. The query event is used to send text query right the binlog.

https://dev.mysql.com/doc/internals/en/query-event.html

type RandEvent 

type RandEvent struct {
	Seed1 uint64
	Seed2 uint64
}

RandEvent is written every time a statement uses the RAND() function. It precedes other events for the statement. Indicates the seed values to use for generating a random number with RAND() in the next statement. This is written only before a QUERY_EVENT and is not used with row-based logging.

https://dev.mysql.com/doc/internals/en/rand-event.html

type Remote 

type Remote struct {
	// contains filtered or unexported fields
}

Remote represents connection to MySQL server.

func Dial 

func Dial(network, address string, timeout time.Duration) (*Remote, error)

Dial connects to the MySQL server specified.

func (*Remote) Authenticate 

func (bl *Remote) Authenticate(username, password string) error

Authenticate sends the credentials to MySQL.

func (*Remote) Close 

func (bl *Remote) Close() error

Close closes connection.

func (*Remote) Dump 

func (bl *Remote) Dump(dir string) error

func (*Remote) IsSSLSupported 

func (bl *Remote) IsSSLSupported() bool

IsSSLSupported tells whether MySQL server supports SSL.

func (*Remote) ListFiles 

func (bl *Remote) ListFiles() ([]string, error)

ListFiles lists the binary log files on the server, in the order they were created. It is equivalent to `SHOW BINARY LOGS` statement.

func (*Remote) MasterStatus 

func (bl *Remote) MasterStatus() (file string, pos uint32, err error)

MasterStatus provides status information about the binary log files of the server. It is equivalent to `SHOW MASTER STATUS` statement.

func (*Remote) NextEvent 

func (bl *Remote) NextEvent() (Event, error)

NextEvent return next binlog event.

return io.EOF when there are no more Events

func (*Remote) NextRow 

func (bl *Remote) NextRow() (values []interface{}, valuesBeforeUpdate []interface{}, err error)

NextRow returns next row for RowsEvent. Returns io.EOF when there are no more rows. valuesBeforeUpdate should be used only for events UPDATE_ROWS_EVENTv1, UPDATE_ROWS_EVENTv2.

func (*Remote) Seek 

func (bl *Remote) Seek(serverID uint32, fileName string, position uint32) error

Seek requests binlog at fileName and position.

if serverID is zero, NextEvent return io.EOF when there are no more events. if serverID is non-zero, NextEvent waits for new events.

func (*Remote) SetHeartbeatPeriod 

func (bl *Remote) SetHeartbeatPeriod(d time.Duration) error

SetHeartbeatPeriod configures the interval to send HeartBeatEvent in absence of data. This avoids connection timeout occurring in the absence of data. Setting interval to 0 disables heartbeats altogether.

Use this, if you are using non-zero serverID to Seek method. In this case, server sends heartbeatEvents when there are no more events.

func (*Remote) UpgradeSSL 

func (bl *Remote) UpgradeSSL(tlsConfig *tls.Config) error

UpgradeSSL upgrades current connection to SSL. If tlsConfig is nil it will use InsecureSkipVerify true value. This should be called before Authenticate call.

type RotateEvent 

type RotateEvent struct {
	Position   uint64 // position of next event
	NextBinlog string // name of next binlog file
}

RotateEvent is written when mysqld switches to a new binary log file. This occurs when someone issues a FLUSH LOGS statement or the current binary log file becomes too large. The maximum size is determined by max_binlog_size.

https://dev.mysql.com/doc/internals/en/rotate-event.html

type RowsEvent 

type RowsEvent struct {
	TableMap *TableMapEvent // associated TableMapEvent
	// contains filtered or unexported fields
}

RowsEvent captures changed rows in a table.

see https://dev.mysql.com/doc/internals/en/rows-event.html

func (RowsEvent) Columns 

func (e RowsEvent) Columns() []Column

Columns returns columns info after update

func (RowsEvent) ColumnsBeforeUpdate 

func (e RowsEvent) ColumnsBeforeUpdate() []Column

ColumnsBeforeUpdate returns columns after after update. returns nil, if rows were inserted.

type RowsQueryEvent 

type RowsQueryEvent struct {
	Query string
}

RowsQueryEvent captures the query that caused the following ROWS_EVENT. see https://dev.mysql.com/doc/internals/en/rows-query-event.html

system variable binlog_rows_query_log_events must be ON for this event. see https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_binlog_rows_query_log_events

type Set 

type Set struct {
	// set's numerical value with bits set corresponding
	// to the set members that make up the column value.
	// 0 means empty string invalid value.
	Val uint64

	// list of permitted values.
	// will be populated only if system
	// variable binlog_row_metadata==FULL
	Values []string
}

Set represents value of TypeSet.

https://dev.mysql.com/doc/refman/8.0/en/set.html

func (Set) MarshalJSON 

func (s Set) MarshalJSON() ([]byte, error)

func (Set) Members 

func (s Set) Members() []string

Members returns the values in this set.

func (Set) String 

func (s Set) String() string

type StopEvent 

type StopEvent struct{}

StopEvent signals last event in the file.

https://dev.mysql.com/doc/internals/en/stop-event.html

type TableMapEvent 

type TableMapEvent struct {
	SchemaName string
	TableName  string
	Columns    []Column
	// contains filtered or unexported fields
}

TableMapEvent is first event used in Row Based Replication declares how a table that is about to be changed is defined.

Used for row-based binary logging. This event precedes each row operation event. It maps a table definition to a number, where the table definition consists of database and table names and column definitions. The purpose of this event is to enable replication when a table has different definitions on the master and slave.

Row operation events that belong to the same transaction may be grouped into sequences, in which case each such sequence of events begins with a sequence of TABLE_MAP_EVENT events: one per table used by events in the sequence.

see https://dev.mysql.com/doc/internals/en/table-map-event.html

type UnknownEvent 

type UnknownEvent struct{}

UnknownEvent should never occur. It is never written to a binary log. If an event is read from a binary log that cannot be recognized as something else, it is treated as UNKNOWN_EVENT.

type UserVarEvent 

type UserVarEvent struct {
	Name     string
	Null     bool
	Type     uint8
	Charset  uint32
	Value    []byte
	Unsigned bool
}

UserVarEvent is written every time a statement uses a user variable. It precedes other events for the statement. Indicates the value to use for the user variable in the next statement. This is written only before a QUERY_EVENT and is not used with row-based logging.

https://dev.mysql.com/doc/internals/en/user-var-event.html

Source Files

View all Source files

Directories

Path Synopsis
cmd
binlog

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.