mysqlctl

package
v0.0.0-...-9a5932e Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 10, 2014 License: BSD-3-Clause Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source
const (
	MysqlWaitTime = 120 * time.Second // default number of seconds to wait
)
View Source
const (
	SlaveStartDeadline = 30
)
View Source
const (
	SnapshotManifestFile = "snapshot_manifest.json"
)
View Source
const (
	SnapshotURLPath = "/snapshot"
)

Variables

View Source
var (
	ErrNotSlave  = errors.New("no slave status")
	ErrNotMaster = errors.New("no master status")
)

Functions

func ConcurrentMap

func ConcurrentMap(concurrency, n int, fun MapFunc) error

ConcurrentMap applies fun in a concurrent manner on integers from 0 to n-1 (they are assumed to be indexes of some slice containing items to be processed). The first error returned by a fun application will returned (subsequent errors will only be logged). It will use concurrency goroutines.

func DirectoryList

func DirectoryList(cnf *Mycnf) []string

func Init

func Init(mt *Mysqld, mysqlWaitTime time.Duration) error

func MakeMycnf

func MakeMycnf(mycnf *Mycnf, cnfFiles []string) (string, error)

Join cnf files cnfPaths and subsitute in the right values.

func MycnfFile

func MycnfFile(uid uint32) string

func SanityCheckManifests

func SanityCheckManifests(ssms []*SplitSnapshotManifest) error

SanityCheckManifests checks if the ssms can be restored together.

func Shutdown

func Shutdown(mt *Mysqld, waitForMysqld bool, mysqlWaitTime time.Duration) error
waitForMysqld: should the function block until mysqld has stopped?

This can actually take a *long* time if the buffer cache needs to be fully flushed - on the order of 20-30 minutes.

func SnapshotDir

func SnapshotDir(uid uint32) string

func Start

func Start(mt *Mysqld, mysqlWaitTime time.Duration) error

func StartReplicationCommands

func StartReplicationCommands(mysqld *Mysqld, replState *proto.ReplicationState) ([]string, error)

func TabletDir

func TabletDir(uid uint32) string

func Teardown

func Teardown(mt *Mysqld, force bool) error

func TopLevelDirs

func TopLevelDirs() []string

Types

type FakeMysqlDaemon

type FakeMysqlDaemon struct {
	// will be returned by GetMasterAddr(). Set to "" to return
	// ErrNotSlave, or to "ERROR" to return an error.
	MasterAddr string

	// will be returned by GetMysqlPort(). Set to -1 to return an error.
	MysqlPort int
}

FakeMysqlDaemon implements MysqlDaemon and allows the user to fake everything.

func (*FakeMysqlDaemon) GetMasterAddr

func (fmd *FakeMysqlDaemon) GetMasterAddr() (string, error)

func (*FakeMysqlDaemon) GetMysqlPort

func (fmd *FakeMysqlDaemon) GetMysqlPort() (int, error)

type MapFunc

type MapFunc func(index int) error

type Mycnf

type Mycnf struct {
	ServerId              uint32
	MysqlPort             int
	DataDir               string
	InnodbDataHomeDir     string
	InnodbLogGroupHomeDir string
	SocketFile            string
	StartKey              string
	EndKey                string
	ErrorLogPath          string
	SlowLogPath           string
	RelayLogPath          string
	RelayLogIndexPath     string
	RelayLogInfoPath      string
	BinLogPath            string
	MasterInfoFile        string
	PidFile               string
	TmpDir                string
	SlaveLoadTmpDir       string
	// contains filtered or unexported fields
}

func NewMycnf

func NewMycnf(uid uint32, mysqlPort int, vtRepl VtReplParams) *Mycnf

NewMycnf fills the Mycnf structure with vt root paths and derived values. This is used to fill out the cnfTemplate values and generate my.cnf. uid is a unique id for a particular tablet - it must be unique within the tabletservers deployed within a keyspace, lest there be collisions on disk. mysqldPort needs to be unique per instance per machine.

func ReadMycnf

func ReadMycnf(cnfFile string) (mycnf *Mycnf, err error)

type MysqlDaemon

type MysqlDaemon interface {
	// GetMasterAddr returns the mysql master address, as shown by
	// 'show slave status'.
	GetMasterAddr() (string, error)

	// GetMysqlPort returns the current port mysql is listening on.
	GetMysqlPort() (int, error)
}

MysqlDaemon is the interface we use for abstracting Mysqld.

type Mysqld

type Mysqld struct {
	TabletDir   string
	SnapshotDir string
	// contains filtered or unexported fields
}

func NewMysqld

func NewMysqld(config *Mycnf, dba, repl *mysql.ConnectionParams) *Mysqld

func (*Mysqld) Addr

func (mysqld *Mysqld) Addr() string

func (*Mysqld) ApplySchemaChange

func (mysqld *Mysqld) ApplySchemaChange(dbName string, change *proto.SchemaChange) (*proto.SchemaChangeResult, error)

func (*Mysqld) BinlogInfo

func (mysqld *Mysqld) BinlogInfo(groupId int64) (rp *proto.ReplicationPosition, err error)

mysql> show binlog info for 5\G *************************** 1. row *************************** Log_name: vt-0000041983-bin.000001 Pos: 1194 Server_ID: 41983

func (*Mysqld) BreakSlaves

func (mysqld *Mysqld) BreakSlaves() error

Force all slaves to error and stop. This is extreme, but helpful for emergencies and tests. Insert a row, block the propagation of its subsequent delete and reinsert it. This forces a failure on slaves only.

func (*Mysqld) CheckReplication

func (mysqld *Mysqld) CheckReplication(timeCheck int64) error

Check for the magic row inserted under controlled reparenting.

func (*Mysqld) Cnf

func (mt *Mysqld) Cnf() *Mycnf

func (*Mysqld) CreateMultiSnapshot

func (mysqld *Mysqld) CreateMultiSnapshot(keyRanges []key.KeyRange, dbName, keyName string, keyType key.KeyspaceIdType, sourceAddr string, allowHierarchicalReplication bool, snapshotConcurrency int, tables []string, skipSlaveRestart bool, maximumFilesize uint64, hookExtraEnv map[string]string) (snapshotManifestFilenames []string, err error)

CreateMultiSnapshot create snapshots of the data.

  • for a resharding snapshot, keyRanges+keyName+keyType are set, and tables is empty. This action will create multiple snapshots, one per keyRange.
  • for a vertical split, tables is set, keyRanges = [KeyRange{}] and keyName+keyType are empty. It will create a single snapshot of the contents of the tables.

Note combinations of table subset and keyranges are not supported.

func (*Mysqld) CreateSnapshot

func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchicalReplication bool, concurrency int, serverMode bool, hookExtraEnv map[string]string) (snapshotManifestUrlPath string, slaveStartRequired, readOnly bool, err error)

This function runs on the machine acting as the source for the clone.

Check master/slave status and determine restore needs. If this instance is a slave, stop replication, otherwise place in read-only mode. Record replication position. Shutdown mysql Check paths for storing data

Depending on the serverMode flag, we do the following: serverMode = false:

Compress /vt/vt_[0-9a-f]+/data/vt_.+
Compute hash (of compressed files, as we serve .gz files here)
Place in /vt/clone_src where they will be served by http server (not rpc)
Restart mysql

serverMode = true:

Make symlinks for /vt/vt_[0-9a-f]+/data/vt_.+ to innodb files
Compute hash (of uncompressed files, as we serve uncompressed files)
Place symlinks in /vt/clone_src where they will be served by http server
Leave mysql stopped, return slaveStartRequired, readOnly

func (*Mysqld) DemoteMaster

func (mysqld *Mysqld) DemoteMaster() (*proto.ReplicationPosition, error)

if the master is still alive, then we need to demote it gracefully make it read-only, flush the writes and get the position

func (*Mysqld) ExecuteMysqlCommand

func (mysqld *Mysqld) ExecuteMysqlCommand(sql string) error

executes some SQL commands using a mysql command line interface process

func (*Mysqld) FindSlaves

func (mysqld *Mysqld) FindSlaves() ([]string, error)

Get IP addresses for all currently connected slaves.

func (*Mysqld) FindVtDatabases

func (mysqld *Mysqld) FindVtDatabases() ([]string, error)

func (*Mysqld) GetColumns

func (mysqld *Mysqld) GetColumns(dbName, table string) ([]string, error)

GetColumns returns the columns of table.

func (*Mysqld) GetMasterAddr

func (mysqld *Mysqld) GetMasterAddr() (string, error)

func (*Mysqld) GetMysqlPort

func (mysqld *Mysqld) GetMysqlPort() (int, error)

func (*Mysqld) GetPermissions

func (mysqld *Mysqld) GetPermissions() (*proto.Permissions, error)

func (*Mysqld) GetPrimaryKeyColumns

func (mysqld *Mysqld) GetPrimaryKeyColumns(dbName, table string) ([]string, error)

GetPrimaryKeyColumns returns the primary key columns of table.

func (*Mysqld) GetSchema

func (mysqld *Mysqld) GetSchema(dbName string, tables []string, includeViews bool) (*proto.SchemaDefinition, error)

GetSchema returns the schema for database for tables listed in tables. If tables is empty, return the schema for all tables.

func (*Mysqld) IpAddr

func (mysqld *Mysqld) IpAddr() string

func (*Mysqld) IsReadOnly

func (mysqld *Mysqld) IsReadOnly() (bool, error)

func (*Mysqld) MasterStatus

func (mysqld *Mysqld) MasterStatus() (rp *proto.ReplicationPosition, err error)

mysql> show master status\G **************************** 1. row *************************** File: vt-000001c6-bin.000003 Position: 106 Binlog_Do_DB: Binlog_Ignore_DB: Group_ID:

func (*Mysqld) MultiRestore

func (mysqld *Mysqld) MultiRestore(destinationDbName string, keyRanges []key.KeyRange, sourceAddrs []*url.URL, snapshotConcurrency, fetchConcurrency, insertTableConcurrency, fetchRetryCount int, strategy string) (err error)

MultiRestore is the main entry point for multi restore.

  • If the strategy contains the string 'writeBinLogs' then we will also write to the binary logs.
  • If the strategy contains the command 'populateBlpCheckpoint' then we will populate the blp_checkpoint table with master positions to start from

func (*Mysqld) MycnfPath

func (mysqld *Mysqld) MycnfPath() string

func (*Mysqld) Port

func (mysqld *Mysqld) Port() int

func (*Mysqld) PreflightSchemaChange

func (mysqld *Mysqld) PreflightSchemaChange(dbName string, change string) (*proto.SchemaChangeResult, error)

func (*Mysqld) PromoteSlave

func (mysqld *Mysqld) PromoteSlave(setReadWrite bool, hookExtraEnv map[string]string) (replicationState *proto.ReplicationState, waitPosition *proto.ReplicationPosition, timePromoted int64, err error)

setReadWrite: set the new master in read-write mode.

replicationState: info slaves need to reparent themselves waitPosition: slaves can wait for this position when restarting replication timePromoted: this timestamp (unix nanoseconds) is inserted into _vt.replication_log to verify the replication config

func (*Mysqld) ReparentPosition

func (mysqld *Mysqld) ReparentPosition(slavePosition *proto.ReplicationPosition) (rs *proto.ReplicationState, waitPosition *proto.ReplicationPosition, reparentTime int64, err error)

Return a replication state that will reparent a slave to the correct master for a specified position.

func (*Mysqld) RestartSlave

func (mysqld *Mysqld) RestartSlave(replicationState *proto.ReplicationState, waitPosition *proto.ReplicationPosition, timeCheck int64) error

func (*Mysqld) RestoreFromSnapshot

func (mysqld *Mysqld) RestoreFromSnapshot(snapshotManifest *SnapshotManifest, fetchConcurrency, fetchRetryCount int, dontWaitForSlaveStart bool, hookExtraEnv map[string]string) error

This piece runs on the presumably empty machine acting as the target in the create replica action.

validate target (self) shutdown_mysql() create temp data directory /vt/target/vt_<keyspace> copy compressed data files via HTTP verify hash of compressed files uncompress into /vt/vt_<target-uid>/data/vt_<keyspace> start_mysql() clean up compressed files

func (*Mysqld) SetReadOnly

func (mysqld *Mysqld) SetReadOnly(on bool) error

func (*Mysqld) SlaveStatus

func (mysqld *Mysqld) SlaveStatus() (*proto.ReplicationPosition, error)

func (*Mysqld) SnapshotSourceEnd

func (mysqld *Mysqld) SnapshotSourceEnd(slaveStartRequired, readOnly, deleteSnapshot bool, hookExtraEnv map[string]string) error

func (*Mysqld) StartSlave

func (mysqld *Mysqld) StartSlave(hookExtraEnv map[string]string) error

func (*Mysqld) StopSlave

func (mysqld *Mysqld) StopSlave(hookExtraEnv map[string]string) error

func (*Mysqld) ValidateCloneTarget

func (mysqld *Mysqld) ValidateCloneTarget(hookExtraEnv map[string]string) error

func (*Mysqld) ValidateSnapshotPath

func (mysqld *Mysqld) ValidateSnapshotPath() error

Helper function to make sure we can write to the local snapshot area, before we actually do any action (can be used for both partial and full snapshots)

func (*Mysqld) WaitBlpPos

func (mysqld *Mysqld) WaitBlpPos(bp *proto.BlpPosition, waitTimeout time.Duration) error

func (*Mysqld) WaitForMinimumReplicationPosition

func (mysqld *Mysqld) WaitForMinimumReplicationPosition(groupId int64, waitTimeout time.Duration) (err error)

func (*Mysqld) WaitForSlave

func (mysqld *Mysqld) WaitForSlave(maxLag int) (err error)

func (*Mysqld) WaitForSlaveStart

func (mysqld *Mysqld) WaitForSlaveStart(slaveStartDeadline int) (err error)

func (*Mysqld) WaitMasterPos

func (mysqld *Mysqld) WaitMasterPos(rp *proto.ReplicationPosition, waitTimeout time.Duration) error

type SnapshotFile

type SnapshotFile struct {
	Path      string
	Size      int64
	Hash      string
	TableName string
}

SnapshotFile describes a file to serve. 'Path' is the path component of the URL. SnapshotManifest.Addr is the host+port component of the URL. If path ends in '.gz', it is compressed. Size and Hash are computed on the Path itself if TableName is set, this file belongs to that table

type SnapshotFiles

type SnapshotFiles []SnapshotFile

func (SnapshotFiles) Len

func (s SnapshotFiles) Len() int

sort.Interface we sort by descending file size

func (SnapshotFiles) Less

func (s SnapshotFiles) Less(i, j int) bool

func (SnapshotFiles) Swap

func (s SnapshotFiles) Swap(i, j int)

type SnapshotManifest

type SnapshotManifest struct {
	Addr string // this is the address of the tabletserver, not mysql

	DbName string
	Files  SnapshotFiles

	ReplicationState *proto.ReplicationState
	MasterState      *proto.ReplicationState
}

a SnapshotManifest describes multiple SnapshotFiles and where to get them from.

func ReadSnapshotManifest

func ReadSnapshotManifest(filename string) (*SnapshotManifest, error)

type SplitSnapshotManifest

type SplitSnapshotManifest struct {
	// Source describes the files and our tablet
	Source *SnapshotManifest

	// KeyRange describes the data present in this snapshot
	// When splitting 40-80 into 40-60 and 60-80, this would
	// have 40-60 for instance.
	KeyRange key.KeyRange

	// The schema for this server
	SchemaDefinition *proto.SchemaDefinition
}

func NewSplitSnapshotManifest

func NewSplitSnapshotManifest(myAddr, myMysqlAddr, masterAddr, dbName string, files []SnapshotFile, pos, myMasterPos *proto.ReplicationPosition, keyRange key.KeyRange, sd *proto.SchemaDefinition) (*SplitSnapshotManifest, error)

NewSplitSnapshotManifest creates a new SplitSnapshotManifest. myAddr and myMysqlAddr are the local server addresses. masterAddr is the address of the server to use as master. pos is the replication position to use on that master. myMasterPos is the local server master position

type VtReplParams

type VtReplParams struct {
	StartKey string
	EndKey   string
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL