datamove

type Column struct {
	*dialect.ColumnModel
	Value       string `json:"value,omitempty"`
	SubProperty bool   `json:"subProperty,omitempty"`
}

type Data struct {
	Total        int64           `json:"total,omitempty"`
	DataType     DataType        `json:"dataType,omitempty"`
	ColsList     [][]interface{} `json:"colsList,omitempty"`
	SqlList      []string        `json:"sqlList,omitempty"`
	SqlAndParams []*SqlAndParam  `json:"sqlAndParams,omitempty"`
	// contains filtered or unexported fields
}

type DataSource interface {
	Stop(progress *Progress)
	ReadStart(progress *Progress) (err error)
	Read(progress *Progress, dataChan chan *Data) (err error)
	ReadEnd(progress *Progress) (err error)
	WriteStart(progress *Progress) (err error)
	Write(progress *Progress, data *Data) (err error)
	WriteEnd(progress *Progress) (err error)
}

type DataSourceBase struct {
	ColumnList []*Column

	ScriptContext map[string]interface{}
	FillColumn    bool `json:"fillColumn,omitempty"`
	// contains filtered or unexported fields
}

type DataSourceConfig struct {
	*dialect.ParamModel
	Type             string `json:"type,omitempty"`
	SqlFileMergeType string `json:"sqlFileMergeType,omitempty"` // SQL 的文件合并类型 如：one：一个文件， owner：每个库一个文件，table：每个表一个文件

	DataSourceSqlParam

	DataSourceTxtParam

	DataSourceScriptParam

	DataSourceRedisParam

	DataSourceExcelParam

	DataSourceKafkaParam

	DataSourceEsParam

	BySql bool `json:"bySql,omitempty"` // 根据 SQL 语句导出

	DataSourceDbParam

	DataSourceDataParam

	TxtFileType string `json:"txtFileType,omitempty"` //

	// 数据库 配置
	DbConfig *db.Config `json:"-"`

	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`

	EsConfig *elasticsearch.Config `json:"-"`

	RedisConfig *redis.Config `json:"-"`

	KafkaConfig *kafka.Config `json:"-"`

	FillColumn bool `json:"fillColumn,omitempty"` // 自动填充列

	ColumnList []*Column `json:"columnList,omitempty"`

	AllOwner       bool       `json:"allOwner,omitempty"`
	Owners         []*DbOwner `json:"owners,omitempty"`
	SkipOwnerNames []string   `json:"skipOwnerNames,omitempty"`

	FilePath    string `json:"filePath,omitempty"`
	ShouldOwner bool   `json:"shouldOwner,omitempty"` // 需要 建库
	ShouldTable bool   `json:"shouldTable,omitempty"` // 需要 建表

	FileNameSplice string `json:"fileNameSplice,omitempty"` // 文件名拼接字符 如：/ :库作为目录 表作为名称 默认
	FileName       string `json:"fileName,omitempty"`
	RowNumber      int64  `json:"rowNumber,omitempty"`
	// contains filtered or unexported fields
}

type DataSourceData struct {
	*DataSourceBase
	*DataSourceDataParam
}

type DataSourceDataParam struct {
	DataList []map[string]interface{} `json:"dataList,omitempty"`
}

type DataSourceDb struct {
	*dialect.ParamModel
	*DataSourceBase
	*DataSourceDbParam
	*DataSourceSqlParam

	Service db.IService
}

type DataSourceDbParam struct {
	SelectSql        string `json:"selectSql,omitempty"`
	ShouldSelectPage bool   `json:"shouldSelectPage,omitempty"`
}

type DataSourceEs struct {
	*DataSourceBase
	*DataSourceEsParam

	Service elasticsearch.IService
}

type DataSourceEsParam struct {
	IndexName     string `json:"indexName,omitempty"`
	IndexIdName   string `json:"indexIdName,omitempty"`
	IndexIdScript string `json:"indexIdScript,omitempty"`
}

type DataSourceExcel struct {
	*DataSourceBase
	*DataSourceExcelParam
	FilePath string `json:"filePath,omitempty"`
	// contains filtered or unexported fields
}

type DataSourceExcelParam struct {
	SheetName string `json:"sheetName,omitempty"`
}

type DataSourceFile struct {
	FilePath string
	// contains filtered or unexported fields
}

type DataSourceKafka struct {
	*DataSourceBase
	*DataSourceKafkaParam

	Service kafka.IService
	// contains filtered or unexported fields
}

type DataSourceKafkaParam struct {
	TopicName        string `json:"topicName,omitempty"`
	TopicGroupName   string `json:"topicGroupName,omitempty"`
	TopicKey         string `json:"topicKey,omitempty"`
	TopicValue       string `json:"topicValue,omitempty"`
	TopicValueByData bool   `json:"topicValueByData,omitempty"`
	PullWait         int64  `json:"pullWait,omitempty"`
}

type DataSourceRedis struct {
	*DataSourceBase
	*DataSourceRedisParam
	Service redis.IService
	// contains filtered or unexported fields
}

type DataSourceRedisParam struct {
	RedisDatabase      int    `json:"redisDatabase,omitempty"`
	RedisKeyPattern    string `json:"redisKeyPattern,omitempty"`
	RedisKeyName       string `json:"redisKeyName,omitempty"`
	RedisKeyScript     string `json:"redisKeyScript,omitempty"`
	RedisFieldName     string `json:"redisFieldName,omitempty"`
	RedisFieldScript   string `json:"redisFieldScript,omitempty"`
	RedisValueName     string `json:"redisValueName,omitempty"`
	RedisValueTypeName string `json:"redisValueTypeName,omitempty"`
	RedisValueByData   bool   `json:"redisValueByData,omitempty"`
	RedisValueType     string `json:"redisValueType,omitempty"`
}

type DataSourceScript struct {
	*DataSourceBase
	*DataSourceScriptParam
}

type DataSourceScriptParam struct {
	Total int64 `json:"total,omitempty"`
}

type DataSourceSql struct {
	*DataSourceBase
	*DataSourceFile
	*DataSourceSqlParam
	*dialect.ParamModel
	// contains filtered or unexported fields
}

type DataSourceSqlParam struct {
	DialectType string `json:"dialectType,omitempty"`
	OwnerName   string `json:"ownerName,omitempty"`
	TableName   string `json:"tableName,omitempty"`
}

type DataSourceTxt struct {
	*DataSourceBase
	*DataSourceFile
	*DataSourceTxtParam
	// contains filtered or unexported fields
}

type DataSourceTxtParam struct {
	ColSeparator    string `json:"colSeparator,omitempty"`    // 列 分隔符 默认 `,`
	ReplaceCol      string `json:"replaceCol,omitempty"`      //
	ReplaceLine     string `json:"replaceLine,omitempty"`     //
	ShouldTrimSpace bool   `json:"shouldTrimSpace,omitempty"` // 是否需要去除空白字符
}

type DataType int8

type DbColumn struct {
	From  *dialect.ColumnModel `json:"from,omitempty"`
	To    *dialect.ColumnModel `json:"to,omitempty"`
	Value string               `json:"value,omitempty"`
}

type DbOwner struct {
	From           *dialect.OwnerModel `json:"from,omitempty"`
	To             *dialect.OwnerModel `json:"to,omitempty"`
	SkipTableNames []string            `json:"skipTableNames,omitempty"`
	AllTable       bool                `json:"allTable,omitempty"`
	Tables         []*DbTable          `json:"tables,omitempty"`
	// contains filtered or unexported fields
}

type DbTable struct {
	From            *dialect.TableModel `json:"from,omitempty"`
	To              *dialect.TableModel `json:"to,omitempty"`
	Columns         []*DbColumn         `json:"columns,omitempty"`
	SkipColumnNames []string            `json:"skipColumnNames,omitempty"`
	AllColumn       bool                `json:"allColumn,omitempty"`
	// contains filtered or unexported fields
}

type Executor struct {
	*Progress
}

type Options struct {
	Key  string            `json:"key,omitempty"`  // 任务的 key
	Dir  string            `json:"dir,omitempty"`  // 任务过程中 生成文件的目录
	From *DataSourceConfig `json:"from,omitempty"` // 源 数据配置
	To   *DataSourceConfig `json:"to,omitempty"`   // 目标 数据配置

	ErrorContinue bool  `json:"errorContinue,omitempty"`
	BatchNumber   int64 `json:"batchNumber,omitempty"`
}

type Progress struct {
	*Options

	DataTotal int64 `json:"dataTotal"`

	OwnerTotal int64          `json:"ownerTotal"`
	OwnerCount *ProgressCount `json:"ownerCount"`

	TableTotal int64          `json:"tableTotal"`
	TableCount *ProgressCount `json:"tableCount"`

	ReadCount  *ProgressCount `json:"readCount"`
	WriteCount *ProgressCount `json:"writeCount"`

	IndexTotal int64          `json:"indexTotal"`
	IndexCount *ProgressCount `json:"indexCount"`

	TopicTotal int64          `json:"topicTotal"`
	TopicCount *ProgressCount `json:"topicCount"`
	// contains filtered or unexported fields
}

type ProgressCount struct {
	Total   int64    `json:"total"`
	Error   int64    `json:"error"`
	Success int64    `json:"success"`
	Errors  []string `json:"errors"`
}

type SqlAndParam struct {
	Sql    string        `json:"sql,omitempty"`
	Params []interface{} `json:"params,omitempty"`
}