global

package
v0.0.0-...-736094d Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 31, 2023 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Overview

* Copyright 2020-2021 the original author(https://github.com/wj596) * * <p> * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * </p>

* Copyright 2020-2021 the original author(https://github.com/wj596) * * <p> * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * </p>

Index

Constants

View Source
const (
	RedisGroupTypeSentinel = "sentinel"
	RedisGroupTypeCluster  = "cluster"

	// update or insert
	UpsertAction = "upsert"
)
View Source
const (
	RedisStructureString    = "String"
	RedisStructureHash      = "Hash"
	RedisStructureList      = "List"
	RedisStructureSet       = "Set"
	RedisStructureSortedSet = "SortedSet"

	ValEncoderJson     = "json"
	ValEncoderKVCommas = "kv-commas"
	ValEncoderVCommas  = "v-commas"
)

Variables

This section is empty.

Functions

func AddRuleIns

func AddRuleIns(ruleKey string, r *Rule)

func BootTime

func BootTime() time.Time

func CurrentNode

func CurrentNode() string

func Initialize

func Initialize(configPath string) error

func IsFollower

func IsFollower() bool

func IsLeader

func IsLeader() bool

func LeaderNode

func LeaderNode() string

func RuleInsExist

func RuleInsExist(ruleKey string) bool

func RuleInsTotal

func RuleInsTotal() int

func RuleKey

func RuleKey(schema string, table string) string

func RuleKeyList

func RuleKeyList() []string

func SetLeaderFlag

func SetLeaderFlag(flag bool)

func SetLeaderNode

func SetLeaderNode(leader string)

Types

type Cluster

type Cluster struct {
	Name             string `yaml:"name"`
	BindIp           string `yaml:"bind_ip"` //绑定IP
	ZkAddrs          string `yaml:"zk_addrs"`
	ZkAuthentication string `yaml:"zk_authentication"`
	EtcdAddrs        string `yaml:"etcd_addrs"`
	EtcdUser         string `yaml:"etcd_user"`
	EtcdPassword     string `yaml:"etcd_password"`
}

type Config

type Config struct {
	Target string `yaml:"target"` // 目标类型,支持redis、mongodb

	Addr     string `yaml:"addr"`
	User     string `yaml:"user"`
	Password string `yaml:"pass"`
	Charset  string `yaml:"charset"`

	SlaveID uint32 `yaml:"slave_id"`
	Flavor  string `yaml:"flavor"`
	DataDir string `yaml:"data_dir"`

	DumpExec       string `yaml:"mysqldump"`
	SkipMasterData bool   `yaml:"skip_master_data"`

	Maxprocs int   `yaml:"maxprocs"` // 最大协程数,默认CPU核心数*2
	BulkSize int64 `yaml:"bulk_size"`

	FlushBulkInterval int `yaml:"flush_bulk_interval"`

	SkipNoPkTable bool `yaml:"skip_no_pk_table"`

	RuleConfigs []*Rule `yaml:"rule"`

	LoggerConfig *logs.Config `yaml:"logger"` // 日志配置

	EnableExporter bool `yaml:"enable_exporter"` // 启用prometheus exporter,默认false
	ExporterPort   int  `yaml:"exporter_addr"`   // prometheus exporter端口

	EnableWebAdmin bool `yaml:"enable_web_admin"` // 启用Web监控,默认false
	WebAdminPort   int  `yaml:"web_admin_port"`   // web监控端口,默认8060

	Cluster *Cluster `yaml:"cluster"` // 集群配置
	// ------------------- REDIS -----------------
	RedisAddr       string `yaml:"redis_addrs"`       //redis地址
	RedisGroupType  string `yaml:"redis_group_type"`  //集群类型 sentinel或者cluster
	RedisMasterName string `yaml:"redis_master_name"` //Master节点名称
	RedisPass       string `yaml:"redis_pass"`        //redis密码
	RedisDatabase   int    `yaml:"redis_database"`    //redis数据库
	RedisTls        bool   `yaml:"redis_tls"`         //redis是否开启tls
	RedisCaCrt      string `yaml:"redis_ca_crt"`
	RedisClientCrt  string `yaml:"redis_client_crt"`
	RedisClientKey  string `yaml:"redis_client_key"`

	// ------------------- ROCKETMQ -----------------
	RocketmqNameServers  string `yaml:"rocketmq_name_servers"`  //rocketmq命名服务地址,多个用逗号分隔
	RocketmqGroupName    string `yaml:"rocketmq_group_name"`    //rocketmq group name,默认为空
	RocketmqInstanceName string `yaml:"rocketmq_instance_name"` //rocketmq instance name,默认为空
	RocketmqAccessKey    string `yaml:"rocketmq_access_key"`    //访问控制 accessKey,默认为空
	RocketmqSecretKey    string `yaml:"rocketmq_secret_key"`    //访问控制 secretKey,默认为空

	// ------------------- MONGODB -----------------
	MongodbAddr     string `yaml:"mongodb_addrs"`    //mongodb地址,多个用逗号分隔
	MongodbUsername string `yaml:"mongodb_username"` //mongodb用户名,默认为空
	MongodbPassword string `yaml:"mongodb_password"` //mongodb密码,默认为空

	// ------------------- RABBITMQ -----------------
	RabbitmqAddr string `yaml:"rabbitmq_addr"` //连接字符串,如: amqp://guest:guest@localhost:5672/

	// ------------------- KAFKA -----------------
	KafkaAddr         string `yaml:"kafka_addrs"`         //kafka连接地址,多个用逗号分隔
	KafkaSASLUser     string `yaml:"kafka_sasl_user"`     //kafka SASL_PLAINTEXT认证模式 用户名
	KafkaSASLPassword string `yaml:"kafka_sasl_password"` //kafka SASL_PLAINTEXT认证模式 密码

	// ------------------- ES -----------------
	ElsAddr     string `yaml:"es_addrs"`    //Elasticsearch连接地址,多个用逗号分隔
	ElsUser     string `yaml:"es_user"`     //Elasticsearch用户名
	ElsPassword string `yaml:"es_password"` //Elasticsearch密码
	ElsVersion  int    `yaml:"es_version"`  //Elasticsearch版本,支持6和7、默认为7
	// contains filtered or unexported fields
}

func Cfg

func Cfg() *Config

func (*Config) DestAddr

func (c *Config) DestAddr() string

func (*Config) DestStdName

func (c *Config) DestStdName() string

func (*Config) Destination

func (c *Config) Destination() string

func (*Config) IsCluster

func (c *Config) IsCluster() bool

func (*Config) IsEls

func (c *Config) IsEls() bool

func (*Config) IsEtcd

func (c *Config) IsEtcd() bool

func (*Config) IsExporterEnable

func (c *Config) IsExporterEnable() bool

func (*Config) IsKafka

func (c *Config) IsKafka() bool

func (*Config) IsMQ

func (c *Config) IsMQ() bool

func (*Config) IsMongodb

func (c *Config) IsMongodb() bool

func (*Config) IsRabbitmq

func (c *Config) IsRabbitmq() bool

func (*Config) IsRedis

func (c *Config) IsRedis() bool

func (*Config) IsReserveRawData

func (c *Config) IsReserveRawData() bool

func (*Config) IsRocketmq

func (c *Config) IsRocketmq() bool

func (*Config) IsScript

func (c *Config) IsScript() bool

func (*Config) IsZk

func (c *Config) IsZk() bool

func (*Config) ZkClusterDir

func (c *Config) ZkClusterDir() string

func (*Config) ZkElectedDir

func (c *Config) ZkElectedDir() string

func (*Config) ZkElectionDir

func (c *Config) ZkElectionDir() string

func (*Config) ZkNodesDir

func (c *Config) ZkNodesDir() string

func (*Config) ZkPositionDir

func (c *Config) ZkPositionDir() string

func (*Config) ZkRootDir

func (c *Config) ZkRootDir() string

type EsMapping

type EsMapping struct {
	Column   string `yaml:"column"`   // 数据库列名称
	Field    string `yaml:"field"`    // 映射后的ES字段名称
	Type     string `yaml:"type"`     // ES字段类型
	Analyzer string `yaml:"analyzer"` // ES分词器
	Format   string `yaml:"format"`   // 日期格式
}

type Rule

type Rule struct {
	Schema                   string `yaml:"schema"`
	Table                    string `yaml:"table"`
	OrderByColumn            string `yaml:"order_by_column"`
	ColumnLowerCase          bool   `yaml:"column_lower_case"`          // 列名称转为小写
	ColumnUpperCase          bool   `yaml:"column_upper_case"`          // 列名称转为大写
	ColumnUnderscoreToCamel  bool   `yaml:"column_underscore_to_camel"` // 列名称下划线转驼峰
	IncludeColumnConfig      string `yaml:"include_columns"`            // 包含的列
	ExcludeColumnConfig      string `yaml:"exclude_columns"`            // 排除掉的列
	ColumnMappingConfigs     string `yaml:"column_mappings"`            // 列名称映射
	DefaultColumnValueConfig string `yaml:"default_column_values"`      // 默认的字段和值
	// #值编码,支持json、kv-commas、v-commas;默认为json;json形如:{"id":123,"name":"wangjie"} 、kv-commas形如:id=123,name="wangjie"、v-commas形如:123,wangjie
	ValueEncoder      string `yaml:"value_encoder"`
	ValueFormatter    string `yaml:"value_formatter"`    //格式化定义key,{id}表示字段id的值、{name}表示字段name的值
	LuaScript         string `yaml:"lua_script"`         //lua 脚本
	LuaFilePath       string `yaml:"lua_file_path"`      //lua 文件地址
	DateFormatter     string `yaml:"date_formatter"`     //date类型格式化, 不填写默认2006-01-02
	DatetimeFormatter string `yaml:"datetime_formatter"` //datetime、timestamp类型格式化,不填写默认RFC3339(2006-01-02T15:04:05Z07:00)

	ReserveRawData bool `yaml:"reserve_raw_data"` // 保留update之前的数据,针对KAFKA、RABBITMQ、ROCKETMQ有效

	// ------------------- REDIS -----------------
	//对应redis的5种数据类型 String、Hash(字典) 、List(列表) 、Set(集合)、Sorted Set(有序集合)
	RedisStructure string `yaml:"redis_structure"`
	RedisKeyPrefix string `yaml:"redis_key_prefix"` //key的前缀
	RedisKeyColumn string `yaml:"redis_key_column"` //使用哪个列的值作为key,不填写默认使用主键
	// 格式化定义key,如{id}-{name};{id}表示字段id的值、{name}表示字段name的值
	RedisKeyFormatter string `yaml:"redis_key_formatter"`
	RedisKeyValue     string `yaml:"redis_key_value"` // key的值,固定值
	// hash的field前缀,仅redis_structure为hash时起作用
	RedisHashFieldPrefix string `yaml:"redis_hash_field_prefix"`
	// 使用哪个列的值作为hash的field,仅redis_structure为hash时起作用
	RedisHashFieldColumn string `yaml:"redis_hash_field_column"`
	// Sorted Set(有序集合)的Score
	RedisSortedSetScoreColumn      string `yaml:"redis_sorted_set_score_column"`
	RedisKeyColumnIndex            int
	RedisKeyColumnIndexs           []int
	RedisHashFieldColumnIndex      int
	RedisHashFieldColumnIndexs     []int
	RedisSortedSetScoreColumnIndex int
	RedisKeyTmpl                   *template.Template

	// mysql 一条数据转化成 redis hash 的方式:
	// 1(""(空||不设置)).默认 key:{column:json{}}
	// 一下两种是改造的
	// 2(todo).key:{column1:v1,column2:v2,...}
	RedisHashTransformationMode string `yaml:"redis_hash_transformation_mode"`

	// ------------------- ROCKETMQ -----------------
	RocketmqTopic string `yaml:"rocketmq_topic"` //rocketmq topic名称,可以为空,为空时使用表名称

	// ------------------- MONGODB -----------------
	MongodbDatabase   string `yaml:"mongodb_database"`   //mongodb database 不能为空
	MongodbCollection string `yaml:"mongodb_collection"` //mongodb collection,可以为空,默认使用表(Table)名称

	// ------------------- RABBITMQ -----------------
	RabbitmqQueue string `yaml:"rabbitmq_queue"` //queue名称,可以为空,默认使用表(Table)名称

	// ------------------- KAFKA -----------------
	KafkaTopic string `yaml:"kafka_topic"` //TOPIC名称,可以为空,默认使用表(Table)名称

	// ------------------- ES -----------------
	ElsIndex   string       `yaml:"es_index"`    //Elasticsearch Index,可以为空,默认使用表(Table)名称
	ElsType    string       `yaml:"es_type"`     //es6.x以后一个Index只能拥有一个Type,可以为空,默认使用_doc; es7.x版本此属性无效
	EsMappings []*EsMapping `yaml:"es_mappings"` //Elasticsearch mappings映射关系,可以为空,为空时根据数据类型自己推导

	// --------------- no config ----------------
	TableInfo             *schema.Table
	TableColumnSize       int
	IsCompositeKey        bool //是否联合主键
	DefaultColumnValueMap map[string]string
	PaddingMap            map[string]*model.Padding
	LuaProto              *lua.FunctionProto
	LuaFunction           *lua.LFunction
	ValueTmpl             *template.Template
}

func RuleDeepClone

func RuleDeepClone(res *Rule) (*Rule, error)

func RuleIns

func RuleIns(ruleKey string) (*Rule, bool)

func RuleInsList

func RuleInsList() []*Rule

func (*Rule) AfterUpdateTableInfo

func (s *Rule) AfterUpdateTableInfo() error

func (*Rule) CompileLuaScript

func (s *Rule) CompileLuaScript(dataDir string) error

编译Lua

func (*Rule) Initialize

func (s *Rule) Initialize() error

func (*Rule) LuaEnable

func (s *Rule) LuaEnable() bool

func (*Rule) TableColumn

func (s *Rule) TableColumn(field string) (*schema.TableColumn, int)

func (*Rule) WrapName

func (s *Rule) WrapName(fieldName string) string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL