metadata

package
v1.2.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 10, 2023 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const (
	AllocSchemaIDPrefix = "SchemaID"
	AllocTableIDPrefix  = "TableID"
)
View Source
const (
	MinShardID = 0
)

Variables

View Source
var (
	ErrCreateCluster        = coderr.NewCodeError(coderr.BadRequest, "create cluster")
	ErrUpdateCluster        = coderr.NewCodeError(coderr.Internal, "update cluster")
	ErrStartCluster         = coderr.NewCodeError(coderr.Internal, "start cluster")
	ErrClusterAlreadyExists = coderr.NewCodeError(coderr.ClusterAlreadyExists, "cluster already exists")
	ErrClusterNotFound      = coderr.NewCodeError(coderr.NotFound, "cluster not found")
	ErrClusterStateInvalid  = coderr.NewCodeError(coderr.Internal, "cluster state invalid")
	ErrSchemaNotFound       = coderr.NewCodeError(coderr.NotFound, "schema not found")
	ErrTableNotFound        = coderr.NewCodeError(coderr.NotFound, "table not found")
	ErrShardNotFound        = coderr.NewCodeError(coderr.NotFound, "shard not found")
	ErrNodeNotFound         = coderr.NewCodeError(coderr.NotFound, "NodeName not found")
	ErrTableAlreadyExists   = coderr.NewCodeError(coderr.Internal, "table already exists")
	ErrOpenTable            = coderr.NewCodeError(coderr.Internal, "open table")
	ErrParseTopologyType    = coderr.NewCodeError(coderr.Internal, "parse topology type")
)

Functions

func ConvertShardsInfoToPB

func ConvertShardsInfoToPB(shard ShardInfo) *metaservicepb.ShardInfo

func ConvertTableInfoToPB

func ConvertTableInfoToPB(table TableInfo) *metaservicepb.TableInfo

func ParseTopologyType

func ParseTopologyType(rawString string) (storage.TopologyType, error)

Types

type CloseTableRequest

type CloseTableRequest struct {
	SchemaName string
	TableName  string
	ShardID    storage.ShardID
	NodeName   string
}

type ClusterMetadata

type ClusterMetadata struct {
	// contains filtered or unexported fields
}

func NewClusterMetadata

func NewClusterMetadata(logger *zap.Logger, meta storage.Cluster, storage storage.Storage, kv clientv3.KV, rootPath string, idAllocatorStep uint) *ClusterMetadata

func (*ClusterMetadata) AddTableTopology

func (c *ClusterMetadata) AddTableTopology(ctx context.Context, shardID storage.ShardID, table storage.Table) (CreateTableResult, error)

func (*ClusterMetadata) AllocShardID

func (c *ClusterMetadata) AllocShardID(ctx context.Context) (uint32, error)

func (*ClusterMetadata) CreateShardViews

func (c *ClusterMetadata) CreateShardViews(ctx context.Context, views []CreateShardView) error

func (*ClusterMetadata) CreateTable

func (c *ClusterMetadata) CreateTable(ctx context.Context, request CreateTableRequest) (CreateTableResult, error)

func (*ClusterMetadata) CreateTableMetadata

func (*ClusterMetadata) DropShardNode

func (c *ClusterMetadata) DropShardNode(ctx context.Context, shardNodes []storage.ShardNode) error

func (*ClusterMetadata) DropTable

func (c *ClusterMetadata) DropTable(ctx context.Context, schemaName, tableName string) (DropTableResult, error)

DropTable will drop table metadata and all mapping of this table. If the table to be dropped has been opened multiple times, all its mapping will be dropped.

func (*ClusterMetadata) DropTableMetadata

func (c *ClusterMetadata) DropTableMetadata(ctx context.Context, schemaName, tableName string) (DropTableMetadataResult, error)

func (*ClusterMetadata) GetClusterID

func (c *ClusterMetadata) GetClusterID() storage.ClusterID

func (*ClusterMetadata) GetClusterMinNodeCount

func (c *ClusterMetadata) GetClusterMinNodeCount() uint32

func (*ClusterMetadata) GetClusterSnapshot

func (c *ClusterMetadata) GetClusterSnapshot() Snapshot

func (*ClusterMetadata) GetClusterState

func (c *ClusterMetadata) GetClusterState() storage.ClusterState

func (*ClusterMetadata) GetClusterView

func (c *ClusterMetadata) GetClusterView() storage.ClusterView

func (*ClusterMetadata) GetClusterViewVersion

func (c *ClusterMetadata) GetClusterViewVersion() uint64

func (*ClusterMetadata) GetCreateTime added in v1.2.1

func (c *ClusterMetadata) GetCreateTime() uint64

func (*ClusterMetadata) GetEnableSchedule added in v1.2.1

func (c *ClusterMetadata) GetEnableSchedule() bool

func (*ClusterMetadata) GetNodeShards

func (c *ClusterMetadata) GetNodeShards(_ context.Context) (GetNodeShardsResult, error)

func (*ClusterMetadata) GetOrCreateSchema

func (c *ClusterMetadata) GetOrCreateSchema(ctx context.Context, schemaName string) (storage.Schema, bool, error)

GetOrCreateSchema the second output parameter bool: returns true if the schema was newly created.

func (*ClusterMetadata) GetProcedureExecutingBatchSize added in v1.2.4

func (c *ClusterMetadata) GetProcedureExecutingBatchSize() uint32

func (*ClusterMetadata) GetRegisteredNodeByName

func (c *ClusterMetadata) GetRegisteredNodeByName(nodeName string) (RegisteredNode, bool)

func (*ClusterMetadata) GetRegisteredNodes

func (c *ClusterMetadata) GetRegisteredNodes() []RegisteredNode

func (*ClusterMetadata) GetShardNodeByTableIDs

func (c *ClusterMetadata) GetShardNodeByTableIDs(tableIDs []storage.TableID) (GetShardNodesByTableIDsResult, error)

func (*ClusterMetadata) GetShardNodes added in v1.2.7

func (c *ClusterMetadata) GetShardNodes() GetShardNodesResult

func (*ClusterMetadata) GetShardNodesByShardID

func (c *ClusterMetadata) GetShardNodesByShardID(id storage.ShardID) ([]storage.ShardNode, error)

func (*ClusterMetadata) GetShardTables

func (c *ClusterMetadata) GetShardTables(shardIDs []storage.ShardID) map[storage.ShardID]ShardTables

func (*ClusterMetadata) GetShards added in v1.2.7

func (c *ClusterMetadata) GetShards() []storage.ShardID

func (*ClusterMetadata) GetStorageMetadata added in v1.2.1

func (c *ClusterMetadata) GetStorageMetadata() storage.Cluster

func (*ClusterMetadata) GetTable

func (c *ClusterMetadata) GetTable(schemaName, tableName string) (storage.Table, bool, error)

GetTable the second output parameter bool: returns true if the table exists.

func (*ClusterMetadata) GetTables added in v1.2.7

func (c *ClusterMetadata) GetTables(schemaName string, tableNames []string) ([]storage.Table, error)

func (*ClusterMetadata) GetTablesByIDs added in v1.2.7

func (c *ClusterMetadata) GetTablesByIDs(tableIDs []storage.TableID) []storage.Table

func (*ClusterMetadata) GetTopologyType added in v1.2.1

func (c *ClusterMetadata) GetTopologyType() storage.TopologyType

func (*ClusterMetadata) GetTotalShardNum

func (c *ClusterMetadata) GetTotalShardNum() uint32

func (*ClusterMetadata) Init

func (c *ClusterMetadata) Init(ctx context.Context) error

Initialize the cluster view and shard view of the cluster. It will be used when we create the cluster.

func (*ClusterMetadata) Load

func (c *ClusterMetadata) Load(ctx context.Context) error

Load cluster NodeName from storage into memory.

func (*ClusterMetadata) LoadMetadata added in v1.2.1

func (c *ClusterMetadata) LoadMetadata(ctx context.Context) error

LoadMetadata load cluster metadata from storage.

func (*ClusterMetadata) MigrateTable

func (c *ClusterMetadata) MigrateTable(ctx context.Context, request MigrateTableRequest) error

MigrateTable used to migrate tables from old shard to new shard. The mapping relationship between table and shard will be modified.

func (*ClusterMetadata) Name

func (c *ClusterMetadata) Name() string

func (*ClusterMetadata) RegisterNode

func (c *ClusterMetadata) RegisterNode(ctx context.Context, registeredNode RegisteredNode) error

func (*ClusterMetadata) RouteTables

func (c *ClusterMetadata) RouteTables(_ context.Context, schemaName string, tableNames []string) (RouteTablesResult, error)

func (*ClusterMetadata) UpdateClusterView

func (c *ClusterMetadata) UpdateClusterView(ctx context.Context, state storage.ClusterState, shardNodes []storage.ShardNode) error

func (*ClusterMetadata) UpdateClusterViewByNode

func (c *ClusterMetadata) UpdateClusterViewByNode(ctx context.Context, shardNodes map[string][]storage.ShardNode) error

type CreateClusterOpts

type CreateClusterOpts struct {
	NodeCount                   uint32
	ReplicationFactor           uint32
	ShardTotal                  uint32
	EnableSchedule              bool
	TopologyType                storage.TopologyType
	ProcedureExecutingBatchSize uint32
}

type CreateShardView

type CreateShardView struct {
	ShardID storage.ShardID
	Tables  []storage.TableID
}

type CreateTableMetadataRequest

type CreateTableMetadataRequest struct {
	SchemaName    string
	TableName     string
	PartitionInfo storage.PartitionInfo
}

type CreateTableMetadataResult

type CreateTableMetadataResult struct {
	Table storage.Table
}

type CreateTableRequest

type CreateTableRequest struct {
	ShardID       storage.ShardID
	SchemaName    string
	TableName     string
	PartitionInfo storage.PartitionInfo
}

type CreateTableResult

type CreateTableResult struct {
	Table              storage.Table
	ShardVersionUpdate ShardVersionUpdate
}

type DropTableMetadataResult

type DropTableMetadataResult struct {
	Table storage.Table
}

type DropTableResult

type DropTableResult struct {
	ShardVersionUpdate []ShardVersionUpdate
}

type GetNodeShardsResult

type GetNodeShardsResult struct {
	ClusterTopologyVersion uint64
	NodeShards             []ShardNodeWithVersion
}

type GetShardNodesByTableIDsResult

type GetShardNodesByTableIDsResult struct {
	ShardNodes map[storage.TableID][]storage.ShardNode
	Version    map[storage.ShardID]uint64
}

type GetShardNodesResult

type GetShardNodesResult struct {
	ShardNodes []storage.ShardNode
	Versions   map[storage.ShardID]uint64
}

type GetShardTablesByNodeResult

type GetShardTablesByNodeResult struct {
	ShardTableIDs map[storage.ShardID]ShardTableIDs
}

type MigrateTableRequest

type MigrateTableRequest struct {
	SchemaName string
	TableNames []string
	OldShardID storage.ShardID
	NewShardID storage.ShardID
}

type OpenTableRequest

type OpenTableRequest struct {
	SchemaName string
	TableName  string
	ShardID    storage.ShardID
	NodeName   string
}

type RegisteredNode

type RegisteredNode struct {
	Node       storage.Node
	ShardInfos []ShardInfo
}

func NewRegisteredNode

func NewRegisteredNode(meta storage.Node, shardInfos []ShardInfo) RegisteredNode

func (RegisteredNode) IsExpired

func (n RegisteredNode) IsExpired(now time.Time) bool

type RouteEntry

type RouteEntry struct {
	Table      TableInfo
	NodeShards []ShardNodeWithVersion
}

type RouteTablesResult

type RouteTablesResult struct {
	ClusterViewVersion uint64
	RouteEntries       map[string]RouteEntry
}

type ShardInfo

type ShardInfo struct {
	ID   storage.ShardID
	Role storage.ShardRole
	// ShardViewVersion
	Version uint64
	// The open state of the shard, which is used to determine whether the shard needs to be opened again.
	Status storage.ShardStatus
}

func ConvertShardsInfoPB

func ConvertShardsInfoPB(shard *metaservicepb.ShardInfo) ShardInfo

type ShardNodeWithVersion

type ShardNodeWithVersion struct {
	ShardInfo ShardInfo
	ShardNode storage.ShardNode
}

type ShardTableIDs

type ShardTableIDs struct {
	TableIDs []storage.TableID
	Version  uint64
}

type ShardTables

type ShardTables struct {
	Shard  ShardInfo
	Tables []TableInfo
}

type ShardVersionUpdate

type ShardVersionUpdate struct {
	ShardID     storage.ShardID
	CurrVersion uint64
	PrevVersion uint64
}

type Snapshot

type Snapshot struct {
	Topology        Topology
	RegisteredNodes []RegisteredNode
}

type TableInfo

type TableInfo struct {
	ID            storage.TableID
	Name          string
	SchemaID      storage.SchemaID
	SchemaName    string
	PartitionInfo storage.PartitionInfo
	CreatedAt     uint64
}

type TableManager

type TableManager interface {
	// Load load table meta data from storage.
	Load(ctx context.Context) error
	// GetTable get table with schemaName and tableName, the second output parameter bool: returns true if the table exists.
	GetTable(schemaName string, tableName string) (storage.Table, bool, error)
	// GetTables get tables with schemaName and tableNames.
	GetTables(schemaName string, tableNames []string) ([]storage.Table, error)
	// GetTablesByIDs get tables with tableIDs.
	GetTablesByIDs(tableIDs []storage.TableID) []storage.Table
	// CreateTable create table with schemaName and tableName.
	CreateTable(ctx context.Context, schemaName string, tableName string, partitionInfo storage.PartitionInfo) (storage.Table, error)
	// DropTable drop table with schemaName and tableName.
	DropTable(ctx context.Context, schemaName string, tableName string) error
	// GetSchema get schema with schemaName.
	GetSchema(schemaName string) (storage.Schema, bool)
	// GetSchemaByID get schema with schemaName.
	GetSchemaByID(schemaID storage.SchemaID) (storage.Schema, bool)
	// GetSchemas get all schemas in cluster.
	GetSchemas() []storage.Schema
	// GetOrCreateSchema get or create schema with schemaName.
	GetOrCreateSchema(ctx context.Context, schemaName string) (storage.Schema, bool, error)
}

TableManager manages table metadata by schema.

func NewTableManagerImpl

func NewTableManagerImpl(logger *zap.Logger, storage storage.Storage, clusterID storage.ClusterID, schemaIDAlloc id.Allocator, tableIDAlloc id.Allocator) TableManager

type TableManagerImpl

type TableManagerImpl struct {
	// contains filtered or unexported fields
}

func (*TableManagerImpl) CreateTable

func (m *TableManagerImpl) CreateTable(ctx context.Context, schemaName string, tableName string, partitionInfo storage.PartitionInfo) (storage.Table, error)

func (*TableManagerImpl) DropTable

func (m *TableManagerImpl) DropTable(ctx context.Context, schemaName string, tableName string) error

func (*TableManagerImpl) GetOrCreateSchema

func (m *TableManagerImpl) GetOrCreateSchema(ctx context.Context, schemaName string) (storage.Schema, bool, error)

func (*TableManagerImpl) GetSchema added in v1.2.7

func (m *TableManagerImpl) GetSchema(schemaName string) (storage.Schema, bool)

func (*TableManagerImpl) GetSchemaByID added in v1.2.7

func (m *TableManagerImpl) GetSchemaByID(schemaID storage.SchemaID) (storage.Schema, bool)

func (*TableManagerImpl) GetSchemas

func (m *TableManagerImpl) GetSchemas() []storage.Schema

func (*TableManagerImpl) GetTable

func (m *TableManagerImpl) GetTable(schemaName, tableName string) (storage.Table, bool, error)

func (*TableManagerImpl) GetTables added in v1.2.7

func (m *TableManagerImpl) GetTables(schemaName string, tableNames []string) ([]storage.Table, error)

func (*TableManagerImpl) GetTablesByIDs

func (m *TableManagerImpl) GetTablesByIDs(tableIDs []storage.TableID) []storage.Table

func (*TableManagerImpl) Load

func (m *TableManagerImpl) Load(ctx context.Context) error

type Tables

type Tables struct {
	// contains filtered or unexported fields
}

type Topology

type Topology struct {
	ShardViewsMapping map[storage.ShardID]storage.ShardView
	ClusterView       storage.ClusterView
}

func (*Topology) IsPrepareFinished

func (t *Topology) IsPrepareFinished() bool

func (*Topology) IsStable

func (t *Topology) IsStable() bool

type TopologyManager

type TopologyManager interface {
	// Load load cluster topology from storage.
	Load(ctx context.Context) error
	// GetVersion get cluster view version.
	GetVersion() uint64
	// GetClusterState get cluster view state.
	GetClusterState() storage.ClusterState
	// GetTableIDs get shardNode and tablesIDs with shardID and nodeName.
	GetTableIDs(shardIDs []storage.ShardID) map[storage.ShardID]ShardTableIDs
	// AddTable add table to cluster topology.
	AddTable(ctx context.Context, shardID storage.ShardID, tables []storage.Table) (ShardVersionUpdate, error)
	// RemoveTable remove table on target shards from cluster topology.
	RemoveTable(ctx context.Context, shardID storage.ShardID, tableIDs []storage.TableID) (ShardVersionUpdate, error)
	// EvictTable evict table from cluster topology.
	EvictTable(ctx context.Context, tableID storage.TableID) ([]ShardVersionUpdate, error)
	// GetShards get all shards in cluster topology.
	GetShards() []storage.ShardID
	// GetShardNodesByID get shardNodes with shardID.
	GetShardNodesByID(shardID storage.ShardID) ([]storage.ShardNode, error)
	// GetShardNodesByTableIDs get shardNodes with tableIDs.
	GetShardNodesByTableIDs(tableID []storage.TableID) (GetShardNodesByTableIDsResult, error)
	// GetShardNodes get all shardNodes in cluster topology.
	GetShardNodes() GetShardNodesResult
	// DropShardNodes drop target shardNodes in cluster topology.
	DropShardNodes(ctx context.Context, shardNodes []storage.ShardNode) error
	// InitClusterView init cluster view when create new cluster.
	InitClusterView(ctx context.Context) error
	// UpdateClusterView update cluster view with shardNodes.
	UpdateClusterView(ctx context.Context, state storage.ClusterState, shardNodes []storage.ShardNode) error
	// UpdateClusterViewByNode update cluster view with target shardNodes, it will only update shardNodes corresponding the node name.
	UpdateClusterViewByNode(ctx context.Context, shardNodes map[string][]storage.ShardNode) error
	// GetClusterView return current cluster view.
	GetClusterView() storage.ClusterView
	// CreateShardViews create shardViews.
	CreateShardViews(ctx context.Context, shardViews []CreateShardView) error
	// GetTopology get current topology snapshot.
	GetTopology() Topology
}

TopologyManager manages the cluster topology, including the mapping relationship between shards, nodes, and tables.

func NewTopologyManagerImpl

func NewTopologyManagerImpl(logger *zap.Logger, storage storage.Storage, clusterID storage.ClusterID, shardIDAlloc id.Allocator) TopologyManager

type TopologyManagerImpl

type TopologyManagerImpl struct {
	// contains filtered or unexported fields
}

func (*TopologyManagerImpl) AddTable

func (m *TopologyManagerImpl) AddTable(ctx context.Context, shardID storage.ShardID, tables []storage.Table) (ShardVersionUpdate, error)

func (*TopologyManagerImpl) CreateShardViews

func (m *TopologyManagerImpl) CreateShardViews(ctx context.Context, createShardViews []CreateShardView) error

func (*TopologyManagerImpl) DropShardNodes

func (m *TopologyManagerImpl) DropShardNodes(ctx context.Context, shardNodes []storage.ShardNode) error

func (*TopologyManagerImpl) EvictTable

func (m *TopologyManagerImpl) EvictTable(ctx context.Context, tableID storage.TableID) ([]ShardVersionUpdate, error)

func (*TopologyManagerImpl) GetClusterState

func (m *TopologyManagerImpl) GetClusterState() storage.ClusterState

func (*TopologyManagerImpl) GetClusterView

func (m *TopologyManagerImpl) GetClusterView() storage.ClusterView

func (*TopologyManagerImpl) GetShardNodes

func (m *TopologyManagerImpl) GetShardNodes() GetShardNodesResult

func (*TopologyManagerImpl) GetShardNodesByID

func (m *TopologyManagerImpl) GetShardNodesByID(shardID storage.ShardID) ([]storage.ShardNode, error)

func (*TopologyManagerImpl) GetShardNodesByTableIDs

func (m *TopologyManagerImpl) GetShardNodesByTableIDs(tableIDs []storage.TableID) (GetShardNodesByTableIDsResult, error)

func (*TopologyManagerImpl) GetShards added in v1.2.7

func (m *TopologyManagerImpl) GetShards() []storage.ShardID

func (*TopologyManagerImpl) GetTableIDs

func (m *TopologyManagerImpl) GetTableIDs(shardIDs []storage.ShardID) map[storage.ShardID]ShardTableIDs

func (*TopologyManagerImpl) GetTopology

func (m *TopologyManagerImpl) GetTopology() Topology

func (*TopologyManagerImpl) GetVersion

func (m *TopologyManagerImpl) GetVersion() uint64

func (*TopologyManagerImpl) InitClusterView

func (m *TopologyManagerImpl) InitClusterView(ctx context.Context) error

func (*TopologyManagerImpl) Load

func (*TopologyManagerImpl) RemoveTable

func (m *TopologyManagerImpl) RemoveTable(ctx context.Context, shardID storage.ShardID, tableIDs []storage.TableID) (ShardVersionUpdate, error)

func (*TopologyManagerImpl) UpdateClusterView

func (m *TopologyManagerImpl) UpdateClusterView(ctx context.Context, state storage.ClusterState, shardNodes []storage.ShardNode) error

func (*TopologyManagerImpl) UpdateClusterViewByNode

func (m *TopologyManagerImpl) UpdateClusterViewByNode(ctx context.Context, shardNodes map[string][]storage.ShardNode) error

type UpdateClusterOpts added in v1.2.1

type UpdateClusterOpts struct {
	EnableSchedule              bool
	TopologyType                storage.TopologyType
	ProcedureExecutingBatchSize uint32
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL