master

package
v2.0.0+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 10, 2020 License: Apache-2.0 Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// the maximum number of tasks that can be handled each time
	MaxTaskNum = 30

	TaskWorkerInterval = time.Second * time.Duration(2)
)

const

View Source
const (
	LRUCacheSize    = 3 << 30
	WriteBufferSize = 4 * util.MB
)
View Source
const (
	EmptyCrcValue   uint32 = 4045511210
	DefaultZoneName        = "default"
)
View Source
const (
	OpSyncAddToken    uint32 = 0x20
	OpSyncDelToken    uint32 = 0x21
	OpSyncUpdateToken uint32 = 0x22
)
View Source
const (
	StatPeriod                 = time.Minute * time.Duration(1)
	MetricDataNodesUsedGB      = "dataNodes_used_GB"
	MetricDataNodesTotalGB     = "dataNodes_total_GB"
	MetricDataNodesIncreasedGB = "dataNodes_increased_GB"
	MetricMetaNodesUsedGB      = "metaNodes_used_GB"
	MetricMetaNodesTotalGB     = "metaNodes_total_GB"
	MetricMetaNodesIncreasedGB = "metaNodes_increased_GB"
	MetricDataNodesCount       = "dataNodes_count"
	MetricMetaNodesCount       = "metaNodes_count"
	MetricVolCount             = "vol_count"
	MetricVolTotalGB           = "vol_total_GB"
	MetricVolUsedGB            = "vol_used_GB"
	MetricVolUsageGB           = "vol_usage_ratio"
	MetricDiskError            = "disk_error"
	MetricDataNodesInactive    = "dataNodes_inactive"
	MetricMetaNodesInactive    = "metaNodes_inactive"
)

metrics

View Source
const (
	ClusterName       = "clusterName"
	ID                = "id"
	IP                = "ip"
	Port              = "port"
	LogLevel          = "logLevel"
	WalDir            = "walDir"
	StoreDir          = "storeDir"
	GroupID           = 1
	ModuleName        = "master"
	CfgRetainLogs     = "retainLogs"
	DefaultRetainLogs = 20000

	SecretKey = "masterServiceKey"
)

configuration keys

View Source
const (
	RootUserID          = "root"
	DefaultRootPasswd   = "ChubaoFSRoot"
	DefaultUserPassword = "ChubaoFSUser"
)
View Source
const (
	NumberOfDataPartitionsToLoad = "NumberOfDataPartitionsToLoad"
)

config key

View Source
const (
	TokenPrefix = keySeparator + tokenAcronym + keySeparator
)

Variables

View Source
var AddrDatabase = make(map[uint64]string)

AddrDatabase is a map that stores the address of a given host (e.g., the leader)

Functions

func Warn

func Warn(clusterID, msg string)

Warn provides warnings when exits

func WarnBySpecialKey

func WarnBySpecialKey(key, msg string)

WarnBySpecialKey provides warnings when exits

Types

type AdminTaskManager

type AdminTaskManager struct {
	TaskMap map[string]*proto.AdminTask
	sync.RWMutex
	// contains filtered or unexported fields
}

AdminTaskManager sends administration commands to the metaNode or dataNode.

func (*AdminTaskManager) AddTask

func (sender *AdminTaskManager) AddTask(t *proto.AdminTask)

AddTask adds a new task to the task map.

func (*AdminTaskManager) DelTask

func (sender *AdminTaskManager) DelTask(t *proto.AdminTask)

DelTask deletes the to-be-deleted tasks.

type Cluster

type Cluster struct {
	Name string

	BadDataPartitionIds *sync.Map
	BadMetaPartitionIds *sync.Map
	DisableAutoAllocate bool

	MasterSecretKey []byte
	// contains filtered or unexported fields
}

Cluster stores all the cluster-level information.

type DataNode

type DataNode struct {
	Total          uint64 `json:"TotalWeight"`
	Used           uint64 `json:"UsedWeight"`
	AvailableSpace uint64
	ID             uint64
	ZoneName       string `json:"Zone"`
	Addr           string
	ReportTime     time.Time

	sync.RWMutex
	UsageRatio                float64 // used / total space
	SelectedTimes             uint64  // number times that this datanode has been selected as the location for a data partition.
	Carry                     float64 // carry is a factor used in cacluate the node's weight
	TaskManager               *AdminTaskManager
	DataPartitionReports      []*proto.PartitionReport
	DataPartitionCount        uint32
	NodeSetID                 uint64
	PersistenceDataPartitions []uint64
	BadDisks                  []string
	// contains filtered or unexported fields
}

DataNode stores all the information about a data node

func (*DataNode) GetAddr

func (dataNode *DataNode) GetAddr() string

func (*DataNode) GetID

func (dataNode *DataNode) GetID() uint64

func (*DataNode) SelectNodeForWrite

func (dataNode *DataNode) SelectNodeForWrite()

SelectNodeForWrite implements "SelectNodeForWrite" in the Node interface

func (*DataNode) SetCarry

func (dataNode *DataNode) SetCarry(carry float64)

SetCarry implements "SetCarry" in the Node interface

type DataPartition

type DataPartition struct {
	PartitionID    uint64
	LastLoadedTime int64
	ReplicaNum     uint8
	Status         int8

	Replicas []*DataReplica
	Hosts    []string // host addresses
	Peers    []proto.Peer
	sync.RWMutex

	MissingNodes map[string]int64 // key: address of the missing node, value: when the node is missing
	VolName      string
	VolID        uint64

	FileInCoreMap           map[string]*FileInCore
	FilesWithMissingReplica map[string]int64 // key: file name, value: last time when a missing replica is found
	// contains filtered or unexported fields
}

DataPartition represents the structure of storing the file contents.

func (*DataPartition) ToProto added in v1.5.0

func (partition *DataPartition) ToProto(c *Cluster) *proto.DataPartitionInfo

type DataPartitionMap

type DataPartitionMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

DataPartitionMap stores all the data partitionMap

type DataReplica

type DataReplica struct {
	proto.DataReplica
	// contains filtered or unexported fields
}

DataReplica represents the replica of a data partition

type FileCrc

type FileCrc struct {
	// contains filtered or unexported fields
}

FileCrc defines the crc of a file

type FileInCore

type FileInCore struct {
	proto.FileInCore
	MetadataArray []*FileMetadata
}

FileInCore define file in data partition

func (FileInCore) ToProto added in v1.5.0

func (fc FileInCore) ToProto() proto.FileInCore

type FileMetadata

type FileMetadata struct {
	proto.FileMetadata
	// contains filtered or unexported fields
}

FileMetadata defines the file metadata on a dataNode

func (*FileMetadata) String

func (fm *FileMetadata) String() (msg string)

type GetCarryNodes

type GetCarryNodes func(maxTotal uint64, excludeHosts []string, nodes *sync.Map) (weightedNodes SortedWeightedNodes, availCount int)

type GetMaxTotal

type GetMaxTotal func(nodes *sync.Map) (maxTotal uint64)

type IDAllocator

type IDAllocator struct {
	// contains filtered or unexported fields
}

IDAllocator generates and allocates ids

type LeaderInfo

type LeaderInfo struct {
	// contains filtered or unexported fields
}

LeaderInfo represents the leader's information

type MetaNode

type MetaNode struct {
	ID                uint64
	Addr              string
	IsActive          bool
	Sender            *AdminTaskManager
	ZoneName          string `json:"Zone"`
	MaxMemAvailWeight uint64 `json:"MaxMemAvailWeight"`
	Total             uint64 `json:"TotalWeight"`
	Used              uint64 `json:"UsedWeight"`
	Ratio             float64
	SelectCount       uint64
	Carry             float64
	Threshold         float32
	ReportTime        time.Time

	MetaPartitionCount int
	NodeSetID          uint64
	sync.RWMutex
	PersistenceMetaPartitions []uint64
	// contains filtered or unexported fields
}

MetaNode defines the structure of a meta node

func (*MetaNode) GetAddr

func (metaNode *MetaNode) GetAddr() string

func (*MetaNode) GetID

func (metaNode *MetaNode) GetID() uint64

func (*MetaNode) SelectNodeForWrite

func (metaNode *MetaNode) SelectNodeForWrite()

SelectNodeForWrite implements the Node interface

func (*MetaNode) SetCarry

func (metaNode *MetaNode) SetCarry(carry float64)

SetCarry implements the Node interface

type MetaPartition

type MetaPartition struct {
	PartitionID uint64
	Start       uint64
	End         uint64
	MaxInodeID  uint64
	Replicas    []*MetaReplica
	ReplicaNum  uint8
	Status      int8
	IsRecover   bool

	Hosts        []string
	Peers        []proto.Peer
	MissNodes    map[string]int64
	LoadResponse []*proto.MetaPartitionLoadResponse
	sync.RWMutex
	// contains filtered or unexported fields
}

MetaPartition defines the structure of a meta partition

type MetaReplica

type MetaReplica struct {
	Addr string

	MaxInodeID uint64
	ReportTime int64
	Status     int8 // unavailable, readOnly, readWrite
	IsLeader   bool
	// contains filtered or unexported fields
}

MetaReplica defines the replica of a meta partition

type MetadataFsm

type MetadataFsm struct {
	// contains filtered or unexported fields
}

MetadataFsm represents the finite state machine of a metadata partition

func (*MetadataFsm) Apply

func (mf *MetadataFsm) Apply(command []byte, index uint64) (resp interface{}, err error)

Apply implements the interface of raft.StateMachine

func (*MetadataFsm) ApplyMemberChange

func (mf *MetadataFsm) ApplyMemberChange(confChange *proto.ConfChange, index uint64) (interface{}, error)

ApplyMemberChange implements the interface of raft.StateMachine

func (*MetadataFsm) ApplySnapshot

func (mf *MetadataFsm) ApplySnapshot(peers []proto.Peer, iterator proto.SnapIterator) (err error)

ApplySnapshot implements the interface of raft.StateMachine

func (*MetadataFsm) HandleFatalEvent

func (mf *MetadataFsm) HandleFatalEvent(err *raft.FatalError)

HandleFatalEvent implements the interface of raft.StateMachine

func (*MetadataFsm) HandleLeaderChange

func (mf *MetadataFsm) HandleLeaderChange(leader uint64)

HandleLeaderChange implements the interface of raft.StateMachine

func (*MetadataFsm) Snapshot

func (mf *MetadataFsm) Snapshot() (proto.Snapshot, error)

Snapshot implements the interface of raft.StateMachine

type MetadataSnapshot

type MetadataSnapshot struct {
	// contains filtered or unexported fields
}

MetadataSnapshot represents the snapshot of a meta partition

func (*MetadataSnapshot) ApplyIndex

func (ms *MetadataSnapshot) ApplyIndex() uint64

ApplyIndex implements the Snapshot interface

func (*MetadataSnapshot) Close

func (ms *MetadataSnapshot) Close()

Close implements the Snapshot interface

func (*MetadataSnapshot) Next

func (ms *MetadataSnapshot) Next() (data []byte, err error)

Next implements the Snapshot interface

type Node

type Node interface {
	SetCarry(carry float64)
	SelectNodeForWrite()
	GetID() uint64
	GetAddr() string
}

Node defines an interface that needs to be implemented by weightedNode

type NodeView

type NodeView struct {
	Addr       string
	Status     bool
	ID         uint64
	IsWritable bool
}

NodeView provides the view of the data or meta node.

type RaftCmd

type RaftCmd struct {
	Op uint32 `json:"op"`
	K  string `json:"k"`
	V  []byte `json:"v"`
}

RaftCmd defines the Raft commands.

func (*RaftCmd) Marshal

func (m *RaftCmd) Marshal() ([]byte, error)

Marshal converts the RaftCmd to a byte array.

func (*RaftCmd) Unmarshal

func (m *RaftCmd) Unmarshal(data []byte) (err error)

Unmarshal converts the byte array to a RaftCmd.

type Server

type Server struct {
	// contains filtered or unexported fields
}

Server represents the server in a cluster

func NewServer

func NewServer() *Server

NewServer creates a new server

func (*Server) Shutdown

func (m *Server) Shutdown()

Shutdown closes the server

func (*Server) Start

func (m *Server) Start(cfg *config.Config) (err error)

Start starts a server

func (*Server) Sync

func (m *Server) Sync()

Sync waits for the execution termination of the server

type SortedWeightedNodes

type SortedWeightedNodes []*weightedNode

SortedWeightedNodes defines an array sorted by carry

func (SortedWeightedNodes) Len

func (nodes SortedWeightedNodes) Len() int

func (SortedWeightedNodes) Less

func (nodes SortedWeightedNodes) Less(i, j int) bool

func (SortedWeightedNodes) Swap

func (nodes SortedWeightedNodes) Swap(i, j int)

type TokenValue

type TokenValue struct {
	VolName   string
	Value     string
	TokenType int8
}

type TopologyView

type TopologyView struct {
	Zones []*ZoneView
}

TopologyView provides the view of the topology view of the cluster

type User

type User struct {
	AKStore sync.Map //K: ak, V: userID

	AKStoreMutex sync.RWMutex
	// contains filtered or unexported fields
}

type Vol

type Vol struct {
	ID           uint64
	Name         string
	Owner        string
	OSSAccessKey string
	OSSSecretKey string

	Status uint8

	Capacity           uint64 // GB
	NeedToLowerReplica bool
	FollowerRead       bool

	MetaPartitions map[uint64]*MetaPartition

	sync.RWMutex
	// contains filtered or unexported fields
}

Vol represents a set of meta partitionMap and data partitionMap

func (*Vol) String

func (vol *Vol) String() string

type Zone

type Zone struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Zone stores all the zone related information

type ZoneView

type ZoneView struct {
	Name    string
	Status  string
	NodeSet map[uint64]*nodeSetView
}

ZoneView define the view of zone

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL