erasure_coding

package
v0.0.0-...-f1e43fe Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 24, 2019 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DataShardsCount             = 10
	ParityShardsCount           = 4
	TotalShardsCount            = DataShardsCount + ParityShardsCount
	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
)

Variables

View Source 
var (
	NotFoundError = errors.New("needle not found")
)

Functions

func EcShardBaseFileName 

func EcShardBaseFileName(collection string, id int) (baseFileName string)

func EcShardFileName 

func EcShardFileName(collection string, dir string, id int) (fileName string)

func RebuildEcFiles 

func RebuildEcFiles(baseFileName string) ([]uint32, error)

func RebuildEcxFile 

func RebuildEcxFile(baseFileName string) error

func ToExt 

func ToExt(ecIndex int) string

func WriteEcFiles 

func WriteEcFiles(baseFileName string) error

WriteEcFiles generates .ec01 ~ .ec14 files

func WriteSortedEcxFile 

func WriteSortedEcxFile(baseFileName string) (e error)

WriteSortedEcxFile generates .ecx file from existing .idx file all keys are sorted in ascending order

Types

type EcVolume 

type EcVolume struct {
	VolumeId   needle.VolumeId
	Collection string

	Shards                    []*EcVolumeShard
	ShardLocations            map[ShardId][]string
	ShardLocationsRefreshTime time.Time
	ShardLocationsLock        sync.RWMutex
	Version                   needle.Version
	// contains filtered or unexported fields
}

func NewEcVolume 

func NewEcVolume(dir string, collection string, vid needle.VolumeId) (ev *EcVolume, err error)

func (*EcVolume) AddEcVolumeShard 

func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool

func (*EcVolume) Close 

func (ev *EcVolume) Close()

func (*EcVolume) CreatedAt 

func (ev *EcVolume) CreatedAt() time.Time

func (*EcVolume) DeleteEcVolumeShard 

func (ev *EcVolume) DeleteEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, deleted bool)

func (*EcVolume) DeleteNeedleFromEcx 

func (ev *EcVolume) DeleteNeedleFromEcx(needleId types.NeedleId) (err error)

func (*EcVolume) Destroy 

func (ev *EcVolume) Destroy()

func (*EcVolume) FileName 

func (ev *EcVolume) FileName() string

func (*EcVolume) FindEcVolumeShard 

func (ev *EcVolume) FindEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, found bool)

func (*EcVolume) FindNeedleFromEcx 

func (ev *EcVolume) FindNeedleFromEcx(needleId types.NeedleId) (offset types.Offset, size uint32, err error)

func (*EcVolume) LocateEcShardNeedle 

func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.Version) (offset types.Offset, size uint32, intervals []Interval, err error)

func (*EcVolume) ShardIdList 

func (ev *EcVolume) ShardIdList() (shardIds []ShardId)

func (*EcVolume) ShardSize 

func (ev *EcVolume) ShardSize() int64

func (*EcVolume) ToVolumeEcShardInformationMessage 

func (ev *EcVolume) ToVolumeEcShardInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage)

type EcVolumeInfo 

type EcVolumeInfo struct {
	VolumeId   needle.VolumeId
	Collection string
	ShardBits  ShardBits
}

data structure used in master

func NewEcVolumeInfo 

func NewEcVolumeInfo(collection string, vid needle.VolumeId, shardBits ShardBits) *EcVolumeInfo

func (*EcVolumeInfo) AddShardId 

func (ecInfo *EcVolumeInfo) AddShardId(id ShardId)

func (*EcVolumeInfo) HasShardId 

func (ecInfo *EcVolumeInfo) HasShardId(id ShardId) bool

func (*EcVolumeInfo) Minus 

func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo

func (*EcVolumeInfo) RemoveShardId 

func (ecInfo *EcVolumeInfo) RemoveShardId(id ShardId)

func (*EcVolumeInfo) ShardIdCount 

func (ecInfo *EcVolumeInfo) ShardIdCount() (count int)

func (*EcVolumeInfo) ShardIds 

func (ecInfo *EcVolumeInfo) ShardIds() (ret []ShardId)

func (*EcVolumeInfo) ToVolumeEcShardInformationMessage 

func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage)

type EcVolumeShard 

type EcVolumeShard struct {
	VolumeId   needle.VolumeId
	ShardId    ShardId
	Collection string
	// contains filtered or unexported fields
}

func NewEcVolumeShard 

func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId ShardId) (v *EcVolumeShard, e error)

func (*EcVolumeShard) Close 

func (shard *EcVolumeShard) Close()

func (*EcVolumeShard) Destroy 

func (shard *EcVolumeShard) Destroy()

func (*EcVolumeShard) FileName 

func (shard *EcVolumeShard) FileName() (fileName string)

func (*EcVolumeShard) ReadAt 

func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error)

func (*EcVolumeShard) Size 

func (shard *EcVolumeShard) Size() int64

func (*EcVolumeShard) String 

func (shard *EcVolumeShard) String() string

type Interval 

type Interval struct {
	BlockIndex          int
	InnerBlockOffset    int64
	Size                uint32
	IsLargeBlock        bool
	LargeBlockRowsCount int
}

func LocateData 

func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset int64, size uint32) (intervals []Interval)

func (Interval) ToShardIdAndOffset 

func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64)

type ShardBits 

type ShardBits uint32 // use bits to indicate the shard id, use 32 bits just for possible future extension

func (ShardBits) AddShardId 

func (b ShardBits) AddShardId(id ShardId) ShardBits

func (ShardBits) HasShardId 

func (b ShardBits) HasShardId(id ShardId) bool

func (ShardBits) Minus 

func (b ShardBits) Minus(other ShardBits) ShardBits

func (ShardBits) Plus 

func (b ShardBits) Plus(other ShardBits) ShardBits

func (ShardBits) RemoveShardId 

func (b ShardBits) RemoveShardId(id ShardId) ShardBits

func (ShardBits) ShardIdCount 

func (b ShardBits) ShardIdCount() (count int)

func (ShardBits) ShardIds 

func (b ShardBits) ShardIds() (ret []ShardId)

type ShardId 

type ShardId uint8

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.