erasure_coding

package
v0.0.0-...-0b345dd Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 6, 2022 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DataShardsCount             = 10
	ParityShardsCount           = 4
	TotalShardsCount            = DataShardsCount + ParityShardsCount
	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
)

Variables

View Source
var (
	MarkNeedleDeleted = func(file *os.File, offset int64) error {
		b := make([]byte, types.SizeSize)
		types.SizeToBytes(b, types.TombstoneFileSize)
		n, err := file.WriteAt(b, offset+types.NeedleIdSize+types.OffsetSize)
		if err != nil {
			return fmt.Errorf("sorted needle write error: %v", err)
		}
		if n != types.SizeSize {
			return fmt.Errorf("sorted needle written %d bytes, expecting %d", n, types.SizeSize)
		}
		return nil
	}
)
View Source
var (
	NotFoundError = errors.New("needle not found")
)

Functions

func EcShardBaseFileName

func EcShardBaseFileName(collection string, id int) (baseFileName string)

func EcShardFileName

func EcShardFileName(collection string, dir string, id int) (fileName string)

func FindDatFileSize

func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64, err error)

FindDatFileSize calculate .dat file size from max offset entry there may be extra deletions after that entry but they are deletions anyway

func RebuildEcFiles

func RebuildEcFiles(baseFileName string) ([]uint32, error)

func RebuildEcxFile

func RebuildEcxFile(baseFileName string) error

func SearchNeedleFromSortedIndex

func SearchNeedleFromSortedIndex(ecxFile *os.File, ecxFileSize int64, needleId types.NeedleId, processNeedleFn func(file *os.File, offset int64) error) (offset types.Offset, size types.Size, err error)

func ToExt

func ToExt(ecIndex int) string

func WriteDatFile

func WriteDatFile(baseFileName string, datFileSize int64) error

WriteDatFile generates .dat from from .ec00 ~ .ec09 files

func WriteEcFiles

func WriteEcFiles(baseFileName string) error

WriteEcFiles generates .ec00 ~ .ec13 files

func WriteIdxFileFromEcIndex

func WriteIdxFileFromEcIndex(baseFileName string) (err error)

write .idx file from .ecx and .ecj files

func WriteSortedFileFromIdx

func WriteSortedFileFromIdx(baseFileName string, ext string) (e error)

WriteSortedFileFromIdx generates .ecx file from existing .idx file all keys are sorted in ascending order

Types

type EcVolume

type EcVolume struct {
	VolumeId   needle.VolumeId
	Collection string

	Shards                    []*EcVolumeShard
	ShardLocations            map[ShardId][]pb.ServerAddress
	ShardLocationsRefreshTime time.Time
	ShardLocationsLock        sync.RWMutex
	Version                   needle.Version
	// contains filtered or unexported fields
}

func NewEcVolume

func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error)

func (*EcVolume) AddEcVolumeShard

func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool

func (*EcVolume) Close

func (ev *EcVolume) Close()

func (*EcVolume) CreatedAt

func (ev *EcVolume) CreatedAt() time.Time

func (*EcVolume) DataBaseFileName

func (ev *EcVolume) DataBaseFileName() string

func (*EcVolume) DeleteEcVolumeShard

func (ev *EcVolume) DeleteEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, deleted bool)

func (*EcVolume) DeleteNeedleFromEcx

func (ev *EcVolume) DeleteNeedleFromEcx(needleId types.NeedleId) (err error)

func (*EcVolume) Destroy

func (ev *EcVolume) Destroy()

func (*EcVolume) FileName

func (ev *EcVolume) FileName(ext string) string

func (*EcVolume) FindEcVolumeShard

func (ev *EcVolume) FindEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, found bool)

func (*EcVolume) FindNeedleFromEcx

func (ev *EcVolume) FindNeedleFromEcx(needleId types.NeedleId) (offset types.Offset, size types.Size, err error)

func (*EcVolume) IndexBaseFileName

func (ev *EcVolume) IndexBaseFileName() string

func (*EcVolume) LocateEcShardNeedle

func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.Version) (offset types.Offset, size types.Size, intervals []Interval, err error)

func (*EcVolume) ShardIdList

func (ev *EcVolume) ShardIdList() (shardIds []ShardId)

func (*EcVolume) ShardSize

func (ev *EcVolume) ShardSize() uint64

func (*EcVolume) Size

func (ev *EcVolume) Size() (size int64)

func (*EcVolume) ToVolumeEcShardInformationMessage

func (ev *EcVolume) ToVolumeEcShardInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage)

type EcVolumeInfo

type EcVolumeInfo struct {
	VolumeId   needle.VolumeId
	Collection string
	ShardBits  ShardBits
	DiskType   string
}

data structure used in master

func NewEcVolumeInfo

func NewEcVolumeInfo(diskType string, collection string, vid needle.VolumeId, shardBits ShardBits) *EcVolumeInfo

func (*EcVolumeInfo) AddShardId

func (ecInfo *EcVolumeInfo) AddShardId(id ShardId)

func (*EcVolumeInfo) HasShardId

func (ecInfo *EcVolumeInfo) HasShardId(id ShardId) bool

func (*EcVolumeInfo) Minus

func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo

func (*EcVolumeInfo) RemoveShardId

func (ecInfo *EcVolumeInfo) RemoveShardId(id ShardId)

func (*EcVolumeInfo) ShardIdCount

func (ecInfo *EcVolumeInfo) ShardIdCount() (count int)

func (*EcVolumeInfo) ShardIds

func (ecInfo *EcVolumeInfo) ShardIds() (ret []ShardId)

func (*EcVolumeInfo) ToVolumeEcShardInformationMessage

func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage)

type EcVolumeShard

type EcVolumeShard struct {
	VolumeId   needle.VolumeId
	ShardId    ShardId
	Collection string

	DiskType types.DiskType
	// contains filtered or unexported fields
}

func NewEcVolumeShard

func NewEcVolumeShard(diskType types.DiskType, dirname string, collection string, id needle.VolumeId, shardId ShardId) (v *EcVolumeShard, e error)

func (*EcVolumeShard) Close

func (shard *EcVolumeShard) Close()

func (*EcVolumeShard) Destroy

func (shard *EcVolumeShard) Destroy()

func (*EcVolumeShard) FileName

func (shard *EcVolumeShard) FileName() (fileName string)

func (*EcVolumeShard) ReadAt

func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error)

func (*EcVolumeShard) Size

func (shard *EcVolumeShard) Size() int64

func (*EcVolumeShard) String

func (shard *EcVolumeShard) String() string

type Interval

type Interval struct {
	BlockIndex          int
	InnerBlockOffset    int64
	Size                types.Size
	IsLargeBlock        bool
	LargeBlockRowsCount int
}

func LocateData

func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset int64, size types.Size) (intervals []Interval)

func (Interval) ToShardIdAndOffset

func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64)

type ShardBits

type ShardBits uint32 // use bits to indicate the shard id, use 32 bits just for possible future extension

func (ShardBits) AddShardId

func (b ShardBits) AddShardId(id ShardId) ShardBits

func (ShardBits) HasShardId

func (b ShardBits) HasShardId(id ShardId) bool

func (ShardBits) Minus

func (b ShardBits) Minus(other ShardBits) ShardBits

func (ShardBits) MinusParityShards

func (b ShardBits) MinusParityShards() ShardBits

func (ShardBits) Plus

func (b ShardBits) Plus(other ShardBits) ShardBits

func (ShardBits) RemoveShardId

func (b ShardBits) RemoveShardId(id ShardId) ShardBits

func (ShardBits) ShardIdCount

func (b ShardBits) ShardIdCount() (count int)

func (ShardBits) ShardIds

func (b ShardBits) ShardIds() (ret []ShardId)

func (ShardBits) ToUint32Slice

func (b ShardBits) ToUint32Slice() (ret []uint32)

type ShardId

type ShardId uint8

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL