storage

package
v2.5.2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 27, 2022 License: Apache-2.0 Imports: 22 Imported by: 5

Documentation

Index

Constants

View Source 
const (
	ExtentOpenOpt  = os.O_CREATE | os.O_RDWR | os.O_EXCL
	ExtentHasClose = -1
	SEEK_DATA      = 3
	SEEK_HOLE      = 4
)
View Source 
const (
	PageSize          = 4 * util.KB
	FallocFLKeepSize  = 1
	FallocFLPunchHole = 2
)
View Source 
const (
	ExtCrcHeaderFileName         = "EXTENT_CRC"
	ExtBaseExtentIDFileName      = "EXTENT_META"
	TinyDeleteFileOpt            = os.O_CREATE | os.O_RDWR | os.O_APPEND
	TinyExtDeletedFileName       = "TINYEXTENT_DELETE"
	NormalExtDeletedFileName     = "NORMALEXTENT_DELETE"
	MaxExtentCount               = 20000
	TinyExtentCount              = 64
	TinyExtentStartID            = 1
	MinExtentID                  = 1024
	DeleteTinyRecordSize         = 24
	UpdateCrcInterval            = 600
	RepairInterval               = 60
	RandomWriteType              = 2
	AppendWriteType              = 1
	NormalExtentDeleteRetainTime = 3600 * 4
)
View Source 
const (
	BaseExtentIDOffset = 0
)
View Source 
const (
	DiskSectorSize = 512
)

Sector size

View Source 
const (
	ExtentMaxSize = 1024 * 1024 * 1024 * 1024 * 4 // 4TB
)

Variables

View Source 
var (
	ExtentHasBeenDeletedError = errors.New("extent has been deleted")
	ParameterMismatchError    = errors.New("parameter mismatch error")
	NoAvailableExtentError    = errors.New("no available extent")
	NoBrokenExtentError       = errors.New("no unavailable extent")
	NoSpaceError              = errors.New("no space left on the device")
	TryAgainError             = errors.New("try again")
	CrcMismatchError          = errors.New("packet Crc is incorrect")
	NoLeaderError             = errors.New("no raft leader")
	ExtentNotFoundError       = errors.New("extent does not exist")
	ExtentExistsError         = errors.New("extent already exists")
	ExtentIsFullError         = errors.New("extent is full")
	BrokenExtentError         = errors.New("extent has been broken")
	BrokenDiskError           = errors.New("disk has broken")
)
View Source 
var (
	RegexpExtentFile, _ = regexp.Compile("^(\\d)+$")
	SnapShotFilePool    = &sync.Pool{New: func() interface{} {
		return new(proto.File)
	}}
)
View Source 
var (
	NormalExtentFilter = func() ExtentFilter {
		now := time.Now()
		return func(ei *ExtentInfo) bool {
			return !IsTinyExtent(ei.FileID) && now.Unix()-ei.ModifyTime > RepairInterval && ei.IsDeleted == false && ei.Size > 0
		}
	}

	TinyExtentFilter = func(filters []uint64) ExtentFilter {
		return func(ei *ExtentInfo) bool {
			if !IsTinyExtent(ei.FileID) {
				return false
			}
			for _, filterID := range filters {
				if filterID == ei.FileID {
					return true
				}
			}
			return false
		}
	}
)

Filters

Functions

func GetSnapShotFileFromPool 

func GetSnapShotFileFromPool() (f *proto.File)

func IsAppendWrite added in v1.4.0 

func IsAppendWrite(writeType int) bool

func IsRandomWrite added in v1.4.0 

func IsRandomWrite(writeType int) bool

func IsTinyExtent 

func IsTinyExtent(extentID uint64) bool

IsTinyExtent checks if the given extent is tiny extent.

func MarshalTinyExtent 

func MarshalTinyExtent(extentID uint64, offset, size int64) (data []byte)

func MkdirAll 

func MkdirAll(name string) (err error)

func NewParameterMismatchErr 

func NewParameterMismatchErr(msg string) (err error)

func PutSnapShotFileToPool 

func PutSnapShotFileToPool(f *proto.File)

func UnMarshalTinyExtent 

func UnMarshalTinyExtent(data []byte) (extentID, offset, size uint64)

Types

type BlockCrc 

type BlockCrc struct {
	BlockNo int
	Crc     uint32
}

type BlockCrcArr 

type BlockCrcArr []*BlockCrc

func (BlockCrcArr) Len 

func (arr BlockCrcArr) Len() int

func (BlockCrcArr) Less 

func (arr BlockCrcArr) Less(i, j int) bool

func (BlockCrcArr) Swap 

func (arr BlockCrcArr) Swap(i, j int)

type Extent 

type Extent struct {
	sync.Mutex
	// contains filtered or unexported fields
}

Extent is an implementation of Extent for local regular extent file data management. This extent implementation manages all header info and data body in one single entry file. Header of extent include inode value of this extent block and Crc blocks of data blocks.

func NewExtentInCore 

func NewExtentInCore(name string, extentID uint64) *Extent

NewExtentInCore create and returns a new extent instance.

func (*Extent) Close 

func (e *Extent) Close() (err error)

Close this extent and release FD.

func (*Extent) DeleteTiny 

func (e *Extent) DeleteTiny(offset, size int64) (hasDelete bool, err error)

DeleteTiny deletes a tiny extent.

func (*Extent) Exist 

func (e *Extent) Exist() (exsit bool)

func (*Extent) Flush 

func (e *Extent) Flush() (err error)

Flush synchronizes data to the disk.

func (*Extent) HasClosed 

func (e *Extent) HasClosed() bool

func (*Extent) InitToFS 

func (e *Extent) InitToFS() (err error)

InitToFS init extent data info filesystem. If entry file exist and overwrite is true, this operation will clear all data of exist entry file and initialize extent header data.

func (*Extent) ModifyTime 

func (e *Extent) ModifyTime() int64

ModifyTime returns the time when this extent was modified recently.

func (*Extent) Read 

func (e *Extent) Read(data []byte, offset, size int64, isRepairRead bool) (crc uint32, err error)

Read reads data from an extent.

func (*Extent) ReadTiny 

func (e *Extent) ReadTiny(data []byte, offset, size int64, isRepairRead bool) (crc uint32, err error)

ReadTiny read data from a tiny extent.

func (*Extent) RestoreFromFS 

func (e *Extent) RestoreFromFS() (err error)

RestoreFromFS restores the entity data and status from the file stored on the filesystem.

func (*Extent) Size 

func (e *Extent) Size() (size int64)

Size returns length of the extent (not including the header).

func (*Extent) TinyExtentRecover added in v1.4.0 

func (e *Extent) TinyExtentRecover(data []byte, offset, size int64, crc uint32, isEmptyPacket bool) (err error)

func (*Extent) Write 

func (e *Extent) Write(data []byte, offset, size int64, crc uint32, writeType int, isSync bool, crcFunc UpdateCrcFunc, ei *ExtentInfo) (err error)

Write writes data to an extent.

func (*Extent) WriteTiny 

func (e *Extent) WriteTiny(data []byte, offset, size int64, crc uint32, writeType int, isSync bool) (err error)

WriteTiny performs write on a tiny extent.

type ExtentCache 

type ExtentCache struct {
	// contains filtered or unexported fields
}

ExtentCache is an implementation of the ExtentCache with LRU support.

func NewExtentCache 

func NewExtentCache(capacity int) *ExtentCache

NewExtentCache creates and returns a new ExtentCache instance.

func (*ExtentCache) Clear 

func (cache *ExtentCache) Clear()

Clear closes all the extents stored in the cache.

func (*ExtentCache) Del 

func (cache *ExtentCache) Del(extentID uint64)

Del deletes the extent stored in the cache.

func (*ExtentCache) Flush 

func (cache *ExtentCache) Flush()

Flush synchronizes the extent stored in the cache to the disk.

func (*ExtentCache) Get 

func (cache *ExtentCache) Get(extentID uint64) (e *Extent, ok bool)

Get gets the extent from the cache.

func (*ExtentCache) Put 

func (cache *ExtentCache) Put(e *Extent)

Put puts an extent object into the cache.

func (*ExtentCache) Size 

func (cache *ExtentCache) Size() int

Size returns number of extents stored in the cache.

type ExtentDeleted 

type ExtentDeleted struct {
	ExtentID uint64 `json:"extentID"`
	Offset   uint64 `json:"offset"`
	Size     uint64 `json:"size"`
}

type ExtentFilter 

type ExtentFilter func(info *ExtentInfo) bool

type ExtentInfo 

type ExtentInfo struct {
	FileID     uint64 `json:"fileId"`
	Size       uint64 `json:"size"`
	Crc        uint32 `json:"Crc"`
	IsDeleted  bool   `json:"deleted"`
	ModifyTime int64  `json:"modTime"` // random write not update modify time
	Source     string `json:"src"`
}

func (*ExtentInfo) String 

func (ei *ExtentInfo) String() (m string)

func (*ExtentInfo) UpdateExtentInfo 

func (ei *ExtentInfo) UpdateExtentInfo(extent *Extent, crc uint32)

type ExtentInfoArr 

type ExtentInfoArr []*ExtentInfo

func (ExtentInfoArr) Len 

func (arr ExtentInfoArr) Len() int

func (ExtentInfoArr) Less 

func (arr ExtentInfoArr) Less(i, j int) bool

func (ExtentInfoArr) Swap 

func (arr ExtentInfoArr) Swap(i, j int)

type ExtentMapItem 

type ExtentMapItem struct {
	// contains filtered or unexported fields
}

ExtentMapItem stores the extent entity pointer and the element pointer of the extent entity in a cache list.

type ExtentStore 

type ExtentStore struct {
	// contains filtered or unexported fields
}

ExtentStore defines fields used in the storage engine. Packets smaller than 128K are stored in the "tinyExtent", a place to persist the small files. packets larger than or equal to 128K are stored in the normal "extent", a place to persist large files. The difference between them is that the extentID of a tinyExtent starts at 5000000 and ends at 5000128. Multiple small files can be appended to the same tinyExtent. In addition, the deletion of small files is implemented by the punch hole from the underlying file system.

func NewExtentStore 

func NewExtentStore(dataDir string, partitionID uint64, storeSize int) (s *ExtentStore, err error)

func (*ExtentStore) AvailableTinyExtentCnt 

func (s *ExtentStore) AvailableTinyExtentCnt() int

AvailableTinyExtentCnt returns the count of the available tiny extents.

func (*ExtentStore) BackendTask 

func (s *ExtentStore) BackendTask()

func (*ExtentStore) BrokenTinyExtentCnt 

func (s *ExtentStore) BrokenTinyExtentCnt() int

BrokenTinyExtentCnt returns the count of the broken tiny extents.

func (*ExtentStore) Close 

func (s *ExtentStore) Close()

Close closes the extent store.

func (*ExtentStore) Create 

func (s *ExtentStore) Create(extentID uint64) (err error)

Create creates an extent.

func (*ExtentStore) DeleteBlockCrc 

func (s *ExtentStore) DeleteBlockCrc(extentID uint64) (err error)

func (*ExtentStore) ExtentID 

func (s *ExtentStore) ExtentID(filename string) (extentID uint64, isExtent bool)

ExtentID return the extent ID.

func (*ExtentStore) GetAllWatermarks 

func (s *ExtentStore) GetAllWatermarks(filter ExtentFilter) (extents []*ExtentInfo, tinyDeleteFileSize int64, err error)

GetAllWatermarks returns all the watermarks.

func (*ExtentStore) GetAvailableTinyExtent 

func (s *ExtentStore) GetAvailableTinyExtent() (extentID uint64, err error)

GetAvailableTinyExtent returns the available tiny extent from the channel.

func (*ExtentStore) GetBrokenTinyExtent 

func (s *ExtentStore) GetBrokenTinyExtent() (extentID uint64, err error)

GetBrokenTinyExtent returns the first broken extent in the channel.

func (*ExtentStore) GetExtentCount 

func (s *ExtentStore) GetExtentCount() (count int)

GetExtentCount returns the number of extents in the extentInfoMap

func (*ExtentStore) GetHasDeleteExtent 

func (s *ExtentStore) GetHasDeleteExtent() (extentDes []ExtentDeleted, err error)

func (*ExtentStore) GetHasDeleteTinyRecords 

func (s *ExtentStore) GetHasDeleteTinyRecords() (extentDes []ExtentDeleted, err error)

func (*ExtentStore) GetMaxExtentIDAndPartitionSize added in v1.4.0 

func (s *ExtentStore) GetMaxExtentIDAndPartitionSize() (maxExtentID, totalSize uint64)

StoreSizeExtentID returns the size of the extent store

func (*ExtentStore) GetPersistenceBaseExtentID 

func (s *ExtentStore) GetPersistenceBaseExtentID() (extentID uint64, err error)

func (*ExtentStore) GetPreAllocSpaceExtentIDOnVerfiyFile 

func (s *ExtentStore) GetPreAllocSpaceExtentIDOnVerfiyFile() (extentID uint64)

func (*ExtentStore) GetStoreUsedSize 

func (s *ExtentStore) GetStoreUsedSize() (used int64)

func (*ExtentStore) GetTinyExtentOffset 

func (s *ExtentStore) GetTinyExtentOffset(extentID uint64) (watermark int64, err error)

GetTinyExtentOffset returns the offset of the given extent.

func (*ExtentStore) HasExtent 

func (s *ExtentStore) HasExtent(extentID uint64) (exist bool)

HasExtent tells if the extent store has the extent with the given ID

func (*ExtentStore) IsDeletedNormalExtent 

func (s *ExtentStore) IsDeletedNormalExtent(extentID uint64) (ok bool)

func (*ExtentStore) LoadTinyDeleteFileOffset 

func (s *ExtentStore) LoadTinyDeleteFileOffset() (offset int64, err error)

func (*ExtentStore) MarkDelete 

func (s *ExtentStore) MarkDelete(extentID uint64, offset, size int64) (err error)

MarkDelete marks the given extent as deleted.

func (*ExtentStore) MoveAllToBrokenTinyExtentC 

func (s *ExtentStore) MoveAllToBrokenTinyExtentC(cnt int)

MoveAllToBrokenTinyExtentC moves all the tiny extents to the channel stores the broken extents.

func (*ExtentStore) NextExtentID 

func (s *ExtentStore) NextExtentID() (extentID uint64, err error)

NextExtentID returns the next extentID. When the client sends the request to create an extent, this function generates an unique extentID within the current partition. This function can only be called by the leader.

func (*ExtentStore) PersistenceBaseExtentID 

func (s *ExtentStore) PersistenceBaseExtentID(extentID uint64) (err error)

func (*ExtentStore) PersistenceBlockCrc 

func (s *ExtentStore) PersistenceBlockCrc(e *Extent, blockNo int, blockCrc uint32) (err error)

func (*ExtentStore) PersistenceHasDeleteExtent 

func (s *ExtentStore) PersistenceHasDeleteExtent(extentID uint64) (err error)

func (*ExtentStore) PreAllocSpaceOnVerfiyFile 

func (s *ExtentStore) PreAllocSpaceOnVerfiyFile(currExtentID uint64)

func (*ExtentStore) PutNormalExtentToDeleteCache 

func (s *ExtentStore) PutNormalExtentToDeleteCache(extentID uint64)

func (*ExtentStore) Read 

func (s *ExtentStore) Read(extentID uint64, offset, size int64, nbuf []byte, isRepairRead bool) (crc uint32, err error)

Read reads the extent based on the given id.

func (*ExtentStore) ReadTinyDeleteRecords 

func (s *ExtentStore) ReadTinyDeleteRecords(offset, size int64, data []byte) (crc uint32, err error)

func (*ExtentStore) RecordTinyDelete 

func (s *ExtentStore) RecordTinyDelete(extentID uint64, offset, size int64) (err error)

func (*ExtentStore) ScanBlocks 

func (s *ExtentStore) ScanBlocks(extentID uint64) (bcs []*BlockCrc, err error)

func (*ExtentStore) SendAllToBrokenTinyExtentC 

func (s *ExtentStore) SendAllToBrokenTinyExtentC(extentIds []uint64)

SendAllToBrokenTinyExtentC sends all the extents to the channel that stores the broken extents.

func (*ExtentStore) SendToAvailableTinyExtentC 

func (s *ExtentStore) SendToAvailableTinyExtentC(extentID uint64)

SendToAvailableTinyExtentC sends the extent to the channel that stores the available tiny extents.

func (*ExtentStore) SendToBrokenTinyExtentC 

func (s *ExtentStore) SendToBrokenTinyExtentC(extentID uint64)

SendToBrokenTinyExtentC sends the given extent id to the channel.

func (*ExtentStore) SnapShot 

func (s *ExtentStore) SnapShot() (files []*proto.File, err error)

SnapShot returns the information of all the extents on the current data partition. When the master sends the loadDataPartition request, the snapshot is used to compare the replicas.

func (*ExtentStore) StoreSizeExtentID added in v1.4.0 

func (s *ExtentStore) StoreSizeExtentID(maxExtentID uint64) (totalSize uint64)

StoreSizeExtentID returns the size of the extent store

func (*ExtentStore) TinyExtentAvaliOffset added in v1.4.0 

func (s *ExtentStore) TinyExtentAvaliOffset(extentID uint64, offset int64) (newOffset, newEnd int64, err error)

func (*ExtentStore) TinyExtentGetFinfoSize added in v1.4.0 

func (s *ExtentStore) TinyExtentGetFinfoSize(extentID uint64) (size uint64, err error)

func (*ExtentStore) TinyExtentRecover added in v1.4.0 

func (s *ExtentStore) TinyExtentRecover(extentID uint64, offset, size int64, data []byte, crc uint32, isEmptyPacket bool) (err error)

func (*ExtentStore) UpdateBaseExtentID 

func (s *ExtentStore) UpdateBaseExtentID(id uint64) (err error)

UpdateBaseExtentID updates the base extent ID.

func (*ExtentStore) Watermark 

func (s *ExtentStore) Watermark(extentID uint64) (ei *ExtentInfo, err error)

Watermark returns the extent info of the given extent on the record.

func (*ExtentStore) Write 

func (s *ExtentStore) Write(extentID uint64, offset, size int64, data []byte, crc uint32, writeType int, isSync bool) (err error)

Write writes the given extent to the disk.

type GetExtentCrcFunc 

type GetExtentCrcFunc func(extentID uint64) (crc uint32, err error)

type UpdateCrcFunc 

type UpdateCrcFunc func(e *Extent, blockNo int, crc uint32) (err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.