Documentation ¶
Index ¶
- Constants
- Variables
- func CheckAndCreateSubdir(name string) (err error)
- func LoopIndexFile(f *os.File, fn func(oid, offset uint64, size, crc uint32) error) (maxOid uint64, err error)
- func NewParamMismatchErr(msg string) (err error)
- func WalkIndexFileAndVerify(indexfp *os.File, datafp *os.File, blobFileInfo *BlobFileInfo) (err error)
- type BlobFile
- type BlobFileInfo
- type BlobStore
- func (s *BlobStore) AllocObjectId(fileId uint32) (uint64, error)
- func (s *BlobStore) ApplyDelObjects(blobfileId uint32, objects []uint64) (err error)
- func (s *BlobStore) CloseAll()
- func (s *BlobStore) DeleteStore()
- func (s *BlobStore) DoCompactWork(blobfileID int) (err error, released uint64)
- func (s *BlobStore) GetAllWatermark() (blobfiles []*FileInfo, err error)
- func (s *BlobStore) GetAvailBlobFile() (blobfileId int, err error)
- func (s *BlobStore) GetAvailChanLen() (chanLen int)
- func (s *BlobStore) GetBlobFileForWrite() (blobfileId int, err error)
- func (s *BlobStore) GetBlobFileInCore(fileID uint32) (*BlobFile, error)
- func (s *BlobStore) GetDelObjects(fileId uint32) (objects []uint64)
- func (s *BlobStore) GetLastOid(fileId uint32) (objectId uint64, err error)
- func (s *BlobStore) GetObject(fileId uint32, objectId uint64) (o *Object, err error)
- func (s *BlobStore) GetStoreBlobFileCount() (files int, err error)
- func (s *BlobStore) GetUnAvailBlobFile() (blobfileId int, err error)
- func (s *BlobStore) GetUnAvailChanLen() (chanLen int)
- func (s *BlobStore) GetWatermark(fileId uint64) (blobfileInfo *FileInfo, err error)
- func (s *BlobStore) IsReadyToCompact(blobfileID, thresh int) (isready bool, fileBytes, deleteBytes uint64, deletePercent float64)
- func (s *BlobStore) MarkDelete(fileId uint32, offset, size int64) error
- func (s *BlobStore) MoveBlobFileToUnavailChan()
- func (s *BlobStore) PutAvailBlobFile(blobfileId int)
- func (s *BlobStore) PutUnAvailBlobFile(blobfileId int)
- func (s *BlobStore) Read(fileId uint32, offset, size int64, nbuf []byte) (crc uint32, err error)
- func (s *BlobStore) Snapshot() ([]*proto.File, error)
- func (s *BlobStore) Sync(fileId uint32) (err error)
- func (s *BlobStore) SyncAll()
- func (s *BlobStore) UseSize() (size int64)
- func (s *BlobStore) VerfiyObjects(chunkid int, needVerify bool) (blobFileInfo *BlobFileInfo)
- func (s *BlobStore) Write(fileId uint32, objectId uint64, size int64, data []byte, crc uint32) (err error)
- func (s *BlobStore) WriteDeleteDentry(objectId uint64, blobfileId int, crc uint32) (err error)
- type Extent
- type ExtentCache
- type ExtentFilter
- type ExtentStore
- func (s *ExtentStore) Close()
- func (s *ExtentStore) Create(extentId uint64, inode uint64, overwrite bool) (err error)
- func (s *ExtentStore) DeleteStore() (err error)
- func (s *ExtentStore) FlushDelete() (err error)
- func (s *ExtentStore) GetAllWatermark(filter ExtentFilter) (extents []*FileInfo, err error)
- func (s *ExtentStore) GetDelObjects() (extents []uint64)
- func (s *ExtentStore) GetWatermark(extentId uint64, reload bool) (extentInfo *FileInfo, err error)
- func (s *ExtentStore) IsExistExtent(extentId uint64) (exist bool)
- func (s *ExtentStore) MarkDelete(extentId uint64) (err error)
- func (s *ExtentStore) NextExtentId() (extentId uint64)
- func (s *ExtentStore) Read(extentId uint64, offset, size int64, nbuf []byte) (crc uint32, err error)
- func (s *ExtentStore) SnapShot() (files []*proto.File, err error)
- func (s *ExtentStore) Sync(extentId uint64) (err error)
- func (s *ExtentStore) UpdateBaseExtentId(id uint64) (err error)
- func (s *ExtentStore) UsedSize() (size int64)
- func (s *ExtentStore) Write(extentId uint64, offset, size int64, data []byte, crc uint32) (err error)
- type FileInfo
- type Object
- type ObjectInfo
- type ObjectTree
Constants ¶
View Source
const ( ExtentOpenOpt = os.O_CREATE | os.O_RDWR | os.O_EXCL ExtentOpenOptOverwrite = os.O_CREATE | os.O_RDWR )
View Source
const ( FALLOC_FL_KEEP_SIZE = 1 FALLOC_FL_PUNCH_HOLE = 2 )
View Source
const ( ObjectHeaderSize = 24 IndexBatchRead = 1024 MarkDeleteObject = math.MaxUint32 )
View Source
const ( BlobFileFileCount = 10 BlobFileOpenOpt = os.O_CREATE | os.O_RDWR | os.O_APPEND CompactThreshold = 40 CompactMaxWait = time.Second * 10 ObjectIdLen = 8 )
View Source
const ( ExtMetaFileName = "EXTENT_META" ExtMetaFileOpt = os.O_CREATE | os.O_RDWR ExtDeleteFileName = "EXTENT_DELETE" ExtDeleteFileOpt = os.O_CREATE | os.O_RDWR | os.O_APPEND ExtMetaBaseIdOffset = 0 ExtMetaBaseIdSize = 8 ExtMetaDeleteIdxOffset = 8 ExtMetaDeleteIdxSize = 8 ExtMetaFileSize = ExtMetaBaseIdSize + ExtMetaDeleteIdxSize )
Variables ¶
View Source
var ( ErrorObjNotFound = errors.New("object not exist") ErrorFileNotFound = errors.New("file not exist") ErrorPartitionReadOnly = errors.New("partition readonly") ErrorHasDelete = errors.New("has delete") ErrorParamMismatch = errors.New("parameter mismatch error") ErrorNoAvaliFile = errors.New("no avail file") ErrorNoUnAvaliFile = errors.New("no Unavail file") ErrorNewStoreMode = errors.New("error new store mode ") ErrExtentNameFormat = errors.New("extent filePath format error") ErrSyscallNoSpace = errors.New("no space left on device") ErrorAgain = errors.New("try again") ErrorCompaction = errors.New("compaction error") ErrorCommit = errors.New("commit error") ErrObjectSmaller = errors.New("object smaller error") ErrPkgCrcMismatch = errors.New("pkg crc is not equal pkg data") )
Error definitions
View Source
var ( GetStableExtentFilter = func() ExtentFilter { now := time.Now() return func(info *FileInfo) bool { return now.Unix()-info.ModTime.Unix() > 30*60 && !info.Deleted && info.Size > 0 } } GetEmptyExtentFilter = func() ExtentFilter { now := time.Now() return func(info *FileInfo) bool { return now.Unix()-info.ModTime.Unix() > 30*60 && !info.Deleted && info.Size == 0 } } )
Filters
View Source
var (
BrokenExtentFileErr = errors.New("broken extent file error")
)
View Source
var (
RegexpExtentFile, _ = regexp.Compile("^(\\d)+$")
)
Functions ¶
func CheckAndCreateSubdir ¶
func LoopIndexFile ¶
func NewParamMismatchErr ¶
func WalkIndexFileAndVerify ¶
Types ¶
type BlobFileInfo ¶
type BlobStore ¶
type BlobStore struct {
// contains filtered or unexported fields
}
BlobStore is a store implement for blob file storage which container 40 blobfile files. This store will choose a available blobfile file and append data to it.
func (*BlobStore) ApplyDelObjects ¶
func (*BlobStore) DeleteStore ¶
func (s *BlobStore) DeleteStore()
func (*BlobStore) DoCompactWork ¶
func (*BlobStore) GetAllWatermark ¶
func (*BlobStore) GetAvailBlobFile ¶
func (*BlobStore) GetAvailChanLen ¶
func (*BlobStore) GetBlobFileForWrite ¶
func (*BlobStore) GetBlobFileInCore ¶
func (*BlobStore) GetDelObjects ¶
func (*BlobStore) GetLastOid ¶
func (*BlobStore) GetStoreBlobFileCount ¶
func (*BlobStore) GetUnAvailBlobFile ¶
func (*BlobStore) GetUnAvailChanLen ¶
func (*BlobStore) GetWatermark ¶
func (*BlobStore) IsReadyToCompact ¶
func (s *BlobStore) IsReadyToCompact(blobfileID, thresh int) (isready bool, fileBytes, deleteBytes uint64, deletePercent float64)
make sure blobfileID is valid
func (*BlobStore) MarkDelete ¶
func (*BlobStore) MoveBlobFileToUnavailChan ¶
func (s *BlobStore) MoveBlobFileToUnavailChan()
func (*BlobStore) PutAvailBlobFile ¶
func (*BlobStore) PutUnAvailBlobFile ¶
func (*BlobStore) VerfiyObjects ¶
func (s *BlobStore) VerfiyObjects(chunkid int, needVerify bool) (blobFileInfo *BlobFileInfo)
type Extent ¶
type Extent interface { // ID returns the identity value (extentId) of this extent entity. ID() uint64 // Ino returns this inode ID of this extent block belong to. Ino() uint64 // Close this extent and release FD. Close() error // InitToFS init extent data info filesystem. If entry file exist and overwrite is true, // this operation will clear all data of exist entry file and initialize extent header data. InitToFS(ino uint64, overwrite bool) error // RestoreFromFS restore entity data and status from entry file stored in filesystem. RestoreFromFS() error // Write data to extent. Write(data []byte, offset, size int64, crc uint32) (err error) // Read data from extent. Read(data []byte, offset, size int64) (crc uint32, err error) // Flush synchronize data to disk immediately. Flush() error // MarkDelete mark this extent as deleted. MarkDelete() error // IsMarkDelete test this extent if has been marked as delete. IsMarkDelete() bool // Size returns length of extent data exclude header. Size() (size int64) // ModTime returns the time when this extent was last modified. ModTime() time.Time // HeaderChecksum returns crc checksum value of extent header data // include inode data and block crc. HeaderChecksum() (crc uint32) }
Extent is used to manage extent block file for extent store engine.
func NewExtentInCore ¶
NewExtentInCore create and returns a new extent instance.
type ExtentCache ¶
type ExtentCache interface { // Put extent object into cache. Put(extent Extent) // Get extent from cache with specified extent identity (extentId). Get(extentId uint64) (extent Extent, ok bool) // Del extent stored in cache this specified extent identity (extentId). Del(extentId uint64) // Size returns number of extents stored in this cache. Size() int // Flush synchronize extent stored in this cache to disk immediately. Flush() // Clear close and synchronize all extent stored in this cache and remove them from cache. Clear() }
ExtentCache defined a cache for Extent entity for access improvement.
func NewExtentCache ¶
func NewExtentCache(capacity int) ExtentCache
NewExtentCache create and returns a new ExtentCache instance.
type ExtentFilter ¶
type ExtentStore ¶
type ExtentStore struct {
// contains filtered or unexported fields
}
func NewExtentStore ¶
func NewExtentStore(dataDir string, storeSize int) (s *ExtentStore, err error)
func (*ExtentStore) Close ¶
func (s *ExtentStore) Close()
func (*ExtentStore) Create ¶
func (s *ExtentStore) Create(extentId uint64, inode uint64, overwrite bool) (err error)
func (*ExtentStore) DeleteStore ¶
func (s *ExtentStore) DeleteStore() (err error)
func (*ExtentStore) FlushDelete ¶
func (s *ExtentStore) FlushDelete() (err error)
func (*ExtentStore) GetAllWatermark ¶
func (s *ExtentStore) GetAllWatermark(filter ExtentFilter) (extents []*FileInfo, err error)
func (*ExtentStore) GetDelObjects ¶
func (s *ExtentStore) GetDelObjects() (extents []uint64)
func (*ExtentStore) GetWatermark ¶
func (s *ExtentStore) GetWatermark(extentId uint64, reload bool) (extentInfo *FileInfo, err error)
func (*ExtentStore) IsExistExtent ¶
func (s *ExtentStore) IsExistExtent(extentId uint64) (exist bool)
func (*ExtentStore) MarkDelete ¶
func (s *ExtentStore) MarkDelete(extentId uint64) (err error)
func (*ExtentStore) NextExtentId ¶
func (s *ExtentStore) NextExtentId() (extentId uint64)
func (*ExtentStore) Sync ¶
func (s *ExtentStore) Sync(extentId uint64) (err error)
func (*ExtentStore) UpdateBaseExtentId ¶
func (s *ExtentStore) UpdateBaseExtentId(id uint64) (err error)
func (*ExtentStore) UsedSize ¶
func (s *ExtentStore) UsedSize() (size int64)
type FileInfo ¶
type FileInfo struct { FileId int `json:"fileId"` Inode uint64 `json:"ino"` Size uint64 `json:"size"` Crc uint32 `json:"crc"` Deleted bool `json:"deleted"` ModTime time.Time `json:"modTime"` Source string `json:"src"` MemberIndex int }
func (*FileInfo) FromExtent ¶
type Object ¶
func (*Object) IsIdentical ¶
type ObjectInfo ¶
type ObjectTree ¶
type ObjectTree struct {
// contains filtered or unexported fields
}
func NewObjectTree ¶
func NewObjectTree(f *os.File) *ObjectTree
func (*ObjectTree) FileBytes ¶
func (tree *ObjectTree) FileBytes() uint64
func (*ObjectTree) Load ¶
func (tree *ObjectTree) Load() (maxOid uint64, err error)
Needle map in this function is not protected, so callers should guarantee there is no write and delete operations on this needle map
Click to show internal directories.
Click to hide internal directories.