storage

package
v0.0.0-...-71dd0ca Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 12, 2018 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ExtentOpenOpt          = os.O_CREATE | os.O_RDWR | os.O_EXCL
	ExtentOpenOptOverwrite = os.O_CREATE | os.O_RDWR
)
View Source
const (
	FALLOC_FL_KEEP_SIZE  = 1
	FALLOC_FL_PUNCH_HOLE = 2
)
View Source
const (
	ObjectHeaderSize = 24
	IndexBatchRead   = 1024
	MarkDeleteObject = math.MaxUint32
)
View Source
const (
	BlobFileFileCount = 10
	BlobFileOpenOpt   = os.O_CREATE | os.O_RDWR | os.O_APPEND
	CompactThreshold  = 40
	CompactMaxWait    = time.Second * 10
	ObjectIdLen       = 8
)
View Source
const (
	ExtMetaFileName        = "EXTENT_META"
	ExtMetaFileOpt         = os.O_CREATE | os.O_RDWR
	ExtDeleteFileName      = "EXTENT_DELETE"
	ExtDeleteFileOpt       = os.O_CREATE | os.O_RDWR | os.O_APPEND
	ExtMetaBaseIdOffset    = 0
	ExtMetaBaseIdSize      = 8
	ExtMetaDeleteIdxOffset = 8
	ExtMetaDeleteIdxSize   = 8
	ExtMetaFileSize        = ExtMetaBaseIdSize + ExtMetaDeleteIdxSize
)

Variables

View Source
var (
	ErrorObjNotFound       = errors.New("object not exist")
	ErrorFileNotFound      = errors.New("file not exist")
	ErrorPartitionReadOnly = errors.New("partition readonly")
	ErrorHasDelete         = errors.New("has delete")
	ErrorParamMismatch     = errors.New("parameter mismatch error")
	ErrorNoAvaliFile       = errors.New("no avail file")
	ErrorNoUnAvaliFile     = errors.New("no Unavail file")
	ErrorNewStoreMode      = errors.New("error new store mode ")
	ErrExtentNameFormat    = errors.New("extent filePath format error")
	ErrSyscallNoSpace      = errors.New("no space left on device")
	ErrorAgain             = errors.New("try again")
	ErrorCompaction        = errors.New("compaction error")
	ErrorCommit            = errors.New("commit error")
	ErrObjectSmaller       = errors.New("object smaller error")
	ErrPkgCrcMismatch      = errors.New("pkg crc is not equal pkg data")
)

Error definitions

View Source
var (
	GetStableExtentFilter = func() ExtentFilter {
		now := time.Now()
		return func(info *FileInfo) bool {
			return now.Unix()-info.ModTime.Unix() > 30*60 && !info.Deleted && info.Size > 0
		}
	}
	GetEmptyExtentFilter = func() ExtentFilter {
		now := time.Now()
		return func(info *FileInfo) bool {
			return now.Unix()-info.ModTime.Unix() > 30*60 && !info.Deleted && info.Size == 0
		}
	}
)

Filters

View Source
var (
	BrokenExtentFileErr = errors.New("broken extent file error")
)
View Source
var (
	RegexpExtentFile, _ = regexp.Compile("^(\\d)+$")
)

Functions

func CheckAndCreateSubdir

func CheckAndCreateSubdir(name string) (err error)

func LoopIndexFile

func LoopIndexFile(f *os.File, fn func(oid, offset uint64, size, crc uint32) error) (maxOid uint64, err error)

func NewParamMismatchErr

func NewParamMismatchErr(msg string) (err error)

func WalkIndexFileAndVerify

func WalkIndexFileAndVerify(indexfp *os.File, datafp *os.File, blobFileInfo *BlobFileInfo) (err error)

Types

type BlobFile

type BlobFile struct {
	// contains filtered or unexported fields
}

func NewBlobFile

func NewBlobFile(dataDir string, blobfileId int) (c *BlobFile, err error)

type BlobFileInfo

type BlobFileInfo struct {
	DataFile      string
	IndexFile     string
	Objects       []*ObjectInfo
	NeedVerify    bool
	ErrMesg       string
	AvailChunkCnt int
	Used          int64
	FileBytes     uint64
	DeleteBytes   uint64
	DeletePercent float64
}

type BlobStore

type BlobStore struct {
	// contains filtered or unexported fields
}

BlobStore is a store implement for blob file storage which container 40 blobfile files. This store will choose a available blobfile file and append data to it.

func NewBlobStore

func NewBlobStore(dataDir string, storeSize int) (s *BlobStore, err error)

func (*BlobStore) AllocObjectId

func (s *BlobStore) AllocObjectId(fileId uint32) (uint64, error)

func (*BlobStore) ApplyDelObjects

func (s *BlobStore) ApplyDelObjects(blobfileId uint32, objects []uint64) (err error)

func (*BlobStore) CloseAll

func (s *BlobStore) CloseAll()

func (*BlobStore) DeleteStore

func (s *BlobStore) DeleteStore()

func (*BlobStore) DoCompactWork

func (s *BlobStore) DoCompactWork(blobfileID int) (err error, released uint64)

func (*BlobStore) GetAllWatermark

func (s *BlobStore) GetAllWatermark() (blobfiles []*FileInfo, err error)

func (*BlobStore) GetAvailBlobFile

func (s *BlobStore) GetAvailBlobFile() (blobfileId int, err error)

func (*BlobStore) GetAvailChanLen

func (s *BlobStore) GetAvailChanLen() (chanLen int)

func (*BlobStore) GetBlobFileForWrite

func (s *BlobStore) GetBlobFileForWrite() (blobfileId int, err error)

func (*BlobStore) GetBlobFileInCore

func (s *BlobStore) GetBlobFileInCore(fileID uint32) (*BlobFile, error)

func (*BlobStore) GetDelObjects

func (s *BlobStore) GetDelObjects(fileId uint32) (objects []uint64)

func (*BlobStore) GetLastOid

func (s *BlobStore) GetLastOid(fileId uint32) (objectId uint64, err error)

func (*BlobStore) GetObject

func (s *BlobStore) GetObject(fileId uint32, objectId uint64) (o *Object, err error)

func (*BlobStore) GetStoreBlobFileCount

func (s *BlobStore) GetStoreBlobFileCount() (files int, err error)

func (*BlobStore) GetUnAvailBlobFile

func (s *BlobStore) GetUnAvailBlobFile() (blobfileId int, err error)

func (*BlobStore) GetUnAvailChanLen

func (s *BlobStore) GetUnAvailChanLen() (chanLen int)

func (*BlobStore) GetWatermark

func (s *BlobStore) GetWatermark(fileId uint64) (blobfileInfo *FileInfo, err error)

func (*BlobStore) IsReadyToCompact

func (s *BlobStore) IsReadyToCompact(blobfileID, thresh int) (isready bool, fileBytes, deleteBytes uint64, deletePercent float64)

make sure blobfileID is valid

func (*BlobStore) MarkDelete

func (s *BlobStore) MarkDelete(fileId uint32, offset, size int64) error

func (*BlobStore) MoveBlobFileToUnavailChan

func (s *BlobStore) MoveBlobFileToUnavailChan()

func (*BlobStore) PutAvailBlobFile

func (s *BlobStore) PutAvailBlobFile(blobfileId int)

func (*BlobStore) PutUnAvailBlobFile

func (s *BlobStore) PutUnAvailBlobFile(blobfileId int)

func (*BlobStore) Read

func (s *BlobStore) Read(fileId uint32, offset, size int64, nbuf []byte) (crc uint32, err error)

func (*BlobStore) Snapshot

func (s *BlobStore) Snapshot() ([]*proto.File, error)

func (*BlobStore) Sync

func (s *BlobStore) Sync(fileId uint32) (err error)

func (*BlobStore) SyncAll

func (s *BlobStore) SyncAll()

func (*BlobStore) UseSize

func (s *BlobStore) UseSize() (size int64)

func (*BlobStore) VerfiyObjects

func (s *BlobStore) VerfiyObjects(chunkid int, needVerify bool) (blobFileInfo *BlobFileInfo)

func (*BlobStore) Write

func (s *BlobStore) Write(fileId uint32, objectId uint64, size int64, data []byte, crc uint32) (err error)

func (*BlobStore) WriteDeleteDentry

func (s *BlobStore) WriteDeleteDentry(objectId uint64, blobfileId int, crc uint32) (err error)

type Extent

type Extent interface {
	// ID returns the identity value (extentId) of this extent entity.
	ID() uint64

	// Ino returns this inode ID of this extent block belong to.
	Ino() uint64

	// Close this extent and release FD.
	Close() error

	// InitToFS init extent data info filesystem. If entry file exist and overwrite is true,
	// this operation will clear all data of exist entry file and initialize extent header data.
	InitToFS(ino uint64, overwrite bool) error

	// RestoreFromFS restore entity data and status from entry file stored in filesystem.
	RestoreFromFS() error

	// Write data to extent.
	Write(data []byte, offset, size int64, crc uint32) (err error)

	// Read data from extent.
	Read(data []byte, offset, size int64) (crc uint32, err error)

	// Flush synchronize data to disk immediately.
	Flush() error

	// MarkDelete mark this extent as deleted.
	MarkDelete() error

	// IsMarkDelete test this extent if has been marked as delete.
	IsMarkDelete() bool

	// Size returns length of extent data exclude header.
	Size() (size int64)

	// ModTime returns the time when this extent was last modified.
	ModTime() time.Time

	// HeaderChecksum returns crc checksum value of extent header data
	// include inode data and block crc.
	HeaderChecksum() (crc uint32)
}

Extent is used to manage extent block file for extent store engine.

func NewExtentInCore

func NewExtentInCore(name string, extentId uint64) Extent

NewExtentInCore create and returns a new extent instance.

type ExtentCache

type ExtentCache interface {
	// Put extent object into cache.
	Put(extent Extent)

	// Get extent from cache with specified extent identity (extentId).
	Get(extentId uint64) (extent Extent, ok bool)

	// Del extent stored in cache this specified extent identity (extentId).
	Del(extentId uint64)

	// Size returns number of extents stored in this cache.
	Size() int

	// Flush synchronize extent stored in this cache to disk immediately.
	Flush()

	// Clear close and synchronize all extent stored in this cache and remove them from cache.
	Clear()
}

ExtentCache defined a cache for Extent entity for access improvement.

func NewExtentCache

func NewExtentCache(capacity int) ExtentCache

NewExtentCache create and returns a new ExtentCache instance.

type ExtentFilter

type ExtentFilter func(info *FileInfo) bool

type ExtentStore

type ExtentStore struct {
	// contains filtered or unexported fields
}

func NewExtentStore

func NewExtentStore(dataDir string, storeSize int) (s *ExtentStore, err error)

func (*ExtentStore) Close

func (s *ExtentStore) Close()

func (*ExtentStore) Create

func (s *ExtentStore) Create(extentId uint64, inode uint64, overwrite bool) (err error)

func (*ExtentStore) DeleteStore

func (s *ExtentStore) DeleteStore() (err error)

func (*ExtentStore) FlushDelete

func (s *ExtentStore) FlushDelete() (err error)

func (*ExtentStore) GetAllWatermark

func (s *ExtentStore) GetAllWatermark(filter ExtentFilter) (extents []*FileInfo, err error)

func (*ExtentStore) GetDelObjects

func (s *ExtentStore) GetDelObjects() (extents []uint64)

func (*ExtentStore) GetWatermark

func (s *ExtentStore) GetWatermark(extentId uint64, reload bool) (extentInfo *FileInfo, err error)

func (*ExtentStore) IsExistExtent

func (s *ExtentStore) IsExistExtent(extentId uint64) (exist bool)

func (*ExtentStore) MarkDelete

func (s *ExtentStore) MarkDelete(extentId uint64) (err error)

func (*ExtentStore) NextExtentId

func (s *ExtentStore) NextExtentId() (extentId uint64)

func (*ExtentStore) Read

func (s *ExtentStore) Read(extentId uint64, offset, size int64, nbuf []byte) (crc uint32, err error)

func (*ExtentStore) SnapShot

func (s *ExtentStore) SnapShot() (files []*proto.File, err error)

func (*ExtentStore) Sync

func (s *ExtentStore) Sync(extentId uint64) (err error)

func (*ExtentStore) UpdateBaseExtentId

func (s *ExtentStore) UpdateBaseExtentId(id uint64) (err error)

func (*ExtentStore) UsedSize

func (s *ExtentStore) UsedSize() (size int64)

func (*ExtentStore) Write

func (s *ExtentStore) Write(extentId uint64, offset, size int64, data []byte, crc uint32) (err error)

type FileInfo

type FileInfo struct {
	FileId      int       `json:"fileId"`
	Inode       uint64    `json:"ino"`
	Size        uint64    `json:"size"`
	Crc         uint32    `json:"crc"`
	Deleted     bool      `json:"deleted"`
	ModTime     time.Time `json:"modTime"`
	Source      string    `json:"src"`
	MemberIndex int
}

func (*FileInfo) FromExtent

func (ei *FileInfo) FromExtent(extent Extent)

func (*FileInfo) String

func (ei *FileInfo) String() (m string)

type Object

type Object struct {
	Oid    uint64
	Offset uint64
	Size   uint32
	Crc    uint32
}

func (*Object) Check

func (o *Object) Check(offset uint64, size, crc uint32) bool

func (*Object) IsIdentical

func (o *Object) IsIdentical(that *Object) bool

func (Object) Less

func (o Object) Less(than btree.Item) bool

func (*Object) Marshal

func (o *Object) Marshal(out []byte)

func (*Object) Unmarshal

func (o *Object) Unmarshal(in []byte)

type ObjectInfo

type ObjectInfo struct {
	Object        *Object
	SuccessVerify bool
	ErrorMesg     string
}

type ObjectTree

type ObjectTree struct {
	// contains filtered or unexported fields
}

func NewObjectTree

func NewObjectTree(f *os.File) *ObjectTree

func (*ObjectTree) FileBytes

func (tree *ObjectTree) FileBytes() uint64

func (*ObjectTree) Load

func (tree *ObjectTree) Load() (maxOid uint64, err error)

Needle map in this function is not protected, so callers should guarantee there is no write and delete operations on this needle map

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL