Documentation ¶
Overview ¶
nolint
Index ¶
- Constants
- Variables
- func BackGroundReaderWait(n int) error
- func CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error)
- func CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error
- func CreateTime(name string) (*time.Time, error)
- func EnableMmapRead(en bool)
- func EnableReadDataCache(en uint64)
- func EnableReadMetaCache(en uint64)
- func EncodeObsPath(endpoint, bucket, path, ak, sk string) string
- func Fadvise(fd int, offset int64, length int64, advice int) (err error)
- func Fdatasync(file File) (err error)
- func Glob(pattern string) ([]string, error)
- func InitWriterPool(size int)
- func IsObsFile(path string) (bool, error)
- func MUnmap(data []byte) error
- func Mkdir(path string, perm os.FileMode, opt ...FSOption) error
- func MkdirAll(path string, perm os.FileMode, opt ...FSOption) error
- func Mmap(fd int, offset int64, length int) (data []byte, err error)
- func NewFileReader(f File, lock *string) *fileReader
- func NewFileWriter(lw NameReadWriterCloser, bufferSize int, lockPath *string) *fileWriter
- func PutObsClient(conf *obsConf, client ObsClient)
- func ReadDir(dirname string) ([]os.FileInfo, error)
- func ReadFile(filename string, opt ...FSOption) ([]byte, error)
- func RecoverLease(lock string) error
- func Remove(name string, opt ...FSOption) error
- func RemoveAll(path string, opt ...FSOption) error
- func RenameFile(oldPath, newPath string, opt ...FSOption) error
- func SetBackgroundReadLimiter(limiter int)
- func SetPageSize(confPageSize string)
- func Stat(name string) (os.FileInfo, error)
- func Truncate(name string, size int64) error
- func WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error
- type BasicFileReader
- type BasicFileWriter
- type FSOption
- type File
- func Create(name string, opt ...FSOption) (File, error)
- func CreateV1(name string, opt ...FSOption) (File, error)
- func CreateV2(name string, opt ...FSOption) (File, error)
- func Open(name string, opt ...FSOption) (File, error)
- func OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error)
- type FileLockOption
- type FilePriorityOption
- type FileWriter
- type FsType
- type LimitWriter
- type Limiter
- type MetaWriter
- type NameReadWriterCloser
- type ObsClient
- type Range
- type RangeRequest
- type VFS
Constants ¶
const ( FADV_NORMAL = 0x0 FADV_RANDOM = 0x1 FADV_SEQUENTIAL = 0x2 FADV_WILLNEED = 0x3 FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 )
const ( DefaultWriterBufferSize = 1024 * 1024 DefaultBufferSize = 256 * 1024 )
const ( ObsClientCacheSize = 1024 ObsClientCacheTTL = 24 * time.Hour )
const ( KeyAK = "AK" KeySK = "SK" KeyBucket = "BUCKET" ObsReadRetryTimes = 3 )
Variables ¶
var ( IO_PRIORITY_ULTRA_HIGH = 0 IO_PRIORITY_HIGH = 1 IO_PRIORITY_NORMAL = 2 IO_PRIORITY_LOW = 3 IO_PRIORITY_LOW_READ = 4 )
var (
BackgroundReadLimiter = NewLimiter(64*1024*1024, 64*1024*1024)
)
var MmapEn = false
var ObsSingleRequestSize int64 = 9 * 1024 * 1024
var ReadDataCacheEn = false
var ReadMetaCacheEn = false
Functions ¶
func BackGroundReaderWait ¶ added in v1.1.0
func CopyFile ¶
CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs. the optional opt is: (FileLockOption,FilePriorityOption)
func CopyFileFromDFVToOBS ¶ added in v1.2.0
CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs
func EnableMmapRead ¶ added in v1.1.0
func EnableMmapRead(en bool)
func EnableReadDataCache ¶ added in v1.2.0
func EnableReadDataCache(en uint64)
func EnableReadMetaCache ¶ added in v1.2.0
func EnableReadMetaCache(en uint64)
func EncodeObsPath ¶ added in v1.2.0
func Glob ¶
Glob returns the names of all files matching pattern or nil if there is no matching file.
func InitWriterPool ¶ added in v1.1.0
func InitWriterPool(size int)
func IsObsFile ¶ added in v1.2.0
IsObsFile determines whether a file is stored in OBS when use streamfs
func Mkdir ¶
Mkdir creates a directory named path, it's parents directory must exist. the optional opt is: FileLockOption
func MkdirAll ¶
MkdirAll creates a directory named path, along with any necessary parents the optional opt is: FileLockOption
func NewFileReader ¶ added in v1.1.0
func NewFileWriter ¶ added in v1.1.0
func NewFileWriter(lw NameReadWriterCloser, bufferSize int, lockPath *string) *fileWriter
func PutObsClient ¶ added in v1.2.0
func PutObsClient(conf *obsConf, client ObsClient)
func ReadDir ¶
ReadDir reads the directory named by dirname and returns a list of fs.FileInfo for the directory's contents, sorted by filename.
func ReadFile ¶
ReadFile reads the file named by filename and returns the contents. the optional opt is: FilePriorityOption
func RecoverLease ¶ added in v1.0.0
func Remove ¶
Remove removes the named file or (empty) directory. the optional opt is: FileLockOption
func RemoveAll ¶
RemoveAll removes path and any children it contains. the optional opt is: FileLockOption
func RenameFile ¶
RenameFile renames (moves) oldPath to newPath. If newPath already exists and is not a directory, Rename replaces it. the optional opt is: FileLockOption
func SetBackgroundReadLimiter ¶ added in v1.1.0
func SetBackgroundReadLimiter(limiter int)
func SetPageSize ¶ added in v1.2.0
func SetPageSize(confPageSize string)
Types ¶
type BasicFileReader ¶ added in v1.1.0
type BasicFileReader interface { Name() string ReadAt(off int64, size uint32, dst *[]byte, ioPriority int) ([]byte, error) StreamReadBatch(off, length []int64, c chan *request.StreamReader, limit int) Rename(newName string) error RenameOnObs(newName string) error ReOpen() error IsMmapRead() bool IsOpen() bool FreeFileHandle() error Close() error }
type BasicFileWriter ¶ added in v1.1.0
type File ¶
type File interface { io.Closer io.Reader io.Seeker io.Writer io.ReaderAt Name() string Truncate(size int64) error Sync() error Stat() (os.FileInfo, error) SyncUpdateLength() error Fd() uintptr StreamReadBatch([]int64, []int64, int64, chan *request.StreamReader, int) }
func Create ¶
Create creates or truncates the named file. If the file already exists, it is truncated. If the file does not exist, it is created with mode 0666 the optional opt is: (FileLockOption,FilePriorityOption)
type FileLockOption ¶
type FileLockOption string
func (FileLockOption) Parameter ¶
func (opt FileLockOption) Parameter() interface{}
type FilePriorityOption ¶
type FilePriorityOption int
func (FilePriorityOption) Parameter ¶
func (opt FilePriorityOption) Parameter() interface{}
type FileWriter ¶ added in v1.1.0
type FileWriter interface { WriteData(b []byte) (int, error) WriteChunkMeta(b []byte) (int, error) Close() error DataSize() int64 ChunkMetaSize() int64 ChunkMetaBlockSize() int64 GetFileWriter() BasicFileWriter AppendChunkMetaToData() error SwitchMetaBuffer() (int, error) MetaDataBlocks(dst [][]byte) [][]byte Name() string }
type LimitWriter ¶ added in v1.1.0
type LimitWriter struct {
// contains filtered or unexported fields
}
func (*LimitWriter) Close ¶ added in v1.1.0
func (w *LimitWriter) Close() error
func (*LimitWriter) Name ¶ added in v1.1.0
func (w *LimitWriter) Name() string
type Limiter ¶ added in v1.1.0
type Limiter interface { SetBurst(newBurst int) SetLimit(newLimit rate.Limit) WaitN(ctx context.Context, n int) (err error) Limit() rate.Limit Burst() int }
func NewLimiter ¶ added in v1.1.0
type MetaWriter ¶ added in v1.2.0
type MetaWriter interface { WriteMetaIndex(b []byte) (int, error) WritePrimaryKey(b []byte) (int, error) WritePrimaryKeyMeta(b []byte) (int, error) WriteBloomFilter(bfIdx int, b []byte) (int, error) GetMetaIndexSize() int64 GetPrimaryKeySize() int64 GetPrimaryKeyMetaSize() int64 GetBloomFilterSize(bfIdx int) int64 GetPrimaryKeyHandler() File GetPrimaryKeyMetaHandler() File Close() error }
type NameReadWriterCloser ¶ added in v1.1.0
type NameReadWriterCloser interface { Name() string io.ReadWriteCloser }
func NewLimitWriter ¶ added in v1.1.0
func NewLimitWriter(w NameReadWriterCloser, l Limiter) NameReadWriterCloser
type ObsClient ¶ added in v1.2.0
type ObsClient interface { ListBuckets(input *obs.ListBucketsInput) (output *obs.ListBucketsOutput, err error) ListObjects(input *obs.ListObjectsInput) (output *obs.ListObjectsOutput, err error) GetObject(input *obs.GetObjectInput) (output *obs.GetObjectOutput, err error) DeleteObject(input *obs.DeleteObjectInput) (output *obs.DeleteObjectOutput, err error) DeleteObjects(input *obs.DeleteObjectsInput) (output *obs.DeleteObjectsOutput, err error) ModifyObject(input *obs.ModifyObjectInput) (output *obs.ModifyObjectOutput, err error) PutObject(input *obs.PutObjectInput) (output *obs.PutObjectOutput, err error) GetObjectMetadata(input *obs.GetObjectMetadataInput) (output *obs.GetObjectMetadataOutput, err error) Do(r *http.Request) (*http.Response, error) }
func GetObsClient ¶ added in v1.2.0
type RangeRequest ¶ added in v1.2.0
type RangeRequest struct {
// contains filtered or unexported fields
}
func NewObsReadRequest ¶ added in v1.2.0
func NewObsRetryReadRequest ¶ added in v1.2.0
func NewObsRetryReadRequest(retryRange map[int64]int64, obsRangeSize int) []*RangeRequest
func NewRangeRequest ¶ added in v1.2.0
func NewRangeRequest() *RangeRequest
func (*RangeRequest) AddReadAction ¶ added in v1.2.0
func (r *RangeRequest) AddReadAction(offset int64, length int64)
func (*RangeRequest) GetRangeString ¶ added in v1.2.0
func (r *RangeRequest) GetRangeString() string
type VFS ¶
type VFS interface { // Open opens the named file with specified options. // the optional opt is: (FileLockOption,FilePriorityOption) Open(name string, opt ...FSOption) (File, error) // OpenFile opens the named file with specified flag and other options. // the optional opt is: (FileLockOption,FilePriorityOption) OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error) // Create creates or truncates the named file. If the file already exists, it is truncated. // If the file does not exist, it is created with mode 0666 // the optional opt is: (FileLockOption,FilePriorityOption) Create(name string, opt ...FSOption) (File, error) CreateV1(name string, opt ...FSOption) (File, error) // CreateV2 create a new file in OBS when use streamfs CreateV2(name string, opt ...FSOption) (File, error) // Remove removes the named file or (empty) directory. // the optional opt is: FileLockOption Remove(name string, opt ...FSOption) error // RemoveAll removes path and any children it contains. // the optional opt is: FileLockOption RemoveAll(path string, opt ...FSOption) error // Mkdir creates a directory named path, it's parents directory must exist. // the optional opt is: FileLockOption Mkdir(path string, perm os.FileMode, opt ...FSOption) error // MkdirAll creates a directory named path, along with any necessary parents // the optional opt is: FileLockOption MkdirAll(path string, perm os.FileMode, opt ...FSOption) error // ReadDir reads the directory named by dirname and returns // a list of fs.FileInfo for the directory's contents, sorted by filename. ReadDir(dirname string) ([]os.FileInfo, error) // Glob returns the names of all files matching pattern or nil if there is no matching file. Glob(pattern string) ([]string, error) // RenameFile renames (moves) oldPath to newPath. // If newPath already exists and is not a directory, Rename replaces it. // the optional opt is: FileLockOption RenameFile(oldPath, newPath string, opt ...FSOption) error // Stat returns a FileInfo describing the named file. Stat(name string) (os.FileInfo, error) // WriteFile writes data to a file named by filename. // If the file does not exist, WriteFile creates it with permissions perm // the optional opt is: (FileLockOption,FilePriorityOption) WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error // ReadFile reads the file named by filename and returns the contents. // the optional opt is: FilePriorityOption ReadFile(filename string, opt ...FSOption) ([]byte, error) // CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs. // the optional opt is: (FileLockOption,FilePriorityOption) CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error) CreateTime(name string) (*time.Time, error) // Truncate changes the size of the file to size. // the optional opt is: (FileLockOption) Truncate(name string, size int64, opt ...FSOption) error // IsObsFile determines whether a file is stored in OBS when use streamfs IsObsFile(path string) (bool, error) // CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error }