fileops

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 29, 2024 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Overview

nolint

Index

Constants

View Source
const (
	FADV_NORMAL     = 0x0
	FADV_RANDOM     = 0x1
	FADV_SEQUENTIAL = 0x2
	FADV_WILLNEED   = 0x3
	FADV_DONTNEED   = 0x4
	FADV_NOREUSE    = 0x5
)
View Source
const (
	DefaultWriterBufferSize = 1024 * 1024
	DefaultBufferSize       = 256 * 1024
)
View Source
const (
	ObsClientCacheSize = 1024
	ObsClientCacheTTL  = 24 * time.Hour
)
View Source
const (
	KeyAK             = "AK"
	KeySK             = "SK"
	KeyBucket         = "BUCKET"
	ObsReadRetryTimes = 3
)

Variables

View Source
var (
	IO_PRIORITY_ULTRA_HIGH = 0
	IO_PRIORITY_HIGH       = 1
	IO_PRIORITY_NORMAL     = 2
	IO_PRIORITY_LOW        = 3
	IO_PRIORITY_LOW_READ   = 4
)
View Source
var (
	BackgroundReadLimiter = NewLimiter(64*1024*1024, 64*1024*1024)
)
View Source
var MmapEn = false
View Source
var ObsSingleRequestSize int64 = 9 * 1024 * 1024
View Source
var ReadDataCacheEn = false
View Source
var ReadMetaCacheEn = false

Functions

func BackGroundReaderWait added in v1.1.0

func BackGroundReaderWait(n int) error

func CopyFile

func CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error)

CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs. the optional opt is: (FileLockOption,FilePriorityOption)

func CopyFileFromDFVToOBS added in v1.2.0

func CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error

CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs

func CreateTime

func CreateTime(name string) (*time.Time, error)

func EnableMmapRead added in v1.1.0

func EnableMmapRead(en bool)

func EnableReadDataCache added in v1.2.0

func EnableReadDataCache(en uint64)

func EnableReadMetaCache added in v1.2.0

func EnableReadMetaCache(en uint64)

func EncodeObsPath added in v1.2.0

func EncodeObsPath(endpoint, bucket, path, ak, sk string) string

func Fadvise added in v1.0.0

func Fadvise(fd int, offset int64, length int64, advice int) (err error)

func Fdatasync added in v1.0.0

func Fdatasync(file File) (err error)

func Glob

func Glob(pattern string) ([]string, error)

Glob returns the names of all files matching pattern or nil if there is no matching file.

func InitWriterPool added in v1.1.0

func InitWriterPool(size int)

func IsObsFile added in v1.2.0

func IsObsFile(path string) (bool, error)

IsObsFile determines whether a file is stored in OBS when use streamfs

func MUnmap

func MUnmap(data []byte) error

func Mkdir

func Mkdir(path string, perm os.FileMode, opt ...FSOption) error

Mkdir creates a directory named path, it's parents directory must exist. the optional opt is: FileLockOption

func MkdirAll

func MkdirAll(path string, perm os.FileMode, opt ...FSOption) error

MkdirAll creates a directory named path, along with any necessary parents the optional opt is: FileLockOption

func Mmap

func Mmap(fd int, offset int64, length int) (data []byte, err error)

func NewFileReader added in v1.1.0

func NewFileReader(f File, lock *string) *fileReader

func NewFileWriter added in v1.1.0

func NewFileWriter(lw NameReadWriterCloser, bufferSize int, lockPath *string) *fileWriter

func PutObsClient added in v1.2.0

func PutObsClient(conf *obsConf, client ObsClient)

func ReadDir

func ReadDir(dirname string) ([]os.FileInfo, error)

ReadDir reads the directory named by dirname and returns a list of fs.FileInfo for the directory's contents, sorted by filename.

func ReadFile

func ReadFile(filename string, opt ...FSOption) ([]byte, error)

ReadFile reads the file named by filename and returns the contents. the optional opt is: FilePriorityOption

func RecoverLease added in v1.0.0

func RecoverLease(lock string) error

func Remove

func Remove(name string, opt ...FSOption) error

Remove removes the named file or (empty) directory. the optional opt is: FileLockOption

func RemoveAll

func RemoveAll(path string, opt ...FSOption) error

RemoveAll removes path and any children it contains. the optional opt is: FileLockOption

func RenameFile

func RenameFile(oldPath, newPath string, opt ...FSOption) error

RenameFile renames (moves) oldPath to newPath. If newPath already exists and is not a directory, Rename replaces it. the optional opt is: FileLockOption

func SetBackgroundReadLimiter added in v1.1.0

func SetBackgroundReadLimiter(limiter int)

func SetPageSize added in v1.2.0

func SetPageSize(confPageSize string)

func Stat

func Stat(name string) (os.FileInfo, error)

Stat returns a FileInfo describing the named file.

func Truncate

func Truncate(name string, size int64) error

func WriteFile

func WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error

WriteFile writes data to a file named by filename. If the file does not exist, WriteFile creates it with permissions perm the optional opt is: (FileLockOption,FilePriorityOption)

Types

type BasicFileReader added in v1.1.0

type BasicFileReader interface {
	Name() string
	ReadAt(off int64, size uint32, dst *[]byte, ioPriority int) ([]byte, error)
	StreamReadBatch(off, length []int64, c chan *request.StreamReader, limit int)
	Rename(newName string) error
	RenameOnObs(newName string) error
	ReOpen() error
	IsMmapRead() bool
	IsOpen() bool
	FreeFileHandle() error
	Close() error
}

type BasicFileWriter added in v1.1.0

type BasicFileWriter interface {
	Write(b []byte) (int, error)
	Close() error
	Size() int
	Reset(lw NameReadWriterCloser)
	Bytes() []byte
	CopyTo(w io.Writer) (int, error)
	SwitchMetaBuffer()
	MetaDataBlocks(dst [][]byte) [][]byte
	GetWriter() *bufio.Writer
}

type FSOption

type FSOption interface {
	Parameter() interface{}
}

type File

type File interface {
	io.Closer
	io.Reader
	io.Seeker
	io.Writer
	io.ReaderAt
	Name() string
	Truncate(size int64) error
	Sync() error
	Stat() (os.FileInfo, error)
	SyncUpdateLength() error
	Fd() uintptr
	StreamReadBatch([]int64, []int64, int64, chan *request.StreamReader, int)
}

func Create

func Create(name string, opt ...FSOption) (File, error)

Create creates or truncates the named file. If the file already exists, it is truncated. If the file does not exist, it is created with mode 0666 the optional opt is: (FileLockOption,FilePriorityOption)

func CreateV1 added in v1.0.0

func CreateV1(name string, opt ...FSOption) (File, error)

func CreateV2 added in v1.2.0

func CreateV2(name string, opt ...FSOption) (File, error)

CreateV2 create a new file in OBS when use streamfs

func Open

func Open(name string, opt ...FSOption) (File, error)

Open opens the named file with specified options. the optional opt is: (FileLockOption,FilePriorityOption)

func OpenFile

func OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error)

OpenFile opens the named file with specified flag and other options. the optional opt is: (FileLockOption,FilePriorityOption)

type FileLockOption

type FileLockOption string

func (FileLockOption) Parameter

func (opt FileLockOption) Parameter() interface{}

type FilePriorityOption

type FilePriorityOption int

func (FilePriorityOption) Parameter

func (opt FilePriorityOption) Parameter() interface{}

type FileWriter added in v1.1.0

type FileWriter interface {
	WriteData(b []byte) (int, error)
	WriteChunkMeta(b []byte) (int, error)
	Close() error
	DataSize() int64
	ChunkMetaSize() int64
	ChunkMetaBlockSize() int64
	GetFileWriter() BasicFileWriter
	AppendChunkMetaToData() error
	SwitchMetaBuffer() (int, error)
	MetaDataBlocks(dst [][]byte) [][]byte
	Name() string
}

type FsType added in v1.2.0

type FsType uint32
const (
	Unknown FsType = 0
	Local   FsType = 1
	Obs     FsType = 2
	Hdfs    FsType = 3

	ObsPrefix  = "obs://"
	HdfsPrefix = "hdfs://"
)

func GetFsType added in v1.2.0

func GetFsType(path string) FsType

type LimitWriter added in v1.1.0

type LimitWriter struct {
	// contains filtered or unexported fields
}

func (*LimitWriter) Close added in v1.1.0

func (w *LimitWriter) Close() error

func (*LimitWriter) Name added in v1.1.0

func (w *LimitWriter) Name() string

func (*LimitWriter) Read added in v1.1.0

func (w *LimitWriter) Read(b []byte) (int, error)

func (*LimitWriter) Write added in v1.1.0

func (w *LimitWriter) Write(p []byte) (int, error)

type Limiter added in v1.1.0

type Limiter interface {
	SetBurst(newBurst int)
	SetLimit(newLimit rate.Limit)
	WaitN(ctx context.Context, n int) (err error)
	Limit() rate.Limit
	Burst() int
}

func NewLimiter added in v1.1.0

func NewLimiter(bytesPerSec, burstLimit int) Limiter

type MetaWriter added in v1.2.0

type MetaWriter interface {
	WriteMetaIndex(b []byte) (int, error)
	WritePrimaryKey(b []byte) (int, error)
	WritePrimaryKeyMeta(b []byte) (int, error)
	WriteBloomFilter(bfIdx int, b []byte) (int, error)
	GetMetaIndexSize() int64
	GetPrimaryKeySize() int64
	GetPrimaryKeyMetaSize() int64
	GetBloomFilterSize(bfIdx int) int64
	GetPrimaryKeyHandler() File
	GetPrimaryKeyMetaHandler() File
	Close() error
}

type NameReadWriterCloser added in v1.1.0

type NameReadWriterCloser interface {
	Name() string
	io.ReadWriteCloser
}

func NewLimitWriter added in v1.1.0

type ObsClient added in v1.2.0

type ObsClient interface {
	ListBuckets(input *obs.ListBucketsInput) (output *obs.ListBucketsOutput, err error)
	ListObjects(input *obs.ListObjectsInput) (output *obs.ListObjectsOutput, err error)
	GetObject(input *obs.GetObjectInput) (output *obs.GetObjectOutput, err error)
	DeleteObject(input *obs.DeleteObjectInput) (output *obs.DeleteObjectOutput, err error)
	DeleteObjects(input *obs.DeleteObjectsInput) (output *obs.DeleteObjectsOutput, err error)
	ModifyObject(input *obs.ModifyObjectInput) (output *obs.ModifyObjectOutput, err error)
	PutObject(input *obs.PutObjectInput) (output *obs.PutObjectOutput, err error)
	GetObjectMetadata(input *obs.GetObjectMetadataInput) (output *obs.GetObjectMetadataOutput, err error)
	Do(r *http.Request) (*http.Response, error)
}

func GetObsClient added in v1.2.0

func GetObsClient(conf *obsConf) (ObsClient, error)

type Range added in v1.2.0

type Range struct {
	// contains filtered or unexported fields
}

func NewRange added in v1.2.0

func NewRange(start, end int64) *Range

func (Range) String added in v1.2.0

func (r Range) String() string

type RangeRequest added in v1.2.0

type RangeRequest struct {
	// contains filtered or unexported fields
}

func NewObsReadRequest added in v1.2.0

func NewObsReadRequest(offs []int64, sizes []int64, minBlockSize int64, obsRangeSize int) ([]*RangeRequest, error)

func NewObsRetryReadRequest added in v1.2.0

func NewObsRetryReadRequest(retryRange map[int64]int64, obsRangeSize int) []*RangeRequest

func NewRangeRequest added in v1.2.0

func NewRangeRequest() *RangeRequest

func (*RangeRequest) AddReadAction added in v1.2.0

func (r *RangeRequest) AddReadAction(offset int64, length int64)

func (*RangeRequest) GetRangeString added in v1.2.0

func (r *RangeRequest) GetRangeString() string

type VFS

type VFS interface {
	// Open opens the named file with specified options.
	// the optional opt is: (FileLockOption,FilePriorityOption)
	Open(name string, opt ...FSOption) (File, error)
	// OpenFile opens the named file with specified flag and other options.
	// the optional opt is: (FileLockOption,FilePriorityOption)
	OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error)
	// Create creates or truncates the named file. If the file already exists, it is truncated.
	// If the file does not exist, it is created with mode 0666
	// the optional opt is: (FileLockOption,FilePriorityOption)
	Create(name string, opt ...FSOption) (File, error)

	CreateV1(name string, opt ...FSOption) (File, error)

	// CreateV2 create a new file in OBS when use streamfs
	CreateV2(name string, opt ...FSOption) (File, error)

	// Remove removes the named file or (empty) directory.
	// the optional opt is: FileLockOption
	Remove(name string, opt ...FSOption) error
	// RemoveAll removes path and any children it contains.
	// the optional opt is: FileLockOption
	RemoveAll(path string, opt ...FSOption) error
	// Mkdir creates a directory named path, it's parents directory must exist.
	// the optional opt is: FileLockOption
	Mkdir(path string, perm os.FileMode, opt ...FSOption) error
	// MkdirAll creates a directory named path, along with any necessary parents
	// the optional opt is: FileLockOption
	MkdirAll(path string, perm os.FileMode, opt ...FSOption) error
	// ReadDir reads the directory named by dirname and returns
	// a list of fs.FileInfo for the directory's contents, sorted by filename.
	ReadDir(dirname string) ([]os.FileInfo, error)
	// Glob returns the names of all files matching pattern or nil if there is no matching file.
	Glob(pattern string) ([]string, error)
	// RenameFile renames (moves) oldPath to newPath.
	// If newPath already exists and is not a directory, Rename replaces it.
	// the optional opt is: FileLockOption
	RenameFile(oldPath, newPath string, opt ...FSOption) error

	// Stat returns a FileInfo describing the named file.
	Stat(name string) (os.FileInfo, error)

	// WriteFile writes data to a file named by filename.
	// If the file does not exist, WriteFile creates it with permissions perm
	// the optional opt is: (FileLockOption,FilePriorityOption)
	WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error
	// ReadFile reads the file named by filename and returns the contents.
	// the optional opt is: FilePriorityOption
	ReadFile(filename string, opt ...FSOption) ([]byte, error)
	// CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs.
	// the optional opt is: (FileLockOption,FilePriorityOption)
	CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error)

	CreateTime(name string) (*time.Time, error)

	// Truncate changes the size of the file to size.
	// the optional opt is: (FileLockOption)
	Truncate(name string, size int64, opt ...FSOption) error

	// IsObsFile determines whether a file is stored in OBS when use streamfs
	IsObsFile(path string) (bool, error)

	// CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs
	CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error
}

func GetFs added in v1.2.0

func GetFs(t FsType) VFS

func NewFS

func NewFS() VFS

func NewObsFs added in v1.2.0

func NewObsFs() VFS

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL