storage

package

v0.1.1 Latest Latest Go to latest Published: Dec 16, 2020 License: GPL-3.0 Imports: 13 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/ToolPackage/fse

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
type EntrySequenceFile
- func NewEntrySequenceFile(path string, mode int) (*EntrySequenceFile, error)
- func (f *EntrySequenceFile) Close() error
- func (f *EntrySequenceFile) Delete() error
- func (f *EntrySequenceFile) ReadEntry() ([]byte, error)
- func (f *EntrySequenceFile) WriteEntry(entry []byte) error
type File
- func (f *File) OpenStream() io.Reader
type FileChunk
- func (c *FileChunk) Validate() bool
type FileDataReader
- func (r *FileDataReader) Read(p []byte) (n int, err error)
type FileStorage
- func NewFileStorage() *FileStorage
- func (fs *FileStorage) DeleteFile(id string) bool
- func (fs *FileStorage) Destroy()
- func (fs *FileStorage) GetAllFiles() []*File
- func (fs *FileStorage) GetFile(id string) (f *File, ok bool)
- func (fs *FileStorage) SaveFile(fileName string, contentType string, reader io.Reader) (*File, error)
type LRUPartitionCache
- func (c *LRUPartitionCache) Destroy()
- func (c *LRUPartitionCache) GetChunk(id PartitionId, onFail func() FileChunk) FileChunk
type PartitionCache
type PartitionId
type Partitions
type SequentialFile
- func NewSequentialFile(path string, chunkSize uint16, chunkCap uint16) (s *SequentialFile, err error)
- func (s *SequentialFile) AppendChunk(data []byte) (chunkId uint16, err error)
- func (s *SequentialFile) Close() (err error)
- func (s *SequentialFile) Delete() error
- func (s *SequentialFile) DeleteChunk(chunkId uint16) error
- func (s *SequentialFile) IsWritable() bool
- func (s *SequentialFile) ReadChunk(chunkId uint16) (chunk *FileChunk, err error)

Constants ¶

View Source

const (
	ReadMode = iota
	WriteMode
)

View Source

const MaxFileChunkDataSize = 64*1024 - 1 // (64kb = 65536) overflow uint16

all bytes

View Source

const MaxFileChunkNum = 8192

View Source

const MaxSequentialFileDataSize = 537010176 // almost 512MB, = (MaxFileChunkDataSize + SequentialFileChunkMetadataSize) * MaxFileChunkNum

View Source

const SequentialFileChunkMetadataSize = 18 // md5 + chunkSize

View Source

const SequentialFileMetadataSize = 8 // chunkSize + chunkCap + chunkNum

Variables ¶

View Source

var (
	DataOutOfChunkError     = errors.New("not enough space to write chunk data")
	DataOutOfFileError      = errors.New("not enough space to write data to file")
	InvalidOperationError   = errors.New("invalid operation")
	EntryTooLargeError      = errors.New("entry is too large")
	InvalidPartitionIdError = errors.New("invalid partition id")
	InvalidChunkIdError     = errors.New("invalid chunk id")
	InvalidRetValue         = errors.New("invalid ret value")
	PartitionNumLimitError  = errors.New("partition num limit error")
	DuplicateFileNameError  = errors.New("duplicate file name error")
)

View Source

var S = NewFileStorage()

Functions ¶

This section is empty.

Types ¶

type EntrySequenceFile ¶

type EntrySequenceFile struct {
	// contains filtered or unexported fields
}

func NewEntrySequenceFile ¶

func NewEntrySequenceFile(path string, mode int) (*EntrySequenceFile, error)

func (*EntrySequenceFile) Close ¶

func (f *EntrySequenceFile) Close() error

func (*EntrySequenceFile) Delete ¶

func (f *EntrySequenceFile) Delete() error

func (*EntrySequenceFile) ReadEntry ¶

func (f *EntrySequenceFile) ReadEntry() ([]byte, error)

InvalidOperationError io.EOF others InvalidRetValue

func (*EntrySequenceFile) WriteEntry ¶

func (f *EntrySequenceFile) WriteEntry(entry []byte) error

type File ¶

type File struct {
	Id          string // 32
	Name        string // 128
	Size        uint32 // 8
	ContentType string // 32
	CreatedAt   int64  // 8
	Partitions  Partitions
	// contains filtered or unexported fields
}

func (*File) OpenStream ¶

func (f *File) OpenStream() io.Reader

type FileChunk ¶

type FileChunk struct {
	// contains filtered or unexported fields
}

func (*FileChunk) Validate ¶

func (c *FileChunk) Validate() bool

type FileDataReader ¶

type FileDataReader struct {
	// contains filtered or unexported fields
}

func (*FileDataReader) Read ¶

func (r *FileDataReader) Read(p []byte) (n int, err error)

type FileStorage ¶

type FileStorage struct {
	// contains filtered or unexported fields
}

TODO: 并发

func NewFileStorage ¶

func NewFileStorage() *FileStorage

func (*FileStorage) DeleteFile ¶

func (fs *FileStorage) DeleteFile(id string) bool

func (*FileStorage) Destroy ¶

func (fs *FileStorage) Destroy()

func (*FileStorage) GetAllFiles ¶

func (fs *FileStorage) GetAllFiles() []*File

func (*FileStorage) GetFile ¶

func (fs *FileStorage) GetFile(id string) (f *File, ok bool)

func (*FileStorage) SaveFile ¶

func (fs *FileStorage) SaveFile(fileName string, contentType string, reader io.Reader) (*File, error)

DuplicateFileNameError write chunk error PartitionNumLimitError read input error

type LRUPartitionCache ¶

type LRUPartitionCache struct {
	// contains filtered or unexported fields
}

func (*LRUPartitionCache) Destroy ¶

func (c *LRUPartitionCache) Destroy()

func (*LRUPartitionCache) GetChunk ¶

func (c *LRUPartitionCache) GetChunk(id PartitionId, onFail func() FileChunk) FileChunk

type PartitionCache ¶

type PartitionCache interface {
	GetChunk(id PartitionId, onFail func() FileChunk) FileChunk
	Destroy()
}

type PartitionId ¶

type PartitionId uint32

PartitionId = sequential file id + file chunk id

type Partitions ¶

type Partitions []PartitionId

type SequentialFile ¶

type SequentialFile struct {
	// contains filtered or unexported fields
}

func NewSequentialFile ¶

func NewSequentialFile(path string, chunkSize uint16, chunkCap uint16) (s *SequentialFile, err error)

Open or create sequential file, if target file exists, chunkSize and chunkNum will be read from file's metadata instead of using function arguments.

func (*SequentialFile) AppendChunk ¶

func (s *SequentialFile) AppendChunk(data []byte) (chunkId uint16, err error)

Append data to new chunk DataOutOfFileError, DataOutOfChunkError, seek failure, write failure, InvalidRetValue TODO: bug 写文件空间不足

func (*SequentialFile) Close ¶

func (s *SequentialFile) Close() (err error)

func (*SequentialFile) Delete ¶

func (s *SequentialFile) Delete() error

func (*SequentialFile) DeleteChunk ¶

func (s *SequentialFile) DeleteChunk(chunkId uint16) error

func (*SequentialFile) IsWritable ¶

func (s *SequentialFile) IsWritable() bool

func (*SequentialFile) ReadChunk ¶

func (s *SequentialFile) ReadChunk(chunkId uint16) (chunk *FileChunk, err error)

read block at specified position in the file

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL