Documentation ¶
Index ¶
- Constants
- Variables
- type BlockConfig
- type BlockStorage
- func (st *BlockStorage) Clean() error
- func (st *BlockStorage) Close() error
- func (st *BlockStorage) ReadBytes(key string) ([]byte, error)
- func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error)
- func (st *BlockStorage) Remove(key string) error
- func (st *BlockStorage) Stat(key string) (bool, error)
- func (st *BlockStorage) WalkKeys(opts WalkKeysOptions) error
- func (st *BlockStorage) WriteBytes(key string, value []byte) error
- func (st *BlockStorage) WriteStream(key string, r io.Reader) error
- type Compressor
- type DiskConfig
- type DiskStorage
- func (st *DiskStorage) Clean() error
- func (st *DiskStorage) Close() error
- func (st *DiskStorage) ReadBytes(key string) ([]byte, error)
- func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error)
- func (st *DiskStorage) Remove(key string) error
- func (st *DiskStorage) Stat(key string) (bool, error)
- func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error
- func (st *DiskStorage) WriteBytes(key string, value []byte) error
- func (st *DiskStorage) WriteStream(key string, r io.Reader) error
- type KeyTransform
- type Lock
- type MemoryStorage
- func (st *MemoryStorage) Clean() error
- func (st *MemoryStorage) Close() error
- func (st *MemoryStorage) ReadBytes(key string) ([]byte, error)
- func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error)
- func (st *MemoryStorage) Remove(key string) error
- func (st *MemoryStorage) Stat(key string) (bool, error)
- func (st *MemoryStorage) WalkKeys(opts WalkKeysOptions) error
- func (st *MemoryStorage) WriteBytes(key string, b []byte) error
- func (st *MemoryStorage) WriteStream(key string, r io.Reader) error
- type Storage
- type StorageEntry
- type WalkKeysOptions
Constants ¶
const LockFile = "store.lock"
LockFile is our standard lockfile name.
Variables ¶
var ( // ErrClosed is returned on operations on a closed storage ErrClosed = errors.New("store/storage: closed") // ErrNotFound is the error returned when a key cannot be found in storage ErrNotFound = errors.New("store/storage: key not found") // ErrAlreadyExist is the error returned when a key already exists in storage ErrAlreadyExists = errors.New("store/storage: key already exists") // ErrInvalidkey is the error returned when an invalid key is passed to storage ErrInvalidKey = errors.New("store/storage: invalid key") // ErrAlreadyLocked is returned on fail opening a storage lockfile ErrAlreadyLocked = errors.New("store/storage: storage lock already open") )
var DefaultBlockConfig = &BlockConfig{ BlockSize: 1024 * 16, WriteBufSize: 4096, Overwrite: false, Compression: NoCompression(), }
DefaultBlockConfig is the default BlockStorage configuration
var DefaultDiskConfig = &DiskConfig{ Overwrite: true, WriteBufSize: 4096, Transform: NopTransform(), Compression: NoCompression(), }
DefaultDiskConfig is the default DiskStorage configuration
Functions ¶
This section is empty.
Types ¶
type BlockConfig ¶
type BlockConfig struct { // BlockSize is the chunking size to use when splitting and storing blocks of data BlockSize int // ReadBufSize is the buffer size to use when reading node files ReadBufSize int // WriteBufSize is the buffer size to use when writing file streams (PutStream) WriteBufSize int // Overwrite allows overwriting values of stored keys in the storage Overwrite bool // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor }
BlockConfig defines options to be used when opening a BlockStorage
type BlockStorage ¶
type BlockStorage struct {
// contains filtered or unexported fields
}
BlockStorage is a Storage implementation that stores input data as chunks on a filesystem. Each value is chunked into blocks of configured size and these blocks are stored with name equal to their base64-encoded SHA256 hash-sum. A "node" file is finally created containing an array of hashes contained within this value
func OpenBlock ¶
func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error)
OpenBlock opens a BlockStorage instance for given folder path and configuration
func (*BlockStorage) Close ¶ added in v1.2.0
func (st *BlockStorage) Close() error
Close implements Storage.Close()
func (*BlockStorage) ReadBytes ¶
func (st *BlockStorage) ReadBytes(key string) ([]byte, error)
ReadBytes implements Storage.ReadBytes()
func (*BlockStorage) ReadStream ¶
func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error)
ReadStream implements Storage.ReadStream()
func (*BlockStorage) Remove ¶
func (st *BlockStorage) Remove(key string) error
Remove implements Storage.Remove()
func (*BlockStorage) Stat ¶
func (st *BlockStorage) Stat(key string) (bool, error)
Stat implements Storage.Stat()
func (*BlockStorage) WalkKeys ¶
func (st *BlockStorage) WalkKeys(opts WalkKeysOptions) error
WalkKeys implements Storage.WalkKeys()
func (*BlockStorage) WriteBytes ¶
func (st *BlockStorage) WriteBytes(key string, value []byte) error
WriteBytes implements Storage.WriteBytes()
func (*BlockStorage) WriteStream ¶
func (st *BlockStorage) WriteStream(key string, r io.Reader) error
WriteStream implements Storage.WriteStream()
type Compressor ¶
type Compressor interface { // Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader Reader(io.Reader) (io.ReadCloser, error) // Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer Writer(io.Writer) (io.WriteCloser, error) }
Compressor defines a means of compressing/decompressing values going into a key-value store
func GZipCompressor ¶
func GZipCompressor() Compressor
GZipCompressor returns a new Compressor that implements GZip at default compression level
func GZipCompressorLevel ¶
func GZipCompressorLevel(level int) Compressor
GZipCompressorLevel returns a new Compressor that implements GZip at supplied compression level
func NoCompression ¶
func NoCompression() Compressor
NoCompression is a Compressor that simply does nothing
func SnappyCompressor ¶
func SnappyCompressor() Compressor
SnappyCompressor returns a new Compressor that implements Snappy
func ZLibCompressor ¶
func ZLibCompressor() Compressor
ZLibCompressor returns a new Compressor that implements ZLib at default compression level
func ZLibCompressorLevel ¶
func ZLibCompressorLevel(level int) Compressor
ZLibCompressorLevel returns a new Compressor that implements ZLib at supplied compression level
func ZLibCompressorLevelDict ¶
func ZLibCompressorLevelDict(level int, dict []byte) Compressor
ZLibCompressorLevelDict returns a new Compressor that implements ZLib at supplied compression level with supplied dict
type DiskConfig ¶
type DiskConfig struct { // Transform is the supplied key<-->path KeyTransform Transform KeyTransform // WriteBufSize is the buffer size to use when writing file streams (PutStream) WriteBufSize int // Overwrite allows overwriting values of stored keys in the storage Overwrite bool // LockFile allows specifying the filesystem path to use for the lockfile, // providing only a filename it will store the lockfile within provided store // path and nest the store under `path/store` to prevent access to lockfile LockFile string // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor }
DiskConfig defines options to be used when opening a DiskStorage
type DiskStorage ¶
type DiskStorage struct {
// contains filtered or unexported fields
}
DiskStorage is a Storage implementation that stores directly to a filesystem
func OpenFile ¶
func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error)
OpenFile opens a DiskStorage instance for given folder path and configuration
func (*DiskStorage) Close ¶ added in v1.2.0
func (st *DiskStorage) Close() error
Close implements Storage.Close()
func (*DiskStorage) ReadBytes ¶
func (st *DiskStorage) ReadBytes(key string) ([]byte, error)
ReadBytes implements Storage.ReadBytes()
func (*DiskStorage) ReadStream ¶
func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error)
ReadStream implements Storage.ReadStream()
func (*DiskStorage) Remove ¶
func (st *DiskStorage) Remove(key string) error
Remove implements Storage.Remove()
func (*DiskStorage) Stat ¶
func (st *DiskStorage) Stat(key string) (bool, error)
Stat implements Storage.Stat()
func (*DiskStorage) WalkKeys ¶
func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error
WalkKeys implements Storage.WalkKeys()
func (*DiskStorage) WriteBytes ¶
func (st *DiskStorage) WriteBytes(key string, value []byte) error
WriteBytes implements Storage.WriteBytes()
func (*DiskStorage) WriteStream ¶
func (st *DiskStorage) WriteStream(key string, r io.Reader) error
WriteStream implements Storage.WriteStream()
type KeyTransform ¶
type KeyTransform interface { // KeyToPath converts a supplied key to storage path KeyToPath(string) string // PathToKey converts a supplied storage path to key PathToKey(string) string }
KeyTransform defines a method of converting store keys to storage paths (and vice-versa)
func NopTransform ¶
func NopTransform() KeyTransform
NopTransform returns a nop key transform (i.e. key = path)
type Lock ¶ added in v1.3.0
type Lock struct {
// contains filtered or unexported fields
}
Lock represents a filesystem lock to ensure only one storage instance open per path.
func (*Lock) Add ¶ added in v1.3.0
func (f *Lock) Add()
Add will add '1' to the underlying sync.WaitGroup.
type MemoryStorage ¶ added in v1.1.0
type MemoryStorage struct {
// contains filtered or unexported fields
}
MemoryStorage is a storage implementation that simply stores key-value pairs in a Go map in-memory. The map is protected by a mutex.
func OpenMemory ¶ added in v1.1.0
func OpenMemory(size int, overwrites bool) *MemoryStorage
OpenMemory opens a new MemoryStorage instance with internal map of 'size'.
func (*MemoryStorage) Clean ¶ added in v1.1.0
func (st *MemoryStorage) Clean() error
Clean implements Storage.Clean().
func (*MemoryStorage) Close ¶ added in v1.2.0
func (st *MemoryStorage) Close() error
Close implements Storage.Close().
func (*MemoryStorage) ReadBytes ¶ added in v1.1.0
func (st *MemoryStorage) ReadBytes(key string) ([]byte, error)
ReadBytes implements Storage.ReadBytes().
func (*MemoryStorage) ReadStream ¶ added in v1.1.0
func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error)
ReadStream implements Storage.ReadStream().
func (*MemoryStorage) Remove ¶ added in v1.1.0
func (st *MemoryStorage) Remove(key string) error
Remove implements Storage.Remove().
func (*MemoryStorage) Stat ¶ added in v1.1.0
func (st *MemoryStorage) Stat(key string) (bool, error)
Stat implements Storage.Stat().
func (*MemoryStorage) WalkKeys ¶ added in v1.1.0
func (st *MemoryStorage) WalkKeys(opts WalkKeysOptions) error
WalkKeys implements Storage.WalkKeys().
func (*MemoryStorage) WriteBytes ¶ added in v1.1.0
func (st *MemoryStorage) WriteBytes(key string, b []byte) error
WriteBytes implements Storage.WriteBytes().
func (*MemoryStorage) WriteStream ¶ added in v1.1.0
func (st *MemoryStorage) WriteStream(key string, r io.Reader) error
WriteStream implements Storage.WriteStream().
type Storage ¶
type Storage interface { // ReadBytes returns the byte value for key in storage ReadBytes(key string) ([]byte, error) // ReadStream returns an io.ReadCloser for the value bytes at key in the storage ReadStream(key string) (io.ReadCloser, error) // WriteBytes writes the supplied value bytes at key in the storage WriteBytes(key string, value []byte) error // WriteStream writes the bytes from supplied reader at key in the storage WriteStream(key string, r io.Reader) error // Stat checks if the supplied key is in the storage Stat(key string) (bool, error) // Remove attempts to remove the supplied key-value pair from storage Remove(key string) error // Close will close the storage, releasing any file locks Close() error // Clean removes unused values and unclutters the storage (e.g. removing empty folders) Clean() error // WalkKeys walks the keys in the storage WalkKeys(opts WalkKeysOptions) error }
Storage defines a means of storing and accessing key value pairs
type StorageEntry ¶
type StorageEntry interface { // Key returns the storage entry's key Key() string }
StorageEntry defines a key in Storage
type WalkKeysOptions ¶
type WalkKeysOptions struct { // WalkFn is the function to apply on each StorageEntry WalkFn func(StorageEntry) }
WalkKeysOptions defines how to walk the keys in a storage implementation