storage

type Alias string

type Backend struct {
	Metadata    Alias // The store that should be used for metadata storage.
	DefaultLog  Alias // The log that should be used by default.
	DefaultKVDB Alias // The key-value datastore that should be used by default.
	Stores      map[Alias]dvid.StoreConfig
	KVAssign    DataMap
	LogAssign   DataMap
	Groupcache  GroupcacheConfig
}

type Batch interface {
	// Delete removes from the batch a put using the given key.
	Delete(TKey)

	// Put adds to the batch a put using the given key-value.
	Put(k TKey, v []byte)

	// Commits a batch of operations and closes the write batch.
	Commit() error
}

type BlobStore interface {
	PutBlob([]byte) (ref string, err error)
	GetBlob(ref string) ([]byte, error)
}

type Block struct {
	*BlockOp
	Coord dvid.ChunkPoint3d
	Value []byte
}

type BlockFunc func(*Block) error

type BlockOp struct {
	Op interface{}
	Wg *sync.WaitGroup
}

type BufferableOps interface {
	KeyValueSetter

	// Put key-value pairs.  Note that it could be more efficient to use the Batcher
	// interface so you don't have to create and keep a slice of KeyValue.  Some
	// databases like leveldb will copy on batch put anyway.
	PutRange(Context, []TKeyValue) error

	// DeleteRange removes all key-value pairs with keys in the given range.
	// If versioned data in mutable stores, this will create tombstones in the version
	// unlike RawDelete or DeleteAll.
	DeleteRange(ctx Context, kStart, kEnd TKey) error

	// ProcessRange will process all gets when flush is called
	ProcessRange(ctx Context, kStart, kEnd TKey, op *ChunkOp, f ChunkFunc) error
}

type Chunk struct {
	*ChunkOp
	*TKeyValue
}

type ChunkFunc func(*Chunk) error

type ChunkOp struct {
	Op interface{}
	Wg *sync.WaitGroup
}

type Context interface {
	// VersionID returns the local version ID of the DAG node being operated on.
	// If not versioned, the version is the root ID.
	VersionID() dvid.VersionID

	// RepoRoot returns the root uuid.
	RepoRoot() (dvid.UUID, error)

	// ConstructKey takes a type-specific key component, and generates a
	// namespaced key that fits with the DVID-wide key space partitioning.
	ConstructKey(TKey) Key

	// ConstructKeyVersion constructs a key like ConstructKey
	// but using specified version
	ConstructKeyVersion(TKey, dvid.VersionID) Key

	// KeyRange returns the minimum and maximum keys for this context.
	KeyRange() (min, max Key)

	// String prints a description of the Context
	String() string

	// Returns a sync.Mutex specific to this context.
	Mutex() *sync.Mutex

	// Versioned is true if this Context is also a VersionedCtx.
	Versioned() bool

	// SplitKey returns key components useful to store all versiones in a familyColumn if the storage engine supports it
	SplitKey(tk TKey) (Key, []byte, error)
}

type DAGStoreGetter interface {
	// Get returns a value given a key.
	Get(ctx Context, k TKey) ([]byte, error)

	// GetRange returns a range of values spanning (kStart, kEnd) keys.
	GetRange(ctx Context, kStart, kEnd TKey) ([]*TKeyValue, error)

	// GetRangeStream sends range of values spanning (kStart, kEnd) key-values.
	GetRangeStream(ctx Context, kStart, kEnd TKey, ordered, keysOnly bool, out chan *KeyValue) error

	// ProcessRange sends a range of type key-value pairs to type-specific chunk handlers,
	// allowing chunk processing to be concurrent with key-value reads.
	// If the ChunkFunc returns an error, it is expected that the ProcessRange should
	// immediately terminate and propagate the error.
	ProcessRange(ctx Context, kStart, kEnd TKey, ordered bool, op *ChunkOp, f ChunkFunc) error
}

type DataContext struct {
	// contains filtered or unexported fields
}

type DataMap map[dvid.DataSpecifier]Alias

type DataSpec interface {
	DataName() dvid.InstanceName
	DataUUID() dvid.UUID
	RootUUID() dvid.UUID
	Tags() map[string]string
	TypeName() dvid.TypeString
}

type Engine interface {
	fmt.Stringer

	// GetName returns a simple driver identifier like "basholeveldb", "kvautobus" or "bigtable".
	GetName() string

	// IsDistributed returns whether the engine is a distributed DB (engine should manage request throttling)
	IsDistributed() bool

	// GetSemVer returns the semantic versioning info.
	GetSemVer() semver.Version

	// NewStore returns a new storage engine given the passed configuration.
	// Should return true for initMetadata if the store needs initialization of metadata.
	NewStore(dvid.StoreConfig) (db dvid.Store, initMetadata bool, err error)
}

type Filter interface {
	// Check filters type-specific key-value pairs.
	Check(tkv *TKeyValue) (skip bool, err error)
}

type FilterSpec string

type Filterer interface {
	NewFilter(FilterSpec) (Filter, error)
}

type GridProps struct {
	VolumeSize dvid.Point3d
	ChunkSize  dvid.Point3d
	Encoding   string     // "raw", "jpeg", or "compressed_segmentation"
	Resolution [3]float64 // resolution in nm for a voxel along dimensions
}

type GridStoreGetter interface {
	GridProperties(scaleLevel int) (GridProps, error)
	GridGet(scaleLevel int, blockCoord dvid.ChunkPoint3d) ([]byte, error)
	GridGetVolume(scaleLevel int, minBlock, maxBlock dvid.ChunkPoint3d, ordered bool, op *BlockOp, f BlockFunc) error
}

type GroupcacheConfig struct {
	GB        int
	Host      string   // The http address of this DVID server's groupcache port.
	Peers     []string // The http addresses of the peer groupcache group.
	Instances []string // Data instances that use groupcache in form "<name>:<uuid>""
}

type GroupcacheStats struct {
	Gets           int64 // any GET request, including from peers
	CacheHits      int64 // either cache was good
	PeerLoads      int64 // either remote load or remote cache hit (not an error)
	PeerErrors     int64
	Loads          int64 // gets - cacheHits
	LoadsDeduped   int64 // after singleflight
	LocalLoads     int64 // total good local loads
	LocalLoadErrs  int64 // total bad local loads
	ServerRequests int64 // gets that came over network from peers

	// Cache stats for items owned by the host.
	MainCache groupcache.CacheStats

	// Cache stats for replicated items from a peer.
	HotCache groupcache.CacheStats
}

type KafkaConfig struct {
	TopicActivity string   // if supplied, will be override topic for activity log
	TopicPrefix   string   // if supplied, will be prefixed to any mutation logging
	TopicSuffixes []string // optional topic suffixes per data UUID
	Servers       []string
	BufferSize    int // queue.buffering.max.messages
}

type Key []byte

type KeyChan chan Key

type KeyRange struct {
	Start   Key // Range includes this Key.
	OpenEnd Key // Range extend to but does not include this Key.
}

type KeyValue struct {
	K Key
	V []byte
}

type KeyValueBatcher interface {
	NewBatch(ctx Context) Batch
}

type KeyValueDB interface {
	dvid.Store
	KeyValueGetter
	KeyValueSetter
}

type KeyValueGetter interface {
	// Get returns a value given a key.
	Get(ctx Context, k TKey) ([]byte, error)

	// Exists returns true if a key has been set.
	Exists(ctx Context, k TKey) (bool, error)
}

type KeyValueIngestable interface {
	// KeyValueIngest accepts mutations without any guarantee that the ingested key value
	// will be immediately readable.  This allows immutable stores to accept data that will
	// be later processed into its final readable, immutable form.
	KeyValueIngest(Context, TKey, []byte) error
}

type KeyValueRequester interface {
	NewBuffer(ctx Context) RequestBuffer
}

type KeyValueSetter interface {
	// Put writes a value with given key in a possibly versioned context.
	Put(Context, TKey, []byte) error

	// Delete deletes a key-value pair so that subsequent Get on the key returns nil.
	// For versioned data in mutable stores, Delete() will create a tombstone for the version
	// unlike RawDelete or DeleteAll.
	Delete(Context, TKey) error

	// RawPut is a low-level function that puts a key-value pair using full keys.
	// This can be used in conjunction with RawRangeQuery.  It does not automatically
	// delete any associated tombstone, unlike the Delete() function, so tombstone
	// deletion must be handled via RawDelete().
	RawPut(Key, []byte) error

	// RawDelete is a low-level function.  It deletes a key-value pair using full keys
	// without any context.  This can be used in conjunction with RawRangeQuery.
	RawDelete(Key) error
}

type KeyValueTimestampGetter interface {
	// Get returns a value as well as last modification time
	GetWithTimestamp(ctx Context, k TKey) ([]byte, time.Time, error)
}

type KeyValues []KeyValue

type LogMessage struct {
	EntryType uint16
	Data      []byte
}

type LogReadable interface {
	ReadLogRequired() bool
	GetReadLog() ReadLog
}

type LogWritable interface {
	WriteLogRequired() bool
	GetWriteLog() WriteLog
}

type Logable interface {
	LogReadable
	LogWritable
}

type MetadataContext struct{}

type Op uint8

type OrderedKeyValueDB interface {
	dvid.Store
	OrderedKeyValueGetter
	OrderedKeyValueSetter
}

type OrderedKeyValueGetter interface {
	KeyValueGetter

	// GetRange returns a range of values spanning (kStart, kEnd) keys.
	GetRange(ctx Context, kStart, kEnd TKey) ([]*TKeyValue, error)

	// KeysInRange returns a range of type-specific key components spanning (kStart, kEnd).
	KeysInRange(ctx Context, kStart, kEnd TKey) ([]TKey, error)

	// SendKeysInRange sends a range of keys down a key channel.
	SendKeysInRange(ctx Context, kStart, kEnd TKey, ch KeyChan) error

	// ProcessRange sends a range of type key-value pairs to type-specific chunk handlers,
	// allowing chunk processing to be concurrent with key-value sequential reads.
	// Since the chunks are typically sent during sequential read iteration, the
	// receiving function can be organized as a pool of chunk handling goroutines.
	// See datatype/imageblk.ProcessChunk() for an example.  If the ChunkFunc returns
	// an error, it is expected that the ProcessRange should immediately terminate and
	// propagate the error.
	ProcessRange(ctx Context, kStart, kEnd TKey, op *ChunkOp, f ChunkFunc) error

	// RawRangeQuery sends a range of full keys.  This is to be used for low-level data
	// retrieval like DVID-to-DVID communication and should not be used by data type
	// implementations if possible because each version's key-value pairs are sent
	// without filtering by the current version and its ancestor graph.  A nil is sent
	// down the channel when the range is complete.  The query can be cancelled by sending
	// a value down the cancel channel.
	RawRangeQuery(kStart, kEnd Key, keysOnly bool, out chan *KeyValue, cancel <-chan struct{}) error
}

type OrderedKeyValueSetter interface {
	KeyValueSetter

	// Put key-value pairs.  Note that it could be more efficient to use the Batcher
	// interface so you don't have to create and keep a slice of KeyValue.  Some
	// databases like leveldb will copy on batch put anyway.
	PutRange(Context, []TKeyValue) error

	// DeleteRange removes all key-value pairs with keys in the given range.
	// If versioned data in mutable stores, this will create tombstones in the version
	// unlike RawDelete or DeleteAll.
	DeleteRange(ctx Context, kStart, kEnd TKey) error

	// DeleteAll removes all key-value pairs for the context.
	DeleteAll(ctx Context) error
}

type PatchFunc func([]byte) ([]byte, error)

type ReadLog interface {
	dvid.Store
	ReadBinary(dataID, version dvid.UUID) ([]byte, error)
	ReadAll(dataID, version dvid.UUID) ([]LogMessage, error)
	StreamAll(dataID, version dvid.UUID, ch chan LogMessage) error
}

type RepairableEngine interface {
	Engine
	Repair(path string) error
}

type RequestBuffer interface {
	BufferableOps

	// ProcessList will process all gets when flush is called
	ProcessList(ctx Context, tkeys []TKey, op *ChunkOp, f ChunkFunc) error

	// PutCallback writes a value with given key in a possibly versioned context and signals the callback
	PutCallback(Context, TKey, []byte, chan error) error

	// Flush processes all the queued jobs
	Flush() error
}

type RequestCtx interface {
	// GetRequestID returns a string identifier or the empty string if none have been set.
	GetRequestID() string

	// SetRequestID sets a string identifier.
	SetRequestID(id string)
}

type Requirements struct {
	BulkIniter bool
	BulkWriter bool
	Batcher    bool
}

type SizeViewer interface {
	GetApproximateSizes(ranges []KeyRange) ([]uint64, error)
}

type TKey []byte

type TKeyClass byte

type TKeyValue struct {
	K TKey
	V []byte
}

type TKeyValues []TKeyValue

type TestableEngine interface {
	Engine
	AddTestConfig(*Backend) (Alias, error)
	Delete(dvid.StoreConfig) error
}

type VersionedCtx interface {
	Context

	// UnversionedKeyPrefix returns a byte slice that is the common prefix
	// for all versions of this particular TKey.
	UnversionedKeyPrefix(TKey) Key

	// UnversionedKey returns a unversioned Key and the version id
	// as separate components.  This can be useful for storage systems
	// like column stores where the row key is the unversioned Key and
	// the column qualifier is the version id.
	UnversionedKey(TKey) (Key, dvid.VersionID, error)

	// VersionFromKey returns a version ID from a full key.  Any VersionedContext is sufficient as receiver
	VersionFromKey(Key) (dvid.VersionID, error)

	// TombstoneKey takes a type-specific key component and returns a key that
	// signals a deletion of any ancestor values.  The returned key must have
	// as its last byte storage.MarkTombstone.
	TombstoneKey(TKey) Key

	// TombstoneKeyVersion implement TombstoneKey but for the specified version
	TombstoneKeyVersion(TKey, dvid.VersionID) Key

	// Head checks whether this is the open head of the master branch
	Head() bool

	// MasterVersion checks whether current version is on master branch
	MasterVersion(dvid.VersionID) bool

	// NumVersions returns the number of version in the current DAG
	NumVersions() int32

	// Returns lower bound key for versions.
	MinVersionKey(TKey) (Key, error)

	// Returns upper bound key for versions.
	MaxVersionKey(TKey) (Key, error)

	// GetBestKeyVersion returns the key that most closely matches this context's version.
	// If no suitable key or a tombstone is encountered closed to the version, nil is returned.
	GetBestKeyVersion([]Key) (Key, error)

	// VersionedKeyValue returns the key-value pair corresponding to this key's version
	// given a list of key-value pairs across many versions.  If no suitable key-value
	// pair is found, nil is returned.
	VersionedKeyValue([]*KeyValue) (*KeyValue, error)

	// Data returns the data associated with this context
	Data() dvid.Data
}

type WriteLog interface {
	dvid.Store
	Append(dataID, version dvid.UUID, msg LogMessage) error
	CloseLog(dataID, version dvid.UUID) error
	TopicAppend(topic string, msg LogMessage) error
	TopicClose(topic string) error
}