store

package
v1.5.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 20, 2023 License: Apache-2.0 Imports: 26 Imported by: 4

Documentation

Overview

Copyright 2022 Codenotary Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source
const DefaultBulkPreparationTimeout = DefaultSyncFrequency
View Source
const DefaultCommitLogMaxOpenedFiles = 10
View Source
const DefaultCompressionFormat = appendable.DefaultCompressionFormat
View Source
const DefaultCompressionLevel = appendable.DefaultCompressionLevel
View Source
const DefaultEmbeddedValues = false
View Source
const DefaultFileMode = os.FileMode(0755)
View Source
const DefaultFileSize = multiapp.DefaultFileSize
View Source
const DefaultIndexingMaxBulkSize = 1
View Source
const DefaultMVCCReadSetLimit = 100_000
View Source
const DefaultMaxActiveTransactions = 1000
View Source
const DefaultMaxConcurrency = 30
View Source
const DefaultMaxIOConcurrency = 1
View Source
const DefaultMaxKeyLen = 1024
View Source
const DefaultMaxTxEntries = 1 << 10 // 1024
View Source
const DefaultMaxValueLen = 4096 // 4Kb
View Source
const DefaultMaxWaitees = 1000
View Source
const DefaultPreallocFiles = false
View Source
const DefaultSyncFrequency = 20 * time.Millisecond
View Source
const DefaultTruncationFrequency = 24 * time.Hour
View Source
const DefaultTxLogCacheSize = 1000
View Source
const DefaultTxLogMaxOpenedFiles = 10
View Source
const DefaultVLogCacheSize = 0
View Source
const DefaultVLogMaxOpenedFiles = 10
View Source
const DefaultWriteBufferSize = 1 << 22 //4Mb
View Source
const DefaultWriteTxHeaderVersion = MaxTxHeaderVersion
View Source
const MaxFileSize = (1 << 31) - 1 // 2Gb
View Source
const MaxKeyLen = 1024 // assumed to be not lower than hash size
View Source
const MaxParallelIO = 127
View Source
const MaxTxHeaderVersion = 1
View Source
const MinimumRetentionPeriod = 24 * time.Hour
View Source
const MinimumTruncationFrequency = 1 * time.Hour
View Source
const NotificationWindow = 60 * time.Second
View Source
const Version = 2

Version 2 includes `metaEmbeddedValues` and `metaPreallocFiles` into clog metadata

Variables

View Source
var ErrAlreadyClosed = embedded.ErrAlreadyClosed
View Source
var ErrBufferFullyConsumed = errors.New("buffer fully consumed")
View Source
var ErrBufferIsFull = errors.New("buffer is full")
View Source
var ErrCompactionUnsupported = errors.New("compaction is unsupported when remote storage is used")
View Source
var ErrCorruptedAHtree = errors.New("appendable hash tree is corrupted")
View Source
var ErrCorruptedCLog = errors.New("commit log is corrupted")
View Source
var ErrCorruptedData = errors.New("data is corrupted")
View Source
var ErrCorruptedIndex = errors.New("corrupted index")
View Source
var ErrCorruptedTxData = errors.New("tx data is corrupted")
View Source
var ErrCorruptedTxDataDuplicateKey = fmt.Errorf("%w: duplicate key in a single TX", ErrCorruptedTxData)
View Source
var ErrCorruptedTxDataMaxKeyLenExceeded = fmt.Errorf("%w: maximum key length exceeded", ErrCorruptedTxData)
View Source
var ErrCorruptedTxDataMaxTxEntriesExceeded = fmt.Errorf("%w: maximum number of TX entries exceeded", ErrCorruptedTxData)
View Source
var ErrCorruptedTxDataUnknownHeaderVersion = fmt.Errorf("%w: unknown TX header version", ErrCorruptedTxData)
View Source
var ErrDuplicatedKey = errors.New("duplicated key")
View Source
var ErrExpiredEntry = fmt.Errorf("%w: expired entry", ErrKeyNotFound)
View Source
var ErrIllegalArguments = embedded.ErrIllegalArguments
View Source
var ErrIllegalState = tbtree.ErrIllegalState // TODO: define error in store layer
View Source
var ErrIllegalTruncationArgument = fmt.Errorf("%w: invalid truncation info", ErrIllegalArguments)
View Source
var ErrInvalidOptions = fmt.Errorf("%w: invalid options", ErrIllegalArguments)
View Source
var ErrInvalidPrecondition = errors.New("invalid precondition")
View Source
var ErrInvalidPreconditionInvalidTxID = fmt.Errorf("%w: invalid transaction ID", ErrInvalidPrecondition)
View Source
var ErrInvalidPreconditionMaxKeyLenExceeded = fmt.Errorf("%w: %v", ErrInvalidPrecondition, ErrMaxKeyLenExceeded)
View Source
var ErrInvalidPreconditionNull = fmt.Errorf("%w: null", ErrInvalidPrecondition)
View Source
var ErrInvalidPreconditionNullKey = fmt.Errorf("%w: %v", ErrInvalidPrecondition, ErrNullKey)
View Source
var ErrInvalidPreconditionTooMany = fmt.Errorf("%w: too many preconditions", ErrInvalidPrecondition)
View Source
var ErrInvalidProof = errors.New("invalid proof")
View Source
var ErrKeyAlreadyExists = errors.New("key already exists")
View Source
var ErrKeyNotFound = tbtree.ErrKeyNotFound // TODO: define error in store layer
View Source
var ErrMVCCReadSetLimitExceeded = errors.New("MVCC read-set limit exceeded")
View Source
var ErrMaxActiveTransactionsLimitExceeded = errors.New("max active transactions limit exceeded")
View Source
var ErrMaxConcurrencyLimitExceeded = errors.New("max concurrency limit exceeded")
View Source
var ErrMaxKeyLenExceeded = errors.New("max key length exceeded")
View Source
var ErrMaxTxEntriesLimitExceeded = errors.New("max number of entries per tx exceeded")
View Source
var ErrMaxValueLenExceeded = errors.New("max value length exceeded")
View Source
var ErrMetadataUnsupported = errors.New(
	"metadata is unsupported when in 1.1 compatibility mode, " +
		"do not use metadata-related features such as expiration and logical deletion",
)
View Source
var ErrNewerVersionOrCorruptedData = errors.New("tx created with a newer version or data is corrupted")
View Source
var ErrNoEntriesProvided = errors.New("no entries provided")
View Source
var ErrNoMoreEntries = tbtree.ErrNoMoreEntries // TODO: define error in store layer
View Source
var ErrNonExpirable = errors.New("non expirable")
View Source
var ErrNotEnoughData = fmt.Errorf("%w: not enough data", ErrBufferFullyConsumed)
View Source
var ErrNullKey = errors.New("null key")
View Source
var ErrOffsetOutOfRange = tbtree.ErrOffsetOutOfRange // TODO: define error in store layer
View Source
var ErrPathIsNotADirectory = errors.New("path is not a directory")
View Source
var ErrPreconditionFailed = errors.New("precondition failed")
View Source
var ErrReadOnly = errors.New("read-only")
View Source
var ErrReadOnlyTx = errors.New("read-only transaction")
View Source
var ErrSourceTxNewerThanTargetTx = fmt.Errorf("%w: source tx is newer than target tx", ErrIllegalArguments)
View Source
var ErrTxAlreadyCommitted = errors.New("tx already committed")
View Source
var ErrTxEntryIndexOutOfRange = errors.New("tx entry index out of range")
View Source
var ErrTxNotFound = errors.New("tx not found")
View Source
var ErrTxNotPresentInMetadata = errors.New("tx not present in metadata")
View Source
var ErrTxPoolExhausted = errors.New("transaction pool exhausted")
View Source
var ErrTxReadConflict = errors.New("tx read conflict")
View Source
var ErrTxSizeGreaterThanMaxTxSize = errors.New("tx size greater than max tx size")
View Source
var ErrUnexpectedError = errors.New("unexpected error")
View Source
var ErrUnexpectedLinkingError = errors.New("internal inconsistency between linear and binary linking")
View Source
var ErrUnsupportedTxHeaderVersion = errors.New("missing tx header serialization method")
View Source
var ErrUnsupportedTxVersion = errors.New("unsupported tx version")
View Source
var ErrWriteOnlyTx = errors.New("write-only transaction")

Functions

func EntrySpecDigest_v0 added in v1.2.0

func EntrySpecDigest_v0(kv *EntrySpec) [sha256.Size]byte

func EntrySpecDigest_v1 added in v1.2.0

func EntrySpecDigest_v1(kv *EntrySpec) [sha256.Size]byte

func TxEntryDigest_v1_1 added in v1.2.0

func TxEntryDigest_v1_1(e *TxEntry) ([sha256.Size]byte, error)

func TxEntryDigest_v1_2 added in v1.2.0

func TxEntryDigest_v1_2(e *TxEntry) ([sha256.Size]byte, error)

func VerifyDualProof

func VerifyDualProof(proof *DualProof, sourceTxID, targetTxID uint64, sourceAlh, targetAlh [sha256.Size]byte) bool

func VerifyDualProofV2 added in v1.5.0

func VerifyDualProofV2(proof *DualProofV2, sourceTxID, targetTxID uint64, sourceAlh, targetAlh [sha256.Size]byte) error

func VerifyInclusion added in v0.9.0

func VerifyInclusion(proof *htree.InclusionProof, entryDigest, root [sha256.Size]byte) bool

func VerifyLinearAdvanceProof added in v1.4.1

func VerifyLinearAdvanceProof(
	proof *LinearAdvanceProof,
	startTxID uint64,
	endTxID uint64,
	endAlh [sha256.Size]byte,
	treeRoot [sha256.Size]byte,
	treeSize uint64,
) bool

func VerifyLinearProof

func VerifyLinearProof(proof *LinearProof, sourceTxID, targetTxID uint64, sourceAlh, targetAlh [sha256.Size]byte) bool

Types

type AHTOptions added in v1.3.2

type AHTOptions struct {
	// Number of new leaves in the tree between synchronous flush to disk
	SyncThld int

	// Size of the in-memory write buffer
	WriteBufferSize int
}

func DefaultAHTOptions added in v1.3.2

func DefaultAHTOptions() *AHTOptions

func (*AHTOptions) Validate added in v1.3.2

func (opts *AHTOptions) Validate() error

func (*AHTOptions) WithSyncThld added in v1.3.2

func (opts *AHTOptions) WithSyncThld(syncThld int) *AHTOptions

func (*AHTOptions) WithWriteBufferSize added in v1.3.2

func (opts *AHTOptions) WithWriteBufferSize(writeBufferSize int) *AHTOptions

type AppFactoryFunc added in v1.0.5

type AppFactoryFunc func(
	rootPath string,
	subPath string,
	opts *multiapp.Options,
) (appendable.Appendable, error)

type DualProof

type DualProof struct {
	SourceTxHeader     *TxHeader
	TargetTxHeader     *TxHeader
	InclusionProof     [][sha256.Size]byte
	ConsistencyProof   [][sha256.Size]byte
	TargetBlTxAlh      [sha256.Size]byte
	LastInclusionProof [][sha256.Size]byte
	LinearProof        *LinearProof
	LinearAdvanceProof *LinearAdvanceProof
}

type DualProofV2 added in v1.5.0

type DualProofV2 struct {
	SourceTxHeader   *TxHeader
	TargetTxHeader   *TxHeader
	InclusionProof   [][sha256.Size]byte
	ConsistencyProof [][sha256.Size]byte
}

type EntrySpec added in v1.2.0

type EntrySpec struct {
	Key      []byte
	Metadata *KVMetadata
	Value    []byte
	// hashValue is the hash of the value
	// if the actual value is truncated. This is
	// used during replication.
	HashValue [sha256.Size]byte
	// isValueTruncated is true if the value is
	// truncated. This is used during replication.
	IsValueTruncated bool
}

type EntrySpecDigest added in v1.2.0

type EntrySpecDigest func(kv *EntrySpec) [sha256.Size]byte

func EntrySpecDigestFor added in v1.2.0

func EntrySpecDigestFor(version int) (EntrySpecDigest, error)

type FilterFn added in v1.2.0

type FilterFn func(valRef ValueRef, t time.Time) error

filter out entries when filter evaluates to a non-nil error

var (
	IgnoreDeleted FilterFn = func(valRef ValueRef, t time.Time) error {
		md := valRef.KVMetadata()
		if md != nil && md.Deleted() {
			return ErrKeyNotFound
		}
		return nil
	}

	IgnoreExpired FilterFn = func(valRef ValueRef, t time.Time) error {
		md := valRef.KVMetadata()
		if md != nil && md.ExpiredAt(t) {
			return ErrExpiredEntry
		}
		return nil
	}
)

type ImmuStore

type ImmuStore struct {
	// contains filtered or unexported fields
}

func Open

func Open(path string, opts *Options) (*ImmuStore, error)

func OpenWith

func OpenWith(path string, vLogs []appendable.Appendable, txLog, cLog appendable.Appendable, opts *Options) (*ImmuStore, error)

func (*ImmuStore) AllowCommitUpto added in v1.4.0

func (s *ImmuStore) AllowCommitUpto(txID uint64) error

func (*ImmuStore) Close

func (s *ImmuStore) Close() error

func (*ImmuStore) CommitWith added in v0.9.1

func (s *ImmuStore) CommitWith(ctx context.Context, callback func(txID uint64, index KeyIndex) ([]*EntrySpec, []Precondition, error), waitForIndexing bool) (*TxHeader, error)

func (*ImmuStore) CommittedAlh added in v1.4.0

func (s *ImmuStore) CommittedAlh() (uint64, [sha256.Size]byte)

func (*ImmuStore) CompactIndex added in v0.9.2

func (s *ImmuStore) CompactIndex() error

func (*ImmuStore) DiscardPrecommittedTxsSince added in v1.4.0

func (s *ImmuStore) DiscardPrecommittedTxsSince(txID uint64) (int, error)

DiscardPrecommittedTxsSince discard precommitted txs No truncation is made into txLog which means, if the store is reopened some precommitted transactions may be reloaded. Discarding may need to be redone after re-opening the store.

func (*ImmuStore) DualProof

func (s *ImmuStore) DualProof(sourceTxHdr, targetTxHdr *TxHeader) (proof *DualProof, err error)

DualProof combines linear cryptographic linking i.e. transactions include the linear accumulative hash up to the previous one, with binary cryptographic linking generated by appending the linear accumulative hash values into an incremental hash tree, whose root is also included as part of each transaction and thus considered when calculating the linear accumulative hash. The objective of this proof is the same as the linear proof, that is, generate data for the calculation of the accumulative hash value of the target transaction from the linear accumulative hash value up to source transaction.

func (*ImmuStore) DualProofV2 added in v1.5.0

func (s *ImmuStore) DualProofV2(sourceTxHdr, targetTxHdr *TxHeader) (proof *DualProofV2, err error)

func (*ImmuStore) ExportTx added in v1.0.5

func (s *ImmuStore) ExportTx(txID uint64, allowPrecommitted bool, skipIntegrityCheck bool, tx *Tx) ([]byte, error)

func (*ImmuStore) FirstTxSince added in v1.3.0

func (s *ImmuStore) FirstTxSince(ts time.Time) (*TxHeader, error)

func (*ImmuStore) FlushIndex added in v1.2.3

func (s *ImmuStore) FlushIndex(cleanupPercentage float32, synced bool) error

func (*ImmuStore) Get added in v0.9.0

func (s *ImmuStore) Get(key []byte) (valRef ValueRef, err error)

func (*ImmuStore) GetWithFilters added in v1.5.0

func (s *ImmuStore) GetWithFilters(key []byte, filters ...FilterFn) (valRef ValueRef, err error)

func (*ImmuStore) GetWithPrefix added in v1.5.0

func (s *ImmuStore) GetWithPrefix(prefix []byte, neq []byte) (key []byte, valRef ValueRef, err error)

func (*ImmuStore) GetWithPrefixAndFilters added in v1.5.0

func (s *ImmuStore) GetWithPrefixAndFilters(prefix []byte, neq []byte, filters ...FilterFn) (key []byte, valRef ValueRef, err error)

func (*ImmuStore) History added in v0.9.1

func (s *ImmuStore) History(key []byte, offset uint64, descOrder bool, limit int) (txs []uint64, hCount uint64, err error)

func (*ImmuStore) IndexInfo

func (s *ImmuStore) IndexInfo() uint64

func (*ImmuStore) IsClosed added in v1.2.3

func (s *ImmuStore) IsClosed() bool

func (*ImmuStore) LastCommittedTxID added in v1.4.0

func (s *ImmuStore) LastCommittedTxID() uint64

func (*ImmuStore) LastPrecommittedTxID added in v1.5.0

func (s *ImmuStore) LastPrecommittedTxID() uint64

func (*ImmuStore) LastTxUntil added in v1.3.0

func (s *ImmuStore) LastTxUntil(ts time.Time) (*TxHeader, error)

func (*ImmuStore) LinearAdvanceProof added in v1.4.1

func (s *ImmuStore) LinearAdvanceProof(sourceTxID, targetTxID uint64, targetBlTxID uint64) (*LinearAdvanceProof, error)

LinearAdvanceProof returns additional inclusion proof for part of the old linear proof consumed by the new Merkle Tree

func (*ImmuStore) LinearProof

func (s *ImmuStore) LinearProof(sourceTxID, targetTxID uint64) (*LinearProof, error)

LinearProof returns a list of hashes to calculate Alh@targetTxID from Alh@sourceTxID

func (*ImmuStore) MVCCReadSetLimit added in v1.5.0

func (s *ImmuStore) MVCCReadSetLimit() int

func (*ImmuStore) MandatoryMVCCUpToTxID added in v1.5.0

func (s *ImmuStore) MandatoryMVCCUpToTxID() uint64

func (*ImmuStore) MaxActiveTransactions added in v1.3.2

func (s *ImmuStore) MaxActiveTransactions() int

func (*ImmuStore) MaxConcurrency

func (s *ImmuStore) MaxConcurrency() int

func (*ImmuStore) MaxIOConcurrency

func (s *ImmuStore) MaxIOConcurrency() int

func (*ImmuStore) MaxKeyLen

func (s *ImmuStore) MaxKeyLen() int

func (*ImmuStore) MaxTxEntries

func (s *ImmuStore) MaxTxEntries() int

func (*ImmuStore) MaxValueLen

func (s *ImmuStore) MaxValueLen() int

func (*ImmuStore) NewTx

func (s *ImmuStore) NewTx(ctx context.Context, opts *TxOptions) (*OngoingTx, error)

func (*ImmuStore) NewTxHolderPool added in v1.3.2

func (s *ImmuStore) NewTxHolderPool(poolSize int, preallocated bool) (TxPool, error)

func (*ImmuStore) NewTxReader

func (s *ImmuStore) NewTxReader(initialTxID uint64, desc bool, tx *Tx) (*TxReader, error)

func (*ImmuStore) NewWriteOnlyTx added in v1.2.0

func (s *ImmuStore) NewWriteOnlyTx(ctx context.Context) (*OngoingTx, error)

func (*ImmuStore) PrecommittedAlh added in v1.4.0

func (s *ImmuStore) PrecommittedAlh() (uint64, [sha256.Size]byte)

func (*ImmuStore) ReadOnly

func (s *ImmuStore) ReadOnly() bool

func (*ImmuStore) ReadTx

func (s *ImmuStore) ReadTx(txID uint64, skipIntegrityCheck bool, tx *Tx) error

func (*ImmuStore) ReadTxEntry added in v1.3.2

func (s *ImmuStore) ReadTxEntry(txID uint64, key []byte, skipIntegrityCheck bool) (*TxEntry, *TxHeader, error)

func (*ImmuStore) ReadTxHeader added in v1.3.2

func (s *ImmuStore) ReadTxHeader(txID uint64, allowPrecommitted bool, skipIntegrityCheck bool) (*TxHeader, error)

func (*ImmuStore) ReadValue

func (s *ImmuStore) ReadValue(entry *TxEntry) ([]byte, error)

ReadValue returns the actual associated value to a key at a specific transaction ErrExpiredEntry is be returned if the specified time has already elapsed

func (*ImmuStore) ReplicateTx added in v1.0.5

func (s *ImmuStore) ReplicateTx(ctx context.Context, exportedTx []byte, skipIntegrityCheck bool, waitForIndexing bool) (*TxHeader, error)

func (*ImmuStore) SetExternalCommitAllowance added in v1.4.0

func (s *ImmuStore) SetExternalCommitAllowance(enabled bool)

func (*ImmuStore) Snapshot

func (s *ImmuStore) Snapshot() (*Snapshot, error)

func (*ImmuStore) SnapshotMustIncludeTxID added in v1.5.0

func (s *ImmuStore) SnapshotMustIncludeTxID(ctx context.Context, txID uint64) (*Snapshot, error)

SnapshotMustIncludeTxID returns a new snapshot based on an existent dumped root (snapshot reuse). Current root may be dumped if there are no previous root already stored on disk or if the dumped one was old enough. If txID is 0, any snapshot may be used.

func (*ImmuStore) SnapshotMustIncludeTxIDWithRenewalPeriod added in v1.5.0

func (s *ImmuStore) SnapshotMustIncludeTxIDWithRenewalPeriod(ctx context.Context, txID uint64, renewalPeriod time.Duration) (*Snapshot, error)

SnapshotMustIncludeTxIDWithRenewalPeriod returns a new snapshot based on an existent dumped root (snapshot reuse). Current root may be dumped if there are no previous root already stored on disk or if the dumped one was old enough. If txID is 0, any snapshot not older than renewalPeriod may be used. If renewalPeriod is 0, renewal period is not taken into consideration

func (*ImmuStore) Sync

func (s *ImmuStore) Sync() error

func (*ImmuStore) Synced

func (s *ImmuStore) Synced() bool

func (*ImmuStore) TruncateUptoTx added in v1.5.0

func (s *ImmuStore) TruncateUptoTx(minTxID uint64) error

TruncateUptoTx deletes the value-log file up to transactions that are strictly below the specified minTxID.

func (*ImmuStore) TxCount

func (s *ImmuStore) TxCount() uint64

func (*ImmuStore) UseTimeFunc added in v1.1.0

func (s *ImmuStore) UseTimeFunc(timeFunc TimeFunc) error

func (*ImmuStore) WaitForIndexingUpto added in v0.9.2

func (s *ImmuStore) WaitForIndexingUpto(ctx context.Context, txID uint64) error

func (*ImmuStore) WaitForTx added in v1.0.5

func (s *ImmuStore) WaitForTx(ctx context.Context, txID uint64, allowPrecommitted bool) error

type IndexOptions

type IndexOptions struct {
	// Size of the Btree node LRU cache
	CacheSize int

	// Number of new index entries between disk flushes
	FlushThld int

	// Number of new index entries between disk flushes with file sync
	SyncThld int

	// Size of the in-memory flush buffer (in bytes)
	FlushBufferSize int

	// Percentage of node files cleaned up during each flush
	CleanupPercentage float32

	// Maximum number of active btree snapshots
	MaxActiveSnapshots int

	// Max size of a single Btree node in bytes
	MaxNodeSize int

	// Time between the most recent DB snapshot is automatically renewed
	RenewSnapRootAfter time.Duration

	// Minimum number of updates entries in the btree to allow for full compaction
	CompactionThld int

	// Additional delay added during indexing when full compaction is in progress
	DelayDuringCompaction time.Duration

	// Maximum number of simultaneously opened nodes files
	NodesLogMaxOpenedFiles int

	// Maximum number of simultaneously opened node history files
	HistoryLogMaxOpenedFiles int

	// Maximum number of simultaneously opened commit log files
	CommitLogMaxOpenedFiles int

	// Maximum number of transactions indexed together
	MaxBulkSize int

	// Maximum time waiting for more transactions to be committed and included into the same bulk
	BulkPreparationTimeout time.Duration
}

func DefaultIndexOptions

func DefaultIndexOptions() *IndexOptions

func (*IndexOptions) Validate added in v1.2.3

func (opts *IndexOptions) Validate() error

func (*IndexOptions) WithBulkPreparationTimeout added in v1.5.0

func (opts *IndexOptions) WithBulkPreparationTimeout(bulkPreparationTimeout time.Duration) *IndexOptions

func (*IndexOptions) WithCacheSize

func (opts *IndexOptions) WithCacheSize(cacheSize int) *IndexOptions

func (*IndexOptions) WithCleanupPercentage added in v1.2.3

func (opts *IndexOptions) WithCleanupPercentage(cleanupPercentage float32) *IndexOptions

func (*IndexOptions) WithCommitLogMaxOpenedFiles added in v1.2.3

func (opts *IndexOptions) WithCommitLogMaxOpenedFiles(commitLogMaxOpenedFiles int) *IndexOptions

func (*IndexOptions) WithCompactionThld added in v0.9.2

func (opts *IndexOptions) WithCompactionThld(compactionThld int) *IndexOptions

func (*IndexOptions) WithDelayDuringCompaction added in v0.9.2

func (opts *IndexOptions) WithDelayDuringCompaction(delayDuringCompaction time.Duration) *IndexOptions

func (*IndexOptions) WithFlushBufferSize added in v1.2.3

func (opts *IndexOptions) WithFlushBufferSize(flushBufferSize int) *IndexOptions

func (*IndexOptions) WithFlushThld

func (opts *IndexOptions) WithFlushThld(flushThld int) *IndexOptions

func (*IndexOptions) WithHistoryLogMaxOpenedFiles added in v1.2.3

func (opts *IndexOptions) WithHistoryLogMaxOpenedFiles(historyLogMaxOpenedFiles int) *IndexOptions

func (*IndexOptions) WithMaxActiveSnapshots

func (opts *IndexOptions) WithMaxActiveSnapshots(maxActiveSnapshots int) *IndexOptions

func (*IndexOptions) WithMaxBulkSize added in v1.5.0

func (opts *IndexOptions) WithMaxBulkSize(maxBulkSize int) *IndexOptions

func (*IndexOptions) WithMaxNodeSize

func (opts *IndexOptions) WithMaxNodeSize(maxNodeSize int) *IndexOptions

func (*IndexOptions) WithNodesLogMaxOpenedFiles added in v1.2.3

func (opts *IndexOptions) WithNodesLogMaxOpenedFiles(nodesLogMaxOpenedFiles int) *IndexOptions

func (*IndexOptions) WithRenewSnapRootAfter

func (opts *IndexOptions) WithRenewSnapRootAfter(renewSnapRootAfter time.Duration) *IndexOptions

func (*IndexOptions) WithSyncThld added in v1.2.3

func (opts *IndexOptions) WithSyncThld(syncThld int) *IndexOptions

type KVMetadata added in v1.2.0

type KVMetadata struct {
	// contains filtered or unexported fields
}

func NewKVMetadata added in v1.2.0

func NewKVMetadata() *KVMetadata

func (*KVMetadata) AsDeleted added in v1.2.0

func (md *KVMetadata) AsDeleted(deleted bool) error

func (*KVMetadata) AsNonIndexable added in v1.2.3

func (md *KVMetadata) AsNonIndexable(nonIndexable bool) error

func (*KVMetadata) Bytes added in v1.2.0

func (md *KVMetadata) Bytes() []byte

func (*KVMetadata) Deleted added in v1.2.0

func (md *KVMetadata) Deleted() bool

func (*KVMetadata) ExpirationTime added in v1.2.0

func (md *KVMetadata) ExpirationTime() (time.Time, error)

func (*KVMetadata) ExpiredAt added in v1.2.0

func (md *KVMetadata) ExpiredAt(mtime time.Time) bool

func (*KVMetadata) ExpiresAt added in v1.2.0

func (md *KVMetadata) ExpiresAt(expiresAt time.Time) error

func (*KVMetadata) IsExpirable added in v1.2.0

func (md *KVMetadata) IsExpirable() bool

func (*KVMetadata) NonExpirable added in v1.2.0

func (md *KVMetadata) NonExpirable() *KVMetadata

func (*KVMetadata) NonIndexable added in v1.2.3

func (md *KVMetadata) NonIndexable() bool

type KeyIndex added in v1.0.0

type KeyIndex interface {
	Get(key []byte) (valRef ValueRef, err error)
	GetWithFilters(key []byte, filters ...FilterFn) (valRef ValueRef, err error)
	GetWithPrefix(prefix []byte, neq []byte) (key []byte, valRef ValueRef, err error)
	GetWithPrefixAndFilters(prefix []byte, neq []byte, filters ...FilterFn) (key []byte, valRef ValueRef, err error)
}

type KeyReader added in v0.9.1

type KeyReader interface {
	Read() (key []byte, val ValueRef, err error)
	ReadBetween(initialTxID uint64, finalTxID uint64) (key []byte, val ValueRef, err error)
	Reset() error
	Close() error
}

type KeyReaderSpec added in v1.0.0

type KeyReaderSpec struct {
	SeekKey       []byte
	EndKey        []byte
	Prefix        []byte
	InclusiveSeek bool
	InclusiveEnd  bool
	DescOrder     bool
	Filters       []FilterFn
	Offset        uint64
}

type LinearAdvanceProof added in v1.4.1

type LinearAdvanceProof struct {
	LinearProofTerms [][sha256.Size]byte
	InclusionProofs  [][][sha256.Size]byte
}

type LinearProof

type LinearProof struct {
	SourceTxID uint64
	TargetTxID uint64
	Terms      [][sha256.Size]byte
}

type NotificationType added in v0.9.2

type NotificationType = int
const (
	Info NotificationType = iota
	Warn
	Error
)

type OngoingTx added in v1.2.0

type OngoingTx struct {
	// contains filtered or unexported fields
}

OngoingTx (no-thread safe) represents an interactive or incremental transaction with support of RYOW. The snapshot may be locally modified but isolated from other transactions

func (*OngoingTx) AddPrecondition added in v1.2.3

func (tx *OngoingTx) AddPrecondition(c Precondition) error

func (*OngoingTx) AsyncCommit added in v1.2.0

func (tx *OngoingTx) AsyncCommit(ctx context.Context) (*TxHeader, error)

func (*OngoingTx) Cancel added in v1.2.0

func (tx *OngoingTx) Cancel() error

func (*OngoingTx) Closed added in v1.5.0

func (tx *OngoingTx) Closed() bool

func (*OngoingTx) Commit added in v1.2.0

func (tx *OngoingTx) Commit(ctx context.Context) (*TxHeader, error)

func (*OngoingTx) Delete added in v1.2.0

func (tx *OngoingTx) Delete(key []byte) error

func (*OngoingTx) Get added in v1.2.0

func (tx *OngoingTx) Get(key []byte) (ValueRef, error)

func (*OngoingTx) GetWithFilters added in v1.5.0

func (tx *OngoingTx) GetWithFilters(key []byte, filters ...FilterFn) (ValueRef, error)

func (*OngoingTx) GetWithPrefix added in v1.5.0

func (tx *OngoingTx) GetWithPrefix(prefix, neq []byte) (key []byte, valRef ValueRef, err error)

func (*OngoingTx) GetWithPrefixAndFilters added in v1.5.0

func (tx *OngoingTx) GetWithPrefixAndFilters(prefix, neq []byte, filters ...FilterFn) (key []byte, valRef ValueRef, err error)

func (*OngoingTx) IsReadOnly added in v1.5.0

func (tx *OngoingTx) IsReadOnly() bool

func (*OngoingTx) IsWriteOnly added in v1.2.0

func (tx *OngoingTx) IsWriteOnly() bool

func (*OngoingTx) Metadata added in v1.2.0

func (tx *OngoingTx) Metadata() *TxMetadata

func (*OngoingTx) NewKeyReader added in v1.2.0

func (tx *OngoingTx) NewKeyReader(spec KeyReaderSpec) (KeyReader, error)

func (*OngoingTx) RequireMVCCOnFollowingTxs added in v1.5.0

func (tx *OngoingTx) RequireMVCCOnFollowingTxs(requireMVCCOnFollowingTxs bool) error

func (*OngoingTx) Set added in v1.2.0

func (tx *OngoingTx) Set(key []byte, md *KVMetadata, value []byte) error

func (*OngoingTx) Timestamp added in v1.3.1

func (tx *OngoingTx) Timestamp() time.Time

func (*OngoingTx) WithMetadata added in v1.2.0

func (tx *OngoingTx) WithMetadata(md *TxMetadata) *OngoingTx

type Options

type Options struct {
	ReadOnly bool

	// Fsync during commit process
	Synced bool

	// Fsync frequency during commit process
	SyncFrequency time.Duration

	// Size of the in-memory buffer for write operations
	WriteBufferSize int

	FileMode os.FileMode

	CompactionDisabled bool

	// Maximum number of pre-committed transactions
	MaxActiveTransactions int

	// Limit the number of read entries per transaction
	MVCCReadSetLimit int

	// Maximum number of simultaneous commits prepared for write
	MaxConcurrency int

	// Maximum number of simultaneous IO writes
	MaxIOConcurrency int

	// Size of the LRU cache for transaction logs
	TxLogCacheSize int

	// Maximum number of simultaneous value files opened
	VLogMaxOpenedFiles int

	// Size of the LRU cache for value logs
	VLogCacheSize int

	// Maximum number of simultaneous transaction log files opened
	TxLogMaxOpenedFiles int

	// Maximum number of simultaneous commit log files opened
	CommitLogMaxOpenedFiles int

	// Version of transaction header to use (limits available features)
	WriteTxHeaderVersion int

	// Maximum number of go-routines waiting for specific transactions to be in a committed or indexed state
	MaxWaitees int

	TimeFunc TimeFunc

	UseExternalCommitAllowance bool

	// options below are only set during initialization and stored as metadata
	MaxTxEntries      int
	MaxKeyLen         int
	MaxValueLen       int
	FileSize          int
	CompressionFormat int
	CompressionLevel  int
	EmbeddedValues    bool
	PreallocFiles     bool

	// options below affect indexing
	IndexOpts *IndexOptions

	// options below affect appendable hash tree
	AHTOpts *AHTOptions
	// contains filtered or unexported fields
}

func DefaultOptions

func DefaultOptions() *Options

func (*Options) Validate added in v1.2.3

func (opts *Options) Validate() error

func (*Options) WithAHTOptions added in v1.3.2

func (opts *Options) WithAHTOptions(ahtOptions *AHTOptions) *Options

func (*Options) WithAppFactory added in v1.0.5

func (opts *Options) WithAppFactory(appFactory AppFactoryFunc) *Options

func (*Options) WithCommitLogMaxOpenedFiles

func (opts *Options) WithCommitLogMaxOpenedFiles(commitLogMaxOpenedFiles int) *Options

func (*Options) WithCompactionDisabled added in v1.0.5

func (opts *Options) WithCompactionDisabled(disabled bool) *Options

func (*Options) WithCompresionLevel

func (opts *Options) WithCompresionLevel(compressionLevel int) *Options

func (*Options) WithCompressionFormat

func (opts *Options) WithCompressionFormat(compressionFormat int) *Options

func (*Options) WithEmbeddedValues added in v1.5.0

func (opts *Options) WithEmbeddedValues(embeddedValues bool) *Options

func (*Options) WithExternalCommitAllowance added in v1.4.0

func (opts *Options) WithExternalCommitAllowance(useExternalCommitAllowance bool) *Options

func (*Options) WithFileMode

func (opts *Options) WithFileMode(fileMode os.FileMode) *Options

func (*Options) WithFileSize

func (opts *Options) WithFileSize(fileSize int) *Options

func (*Options) WithIndexOptions

func (opts *Options) WithIndexOptions(indexOptions *IndexOptions) *Options

func (*Options) WithLogger added in v1.2.4

func (opts *Options) WithLogger(logger logger.Logger) *Options

func (*Options) WithMVCCReadSetLimit added in v1.5.0

func (opts *Options) WithMVCCReadSetLimit(mvccReadSetLimit int) *Options

func (*Options) WithMaxActiveTransactions added in v1.3.2

func (opts *Options) WithMaxActiveTransactions(maxActiveTransactions int) *Options

func (*Options) WithMaxConcurrency

func (opts *Options) WithMaxConcurrency(maxConcurrency int) *Options

func (*Options) WithMaxIOConcurrency

func (opts *Options) WithMaxIOConcurrency(maxIOConcurrency int) *Options

func (*Options) WithMaxKeyLen

func (opts *Options) WithMaxKeyLen(maxKeyLen int) *Options

func (*Options) WithMaxTxEntries

func (opts *Options) WithMaxTxEntries(maxTxEntries int) *Options

func (*Options) WithMaxValueLen

func (opts *Options) WithMaxValueLen(maxValueLen int) *Options

func (*Options) WithMaxWaitees added in v0.9.2

func (opts *Options) WithMaxWaitees(maxWaitees int) *Options

func (*Options) WithPreallocFiles added in v1.5.0

func (opts *Options) WithPreallocFiles(preallocFiles bool) *Options

func (*Options) WithReadOnly

func (opts *Options) WithReadOnly(readOnly bool) *Options

func (*Options) WithSyncFrequency added in v1.3.2

func (opts *Options) WithSyncFrequency(frequency time.Duration) *Options

func (*Options) WithSynced

func (opts *Options) WithSynced(synced bool) *Options

func (*Options) WithTimeFunc added in v1.1.0

func (opts *Options) WithTimeFunc(timeFunc TimeFunc) *Options

func (*Options) WithTxLogCacheSize added in v0.9.2

func (opts *Options) WithTxLogCacheSize(txLogCacheSize int) *Options

func (*Options) WithTxLogMaxOpenedFiles

func (opts *Options) WithTxLogMaxOpenedFiles(txLogMaxOpenedFiles int) *Options

func (*Options) WithVLogCacheSize added in v1.5.0

func (opts *Options) WithVLogCacheSize(vLogCacheSize int) *Options

func (*Options) WithVLogMaxOpenedFiles

func (opts *Options) WithVLogMaxOpenedFiles(vLogMaxOpenedFiles int) *Options

func (*Options) WithWriteBufferSize added in v1.3.2

func (opts *Options) WithWriteBufferSize(writeBufferSize int) *Options

func (*Options) WithWriteTxHeaderVersion added in v1.2.3

func (opts *Options) WithWriteTxHeaderVersion(version int) *Options

type Precondition added in v1.2.3

type Precondition interface {
	String() string

	// Validate performs initial validation check to discard invalid preconditions before even executing them
	Validate(st *ImmuStore) error

	// Check performs the validation on a current state of the database
	Check(idx KeyIndex) (bool, error)
}

type PreconditionKeyMustExist added in v1.2.3

type PreconditionKeyMustExist struct {
	Key []byte
}

func (*PreconditionKeyMustExist) Check added in v1.2.3

func (cs *PreconditionKeyMustExist) Check(idx KeyIndex) (bool, error)

func (*PreconditionKeyMustExist) String added in v1.2.3

func (cs *PreconditionKeyMustExist) String() string

func (*PreconditionKeyMustExist) Validate added in v1.2.3

func (cs *PreconditionKeyMustExist) Validate(st *ImmuStore) error

type PreconditionKeyMustNotExist added in v1.2.3

type PreconditionKeyMustNotExist struct {
	Key []byte
}

func (*PreconditionKeyMustNotExist) Check added in v1.2.3

func (cs *PreconditionKeyMustNotExist) Check(idx KeyIndex) (bool, error)

func (*PreconditionKeyMustNotExist) String added in v1.2.3

func (cs *PreconditionKeyMustNotExist) String() string

func (*PreconditionKeyMustNotExist) Validate added in v1.2.3

func (cs *PreconditionKeyMustNotExist) Validate(st *ImmuStore) error

type PreconditionKeyNotModifiedAfterTx added in v1.2.3

type PreconditionKeyNotModifiedAfterTx struct {
	Key  []byte
	TxID uint64
}

func (*PreconditionKeyNotModifiedAfterTx) Check added in v1.2.3

func (*PreconditionKeyNotModifiedAfterTx) String added in v1.2.3

func (*PreconditionKeyNotModifiedAfterTx) Validate added in v1.2.3

type Snapshot added in v1.0.0

type Snapshot struct {
	// contains filtered or unexported fields
}

func (*Snapshot) Close added in v1.0.0

func (s *Snapshot) Close() error

func (*Snapshot) Get added in v1.0.0

func (s *Snapshot) Get(key []byte) (valRef ValueRef, err error)

func (*Snapshot) GetWithFilters added in v1.5.0

func (s *Snapshot) GetWithFilters(key []byte, filters ...FilterFn) (valRef ValueRef, err error)

func (*Snapshot) GetWithPrefix added in v1.5.0

func (s *Snapshot) GetWithPrefix(prefix []byte, neq []byte) (key []byte, valRef ValueRef, err error)

func (*Snapshot) GetWithPrefixAndFilters added in v1.5.0

func (s *Snapshot) GetWithPrefixAndFilters(prefix []byte, neq []byte, filters ...FilterFn) (key []byte, valRef ValueRef, err error)

func (*Snapshot) History added in v1.0.0

func (s *Snapshot) History(key []byte, offset uint64, descOrder bool, limit int) (tss []uint64, hCount uint64, err error)

func (*Snapshot) NewKeyReader added in v1.0.0

func (s *Snapshot) NewKeyReader(spec KeyReaderSpec) (KeyReader, error)

func (*Snapshot) Ts added in v1.0.0

func (s *Snapshot) Ts() uint64

type TimeFunc added in v1.1.0

type TimeFunc func() time.Time

type Tx

type Tx struct {
	// contains filtered or unexported fields
}

func NewTx added in v0.9.0

func NewTx(nentries int, maxKeyLen int) *Tx

func NewTxWithEntries added in v0.9.0

func NewTxWithEntries(header *TxHeader, entries []*TxEntry) *Tx

func (*Tx) BuildHashTree added in v0.9.0

func (tx *Tx) BuildHashTree() error

func (*Tx) Entries

func (tx *Tx) Entries() []*TxEntry

func (*Tx) EntryOf added in v1.2.0

func (tx *Tx) EntryOf(key []byte) (*TxEntry, error)

func (*Tx) Header added in v1.2.0

func (tx *Tx) Header() *TxHeader

func (*Tx) IndexOf

func (tx *Tx) IndexOf(key []byte) (int, error)

func (*Tx) Proof

func (tx *Tx) Proof(key []byte) (*htree.InclusionProof, error)

type TxEntry added in v0.9.1

type TxEntry struct {
	// contains filtered or unexported fields
}

func NewTxEntry added in v0.9.1

func NewTxEntry(key []byte, md *KVMetadata, vLen int, hVal [sha256.Size]byte, vOff int64) *TxEntry

func (*TxEntry) HVal added in v0.9.1

func (e *TxEntry) HVal() [sha256.Size]byte

func (*TxEntry) Key added in v0.9.1

func (e *TxEntry) Key() []byte

func (*TxEntry) Metadata added in v1.2.0

func (e *TxEntry) Metadata() *KVMetadata

func (*TxEntry) VLen added in v0.9.1

func (e *TxEntry) VLen() int

func (*TxEntry) VOff added in v0.9.1

func (e *TxEntry) VOff() int64

type TxEntryDigest added in v1.2.0

type TxEntryDigest func(e *TxEntry) ([sha256.Size]byte, error)

type TxHeader added in v1.2.0

type TxHeader struct {
	ID      uint64
	Ts      int64
	BlTxID  uint64
	BlRoot  [sha256.Size]byte
	PrevAlh [sha256.Size]byte

	Version int

	Metadata *TxMetadata

	NEntries int
	Eh       [sha256.Size]byte
}

func (*TxHeader) Alh added in v1.2.0

func (hdr *TxHeader) Alh() [sha256.Size]byte

Alh calculates the Accumulative Linear Hash up to this transaction Alh is calculated as hash(txID + prevAlh + hash(ts + nentries + eH + blTxID + blRoot)) Inner hash is calculated so to reduce the length of linear proofs

func (*TxHeader) Bytes added in v1.2.0

func (hdr *TxHeader) Bytes() ([]byte, error)

func (*TxHeader) ReadFrom added in v1.2.0

func (hdr *TxHeader) ReadFrom(b []byte) error

func (*TxHeader) TxEntryDigest added in v1.3.2

func (hdr *TxHeader) TxEntryDigest() (TxEntryDigest, error)

type TxMetadata

type TxMetadata struct {
	// contains filtered or unexported fields
}

TxMetadata is used to store metadata of a transaction.

func NewTxMetadata added in v1.2.0

func NewTxMetadata() *TxMetadata

func (*TxMetadata) Bytes added in v1.2.0

func (md *TxMetadata) Bytes() []byte

func (*TxMetadata) Equal added in v1.2.0

func (md *TxMetadata) Equal(amd *TxMetadata) bool

func (*TxMetadata) GetTruncatedTxID added in v1.5.0

func (md *TxMetadata) GetTruncatedTxID() (uint64, error)

GetTruncatedTxID returns the transaction ID up to which the database was last truncated.

func (*TxMetadata) HasTruncatedTxID added in v1.5.0

func (md *TxMetadata) HasTruncatedTxID() bool

HasTruncatedTxID returns true if the transaction stores the information up to which given transaction ID the database was truncated.

func (*TxMetadata) IsEmpty added in v1.5.0

func (md *TxMetadata) IsEmpty() bool

func (*TxMetadata) ReadFrom added in v1.2.0

func (md *TxMetadata) ReadFrom(b []byte) error

func (*TxMetadata) WithTruncatedTxID added in v1.5.0

func (md *TxMetadata) WithTruncatedTxID(txID uint64) *TxMetadata

WithTruncatedTxID sets the vlog truncated attribute indicating that the transaction stores the information up to which given transaction ID the database was truncated.

type TxMode added in v1.5.0

type TxMode int
const (
	ReadOnlyTx TxMode = iota
	WriteOnlyTx
	ReadWriteTx
)

type TxOptions added in v1.5.0

type TxOptions struct {
	Mode TxMode
	// SnapshotMustIncludeTxID is a function which receives the latest precommitted transaction ID as parameter.
	// It gives the possibility to reuse a snapshot which includes a percentage of the transactions already indexed
	// e.g. func(lastPrecommittedTxID uint64) uint64 { return  lastPrecommittedTxID * 90 / 100 }
	// or just a fixed transaction ID e.g. func(_ uint64) uint64 { return  1_000 }
	SnapshotMustIncludeTxID func(lastPrecommittedTxID uint64) uint64
	// SnapshotRenewalPeriod determines for how long a snaphsot may reuse existent dumped root
	SnapshotRenewalPeriod time.Duration

	// MVCC does not wait for indexing to be up to date
	UnsafeMVCC bool
}

func DefaultTxOptions added in v1.5.0

func DefaultTxOptions() *TxOptions

func (*TxOptions) Validate added in v1.5.0

func (opts *TxOptions) Validate() error

func (*TxOptions) WithMode added in v1.5.0

func (opts *TxOptions) WithMode(mode TxMode) *TxOptions

func (*TxOptions) WithSnapshotMustIncludeTxID added in v1.5.0

func (opts *TxOptions) WithSnapshotMustIncludeTxID(snapshotMustIncludeTxID func(lastPrecommittedTxID uint64) uint64) *TxOptions

func (*TxOptions) WithSnapshotRenewalPeriod added in v1.5.0

func (opts *TxOptions) WithSnapshotRenewalPeriod(snapshotRenewalPeriod time.Duration) *TxOptions

func (*TxOptions) WithUnsafeMVCC added in v1.5.0

func (opts *TxOptions) WithUnsafeMVCC(unsafeMVCC bool) *TxOptions

type TxPool added in v1.3.2

type TxPool interface {
	Alloc() (*Tx, error)
	Release(*Tx)
	Stats() (used, free, max int)
}

type TxReader

type TxReader struct {
	InitialTxID uint64
	Desc        bool

	CurrTxID uint64
	CurrAlh  [sha256.Size]byte
	// contains filtered or unexported fields
}

func (*TxReader) Read

func (txr *TxReader) Read() (*Tx, error)

type ValueRef added in v0.9.1

type ValueRef interface {
	Resolve() (val []byte, err error)
	Tx() uint64
	HC() uint64
	TxMetadata() *TxMetadata
	KVMetadata() *KVMetadata
	HVal() [sha256.Size]byte
	Len() uint32
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL