Documentation ¶
Overview ¶
Package dbadger provies a distributed embeddable key-value database based on BadgerDB persistent storage and Raft consensus algorithm.
- Uses BadgerDB as both data store and log store and simple fsync-ed file for stable store.
- Uses GRPC for inter-node communication and redirects requests to the leader node when necessary.
- Listens on a single port for both Raft and GRPC traffic by multiplexing TCP streams.
The main type in dbadger is DB that must be started with Start.
Usage ¶
config := dbadger.DefaultConfig("./datapath", "127.0.0.1:7001").WithBootstrap(true) db, err := dbadger.Start(config) if err != nil { log.Fatal(err) } // Run your application db.Set(context.TODO(), []byte("key"), []byte("value")) db.Get(context.TODO(), []byte("key"), dbadger.Eventual) if err := db.Close(); err != nil { log.Fatal(err) }
See the README.md, examples and tests for information.
Index ¶
- Constants
- Variables
- type Address
- type Config
- func (c *Config) WithBootstrap(bootstrap bool) *Config
- func (c *Config) WithCommitTimeout(v time.Duration) *Config
- func (c *Config) WithDataStoreConfig(v *StoreConfig) *Config
- func (c *Config) WithElectionTimeout(v time.Duration) *Config
- func (c *Config) WithHeartbeatTimeout(v time.Duration) *Config
- func (c *Config) WithInMemory(inmem bool) *Config
- func (c *Config) WithJoin(join Address) *Config
- func (c *Config) WithLeaderLeaseTimeout(v time.Duration) *Config
- func (c *Config) WithLogStoreConfig(v *StoreConfig) *Config
- func (c *Config) WithLogger(logger Logger) *Config
- func (c *Config) WithRecover(recover bool) *Config
- func (c *Config) WithSnapshotInterval(v time.Duration) *Config
- func (c *Config) WithSnapshotRetention(v int) *Config
- func (c *Config) WithSnapshotThreshold(v uint64) *Config
- func (c *Config) WithTLS(ca, cert, key []byte) *Config
- func (c *Config) WithTLSFiles(caFile, certFile, keyFile string) *Config
- func (c *Config) WithTrailingLogs(v uint64) *Config
- type DB
- func (db *DB) Addr() Address
- func (db *DB) Delete(ctx context.Context, key []byte) error
- func (db *DB) DeleteAll(ctx context.Context) error
- func (db *DB) DeleteMany(ctx context.Context, keys [][]byte) error
- func (db *DB) DeleteManyString(ctx context.Context, keys []string) error
- func (db *DB) DeletePrefix(ctx context.Context, prefix []byte) error
- func (db *DB) DeletePrefixString(ctx context.Context, prefix string) error
- func (db *DB) DeleteRange(ctx context.Context, min, max []byte) (keys [][]byte, _ error)
- func (db *DB) DeleteRangeString(ctx context.Context, min, max string) (keys []string, _ error)
- func (db *DB) DeleteString(ctx context.Context, key string) error
- func (db *DB) Get(ctx context.Context, key []byte, readPreference ReadPreference) ([]byte, error)
- func (db *DB) GetMany(ctx context.Context, keys [][]byte, readPreference ReadPreference) (values [][]byte, _ error)
- func (db *DB) GetManyString(ctx context.Context, keys []string, readPreference ReadPreference) (values []string, _ error)
- func (db *DB) GetPrefix(ctx context.Context, prefix []byte, readPreference ReadPreference) (keys, values [][]byte, _ error)
- func (db *DB) GetPrefixString(ctx context.Context, prefix string, readPreference ReadPreference) (keys, values []string, _ error)
- func (db *DB) GetRange(ctx context.Context, min, max []byte, count uint64, ...) (keys, values [][]byte, _ error)
- func (db *DB) GetRangeString(ctx context.Context, min, max string, count uint64, ...) (keys, values []string, _ error)
- func (db *DB) GetString(ctx context.Context, key string, readPreference ReadPreference) (string, error)
- func (db *DB) IsLeader() bool
- func (db *DB) IsReady() bool
- func (db *DB) Leader() Address
- func (db *DB) Nodes() []Address
- func (db *DB) Restore(id string) error
- func (db *DB) Set(ctx context.Context, key, value []byte) error
- func (db *DB) SetMany(ctx context.Context, keys, values [][]byte) error
- func (db *DB) SetManyString(ctx context.Context, keys, values []string) error
- func (db *DB) SetString(ctx context.Context, key, value string) error
- func (db *DB) Snapshot() (id string, err error)
- func (db *DB) Stats() map[string]string
- func (db *DB) Stop() error
- type Logger
- type ReadPreference
- type StoreConfig
- func (c *StoreConfig) WithBaseLevelSize(v int64) *StoreConfig
- func (c *StoreConfig) WithBaseTableSize(v int64) *StoreConfig
- func (c *StoreConfig) WithBlockCacheSize(v int64) *StoreConfig
- func (c *StoreConfig) WithBlockSize(v int) *StoreConfig
- func (c *StoreConfig) WithCompactL0OnClose(v bool) *StoreConfig
- func (c *StoreConfig) WithCompression(v bool) *StoreConfig
- func (c *StoreConfig) WithGC(interval time.Duration, discardRatio float64) *StoreConfig
- func (c *StoreConfig) WithLevelSizeMultiplier(v int) *StoreConfig
- func (c *StoreConfig) WithMaxLevels(v int) *StoreConfig
- func (c *StoreConfig) WithMemTableSize(v int64) *StoreConfig
- func (c *StoreConfig) WithNumCompactors(v int) *StoreConfig
- func (c *StoreConfig) WithNumLevelZeroTables(v int) *StoreConfig
- func (c *StoreConfig) WithNumLevelZeroTablesStall(v int) *StoreConfig
- func (c *StoreConfig) WithNumMemtables(v int) *StoreConfig
- func (c *StoreConfig) WithSyncWrites(v bool) *StoreConfig
- func (c *StoreConfig) WithValueLogFileSize(v int64) *StoreConfig
- func (c *StoreConfig) WithValueLogMaxEntries(v uint32) *StoreConfig
- type TLSConfig
Constants ¶
const ( // LeaderPreference tells DB to read data from the current cluster leader. // This ensures that no stale data is returned from the read operation. LeaderPreference = rpc.ReadPreference_LEADER // LocalPreference tells DB to read data from the current node. // This is faster than routing request to the leader, but may return stale data. LocalPreference = rpc.ReadPreference_LOCAL )
Variables ¶
var ( // ErrNoLeader is returned if a cluster has no leader or the leader is unknown at the moment. ErrNoLeader = errors.New("cluster has no leader") // ErrEmptyKey is returned if an empty key is passed to a write operation. ErrEmptyKey = errors.New("key can not be empty") // ErrInvalidKey is returned if the key has a special !badger! prefix, reserved for internal usage. ErrInvalidKey = errors.New("key is using a reserved !badger! prefix") // ErrNotFound is returned when key is not found in read operations. ErrNotFound = errors.New("key not found") // ErrInvalidRequest is returned if the user request is invalid. ErrInvalidRequest = errors.New("invalid request") // ErrConflict is returned when a transaction conflicts with another transaction. This can // happen if the read rows had been updated concurrently by another transaction. ErrConflict = errors.New("transaction conflict, please retry") // requested operation, but the operation should be retried with a backoff. // This can mean network errors, blocked writes during DeleteAll, etc. ErrUnavailable = errors.New("operation is currently unavailable, please retry") ErrInternal = errors.New("internal error") )
Errors that can be returned from DB methods.
Functions ¶
This section is empty.
Types ¶
type Config ¶
type Config struct { // Path is the path of the directory where data will be stored in. If directory does not exists it will be created. Path string // Bind is a network address for the node to listen on (for example, "127.0.0.1:7001", "[::1]:7001", ":7001"). Bind Address // TLS configures certificates and keys for the node. // // Zero value means TLS is disabled. TLS TLSConfig // InMemory makes node run in in-memory mode, which means everything is stored in memory and no files are created. // // All data will be lost when node is stopped or crashed. Used primarily for testing purposes. InMemory bool // Bootstrap a new cluster from this node. // // A cluster should only be bootstrapped once from a single primary node. // After that other nodes can join the cluster. Bootstrap bool // Recover cluster from a loss of quorum (e.g. when multiple nodes die at the same time) // by forcing a new cluster configuration. This works by reading all the current state // for this node, creating a snapshot with the new configuration, and then truncating the log. // // Typically to bring the cluster back up you should choose a node to become a new leader, // recover that node and then join new clean-sate nodes as usual. Recover bool // Join an existing cluster via given node. // // It is safe to join a cluster via any node. If the target node is not a leader of the cluster // it will forward the request to the cluster's leader. Join Address // HeartbeatTimeout specifies the time in follower state without contact from a leader before // Raft attempts an election. HeartbeatTimeout time.Duration // ElectionTimeout specifies the time in candidate state without contact from a leader before // Raft attempts an election. ElectionTimeout time.Duration // LeaderLeaseTimeout is used to control how long the lease lasts for being the leader without // being able to contact a quorum of nodes. If we reach this interval without contact, // the node will step down as leader. LeaderLeaseTimeout time.Duration // CommitTimeout specifies the time without an Apply operation before the leader sends an // AppendEntry RPC to followers, to ensure a timely commit of log entries. CommitTimeout time.Duration // SnapshotInterval controls how often Raft checks if it should perform a snapshot. // // Raft randomly staggers between this value and 2x this value to avoid the entire cluster // from performing a snapshot at once. SnapshotInterval time.Duration // SnapshotThreshold controls how many outstanding logs there must be before Raft performs a snapshot. // // This is to prevent excessive snapshotting by replaying a small set of logs instead. SnapshotThreshold uint64 // SnapshotRetention controls how many snapshots are retained. Must be at least 1. SnapshotRetention int // TrailingLogs controls how many logs Raft leaves after a snapshot. // // This is used so that a follower can quickly replay logs instead of being forced // to receive an entire snapshot. TrailingLogs uint64 // LogStoreConfig configures replicated log storage. LogStoreConfig *StoreConfig // DataStoreConfig configures data storage. DataStoreConfig *StoreConfig // Logger configures which logger DB uses. // // Leaving this nil will disable logging. Logger Logger }
Config is DBadger node config.
See DefaultConfig for defaults.
func DefaultConfig ¶
DefaultConfig return default Config.
func DefaultConfigInMemory ¶
DefaultConfigInMemory returns default Config with InMemory mode turned on.
func (*Config) WithBootstrap ¶
WithBootstrap returns Config with Bootstrap set to the given value.
func (*Config) WithCommitTimeout ¶ added in v0.1.1
WithCommitTimeout returns Config with CommitTimeout set to the given value.
func (*Config) WithDataStoreConfig ¶ added in v0.1.1
func (c *Config) WithDataStoreConfig(v *StoreConfig) *Config
WithDataStoreConfig returns Config with DataStoreConfig set to the given value.
func (*Config) WithElectionTimeout ¶ added in v0.1.1
WithElectionTimeout returns Config with ElectionTimeout set to the given value.
func (*Config) WithHeartbeatTimeout ¶ added in v0.1.1
WithHeartbeatTimeout returns Config with HeartbeatTimeout set to the given value.
func (*Config) WithInMemory ¶
WithInMemory returns Config with InMemory set to the given value.
func (*Config) WithLeaderLeaseTimeout ¶ added in v0.1.1
WithLeaderLeaseTimeout returns Config with LeaderLeaseTimeout set to the given value.
func (*Config) WithLogStoreConfig ¶ added in v0.1.1
func (c *Config) WithLogStoreConfig(v *StoreConfig) *Config
WithLogStoreConfig returns Config with LogStoreConfig set to the given value.
func (*Config) WithLogger ¶
WithLogger returns Config with Logger set to the given value.
func (*Config) WithRecover ¶
WithRecover returns Config with Recover set to the given value.
func (*Config) WithSnapshotInterval ¶ added in v0.1.1
WithSnapshotInterval returns Config with SnapshotInterval set to the given value.
func (*Config) WithSnapshotRetention ¶ added in v0.1.1
WithSnapshotRetention returns Config with SnapshotRetention set to the given value.
func (*Config) WithSnapshotThreshold ¶ added in v0.1.1
WithSnapshotThreshold returns Config with SnapshotThreshold set to the given value.
func (*Config) WithTLSFiles ¶
WithTLSFiles returns Config with TLS configuration set to the given values.
type DB ¶
type DB struct {
// contains filtered or unexported fields
}
DB is a DBadger node.
func (*DB) DeleteMany ¶
DeleteMany removes multiple keys.
func (*DB) DeleteManyString ¶
DeleteManyString removes multiple keys.
func (*DB) DeletePrefix ¶
DeletePrefix removes keys with the given prefix.
func (*DB) DeletePrefixString ¶
DeletePrefixString removes keys with the given prefix.
func (*DB) DeleteRange ¶
DeleteRange removes keys in range [min, max] and returns keys that has been removed.
func (*DB) DeleteRangeString ¶
DeleteRangeString removes keys in range [min, max] and returns keys that has been removed.
func (*DB) DeleteString ¶
DeleteString removes a key.
func (*DB) GetMany ¶
func (db *DB) GetMany(ctx context.Context, keys [][]byte, readPreference ReadPreference) (values [][]byte, _ error)
GetMany returns values corresponding to the given keys.
func (*DB) GetManyString ¶
func (db *DB) GetManyString(ctx context.Context, keys []string, readPreference ReadPreference) (values []string, _ error)
GetManyString returns values corresponding to the given keys.
func (*DB) GetPrefix ¶
func (db *DB) GetPrefix(ctx context.Context, prefix []byte, readPreference ReadPreference) (keys, values [][]byte, _ error)
GetPrefix returns values for the keys with the given prefix.
func (*DB) GetPrefixString ¶
func (db *DB) GetPrefixString(ctx context.Context, prefix string, readPreference ReadPreference) (keys, values []string, _ error)
GetPrefixString returns values for the keys with the given prefix.
func (*DB) GetRange ¶
func (db *DB) GetRange(ctx context.Context, min, max []byte, count uint64, readPreference ReadPreference) (keys, values [][]byte, _ error)
GetRange returns maximum of count values for the keys in range [min, max].
Both min and max can be nil.
func (*DB) GetRangeString ¶
func (db *DB) GetRangeString(ctx context.Context, min, max string, count uint64, readPreference ReadPreference) (keys, values []string, _ error)
GetRangeString returns maximum of count values for the keys in range [min, max].
Both min and max can be nil.
func (*DB) GetString ¶
func (db *DB) GetString(ctx context.Context, key string, readPreference ReadPreference) (string, error)
GetString returns value corresponding to the given key.
func (*DB) Restore ¶
Restore forces DB to consume a snapshot, such as if restoring from a backup. This should not be used in normal operation, only for disaster recovery onto a new cluster.
func (*DB) SetManyString ¶
SetManyString adds multiple key-value pairs.
func (*DB) Snapshot ¶
Snapshot forces DB to take a snapshot, e.g. for backup purposes. Returns snapshot id and error, if any.
type ReadPreference ¶
type ReadPreference = rpc.ReadPreference
ReadPreference determines how DB routes read requests to the nodes of the cluster.
type StoreConfig ¶ added in v0.1.1
type StoreConfig struct { // SyncWrites controls whether Badger would call an additional msync after writes to flush mmap // buffer over to disk to survive hard reboots. Most users should not need to do this. SyncWrites bool // MaxLevels is the maximum number of levels of compaction allowed in the LSM. MaxLevels int // LevelSizeMultiplier sets the ratio between the maximum sizes of contiguous levels in the LSM. // Once a level grows to be larger than this ratio, the compaction process will be triggered. LevelSizeMultiplier int // BaseTableSize sets the maximum size in bytes for LSM table or file in the base level. BaseTableSize int64 // BaseLevelSize sets the maximum size target for the base level. BaseLevelSize int64 // ValueLogFileSize sets the maximum size of a single value log file. ValueLogFileSize int64 // ValueLogMaxEntries sets the maximum number of entries a value log file can hold approximately. // // The actual size limit of a value log file is the // minimum of ValueLogFileSize and ValueLogMaxEntries. ValueLogMaxEntries uint32 // NumMemtables sets the maximum number of tables to keep in memory before stalling. NumMemtables int // MemTableSize sets the maximum size in bytes for memtable table. MemTableSize int64 // BlockSize sets the size of any block in SSTable. SSTable is divided into multiple blocks internally. BlockSize int // BlockCacheSize specifies how much data cache should be held in memory. A small size // of cache means lower memory consumption and lookups/iterations would take // longer. It is recommended to use a cache if you're using compression or encryption. // If compression and encryption both are disabled, adding a cache will lead to // unnecessary overhead which will affect the read performance. Setting size to // zero disables the cache altogether. BlockCacheSize int64 // NumLevelZeroTables sets the maximum number of Level 0 tables before compaction starts. NumLevelZeroTables int // NumLevelZeroTablesStall sets the number of Level 0 tables that once reached causes the DB to // stall until compaction succeeds. NumLevelZeroTablesStall int // NumCompactors sets the number of compaction workers to run concurrently. Setting this to // zero stops compactions, which could eventually cause writes to block forever. NumCompactors int // CompactL0OnClose sets whether Level 0 should be compacted before closing the DB. This ensures // that both reads and writes are efficient when the DB is opened later. CompactL0OnClose bool // Compression enables snappy compression for every block. Compression bool // GCEnabled enables value log garbage collection. GCEnabled bool // GCInterval sets value log garbage collection interval. GCInterval time.Duration // GCDiscardRatio sets threshold for rewriting files during garbage collection. // // During value log garbage collection Badger will rewrite files when it can discard // at least discardRatio space of that file. GCDiscardRatio float64 }
StoreConfig is a persistent storage configuration.
func DefaultDataStoreConfig ¶ added in v0.1.1
func DefaultDataStoreConfig() *StoreConfig
DefaultDataStoreConfig returns default data storage config.
func DefaultLogStoreConfig ¶ added in v0.1.1
func DefaultLogStoreConfig() *StoreConfig
DefaultLogStoreConfig returns default replicated log storage config.
func (*StoreConfig) WithBaseLevelSize ¶ added in v0.1.1
func (c *StoreConfig) WithBaseLevelSize(v int64) *StoreConfig
WithBaseLevelSize returns [StorageConfig] with BaseLevelSize set to the given value.
func (*StoreConfig) WithBaseTableSize ¶ added in v0.1.1
func (c *StoreConfig) WithBaseTableSize(v int64) *StoreConfig
WithBaseTableSize returns [StorageConfig] with BaseTableSize set to the given value.
func (*StoreConfig) WithBlockCacheSize ¶ added in v0.1.1
func (c *StoreConfig) WithBlockCacheSize(v int64) *StoreConfig
WithBlockCacheSize returns [StorageConfig] with BlockCacheSize set to the given value.
func (*StoreConfig) WithBlockSize ¶ added in v0.1.1
func (c *StoreConfig) WithBlockSize(v int) *StoreConfig
WithBlockSize returns [StorageConfig] with BlockSize set to the given value.
func (*StoreConfig) WithCompactL0OnClose ¶ added in v0.1.1
func (c *StoreConfig) WithCompactL0OnClose(v bool) *StoreConfig
WithCompactL0OnClose returns [StorageConfig] with CompactL0OnClose set to the given value.
func (*StoreConfig) WithCompression ¶ added in v0.1.1
func (c *StoreConfig) WithCompression(v bool) *StoreConfig
WithCompression returns [StorageConfig] with Compression set to the given value.
func (*StoreConfig) WithGC ¶ added in v0.1.1
func (c *StoreConfig) WithGC(interval time.Duration, discardRatio float64) *StoreConfig
WithGC returns [StorageConfig] with value log garbage collection enabled.
func (*StoreConfig) WithLevelSizeMultiplier ¶ added in v0.1.1
func (c *StoreConfig) WithLevelSizeMultiplier(v int) *StoreConfig
WithLevelSizeMultiplier returns [StorageConfig] with LevelSizeMultiplier set to the given value.
func (*StoreConfig) WithMaxLevels ¶ added in v0.1.1
func (c *StoreConfig) WithMaxLevels(v int) *StoreConfig
WithMaxLevels returns [StorageConfig] with MaxLevels set to the given value.
func (*StoreConfig) WithMemTableSize ¶ added in v0.1.1
func (c *StoreConfig) WithMemTableSize(v int64) *StoreConfig
WithMemTableSize returns [StorageConfig] with MemTableSize set to the given value.
func (*StoreConfig) WithNumCompactors ¶ added in v0.1.1
func (c *StoreConfig) WithNumCompactors(v int) *StoreConfig
WithNumCompactors returns [StorageConfig] with NumCompactors set to the given value.
func (*StoreConfig) WithNumLevelZeroTables ¶ added in v0.1.1
func (c *StoreConfig) WithNumLevelZeroTables(v int) *StoreConfig
WithNumLevelZeroTables returns [StorageConfig] with NumLevelZeroTables set to the given value.
func (*StoreConfig) WithNumLevelZeroTablesStall ¶ added in v0.1.1
func (c *StoreConfig) WithNumLevelZeroTablesStall(v int) *StoreConfig
WithNumLevelZeroTablesStall returns [StorageConfig] with NumLevelZeroTablesStall set to the given value.
func (*StoreConfig) WithNumMemtables ¶ added in v0.1.1
func (c *StoreConfig) WithNumMemtables(v int) *StoreConfig
WithNumMemtables returns [StorageConfig] with NumMemtables set to the given value.
func (*StoreConfig) WithSyncWrites ¶ added in v0.1.1
func (c *StoreConfig) WithSyncWrites(v bool) *StoreConfig
WithSyncWrites returns [StorageConfig] with SyncWrites set to the given value.
func (*StoreConfig) WithValueLogFileSize ¶ added in v0.1.1
func (c *StoreConfig) WithValueLogFileSize(v int64) *StoreConfig
WithValueLogFileSize returns [StorageConfig] with ValueLogFileSize set to the given value.
func (*StoreConfig) WithValueLogMaxEntries ¶ added in v0.1.1
func (c *StoreConfig) WithValueLogMaxEntries(v uint32) *StoreConfig
WithValueLogMaxEntries returns [StorageConfig] with ValueLogMaxEntries set to the given value.
type TLSConfig ¶
type TLSConfig struct { CA []byte // PEM encoded TLS certificate authority. Cert []byte // PEM encoded TLS certificate. Key []byte // PEM encoded TLS private key. CAFile string // TLS certificate authority file path. CertFile string // TLS certificate file path. KeyFile string // TLS private key file path. }
TLSConfig is a TLS configuration.
PEM encoded options are prioritized over file paths. You can mix both file and PEM options in the same config.