storager

package module
v0.0.0-...-c602644 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 16, 2023 License: MIT Imports: 33 Imported by: 1

README

xdis-storager

This library is used to implement the mapping between resp cmd(like redis commands) and openkv store impl

feature

  1. support slot tag key migrate, for (smart client/proxy)'s configSrv admin contoller lay use it~~ use SLOTSMGRTTAGSLOT cmd to migrate slot's key with same tag
127.0.0.1:6660> setex 122{tag} 86400 v3
OK
127.0.0.1:6660> setex 1{tag} 86400 v3
OK
127.0.0.1:6660> sadd 12{tag} m1 m2 m3
(integer) 0
127.0.0.1:6660> hmset 123{tag} f1 v1 f2 v2
OK
127.0.0.1:6660> lpush 123{tag} l1 l2
(integer) 2
127.0.0.1:6660> zadd 123{tag} 100 z1 10 z2
(integer) 2
127.0.0.1:6660> slotshashkey 123{tag}
1) (integer) 899
127.0.0.1:6660> slotsinfo 899 0 withsize
1) 1) (integer) 899
   2) (integer) 6
127.0.0.1:6660> SLOTSMGRTTAGSLOT 127.0.0.1 6666 300000 899
1) (integer) 6
2) (integer) 0
127.0.0.1:6660> SLOTSMGRTTAGSLOT 127.0.0.1 6666 300000 899
1) (integer) 0
2) (integer) 0

127.0.0.1:6666> slotsinfo 0 1024 withsize
1) 1) (integer) 899
   2) (integer) 6
127.0.0.1:6666> get 122{tag}
"v3"
127.0.0.1:6666> ttl 122{tag}
(integer) 86133
127.0.0.1:6666> get 1{tag}
"v3"
127.0.0.1:6666> ttl 1{tag}
(integer) 86120
127.0.0.1:6666> hgetall 123{tag}
1) "f1"
2) "v1"
3) "f2"
4) "v2"
127.0.0.1:6666> lrange 123{tag} 0 -1
1) "l2"
2) "l1"
127.0.0.1:6666> zrange 123{tag} 0 10 withscores
1) "z2"
2) "10"
3) "z1"
4) "100"

Tips: if use codis-proxy, codis-dashboard config set migration_method = "sync"

Build & Inject

see wedis, just simple go build/run to inject it and build/run.

SuperMarioBros

  1. RediXSlots
  2. xdis-tikv

Reference

Documentation

Index

Constants

View Source 
const (
	// max allowed databases
	MaxDatabases int = 10240

	// max key size
	MaxKeySize int = 1024

	// max hash field size
	MaxHashFieldSize int = 1024

	// max zset member size
	MaxZSetMemberSize int = 1024

	// max set member size
	MaxSetMemberSize int = 1024

	// max value size
	MaxValueSize int = 1024 * 1024 * 1024

	// default scan count
	DefaultScanCount int = 10
)

For different const size configuration

View Source 
const (
	StringName = "STRING"
	ListName   = "LIST"
	HashName   = "HASH"
	SetName    = "SET"
	ZSetName   = "ZSET"
	BitmapName = "BITMAP"
)

For different type name

View Source 
const (
	NoneType byte = iota
	StringType

	HSizeType
	HashType

	LMetaType
	ListType

	SSizeType
	SetType

	ZSizeType
	ZSetType
	ZScoreType

	ExpTimeType
	ExpMetaType

	BitmapType
)

for backend store key notice: Please add new type in order

View Source 
const (
	UnionType byte = 51
	DiffType  byte = 52
	InterType byte = 53
)

For set op

View Source 
const (
	MinScore     int64 = math.MinInt64 + 1
	MaxScore     int64 = math.MaxInt64
	InvalidScore int64 = math.MinInt64

	AggregateSum = "sum"
	AggregateMin = "min"
	AggregateMax = "max"
)

For zset op

View Source 
const (
	BitAND = "and"
	BitOR  = "or"
	BitXOR = "xor"
	BitNot = "not"
)

For bit operation

View Source 
const (
	MaxVarintLen16 = 3
	MaxVarintLen32 = 5
	// the most size for varint is 10 bytes
	MaxVarintLen64 = 10
)

MaxVarintLenN is the maximum length of a varint-encoded N-bit integer.

View Source 
const (
	CodeTypeMeta byte = '@'
	CodeTypeData byte = '$'

	Version byte = 0
)
View Source 
const (
	RegisterStoragerName = "xdis-storager"
)

Variables

View Source 
var (
	ErrValueIntOutOfRange = errors.New("ERR value is not an integer or out of range")
	ErrIntNumber          = errors.New("ERR invalid integer")

	ErrKeySize     = errors.New("ERR invalid key size")
	ErrValueSize   = errors.New("ERR invalid value size")
	ErrExpireValue = errors.New("ERR invalid expire value")

	ErrStringKey = errors.New("ERR invalid encode string key")

	ErrLMetaKey  = errors.New("ERR invalid lmeta key")
	ErrListKey   = errors.New("ERR invalid list key")
	ErrListSeq   = errors.New("ERR invalid list sequence, overflow")
	ErrListIndex = errors.New("ERR invalid list index")

	ErrHashKey       = errors.New("ERR invalid hash key")
	ErrHashIntVal    = errors.New("ERR hash value is not an integer")
	ErrHSizeKey      = errors.New("ERR invalid hsize key")
	ErrHashFieldSize = errors.New("ERR invalid hash field size")

	ErrSetKey        = errors.New("ERR invalid set key")
	ErrSSizeKey      = errors.New("ERR invalid ssize key")
	ErrSetMemberSize = errors.New("ERR invalid set member size")

	ErrZSizeKey         = errors.New("ERR invalid zsize key")
	ErrZSetKey          = errors.New("ERR invalid zset key")
	ErrZScoreKey        = errors.New("ERR invalid zscore key")
	ErrScoreOverflow    = errors.New("ERR zset score overflow")
	ErrInvalidAggregate = errors.New("ERR invalid aggregate")
	ErrInvalidWeightNum = errors.New("ERR invalid weight number")
	ErrInvalidSrcKeyNum = errors.New("ERR invalid src key number")
	ErrZSetMemberSize   = errors.New("ERR invalid zset member size")

	ErrExpMetaKey = errors.New("ERR invalid expire meta key")
	ErrExpTimeKey = errors.New("ERR invalid expire time key")

	ErrDataType = errors.New("ERR invalid data type")
	ErrMetaKey  = errors.New("ERR invalid meta key")

	ErrBitmapKey = errors.New("ERR invalid encode bitmap key")

	// For different common errors
	ErrScoreMiss = errors.New("ERR zset score miss")

	// For slots
	ErrUnsupportSlots = errors.New("unsupport slots")
	ErrKeyNotFound    = errors.New("key not found")
)
View Source 
var TypeName = map[byte]string{
	StringType:  "string",
	HashType:    "hash",
	HSizeType:   "hsize",
	ListType:    "list",
	LMetaType:   "lmeta",
	SetType:     "set",
	SSizeType:   "ssize",
	ZSetType:    "zset",
	ZSizeType:   "zsize",
	ZScoreType:  "zscore",
	ExpTimeType: "exptime",
	ExpMetaType: "expmeta",
	BitmapType:  "bitmap",
}

TypeName is the map of type -> name

Functions

func AddDBExpKeyStats 

func AddDBExpKeyStats(store *Storager, stats *DBStats, mk, tk []byte, exp int64)

func AddDBKeyCn 

func AddDBKeyCn(store *Storager, stats *DBStats, key []byte)

func DBHasKey 

func DBHasKey(s *Storager, index int) bool

func HashTag 

func HashTag(key []byte) []byte

HashTag like redis cluster hash tag

func InitAllDbStats 

func InitAllDbStats(s *Storager)

InitDbStats begin init db stats load to map after storager open

func Int64 

func Int64(v []byte, err error) (int64, error)

Int64 gets 64 integer with the little endian format.

func MaxInt32 

func MaxInt32(a int32, b int32) int32

func MinInt32 

func MinInt32(a int32, b int32) int32

func PutInt64 

func PutInt64(v int64) []byte

PutInt64 puts the 64 integer.

func RemoveDBExpKeyStats 

func RemoveDBExpKeyStats(store *Storager, stats *DBStats, mk, tk []byte, exp int64)

func RemoveDBKeyCn 

func RemoveDBKeyCn(store *Storager, stats *DBStats, key []byte)

func Uint64 

func Uint64(v []byte, err error) (uint64, error)

Uint64 gets unsigned 64 integer with the little endian format.

Types

type Batch 

type Batch struct {
	*openkv.WriteBatch

	sync.Locker
	// contains filtered or unexported fields
}

Batch write batch commit

func NewBatch 

func NewBatch(store *Storager, wb *openkv.WriteBatch, locker sync.Locker) *Batch

func (*Batch) Commit 

func (b *Batch) Commit(ctx context.Context) error

func (*Batch) Delete 

func (b *Batch) Delete(key []byte)

func (*Batch) Lock 

func (b *Batch) Lock()

func (*Batch) Put 

func (b *Batch) Put(key []byte, value []byte)

func (*Batch) Unlock 

func (b *Batch) Unlock()

type DB 

type DB struct {

	// IKV impl
	kvDriver.IKV
	// contains filtered or unexported fields
}

DB core sturct impl like redis string, list, hash, set, zset, bitmap struct store db op

func NewDB 

func NewDB(store *Storager, idx int) *DB

func (*DB) Close 

func (m *DB) Close() (err error)

func (*DB) DBBitmap 

func (m *DB) DBBitmap() driver.IBitmapCmd

func (*DB) DBHash 

func (m *DB) DBHash() driver.IHashCmd

func (*DB) DBList 

func (m *DB) DBList() driver.IListCmd

func (*DB) DBSet 

func (m *DB) DBSet() driver.ISetCmd

func (*DB) DBSlot 

func (m *DB) DBSlot() driver.ISlotsCmd

func (*DB) DBString 

func (m *DB) DBString() driver.IStringCmd

func (*DB) DBZSet 

func (m *DB) DBZSet() driver.IZsetCmd

func (*DB) DelKeyMeta 

func (m *DB) DelKeyMeta(t *Batch, key []byte, dataType byte)

func (*DB) FlushDB 

func (db *DB) FlushDB(ctx context.Context) (drop int64, err error)

FlushDB flushes the data.

func (*DB) Index 

func (db *DB) Index() int

Index gets the index of database.

func (*DB) IndexVarBuf 

func (db *DB) IndexVarBuf() []byte

IndexVarBuf gets the index varint buf of database.

func (*DB) SetIndex 

func (db *DB) SetIndex(index int)

SetIndex set the index of database.

func (*DB) SetKeyMeta 

func (m *DB) SetKeyMeta(t *Batch, key []byte, dataType byte)

type DBBitmap 

type DBBitmap struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBBitmap 

func NewDBBitmap(db *DB) *DBBitmap

func (*DBBitmap) BitCount 

func (db *DBBitmap) BitCount(ctx context.Context, key []byte, start int, end int) (int64, error)

BitCount returns the bit count of data.

func (*DBBitmap) BitOP 

func (db *DBBitmap) BitOP(ctx context.Context, op string, destKey []byte, srcKeys ...[]byte) (int64, error)

BitOP does the bit operations in data.

func (*DBBitmap) BitPos 

func (db *DBBitmap) BitPos(ctx context.Context, key []byte, on int, start int, end int) (int64, error)

BitPos returns the pos of the data.

func (*DBBitmap) GetBit 

func (db *DBBitmap) GetBit(ctx context.Context, key []byte, offset int) (int64, error)

GetBit gets the bit of data at offset.

func (*DBBitmap) SetBit 

func (db *DBBitmap) SetBit(ctx context.Context, key []byte, offset int, on int) (int64, error)

SetBit sets the bit to the data.

type DBHash 

type DBHash struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBHash 

func NewDBHash(db *DB) *DBHash

func (*DBHash) BatchDel 

func (db *DBHash) BatchDel(ctx context.Context, t *Batch, keys ...[]byte) (int64, error)

BatchDel cleans multi hash data.

func (*DBHash) BatchExpire 

func (db *DBHash) BatchExpire(ctx context.Context, t *Batch, key []byte, duration int64) (int64, error)

BatchExpire expires the data with duration.

func (*DBHash) BatchHMset 

func (db *DBHash) BatchHMset(ctx context.Context, t *Batch, key []byte, args ...driver.FVPair) error

BatchHMset sets multi field-values.

func (*DBHash) Clear 

func (db *DBHash) Clear(ctx context.Context, key []byte) (int64, error)

Clear clears the data.

func (*DBHash) Del 

func (db *DBHash) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del cleans multi hash data.

func (*DBHash) Dump 

func (db *DBHash) Dump(ctx context.Context, key []byte) (binVal []byte, err error)

Dump hash rdb

func (*DBHash) Exists 

func (db *DBHash) Exists(ctx context.Context, key []byte) (int64, error)

HKeyExists checks whether data exists or not.

func (*DBHash) Expire 

func (db *DBHash) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

Expire expires the data with duration.

func (*DBHash) ExpireAt 

func (db *DBHash) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

ExpireAt expires the data at time when.

func (*DBHash) HDel 

func (db *DBHash) HDel(ctx context.Context, key []byte, args ...[]byte) (int64, error)

HDel deletes the fields.

func (*DBHash) HGet 

func (db *DBHash) HGet(ctx context.Context, key []byte, field []byte) ([]byte, error)

HGet gets the value of the field.

func (*DBHash) HGetAll 

func (db *DBHash) HGetAll(ctx context.Context, key []byte) ([]driver.FVPair, error)

HGetAll returns all field-values.

func (*DBHash) HIncrBy 

func (db *DBHash) HIncrBy(ctx context.Context, key []byte, field []byte, delta int64) (int64, error)

HIncrBy increases the value of field by delta.

func (*DBHash) HKeys 

func (db *DBHash) HKeys(ctx context.Context, key []byte) ([][]byte, error)

HKeys returns the all fields.

func (*DBHash) HLen 

func (db *DBHash) HLen(ctx context.Context, key []byte) (int64, error)

HLen returns the lengh of hash.

func (*DBHash) HMget 

func (db *DBHash) HMget(ctx context.Context, key []byte, args ...[]byte) ([][]byte, error)

HMget gets multi values of fields

func (*DBHash) HMset 

func (db *DBHash) HMset(ctx context.Context, key []byte, args ...driver.FVPair) error

HMset sets multi field-values.

func (*DBHash) HSet 

func (db *DBHash) HSet(ctx context.Context, key []byte, field []byte, value []byte) (int64, error)

uHSet sets the field with value of key.

func (*DBHash) HValues 

func (db *DBHash) HValues(ctx context.Context, key []byte) ([][]byte, error)

HValues returns all values

func (*DBHash) Persist 

func (db *DBHash) Persist(ctx context.Context, key []byte) (int64, error)

Persist removes the TTL of data.

func (*DBHash) Restore 

func (db *DBHash) Restore(ctx context.Context, t *Batch, key []byte, ttl int64, val rdb.Hash) (err error)

Restore hash rdb

func (*DBHash) TTL 

func (db *DBHash) TTL(ctx context.Context, key []byte) (int64, error)

TTL gets the TTL of data.

type DBList 

type DBList struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBList 

func NewDBList(db *DB) *DBList

func (*DBList) BLPop 

func (db *DBList) BLPop(ctx context.Context, keys [][]byte, timeout time.Duration) ([]interface{}, error)

BLPop pops the list with block way.

func (*DBList) BRPop 

func (db *DBList) BRPop(ctx context.Context, keys [][]byte, timeout time.Duration) ([]interface{}, error)

BRPop bpops the list with block way.

func (*DBList) BatchDel 

func (db *DBList) BatchDel(ctx context.Context, t *Batch, keys ...[]byte) (int64, error)

BatchDel clears multi lists.

func (*DBList) BatchExpire 

func (db *DBList) BatchExpire(ctx context.Context, t *Batch, key []byte, duration int64) (int64, error)

func (*DBList) BatchRPush 

func (db *DBList) BatchRPush(ctx context.Context, t *Batch, key []byte, args ...[]byte) (int64, error)

BatchRPush rpushs the value .

func (*DBList) Del 

func (db *DBList) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del clears multi lists.

func (*DBList) Dump 

func (db *DBList) Dump(ctx context.Context, key []byte) (binVal []byte, err error)

Dump list rdb

func (*DBList) Exists 

func (db *DBList) Exists(ctx context.Context, key []byte) (int64, error)

Exists check list existed or not.

func (*DBList) Expire 

func (db *DBList) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

Expire expires the list.

func (*DBList) ExpireAt 

func (db *DBList) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

ExpireAt expires the list at when.

func (*DBList) LIndex 

func (db *DBList) LIndex(ctx context.Context, key []byte, index int32) ([]byte, error)

LIndex returns the value at index.

func (*DBList) LLen 

func (db *DBList) LLen(ctx context.Context, key []byte) (int64, error)

LLen gets the length of the list.

func (*DBList) LPop 

func (db *DBList) LPop(ctx context.Context, key []byte) ([]byte, error)

LPop pops the value.

func (*DBList) LPush 

func (db *DBList) LPush(ctx context.Context, key []byte, args ...[]byte) (int64, error)

LPush push the value to the list.

func (*DBList) LRange 

func (db *DBList) LRange(ctx context.Context, key []byte, start int32, stop int32) ([][]byte, error)

LRange gets the value of list at range.

func (*DBList) LSet 

func (db *DBList) LSet(ctx context.Context, key []byte, index int32, value []byte) error

LSet sets the value at index.

func (*DBList) LTrim 

func (db *DBList) LTrim(ctx context.Context, key []byte, start, stop int64) error

LTrim trims the value from start to stop.

func (*DBList) LTrimBack 

func (db *DBList) LTrimBack(ctx context.Context, key []byte, trimSize int32) (int32, error)

LTrimBack trims the value from back.

func (*DBList) LTrimFront 

func (db *DBList) LTrimFront(ctx context.Context, key []byte, trimSize int32) (int32, error)

LTrimFront trims the value from top.

func (*DBList) Persist 

func (db *DBList) Persist(ctx context.Context, key []byte) (int64, error)

Persist removes the TTL of list.

func (*DBList) RPop 

func (db *DBList) RPop(ctx context.Context, key []byte) ([]byte, error)

RPop rpops the value.

func (*DBList) RPush 

func (db *DBList) RPush(ctx context.Context, key []byte, args ...[]byte) (int64, error)

RPush rpushs the value .

func (*DBList) Restore 

func (db *DBList) Restore(ctx context.Context, t *Batch, key []byte, ttl int64, val rdb.List) (err error)

Restore list rdb

func (*DBList) TTL 

func (db *DBList) TTL(ctx context.Context, key []byte) (int64, error)

TTL gets the TTL of list.

type DBSet 

type DBSet struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBSet 

func NewDBSet(db *DB) *DBSet

func (*DBSet) BatchDel 

func (db *DBSet) BatchDel(ctx context.Context, t *Batch, keys ...[]byte) (int64, error)

BatchDel clears multi sets.

func (*DBSet) BatchExpire 

func (db *DBSet) BatchExpire(ctx context.Context, t *Batch, key []byte, duration int64) (int64, error)

BatchExpire expires the set.

func (*DBSet) BatchSAdd 

func (db *DBSet) BatchSAdd(ctx context.Context, t *Batch, key []byte, args ...[]byte) (int64, error)

BatchSAdd adds the value to the set.

func (*DBSet) Del 

func (db *DBSet) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del clears multi sets.

func (*DBSet) Dump 

func (db *DBSet) Dump(ctx context.Context, key []byte) (binVal []byte, err error)

Dump set rdb

func (*DBSet) Exists 

func (db *DBSet) Exists(ctx context.Context, key []byte) (int64, error)

Exists checks whether set existed or not.

func (*DBSet) Expire 

func (db *DBSet) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

Expire expires the set.

func (*DBSet) ExpireAt 

func (db *DBSet) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

ExpireAt expires the set at when.

func (*DBSet) Persist 

func (db *DBSet) Persist(ctx context.Context, key []byte) (int64, error)

Persist removes the TTL of set.

func (*DBSet) Restore 

func (db *DBSet) Restore(ctx context.Context, t *Batch, key []byte, ttl int64, val rdb.Set) (err error)

Restore set rdb

func (*DBSet) SAdd 

func (db *DBSet) SAdd(ctx context.Context, key []byte, args ...[]byte) (int64, error)

SAdd adds the value to the set.

func (*DBSet) SCard 

func (db *DBSet) SCard(ctx context.Context, key []byte) (int64, error)

SCard gets the size of set.

func (*DBSet) SDiff 

func (db *DBSet) SDiff(ctx context.Context, keys ...[]byte) ([][]byte, error)

SDiff gets the different of sets.

func (*DBSet) SDiffStore 

func (db *DBSet) SDiffStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

SDiffStore gets the different of sets and stores to dest set.

func (*DBSet) SInter 

func (db *DBSet) SInter(ctx context.Context, keys ...[]byte) ([][]byte, error)

SInter intersects the sets.

func (*DBSet) SInterStore 

func (db *DBSet) SInterStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

SInterStore intersects the sets and stores to dest set.

func (*DBSet) SIsMember 

func (db *DBSet) SIsMember(ctx context.Context, key []byte, member []byte) (int64, error)

SIsMember checks member in set.

func (*DBSet) SKeyExists 

func (db *DBSet) SKeyExists(ctx context.Context, key []byte) (int64, error)

SKeyExists checks whether set existed or not.

func (*DBSet) SMembers 

func (db *DBSet) SMembers(ctx context.Context, key []byte) ([][]byte, error)

SMembers gets members of set.

func (*DBSet) SRem 

func (db *DBSet) SRem(ctx context.Context, key []byte, args ...[]byte) (int64, error)

SRem removes the members of set.

func (*DBSet) SUnion 

func (db *DBSet) SUnion(ctx context.Context, keys ...[]byte) ([][]byte, error)

SUnion unions the sets.

func (*DBSet) SUnionStore 

func (db *DBSet) SUnionStore(ctx context.Context, dstKey []byte, keys ...[]byte) (int64, error)

SUnionStore unions the sets and stores to the dest set.

func (*DBSet) TTL 

func (db *DBSet) TTL(ctx context.Context, key []byte) (int64, error)

TTL gets the TTL of set.

type DBSlot 

type DBSlot struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBSlot 

func NewDBSlot(db *DB) *DBSlot

func (*DBSlot) BatchDel 

func (m *DBSlot) BatchDel(ctx context.Context, keys ...*MetaObjKey) (num int64, err error)

func (*DBSlot) Dump 

func (m *DBSlot) Dump(ctx context.Context, key *MetaObjKey) (binVal []byte, err error)

Dump metaObj key

func (*DBSlot) HashKeyToSlot 

func (m *DBSlot) HashKeyToSlot(key []byte) ([]byte, uint32)

func (*DBSlot) HashTagToSlot 

func (m *DBSlot) HashTagToSlot(tag []byte) uint32

func (*DBSlot) MigrateKeyWithSameTag 

func (m *DBSlot) MigrateKeyWithSameTag(ctx context.Context, addr string, timeout time.Duration, key []byte) (migrateCn int64, err error)

MigrateKeyWithSameTag migrate keys/vals which have the same tag with one key to addr with timeout (ms) return n, n migrate success, 0 slot is empty

func (*DBSlot) MigrateOneKey 

func (m *DBSlot) MigrateOneKey(ctx context.Context, addr string, timeout time.Duration, key []byte) (migrateCn int64, err error)

MigrateOneKey migrate one key/val (no hash tag tag=key) to addr with timeout (ms) return n (same key, diff dataType), success, 0 slot is empty

func (*DBSlot) MigrateSlotKeyWithSameTag 

func (m *DBSlot) MigrateSlotKeyWithSameTag(ctx context.Context, addr string, timeout time.Duration, slot uint64) (migrateCn int64, err error)

MigrateSlotKeyWithSameTag migrate slot keys/vals which have the same tag with one key to addr with timeout (ms) return n, success, 0 slot is empty

func (*DBSlot) MigrateSlotOneKey 

func (m *DBSlot) MigrateSlotOneKey(ctx context.Context, addr string, timeout time.Duration, slot uint64) (migrateCn int64, err error)

MigrateSlotOneKey migrate slot one key/val to addr with timeout (ms) return 1, success, 0 slot is empty

func (*DBSlot) SlotsCheck 

func (m *DBSlot) SlotsCheck(ctx context.Context) (err error)

SlotsCheck slots must check below case - The key stored in each slot can find the corresponding val in the db - Keys in each db can be found in the corresponding slot WARNING: just used debug/test, don't use in product,

func (*DBSlot) SlotsDel 

func (m *DBSlot) SlotsDel(ctx context.Context, slots ...uint64) (slotInfos []*driver.SlotInfo, err error)

SlotsDel del slots, return after del slot info just del slot one key if slot have key to del, slotInfo.Size is 1, else is 0

func (*DBSlot) SlotsHashKey 

func (m *DBSlot) SlotsHashKey(ctx context.Context, keys ...[]byte) ([]uint64, error)

SlotsHashKey hash keys to slots, return slot slice

func (*DBSlot) SlotsInfo 

func (m *DBSlot) SlotsInfo(ctx context.Context, startSlot, count uint64, withSize bool) (slotInfos []*driver.SlotInfo, err error)

SlotsInfo show slot info with slots range [start,start+count] return slotInfo slice if withSize is true, slotInfo.Size is slot's keys cn; size>0,show it; else exits key is 1, show it

func (*DBSlot) SlotsRestore 

func (m *DBSlot) SlotsRestore(ctx context.Context, objs ...*driver.SlotsRestoreObj) (err error)

SlotsRestore dest migrate addr restore slot obj [key ttlms serialized-value(rdb) ...]

func (*DBSlot) TTL 

func (m *DBSlot) TTL(ctx context.Context, key *MetaObjKey) (int64, error)

TTL metaObj key return ttl second

type DBStats 

type DBStats struct {
	KeyCn           atomic.Int64
	ExpireKeyCn     atomic.Int64
	TotalExpireTime atomic.Int64
	TotalExpireCn   atomic.Int64
}

func GetDbStats 

func GetDbStats(s *Storager, index int) *DBStats

func LoadDbStats 

func LoadDbStats(dbIndex int) (m *DBStats)

func (*DBStats) String 

func (m *DBStats) String() string

type DBString 

type DBString struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBString 

func NewDBString(db *DB) *DBString

func (*DBString) Append 

func (db *DBString) Append(ctx context.Context, key []byte, value []byte) (int64, error)

Append appends the value to the data.

func (*DBString) BatchDel 

func (db *DBString) BatchDel(ctx context.Context, t *Batch, keys ...[]byte) (int64, error)

BatchDel deletes the data.

func (*DBString) BatchExpire 

func (db *DBString) BatchExpire(ctx context.Context, t *Batch, key []byte, duration int64) (int64, error)

BatchExpire expires the data.

func (*DBString) BatchSet 

func (db *DBString) BatchSet(ctx context.Context, t *Batch, key []byte, value []byte) error

BatchSet sets the data.

func (*DBString) Decr 

func (db *DBString) Decr(ctx context.Context, key []byte) (int64, error)

Decr decreases the data.

func (*DBString) DecrBy 

func (db *DBString) DecrBy(ctx context.Context, key []byte, decrement int64) (int64, error)

DecrBy decreases the data by decrement.

func (*DBString) Del 

func (db *DBString) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del deletes the data.

func (*DBString) Dump 

func (db *DBString) Dump(ctx context.Context, key []byte) (binVal []byte, err error)

Dump string rdb

func (*DBString) Exists 

func (db *DBString) Exists(ctx context.Context, key []byte) (int64, error)

Exists check data exists or not.

func (*DBString) Expire 

func (db *DBString) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

Expire expires the data.

func (*DBString) ExpireAt 

func (db *DBString) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

ExpireAt expires the data at when.

func (*DBString) Get 

func (db *DBString) Get(ctx context.Context, key []byte) ([]byte, error)

Get gets the value.

func (*DBString) GetRange 

func (db *DBString) GetRange(ctx context.Context, key []byte, start int, end int) ([]byte, error)

GetRange gets the range of the data.

func (*DBString) GetSet 

func (db *DBString) GetSet(ctx context.Context, key []byte, value []byte) ([]byte, error)

GetSet gets the value and sets new value.

func (*DBString) GetSlice 

func (db *DBString) GetSlice(ctx context.Context, key []byte) (openkvDriver.ISlice, error)

GetSlice gets the slice of the data to adapt leveldb slice

func (*DBString) Incr 

func (db *DBString) Incr(ctx context.Context, key []byte) (int64, error)

Incr increases the data.

func (*DBString) IncrBy 

func (db *DBString) IncrBy(ctx context.Context, key []byte, increment int64) (int64, error)

IncrBy increases the data by increment.

func (*DBString) MGet 

func (db *DBString) MGet(ctx context.Context, keys ...[]byte) ([][]byte, error)

MGet gets multi data.

func (*DBString) MSet 

func (db *DBString) MSet(ctx context.Context, args ...driver.KVPair) error

MSet sets multi data.

func (*DBString) Persist 

func (db *DBString) Persist(ctx context.Context, key []byte) (int64, error)

Persist removes the TTL of the data.

func (*DBString) Restore 

func (db *DBString) Restore(ctx context.Context, t *Batch, key []byte, ttl int64, val rdb.String) (err error)

Restore string rdb

func (*DBString) Set 

func (db *DBString) Set(ctx context.Context, key []byte, value []byte) error

Set sets the data.

func (*DBString) SetEX 

func (db *DBString) SetEX(ctx context.Context, key []byte, duration int64, value []byte) error

SetEX sets the data with a TTL.

func (*DBString) SetNX 

func (db *DBString) SetNX(ctx context.Context, key []byte, value []byte) (n int64, err error)

SetNX sets the data if not existed.

func (*DBString) SetNXEX 

func (db *DBString) SetNXEX(ctx context.Context, key []byte, duration int64, value []byte) (n int64, err error)

SetNXEX set k v nx ex seconds NX -- Only set the key if it does not already exist. EX seconds -- Set the specified expire time, in seconds.

func (*DBString) SetRange 

func (db *DBString) SetRange(ctx context.Context, key []byte, offset int, value []byte) (int64, error)

SetRange sets the data with new value from offset.

func (*DBString) SetXXEX 

func (db *DBString) SetXXEX(ctx context.Context, key []byte, duration int64, value []byte) (n int64, err error)

SetXXEX set k v xx ex seconds XX -- Only set the key if it already exists. EX seconds -- Set the specified expire time, in seconds.

func (*DBString) StrLen 

func (db *DBString) StrLen(ctx context.Context, key []byte) (int64, error)

StrLen returns the length of the data.

func (*DBString) TTL 

func (db *DBString) TTL(ctx context.Context, key []byte) (int64, error)

TTL returns the TTL of the data.

type DBZSet 

type DBZSet struct {
	*DB
	// contains filtered or unexported fields
}

func NewDBZSet 

func NewDBZSet(db *DB) *DBZSet

func (*DBZSet) BatchDel 

func (db *DBZSet) BatchDel(ctx context.Context, t *Batch, keys ...[]byte) (int64, error)

BatchDel clears multi zsets.

func (*DBZSet) BatchExpire 

func (db *DBZSet) BatchExpire(ctx context.Context, t *Batch, key []byte, duration int64) (int64, error)

BatchExpire expires the zset.

func (*DBZSet) BatchZAdd 

func (db *DBZSet) BatchZAdd(ctx context.Context, t *Batch, key []byte, args ...driver.ScorePair) (int64, error)

BatchZAdd add the members.

func (*DBZSet) Del 

func (db *DBZSet) Del(ctx context.Context, keys ...[]byte) (int64, error)

Del clears multi zsets.

func (*DBZSet) Dump 

func (db *DBZSet) Dump(ctx context.Context, key []byte) (binVal []byte, err error)

Dump zset rdb

func (*DBZSet) Exists 

func (db *DBZSet) Exists(ctx context.Context, key []byte) (int64, error)

Exists checks zset existed or not.

func (*DBZSet) Expire 

func (db *DBZSet) Expire(ctx context.Context, key []byte, duration int64) (int64, error)

Expire expires the zset.

func (*DBZSet) ExpireAt 

func (db *DBZSet) ExpireAt(ctx context.Context, key []byte, when int64) (int64, error)

ExpireAt expires the zset at when.

func (*DBZSet) Persist 

func (db *DBZSet) Persist(ctx context.Context, key []byte) (int64, error)

Persist removes the TTL of zset.

func (*DBZSet) Restore 

func (db *DBZSet) Restore(ctx context.Context, t *Batch, key []byte, ttl int64, val rdb.ZSet) (err error)

Restore zset rdb use int64 for zset score, not float

func (*DBZSet) TTL 

func (db *DBZSet) TTL(ctx context.Context, key []byte) (int64, error)

TTL gets the TTL of zset.

func (*DBZSet) ZAdd 

func (db *DBZSet) ZAdd(ctx context.Context, key []byte, args ...driver.ScorePair) (int64, error)

ZAdd add the members.

func (*DBZSet) ZCard 

func (db *DBZSet) ZCard(ctx context.Context, key []byte) (int64, error)

ZCard gets the size of the zset.

func (*DBZSet) ZCount 

func (db *DBZSet) ZCount(ctx context.Context, key []byte, min int64, max int64) (int64, error)

ZCount gets the number of score in [min, max]

func (*DBZSet) ZIncrBy 

func (db *DBZSet) ZIncrBy(ctx context.Context, key []byte, delta int64, member []byte) (int64, error)

ZIncrBy increases the score of member with delta.

func (*DBZSet) ZInterStore 

func (db *DBZSet) ZInterStore(ctx context.Context, destKey []byte, srcKeys [][]byte, weights []int64, aggregate []byte) (int64, error)

ZInterStore intersects the zsets and stores to dest zset.

func (*DBZSet) ZLexCount 

func (db *DBZSet) ZLexCount(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType) (int64, error)

ZLexCount gets the count of zset lexicographically.

func (*DBZSet) ZRange 

func (db *DBZSet) ZRange(ctx context.Context, key []byte, start int, stop int) ([]driver.ScorePair, error)

ZRange gets the members from start to stop.

func (*DBZSet) ZRangeByLex 

func (db *DBZSet) ZRangeByLex(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType, offset int, count int) ([][]byte, error)

ZRangeByLex scans the zset lexicographically

func (*DBZSet) ZRangeByScore 

func (db *DBZSet) ZRangeByScore(ctx context.Context, key []byte, min int64, max int64,
	offset int, count int) ([]driver.ScorePair, error)

ZRangeByScore gets the data with score in min and max. min and max must be inclusive if no limit, set offset = 0 and count = -1

func (*DBZSet) ZRangeByScoreGeneric 

func (db *DBZSet) ZRangeByScoreGeneric(ctx context.Context, key []byte, min int64, max int64,
	offset int, count int, reverse bool) ([]driver.ScorePair, error)

ZRangeByScoreGeneric is a generic function to scan zset with score. min and max must be inclusive if no limit, set offset = 0 and count<0

func (*DBZSet) ZRangeGeneric 

func (db *DBZSet) ZRangeGeneric(ctx context.Context, key []byte, start int, stop int, reverse bool) ([]driver.ScorePair, error)

ZRangeGeneric is a generic function for scan zset. zrange/zrevrange index pos start,stop

func (*DBZSet) ZRank 

func (db *DBZSet) ZRank(ctx context.Context, key []byte, member []byte) (int64, error)

ZRank gets the rank of member.

func (*DBZSet) ZRem 

func (db *DBZSet) ZRem(ctx context.Context, key []byte, members ...[]byte) (int64, error)

ZRem removes members

func (*DBZSet) ZRemRangeByLex 

func (db *DBZSet) ZRemRangeByLex(ctx context.Context, key []byte, min []byte, max []byte, rangeType driver.RangeType) (int64, error)

ZRemRangeByLex remvoes members in [min, max] lexicographically

func (*DBZSet) ZRemRangeByRank 

func (db *DBZSet) ZRemRangeByRank(ctx context.Context, key []byte, start int, stop int) (int64, error)

ZRemRangeByRank removes the member at range from start to stop.

func (*DBZSet) ZRemRangeByScore 

func (db *DBZSet) ZRemRangeByScore(ctx context.Context, key []byte, min int64, max int64) (int64, error)

ZRemRangeByScore removes the data with score at [min, max]

func (*DBZSet) ZRevRange 

func (db *DBZSet) ZRevRange(ctx context.Context, key []byte, start int, stop int) ([]driver.ScorePair, error)

ZRevRange gets the data reversed.

func (*DBZSet) ZRevRangeByScore 

func (db *DBZSet) ZRevRangeByScore(ctx context.Context, key []byte, min int64, max int64, offset int, count int) ([]driver.ScorePair, error)

ZRevRangeByScore gets the data with score at [min, max] min and max must be inclusive if no limit, set offset = 0 and count = -1

func (*DBZSet) ZRevRank 

func (db *DBZSet) ZRevRank(ctx context.Context, key []byte, member []byte) (int64, error)

ZRevRank gets the rank of member reversed.

func (*DBZSet) ZScore 

func (db *DBZSet) ZScore(ctx context.Context, key []byte, member []byte) (int64, error)

ZScore gets the score of member.

func (*DBZSet) ZUnionStore 

func (db *DBZSet) ZUnionStore(ctx context.Context, destKey []byte, srcKeys [][]byte, weights []int64, aggregate []byte) (int64, error)

ZUnionStore unions the zsets and stores to dest zset.

type DataType 

type DataType byte

DataType is defined for the different types 

const (
	STRING DataType = iota
	LIST
	HASH
	SET
	ZSET
	BITMAP
)

for out use

func (DataType) String 

func (d DataType) String() string

type MetaObjKey 

type MetaObjKey struct {
	Version  byte
	CodeType byte
	DBIndex  uint64
	SlotId   uint64
	Tag      []byte
	DataType byte
	DataKey  []byte
}

type MigrateAsyncTask 

type MigrateAsyncTask struct {
	Ctx    context.Context
	DBSlot *DBSlot
	Cli    *redis.Client
	Keys   []*MetaObjKey
}

func (*MigrateAsyncTask) Run 

func (m *MigrateAsyncTask) Run() (err error)

type SnapshotHead 

type SnapshotHead struct {
	CommitID uint64
}

SnapshotHead is the head of a snapshot.

func (*SnapshotHead) Read 

func (h *SnapshotHead) Read(r io.Reader) error

Read reads meta from the Reader.

func (*SnapshotHead) Write 

func (h *SnapshotHead) Write(w io.Writer) error

Write writes meta to the Writer

type Storager 

type Storager struct {
	// contains filtered or unexported fields
}

Storager core store struct for server use like redis

func New 

func New(opts *config.StorgerOptions) (store *Storager)

func (*Storager) Close 

func (m *Storager) Close() (err error)

Close close ttl check, kv store close, flock close

func (*Storager) ExistKeyDB 

func (s *Storager) ExistKeyDB() (info []driver.InfoPair)

func (*Storager) FlushAll 

func (m *Storager) FlushAll(ctx context.Context) error

FlushAll will clear all data

func (*Storager) GetCommitLock 

func (s *Storager) GetCommitLock() *sync.Mutex

GetCommitLock noCopy return mutex lock for commit log/data

func (*Storager) GetKVStore 

func (s *Storager) GetKVStore() storagerDriver.IKV

GetKVStore get kvstore engine which save data

func (*Storager) GetRWLock 

func (s *Storager) GetRWLock() *sync.RWMutex

GetRWlock noCopy return rw muxtex lock for read only

func (*Storager) InitOpts 

func (m *Storager) InitOpts(opts *config.StorgerOptions)

func (*Storager) KeySpaceStatsInfo 

func (s *Storager) KeySpaceStatsInfo() (info []driver.InfoPair)

func (*Storager) Name 

func (m *Storager) Name() string

func (*Storager) Open 

func (store *Storager) Open(ctx context.Context) (err error)

func (*Storager) RecoverFromSnapshotWithHead 

func (s *Storager) RecoverFromSnapshotWithHead(ctx context.Context, r io.Reader) (h *SnapshotHead, err error)

RecoverFromSnapshotWithHead clears all data and loads dump file to db return snapshot head info,error

func (*Storager) SaveSnapshotWithHead 

func (s *Storager) SaveSnapshotWithHead(ctx context.Context, h *SnapshotHead, w io.Writer) (err error)

SaveSnapshotWithHead dumps data to the Writer with SnapshotHead

func (*Storager) Select 

func (m *Storager) Select(ctx context.Context, index int) (idb driver.IDB, err error)

Select chooses a database.

func (*Storager) SetCommitter 

func (s *Storager) SetCommitter(committer storagerDriver.ICommitter)

SetCommitter

func (*Storager) StatsInfo 

func (s *Storager) StatsInfo(sections ...string) (info map[string][]driver.InfoPair)

StatsInfo get stroage stats info by sections

type TTLChecker 

type TTLChecker struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewTTLChecker 

func NewTTLChecker(db *DB) *TTLChecker

Source Files

View all Source files

Directories

Path Synopsis
mocks

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.