utils

package
v0.0.0-...-8eacd36 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 22, 2022 License: MIT Imports: 23 Imported by: 0

Documentation

Overview

Copyright 2021 hardcore-os Project Authors

Licensed under the Apache License, Version 2.0 (the "License") you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const (
	// MaxLevelNum _
	MaxLevelNum = 7
	// DefaultValueThreshold _
	DefaultValueThreshold = 1024
)
View Source 
const (
	ManifestFilename                  = "MANIFEST"
	ManifestRewriteFilename           = "REWRITEMANIFEST"
	ManifestDeletionsRewriteThreshold = 10000
	ManifestDeletionsRatio            = 10
	DefaultFileFlag                   = os.O_RDWR | os.O_CREATE | os.O_APPEND
	DefaultFileMode                   = 0666
	MaxValueLogSize                   = 10 << 20

	// 基于可变长编码,其最可能的编码
	MaxHeaderSize            = 21
	VlogHeaderSize           = 0
	MaxVlogFileSize   uint32 = math.MaxUint32
	Mi                int64  = 1 << 20
	KVWriteChCapacity        = 1000
)

file

View Source 
const (
	BitDelete       byte = 1 << 0 // Set if the key has been deleted.
	BitValuePointer byte = 1 << 1 // Set if the value is NOT stored directly next to key.
)

meta

View Source 
const (
	MaxNodeSize = int(unsafe.Sizeof(node{}))
)
View Source 
const (
	// size of vlog header.
	// +----------------+------------------+
	// | keyID(8 bytes) |  baseIV(12 bytes)|
	// +----------------+------------------+
	ValueLogHeaderSize = 20
)

Variables

View Source 
var (
	MagicText    = [4]byte{'H', 'A', 'R', 'D'}
	MagicVersion = uint32(1)
	// CastagnoliCrcTable is a CRC32 polynomial table
	CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli)
)

codec

View Source 
var (
	// ErrKeyNotFound is returned when key isn't found on a txn.Get.
	ErrKeyNotFound = errors.New("Key not found")
	// ErrEmptyKey is returned if an empty key is passed on an update function.
	ErrEmptyKey = errors.New("Key cannot be empty")
	// ErrReWriteFailure reWrite failure
	ErrReWriteFailure = errors.New("reWrite failure")
	// ErrBadMagic bad magic
	ErrBadMagic = errors.New("bad magic")
	// ErrBadChecksum bad check sum
	ErrBadChecksum = errors.New("bad check sum")
	// ErrChecksumMismatch is returned at checksum mismatch.
	ErrChecksumMismatch = errors.New("checksum mismatch")

	ErrTruncate = errors.New("Do truncate")
	ErrStop     = errors.New("Stop")

	// compact
	ErrFillTables = errors.New("Unable to fill tables")

	ErrBlockedWrites  = errors.New("Writes are blocked, possibly due to DropAll or Close")
	ErrTxnTooBig      = errors.New("Txn is too big to fit into one request")
	ErrDeleteVlogFile = errors.New("Delete vlog file")
	ErrNoRoom         = errors.New("No room for write")

	// ErrInvalidRequest is returned if the user request is invalid.
	ErrInvalidRequest = errors.New("Invalid request")
	// ErrNoRewrite is returned if a call for value log GC doesn't result in a log file rewrite.
	ErrNoRewrite = errors.New("Value log GC attempt didn't result in any cleanup")

	// ErrRejected is returned if a value log GC is called either while another GC is running, or
	// after DB::Close has been called.
	ErrRejected = errors.New("Value log GC request rejected")
)

NotFoundKey 找不到key

Functions

func AssertTrue 

func AssertTrue(b bool)

AssertTrue asserts that b is true. Otherwise, it would log fatal.

func AssertTruef 

func AssertTruef(b bool, format string, args ...interface{})

AssertTruef is AssertTrue with extra info.

func BloomBitsPerKey 

func BloomBitsPerKey(numEntries int, fp float64) int

BloomBitsPerKey returns the bits per key required by bloomfilter based on the false positive rate.

func BytesToU32 

func BytesToU32(b []byte) uint32

BytesToU32 converts the given byte slice to uint32

func BytesToU32Slice 

func BytesToU32Slice(b []byte) []uint32

BytesToU32Slice converts the given byte slice to uint32 slice

func BytesToU64 

func BytesToU64(b []byte) uint64

BytesToU64 _

func CalculateChecksum 

func CalculateChecksum(data []byte) uint64

CalculateChecksum _

func CompareKeys 

func CompareKeys(key1, key2 []byte) int

CompareKeys checks the key without timestamp and checks the timestamp if keyNoTs is same. a<timestamp> would be sorted higher than aa<timestamp> if we use bytes.compare All keys should have timestamp.

func CondPanic 

func CondPanic(condition bool, err error)

CondPanic e

func Copy 

func Copy(a []byte) []byte

Copy copies a byte slice and returns the copied slice.

func CreateSyncedFile 

func CreateSyncedFile(filename string, sync bool) (*os.File, error)

CreateSyncedFile creates a new file (using O_EXCL), errors if it already existed.

func DiscardEntry 

func DiscardEntry(e, vs *Entry) bool

func Err 

func Err(err error) error

Err err

func EstimateWalCodecSize 

func EstimateWalCodecSize(e *Entry) int

EstimateWalCodecSize 预估当前kv 写入wal文件占用的空间大小

func FID 

func FID(name string) uint64

FID 根据file name 获取其fid

func FastRand 

func FastRand() uint32

FastRand is a fast thread local random function.

func FileNameSSTable 

func FileNameSSTable(dir string, id uint64) string

FileNameSSTable sst 文件名

func Float64 

func Float64() float64

func Hash 

func Hash(b []byte) uint32

Hash implements a hashing algorithm similar to the Murmur hash.

func Int63n 

func Int63n(n int64) int64

func IsDeletedOrExpired 

func IsDeletedOrExpired(meta byte, expiresAt uint64) bool

func IsValuePtr 

func IsValuePtr(e *Entry) bool

func KeyWithTs 

func KeyWithTs(key []byte, ts uint64) []byte

KeyWithTs generates a new key by appending ts to key.

func LoadIDMap 

func LoadIDMap(dir string) map[uint64]struct{}

LoadIDMap Get the id of all sst files in the current folder

func MemHash 

func MemHash(data []byte) uint64

MemHash is the hash function used by go map, it utilizes available hardware instructions(behaves as aeshash if aes instruction is available). NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.

func MemHashString 

func MemHashString(str string) uint64

MemHashString is the hash function used by go map, it utilizes available hardware instructions (behaves as aeshash if aes instruction is available). NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.

func NewCurVersion 

func NewCurVersion() uint64

func Panic 

func Panic(err error)

Panic 如果err 不为nil 则panicc

func Panic2 

func Panic2(_ interface{}, err error)

Panic2 _

func ParseKey 

func ParseKey(key []byte) []byte

ParseKey parses the actual key from the key bytes.

func ParseTs 

func ParseTs(key []byte) uint64

ParseTs parses the timestamp from the key bytes.

func RandN 

func RandN(n int) int

func RemoveDir 

func RemoveDir(dir string)

RemoveDir _

func RunCallback 

func RunCallback(cb func())

RunCallback _

func SafeCopy 

func SafeCopy(a, src []byte) []byte

SafeCopy does append(a[:0], src...).

func SameKey 

func SameKey(src, dst []byte) bool

SameKey checks for key equality ignoring the version timestamp suffix.

func SyncDir 

func SyncDir(dir string) error

SyncDir When you create or delete a file, you have to ensure the directory entry for the file is synced in order to guarantee the file is visible (if the system crashes). (See the man page for fsync, or see https://github.com/coreos/etcd/issues/6368 for an example.)

func U32SliceToBytes 

func U32SliceToBytes(u32s []uint32) []byte

U32SliceToBytes converts the given Uint32 slice to byte slice

func U32ToBytes 

func U32ToBytes(v uint32) []byte

U32ToBytes converts the given Uint32 to bytes

func U64ToBytes 

func U64ToBytes(v uint64) []byte

U64ToBytes converts the given Uint64 to bytes

func ValuePtrCodec 

func ValuePtrCodec(vp *ValuePtr) []byte

ValuePtrCodec _

func ValueSize 

func ValueSize(value []byte) int64

func VerifyChecksum 

func VerifyChecksum(data []byte, expected []byte) error

VerifyChecksum crc32

func VlogFilePath 

func VlogFilePath(dirPath string, fid uint32) string

func WalCodec 

func WalCodec(buf *bytes.Buffer, e *Entry) int

WalCodec 写入wal文件的编码 | header | key | value | crc32 |

func WarpErr 

func WarpErr(format string, err error) error

WarpErr err

Types

type Arena 

type Arena struct {
	// contains filtered or unexported fields
}

Arena should be lock-free.

type Closer 

type Closer struct {
	CloseSignal chan struct{}
	// contains filtered or unexported fields
}

Closer _用于资源回收的信号控制

func NewCloser 

func NewCloser() *Closer

NewCloser _

func (*Closer) Add 

func (c *Closer) Add(n int)

Add 添加wait 计数

func (*Closer) Close 

func (c *Closer) Close()

Close 上游通知下游协程进行资源回收，并等待协程通知回收完毕

func (*Closer) Done 

func (c *Closer) Done()

Done 标示协程已经完成资源回收，通知上游正式关闭

type CoreMap 

type CoreMap struct {
	// contains filtered or unexported fields
}

CoreMap _

func NewMap 

func NewMap() *CoreMap

NewMap _

func (*CoreMap) Del 

func (c *CoreMap) Del(key interface{})

Del _

func (*CoreMap) Get 

func (c *CoreMap) Get(key interface{}) (interface{}, bool)

Get _

func (*CoreMap) Range 

func (c *CoreMap) Range(f func(key, value interface{}) bool)

Range _

func (*CoreMap) Set 

func (c *CoreMap) Set(key, value interface{})

Set _

type Entry 

type Entry struct {
	Key       []byte
	Value     []byte
	ExpiresAt uint64

	Meta         byte
	Version      uint64
	Offset       uint32
	Hlen         int // Length of the header.
	ValThreshold int64
}

Entry _ 最外层写入的结构体

func BuildEntry 

func BuildEntry() *Entry

构建entry对象

func NewEntry 

func NewEntry(key, value []byte) *Entry

NewEntry_

func (*Entry) EncodedSize 

func (e *Entry) EncodedSize() uint32

EncodedSize is the size of the ValueStruct when encoded

func (*Entry) Entry 

func (e *Entry) Entry() *Entry

Entry_

func (*Entry) EstimateSize 

func (e *Entry) EstimateSize(threshold int) int

EstimateSize

func (*Entry) IsDeletedOrExpired 

func (e *Entry) IsDeletedOrExpired() bool

func (*Entry) IsZero 

func (e *Entry) IsZero() bool

IsZero _

func (*Entry) LogHeaderLen 

func (e *Entry) LogHeaderLen() int

LogHeaderLen _

func (*Entry) LogOffset 

func (e *Entry) LogOffset() uint32

LogOffset _

func (*Entry) WithTTL 

func (e *Entry) WithTTL(dur time.Duration) *Entry

WithTTL _

type Filter 

type Filter []byte

Filter is an encoded set of []byte keys.

func NewFilter 

func NewFilter(keys []uint32, bitsPerKey int) Filter

NewFilter returns a new Bloom filter that encodes a set of []byte keys with the given number of bits per key, approximately.

A good bitsPerKey value is 10, which yields a filter with ~ 1% false positive rate.

func (Filter) MayContain 

func (f Filter) MayContain(h uint32) bool

MayContain returns whether the filter may contain given key. False positives are possible, where it returns true for keys not in the original set.

func (Filter) MayContainKey 

func (f Filter) MayContainKey(k []byte) bool

MayContainKey _

type HashReader 

type HashReader struct {
	R         io.Reader
	H         hash.Hash32
	BytesRead int // Number of bytes read.
}

func NewHashReader 

func NewHashReader(r io.Reader) *HashReader

func (*HashReader) Read 

func (t *HashReader) Read(p []byte) (int, error)

Read reads len(p) bytes from the reader. Returns the number of bytes read, error on failure.

func (*HashReader) ReadByte 

func (t *HashReader) ReadByte() (byte, error)

ReadByte reads exactly one byte from the reader. Returns error on failure.

func (*HashReader) Sum32 

func (t *HashReader) Sum32() uint32

Sum32 returns the sum32 of the underlying hash. 

type Header struct {
	KLen      uint32
	VLen      uint32
	ExpiresAt uint64
	Meta      byte
}

header 对象 header is used in value log as a header before Entry.

func (*Header) Decode 

func (h *Header) Decode(buf []byte) int

Decode decodes the given header from the provided byte slice. Returns the number of bytes read.

func (*Header) DecodeFrom 

func (h *Header) DecodeFrom(reader *HashReader) (int, error)

DecodeFrom reads the header from the hashReader. Returns the number of bytes read.

func (Header) Encode 

func (h Header) Encode(out []byte) int

+------+----------+------------+--------------+-----------+ | Meta | UserMeta | Key Length | Value Length | ExpiresAt | +------+----------+------------+--------------+-----------+

type Item 

type Item interface {
	Entry() *Entry
}

Item _

type Iterator 

type Iterator interface {
	Next()
	Valid() bool
	Rewind()
	Item() Item
	Close() error
	Seek(key []byte)
}

Iterator 迭代器

type LogEntry 

type LogEntry func(e *Entry, vp *ValuePtr) error

LogEntry

type Options 

type Options struct {
	Prefix []byte
	IsAsc  bool
}

Options _ TODO 可能被重构

type SkipListIterator 

type SkipListIterator struct {
	// contains filtered or unexported fields
}

Iterator is an iterator over skiplist object. For new objects, you just need to initialize Iterator.list.

func (*SkipListIterator) Close 

func (s *SkipListIterator) Close() error

Close frees the resources held by the iterator

func (*SkipListIterator) Item 

func (s *SkipListIterator) Item() Item

func (*SkipListIterator) Key 

func (s *SkipListIterator) Key() []byte

Key returns the key at the current position.

func (*SkipListIterator) Next 

func (s *SkipListIterator) Next()

Next advances to the next position.

func (*SkipListIterator) Prev 

func (s *SkipListIterator) Prev()

Prev advances to the previous position.

func (*SkipListIterator) Rewind 

func (s *SkipListIterator) Rewind()

func (*SkipListIterator) Seek 

func (s *SkipListIterator) Seek(target []byte)

Seek advances to the first entry with a key >= target.

func (*SkipListIterator) SeekForPrev 

func (s *SkipListIterator) SeekForPrev(target []byte)

SeekForPrev finds an entry with key <= target.

func (*SkipListIterator) SeekToFirst 

func (s *SkipListIterator) SeekToFirst()

SeekToFirst seeks position at the first entry in list. Final state of iterator is Valid() iff list is not empty.

func (*SkipListIterator) SeekToLast 

func (s *SkipListIterator) SeekToLast()

SeekToLast seeks position at the last entry in list. Final state of iterator is Valid() iff list is not empty.

func (*SkipListIterator) Valid 

func (s *SkipListIterator) Valid() bool

Valid returns true iff the iterator is positioned at a valid node.

func (*SkipListIterator) Value 

func (s *SkipListIterator) Value() ValueStruct

Value returns value.

func (*SkipListIterator) ValueUint64 

func (s *SkipListIterator) ValueUint64() uint64

ValueUint64 returns the uint64 value of the current node.

type Skiplist 

type Skiplist struct {
	OnClose func()
	// contains filtered or unexported fields
}

func NewSkiplist 

func NewSkiplist(arenaSize int64) *Skiplist

NewSkiplist makes a new empty skiplist, with a given arena size

func (*Skiplist) Add 

func (s *Skiplist) Add(e *Entry)

Put inserts the key-value pair.

func (*Skiplist) DecrRef 

func (s *Skiplist) DecrRef()

DecrRef decrements the refcount, deallocating the Skiplist when done using it

func (*Skiplist) Draw 

func (s *Skiplist) Draw(align bool)

Draw plot Skiplist, align represents align the same node in different level

func (*Skiplist) Empty 

func (s *Skiplist) Empty() bool

Empty returns if the Skiplist is empty.

func (*Skiplist) IncrRef 

func (s *Skiplist) IncrRef()

IncrRef increases the refcount

func (*Skiplist) MemSize 

func (s *Skiplist) MemSize() int64

MemSize returns the size of the Skiplist in terms of how much memory is used within its internal arena.

func (*Skiplist) NewSkipListIterator 

func (s *Skiplist) NewSkipListIterator() Iterator

NewIterator returns a skiplist iterator. You have to Close() the iterator.

func (*Skiplist) Search 

func (s *Skiplist) Search(key []byte) ValueStruct

Get gets the value associated with the key. It returns a valid value if it finds equal or earlier version of the same key.

type Slice 

type Slice struct {
	// contains filtered or unexported fields
}

Slice holds a reusable buf, will reallocate if you request a larger size than ever before. One problem is with n distinct sizes in random order it'll reallocate log(n) times.

type Throttle 

type Throttle struct {
	// contains filtered or unexported fields
}

Throttle allows a limited number of workers to run at a time. It also provides a mechanism to check for errors encountered by workers and wait for them to finish.

func NewThrottle 

func NewThrottle(max int) *Throttle

NewThrottle creates a new throttle with a max number of workers.

func (*Throttle) Do 

func (t *Throttle) Do() error

Do should be called by workers before they start working. It blocks if there are already maximum number of workers working. If it detects an error from previously Done workers, it would return it.

func (*Throttle) Done 

func (t *Throttle) Done(err error)

Done should be called by workers when they finish working. They can also pass the error status of work done.

func (*Throttle) Finish 

func (t *Throttle) Finish() error

Finish waits until all workers have finished working. It would return any error passed by Done. If Finish is called multiple time, it will wait for workers to finish only once(first time). From next calls, it will return same error as found on first call.

type UniIterator 

type UniIterator struct {
	// contains filtered or unexported fields
}

UniIterator is a unidirectional memtable iterator. It is a thin wrapper around Iterator. We like to keep Iterator as before, because it is more powerful and we might support bidirectional iterators in the future.

type ValuePtr 

type ValuePtr struct {
	Len    uint32
	Offset uint32
	Fid    uint32
}

func (*ValuePtr) Decode 

func (p *ValuePtr) Decode(b []byte)

Decode decodes the value pointer into the provided byte buffer.

func (ValuePtr) Encode 

func (p ValuePtr) Encode() []byte

Encode encodes Pointer into byte buffer.

func (ValuePtr) IsZero 

func (p ValuePtr) IsZero() bool

func (ValuePtr) Less 

func (p ValuePtr) Less(o *ValuePtr) bool

type ValueStruct 

type ValueStruct struct {
	Meta      byte
	Value     []byte
	ExpiresAt uint64

	Version uint64 // This field is not serialized. Only for internal usage.
}

func (*ValueStruct) DecodeValue 

func (vs *ValueStruct) DecodeValue(buf []byte)

DecodeValue

func (*ValueStruct) EncodeValue 

func (vs *ValueStruct) EncodeValue(b []byte) uint32

对value进行编码，并将编码后的字节写入byte 这里将过期时间和value的值一起编码

func (*ValueStruct) EncodedSize 

func (vs *ValueStruct) EncodedSize() uint32

value只持久化具体的value值和过期时间

type WalHeader 

type WalHeader struct {
	KeyLen    uint32
	ValueLen  uint32
	Meta      byte
	ExpiresAt uint64
}

func (*WalHeader) Decode 

func (h *WalHeader) Decode(reader *HashReader) (int, error)

func (WalHeader) Encode 

func (h WalHeader) Encode(out []byte) int

Source Files

View all Source files

Directories

Path Synopsis
Mmap uses the mmap system call to memory-map a file.
 Mmap uses the mmap system call to memory-map a file.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.