filekv

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 4, 2023 License: MIT Imports: 11 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrItemExists   = errors.New("item already exist")
	ErrItemFiltered = errors.New("item filtered")
)
View Source
var (
	BufferSize = 50 * 1024 * 1024 // 50Mb
	Separator  = ";;;"
	NewLine    = "\n"
	FpRatio    = 0.0001
	MaxItems   = uint(250000)
)

Functions

This section is empty.

Types

type FileDB

type FileDB struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

FileDB - represents a file db implementation

func Open

func Open(options Options) (*FileDB, error)

Open a new file based db

func (*FileDB) Close

func (fdb *FileDB) Close()

Close ...

func (*FileDB) Merge

func (f *FileDB) Merge(items ...interface{}) (uint, error)

func (*FileDB) MergeFile

func (f *FileDB) MergeFile(filename string) (uint, error)

func (*FileDB) MergeReader

func (f *FileDB) MergeReader(reader io.Reader) (uint, error)

func (*FileDB) Process

func (fdb *FileDB) Process() error

Process added files/slices/elements

func (*FileDB) Reset

func (fdb *FileDB) Reset() error

Reset the db

func (*FileDB) Scan

func (fdb *FileDB) Scan(handler func([]byte, []byte) error) error

Scan - iterate over the whole store using the handler function

func (*FileDB) Set

func (fdb *FileDB) Set(k, v []byte) error

func (*FileDB) Size

func (fdb *FileDB) Size() int64

Size - returns the size of the database in bytes

type Options

type Options struct {
	Path           string
	Compress       bool
	MaxItems       uint
	Cleanup        bool
	SkipEmpty      bool
	FilterCallback func(k, v []byte) bool
	Dedupe         Strategy
}
var DefaultOptions Options = Options{
	Compress:  false,
	Cleanup:   true,
	Dedupe:    MemoryLRU,
	SkipEmpty: true,
}

type Stats

type Stats struct {
	NumberOfFilteredItems uint
	NumberOfAddedItems    uint
	NumberOfDupedItems    uint
	NumberOfItems         uint
}

type Strategy

type Strategy uint8
const (
	None Strategy = iota
	// Uses go standard map without eviction - grows linearly with the items number
	MemoryMap
	// MemoryLRU keeps in memory the last x items and filter with a look-back probabilistic window with fixed size
	MemoryLRU
	// MemoryFilter uses bitset in-memory filters to remove duplicates
	MemoryFilter
	// Use full disk kv store to remove all duplicates - it should have low heap memory footprint but lots of I/O interactions
	DiskFilter
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL