badgeridx

package

v0.3.0 Latest Latest Go to latest Published: Mar 6, 2023 License: Apache-2.0 Imports: 21 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/nlnwa/gowarcserver

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
func CompressionDecodeHookFunc() mapstructure.DecodeHookFuncType
func Get(db *badger.DB, key []byte) (value []byte, err error)
func Search(db *badger.DB, key string, reverse bool, f PerItemFunc, a AfterIterFunc) error
type AfterIterFunc
type DB
- func NewDB(options ...Option) (db *DB, err error)
type Option
type Options
type PerCdxFunc
type PerItemFunc
type Sorter
- func NewSorter(closest int64, reverse bool) Sorter

Constants ¶

View Source

const (
	SnappyCompression = "snappy"
	NoCompression     = "none"
	ZstdCompression   = "zstd"
)

Variables ¶

This section is empty.

Functions ¶

func CompressionDecodeHookFunc ¶

func CompressionDecodeHookFunc() mapstructure.DecodeHookFuncType

CompressionDecodeHookFunc helps decode a string to a badger compression type using viper.

Example: var c options.CompressionType _ = viper.UnmarshalKey("compression", &c, viper.DecodeHook(CompressionDecodeHookFunc())

func Get ¶

func Get(db *badger.DB, key []byte) (value []byte, err error)

Get value stored at key from db.

func Search ¶

func Search(db *badger.DB, key string, reverse bool, f PerItemFunc, a AfterIterFunc) error

Search iterates over keys in db prefixed with key and applies PerItemFunc f to each item value.

Types ¶

type AfterIterFunc ¶

type AfterIterFunc func(txn *badger.Txn) error

type DB ¶

type DB struct {
	// IdIndex maps record id to storage ref
	IdIndex *badger.DB

	// FileIndex maps filepath to fileinfo
	FileIndex *badger.DB

	// CdxIndex maps cdx key to cdx record
	CdxIndex *badger.DB
	// contains filtered or unexported fields
}

func NewDB ¶

func NewDB(options ...Option) (db *DB, err error)

func (*DB) Close ¶

func (db *DB) Close()

Close stops the gc and batch workers and closes the index databases.

func (*DB) Closest ¶

func (db *DB) Closest(_ context.Context, request index.Request, results chan<- index.CdxResponse) error

Closest returns the first closest cdx value

func (*DB) GetCdx ¶

func (db *DB) GetCdx(key string) (*schema.Cdx, error)

func (*DB) GetFileInfo ¶

func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, error)

func (*DB) GetStorageRef ¶

func (db *DB) GetStorageRef(ctx context.Context, warcId string) (string, error)

func (*DB) Index ¶

func (db *DB) Index(path string) error

func (*DB) List ¶

func (db *DB) List(ctx context.Context, limit int, results chan<- index.CdxResponse) error

func (*DB) ListFileInfo ¶

func (db *DB) ListFileInfo(ctx context.Context, limit int, results chan<- index.FileInfoResponse) error

func (*DB) ListRecords ¶

func (db *DB) ListRecords(fn func(warcId string, cdx *schema.Cdx) (stopIteration bool)) error

func (*DB) ListStorageRef ¶

func (db *DB) ListStorageRef(ctx context.Context, limit int, results chan<- index.IdResponse) error

func (*DB) Resolve ¶

func (db *DB) Resolve(_ context.Context, warcId string) (storageRef string, err error)

Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found.

func (*DB) ResolvePath ¶

func (db *DB) ResolvePath(filename string) (filePath string, err error)

ResolvePath looks up filename in file index and returns the path field.

func (*DB) Search ¶

func (db *DB) Search(ctx context.Context, search index.Request, results chan<- index.CdxResponse) error

func (*DB) Write ¶

func (db *DB) Write(rec index.Record) error

type Option ¶

type Option func(opts *Options)

func WithBatchMaxSize ¶

func WithBatchMaxSize(size int) Option

func WithBatchMaxWait ¶

func WithBatchMaxWait(t time.Duration) Option

func WithCompression ¶

func WithCompression(c badgerOptions.CompressionType) Option

func WithDatabase ¶

func WithDatabase(db string) Option

func WithDir ¶

func WithDir(d string) Option

func WithGcInterval ¶

func WithGcInterval(t time.Duration) Option

func WithIndexer ¶

func WithIndexer(indexer index.Indexer) Option

func WithReadOnly ¶

func WithReadOnly(readOnly bool) Option

type Options ¶

type Options struct {
	Compression  badgerOptions.CompressionType
	BatchMaxSize int
	BatchMaxWait time.Duration
	GcInterval   time.Duration
	Path         string
	ReadOnly     bool
	Database     string
	Index        index.Indexer
}

type PerCdxFunc ¶

type PerCdxFunc func(cdx *schema.Cdx) error

type PerItemFunc ¶

type PerItemFunc func(*badger.Item) (stopIteration bool)

type Sorter ¶

type Sorter struct {
	Closest int64
	Reverse bool
	Values  []value
}

func NewSorter ¶

func NewSorter(closest int64, reverse bool) Sorter

func (*Sorter) Add ¶

func (s *Sorter) Add(k []byte)

func (*Sorter) Sort ¶

func (s *Sorter) Sort()

func (*Sorter) Walk ¶

func (s *Sorter) Walk(txn *badger.Txn, perItemFn PerItemFunc) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL