badgeridx

package

v0.4.1 Latest Latest Go to latest Published: May 20, 2024 License: Apache-2.0 Imports: 20 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/nlnwa/gowarcserver

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
type DB
- func NewDB(options ...Option) (db *DB, err error)
type Option
type Options

Constants ¶

View Source

const (
	SnappyCompression = "snappy"
	NoCompression     = "none"
	ZstdCompression   = "zstd"
)

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type DB ¶

type DB struct {
	// IdIndex maps record id to storage ref
	IdIndex *badger.DB

	// FileIndex maps filepath to fileinfo
	FileIndex *badger.DB

	// CdxIndex maps cdx key to cdx record
	CdxIndex *badger.DB

	// ReportIndex maps report id to report
	ReportIndex *badger.DB
	// contains filtered or unexported fields
}

func NewDB ¶

func NewDB(options ...Option) (db *DB, err error)

func (*DB) AddTask ¶ added in v0.4.0

func (db *DB) AddTask(id string, cancel context.CancelFunc)

func (*DB) CancelReport ¶ added in v0.4.0

func (db *DB) CancelReport(ctx context.Context, id string) error

func (*DB) Close ¶

func (db *DB) Close()

Close stops the gc and batch workers and closes the index databases.

func (*DB) CreateReport ¶ added in v0.4.0

func (db *DB) CreateReport(ctx context.Context, req index.Request) (*schema.Report, error)

func (*DB) Debug ¶ added in v0.4.0

func (db *DB) Debug(ctx context.Context, req keyvalue.DebugRequest, results chan<- keyvalue.CdxResponse) error

func (*DB) Delete ¶ added in v0.4.0

func (db *DB) Delete(ctx context.Context) error

Delete removes all data from the database.

func (*DB) DeleteReport ¶ added in v0.4.0

func (db *DB) DeleteReport(ctx context.Context, id string) error

func (*DB) DeleteTask ¶ added in v0.4.0

func (db *DB) DeleteTask(id string)

func (*DB) FlushBatch ¶ added in v0.4.0

func (db *DB) FlushBatch()

FlushBatch collects all records in the batch channel and updates the id and cdx indices.

func (*DB) GetCdx ¶

func (db *DB) GetCdx(key string) (*schema.Cdx, error)

func (*DB) GetFileInfo ¶

func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.FileInfo, error)

func (*DB) GetReport ¶ added in v0.4.0

func (db *DB) GetReport(ctx context.Context, id string) (*schema.Report, error)

func (*DB) GetStorageRef ¶

func (db *DB) GetStorageRef(ctx context.Context, warcId string) (string, error)

func (*DB) Index ¶

func (db *DB) Index(path string) error

func (*DB) ListFileInfo ¶

func (db *DB) ListFileInfo(ctx context.Context, req index.Request, results chan<- index.FileInfoResponse) error

func (*DB) ListReports ¶ added in v0.4.0

func (db *DB) ListReports(ctx context.Context, req index.Request, results chan<- index.ReportResponse) error

func (*DB) ListStorageRef ¶

func (db *DB) ListStorageRef(ctx context.Context, req index.Request, results chan<- index.IdResponse) error

func (*DB) Resolve ¶

func (db *DB) Resolve(_ context.Context, warcId string) (storageRef string, err error)

Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found.

func (*DB) ResolvePath ¶

func (db *DB) ResolvePath(filename string) (filePath string, err error)

ResolvePath looks up filename in file index and returns the path field.

func (*DB) SaveReport ¶ added in v0.4.0

func (db *DB) SaveReport(ctx context.Context, report *schema.Report) error

func (*DB) Search ¶

func (db *DB) Search(ctx context.Context, search index.Request, results chan<- index.CdxResponse) error

Search searches the index database

func (*DB) Write ¶

func (db *DB) Write(rec index.Record) error

type Option ¶

type Option func(opts *Options)

func WithBatchMaxSize ¶

func WithBatchMaxSize(size int) Option

func WithBatchMaxWait ¶

func WithBatchMaxWait(t time.Duration) Option

func WithCompression ¶

func WithCompression(c string) Option

func WithDatabase ¶

func WithDatabase(db string) Option

func WithDir ¶

func WithDir(d string) Option

func WithGcInterval ¶

func WithGcInterval(t time.Duration) Option

func WithIndexer ¶

func WithIndexer(indexer index.Indexer) Option

func WithReadOnly ¶

func WithReadOnly(readOnly bool) Option

func WithoutBadgerLogging ¶ added in v0.4.0

func WithoutBadgerLogging() Option

type Options ¶

type Options struct {
	Compression  badgerOptions.CompressionType
	BatchMaxSize int
	BatchMaxWait time.Duration
	GcInterval   time.Duration
	Path         string
	ReadOnly     bool
	Database     string
	Index        index.Indexer
	Silent       bool
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL