ingest

package

v0.14.0 Latest Latest Go to latest Published: Aug 27, 2023 License: Apache-2.0 Imports: 15 Imported by: 1

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/wrgl/wrgl

Links

Open Source Insights

Documentation ¶

Index ¶

func IndexTable(db objects.Store, tblSum []byte, tbl *objects.Table, logger logr.Logger) error
func IngestTable(db objects.Store, sorter *sorter.Sorter, f io.ReadCloser, pk []string, ...) ([]byte, error)
func IngestTableFromBlocks(db objects.Store, sorter *sorter.Sorter, columns []string, pk []uint32, ...) ([]byte, error)
func ProfileTable(db objects.Store, sum []byte, tbl *objects.Table) error
func ReingestTable(db objects.Store, s *sorter.Sorter, tbl *objects.Table, useBlockIndex bool, ...) (newTableSum []byte, err error)
type Error
- func (err *Error) Error() string
type Inserter
- func NewInserter(db objects.Store, sorter *sorter.Sorter, logger logr.Logger, ...) *Inserter
- func (i *Inserter) IngestTableFromSorter(columns []string, pk []uint32) ([]byte, error)
type InserterOption
- func WithNumWorkers(n int) InserterOption
- func WithProgressBar(pt pbar.Bar) InserterOption
type ProgressBar

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func IndexTable ¶

func IndexTable(db objects.Store, tblSum []byte, tbl *objects.Table, logger logr.Logger) error

func IngestTable ¶

func IngestTable(db objects.Store, sorter *sorter.Sorter, f io.ReadCloser, pk []string, logger logr.Logger, opts ...InserterOption) ([]byte, error)

func IngestTableFromBlocks ¶

func IngestTableFromBlocks(db objects.Store, sorter *sorter.Sorter, columns []string, pk []uint32, blocks <-chan *sorter.Block, logger logr.Logger, opts ...InserterOption) ([]byte, error)

func ProfileTable ¶ added in v0.8.0

func ProfileTable(db objects.Store, sum []byte, tbl *objects.Table) error

func ReingestTable ¶ added in v0.12.1

func ReingestTable(db objects.Store, s *sorter.Sorter, tbl *objects.Table, useBlockIndex bool, logger logr.Logger, opts ...InserterOption) (newTableSum []byte, err error)

ReingestTable searchs for duplicated rows in a table. If duplicated rows are detected, it re-ingests the table and return the new table sum

Types ¶

type Error ¶ added in v0.11.6

type Error struct {
	Message string
}

func (*Error) Error ¶ added in v0.11.6

func (err *Error) Error() string

type Inserter ¶

type Inserter struct {
	// contains filtered or unexported fields
}

func NewInserter ¶ added in v0.7.0

func NewInserter(db objects.Store, sorter *sorter.Sorter, logger logr.Logger, opts ...InserterOption) *Inserter

func (*Inserter) IngestTableFromSorter ¶ added in v0.12.1

func (i *Inserter) IngestTableFromSorter(columns []string, pk []uint32) ([]byte, error)

type InserterOption ¶

type InserterOption func(*Inserter)

func WithNumWorkers ¶

func WithNumWorkers(n int) InserterOption

func WithProgressBar ¶

func WithProgressBar(pt pbar.Bar) InserterOption

type ProgressBar ¶

type ProgressBar interface {
	Add(int) error
	Finish() error
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL