lake

package
v1.15.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 29, 2024 License: BSD-3-Clause Imports: 34 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	DataTag     = "data"
	BranchesTag = "branches"
	CommitsTag  = "commits"
)
View Source 
const (
	Version         = 3
	PoolsTag        = "pools"
	LakeMagicFile   = "lake.zng"
	LakeMagicString = "ZED LAKE"
)

Variables

View Source 
var (
	ErrCommitFailed      = fmt.Errorf("exceeded max update attempts (%d) to branch tip: commit failed", maxCommitRetries)
	ErrInvalidCommitMeta = errors.New("cannot parse ZSON string")
)
View Source 
var (
	ErrExist    = errors.New("lake already exists")
	ErrNotExist = errors.New("lake does not exist")
)

Functions

func CreateBranch 

func CreateBranch(ctx context.Context, engine storage.Engine, logger *zap.Logger, root *storage.URI, poolConfig *pools.Config, name string, parent ksuid.KSUID) (*branches.Config, error)

func CreatePool 

func CreatePool(ctx context.Context, engine storage.Engine, logger *zap.Logger, root *storage.URI, config *pools.Config) error

func DataPath 

func DataPath(poolPath *storage.URI) *storage.URI

func ImportComparator added in v1.2.0 

func ImportComparator(zctx *zed.Context, pool *Pool) *expr.Comparator

func RemovePool 

func RemovePool(ctx context.Context, engine storage.Engine, root *storage.URI, config *pools.Config) error

Types

type Branch 

type Branch struct {
	branches.Config
	// contains filtered or unexported fields
}

func OpenBranch 

func OpenBranch(ctx context.Context, config *branches.Config, engine storage.Engine, poolPath *storage.URI, pool *Pool) (*Branch, error)

func (*Branch) AddVectors added in v1.3.0 

func (b *Branch) AddVectors(ctx context.Context, ids []ksuid.KSUID, author, message string) (ksuid.KSUID, error)

func (*Branch) CommitCompact added in v1.2.0 

func (b *Branch) CommitCompact(ctx context.Context, src, rollup []*data.Object, rollupVecs []ksuid.KSUID, author, message, meta string) (ksuid.KSUID, error)

func (*Branch) Delete 

func (b *Branch) Delete(ctx context.Context, ids []ksuid.KSUID, author, message string) (ksuid.KSUID, error)

func (*Branch) DeleteVectors added in v1.3.0 

func (b *Branch) DeleteVectors(ctx context.Context, ids []ksuid.KSUID, author, message string) (ksuid.KSUID, error)

func (*Branch) DeleteWhere added in v1.3.0 

func (b *Branch) DeleteWhere(ctx context.Context, c runtime.Compiler, program ast.Seq, author, message, meta string) (ksuid.KSUID, error)

func (*Branch) Load 

func (b *Branch) Load(ctx context.Context, zctx *zed.Context, r zio.Reader, author, message, meta string) (ksuid.KSUID, error)

func (*Branch) LookupTags 

func (b *Branch) LookupTags(ctx context.Context, tags []ksuid.KSUID) ([]ksuid.KSUID, error)

func (*Branch) Pool 

func (b *Branch) Pool() *Pool

func (*Branch) Revert 

func (b *Branch) Revert(ctx context.Context, commit ksuid.KSUID, author, message string) (ksuid.KSUID, error)

type BranchMeta 

type BranchMeta struct {
	Pool   pools.Config    `zed:"pool"`
	Branch branches.Config `zed:"branch"`
}

type BranchTip 

type BranchTip struct {
	Name   string
	Commit ksuid.KSUID
}

type ImportStats 

type ImportStats struct {
	ObjectsWritten     int64
	RecordBytesWritten int64
	RecordsWritten     int64
}

func (*ImportStats) Accumulate 

func (s *ImportStats) Accumulate(b ImportStats)

func (*ImportStats) Copy 

func (s *ImportStats) Copy() ImportStats

type LakeMagic 

type LakeMagic struct {
	Magic   string `zed:"magic"`
	Version int    `zed:"version"`
}

type Pool 

type Pool struct {
	pools.Config

	Path     *storage.URI
	DataPath *storage.URI
	// contains filtered or unexported fields
}

func OpenPool 

func OpenPool(ctx context.Context, engine storage.Engine, logger *zap.Logger, root *storage.URI, config *pools.Config) (*Pool, error)

func (*Pool) BatchifyBranchTips added in v1.2.0 

func (p *Pool) BatchifyBranchTips(ctx context.Context, zctx *zed.Context, f expr.Evaluator) ([]zed.Value, error)

func (*Pool) BatchifyBranches added in v1.2.0 

func (p *Pool) BatchifyBranches(ctx context.Context, zctx *zed.Context, recs []zed.Value, m *zson.MarshalZNGContext, f expr.Evaluator) ([]zed.Value, error)

func (*Pool) ListBranches 

func (p *Pool) ListBranches(ctx context.Context) ([]branches.Config, error)

func (*Pool) LookupBranchByName 

func (p *Pool) LookupBranchByName(ctx context.Context, name string) (*branches.Config, error)

func (*Pool) Main 

func (p *Pool) Main(ctx context.Context) (BranchMeta, error)

func (*Pool) ObjectExists 

func (p *Pool) ObjectExists(ctx context.Context, id ksuid.KSUID) (bool, error)

XXX this is inefficient but is only meant for interactive queries...?

func (*Pool) OpenBranchByName 

func (p *Pool) OpenBranchByName(ctx context.Context, name string) (*Branch, error)

func (*Pool) OpenCommitLog added in v1.2.0 

func (p *Pool) OpenCommitLog(ctx context.Context, zctx *zed.Context, commit ksuid.KSUID) zio.Reader

func (*Pool) OpenCommitLogAsZNG added in v1.2.0 

func (p *Pool) OpenCommitLogAsZNG(ctx context.Context, zctx *zed.Context, commit ksuid.KSUID) (*zngio.Reader, error)

func (*Pool) ResolveRevision added in v1.8.0 

func (p *Pool) ResolveRevision(ctx context.Context, revision string) (ksuid.KSUID, error)

ResolveRevision returns the commit id for revision. revision can be either a commit ID in string form or a branch name.

func (*Pool) Snapshot 

func (p *Pool) Snapshot(ctx context.Context, commit ksuid.KSUID) (commits.View, error)

func (*Pool) Storage added in v1.2.0 

func (p *Pool) Storage() storage.Engine

func (*Pool) Vacuum added in v1.8.0 

func (p *Pool) Vacuum(ctx context.Context, commit ksuid.KSUID, dryrun bool) ([]ksuid.KSUID, error)

type Root 

type Root struct {
	// contains filtered or unexported fields
}

The Root of the lake represents the path prefix and configuration state for all of the data pools in the lake.

func Create 

func Create(ctx context.Context, engine storage.Engine, logger *zap.Logger, path *storage.URI) (*Root, error)

func CreateOrOpen 

func CreateOrOpen(ctx context.Context, engine storage.Engine, logger *zap.Logger, path *storage.URI) (*Root, error)

func Open 

func Open(ctx context.Context, engine storage.Engine, logger *zap.Logger, path *storage.URI) (*Root, error)

func (*Root) BatchifyBranches added in v1.2.0 

func (r *Root) BatchifyBranches(ctx context.Context, zctx *zed.Context, f expr.Evaluator) ([]zed.Value, error)

func (*Root) BatchifyPools added in v1.2.0 

func (r *Root) BatchifyPools(ctx context.Context, zctx *zed.Context, f expr.Evaluator) ([]zed.Value, error)

func (*Root) CommitObject 

func (r *Root) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchName string) (ksuid.KSUID, error)

func (*Root) CreateBranch 

func (r *Root) CreateBranch(ctx context.Context, poolID ksuid.KSUID, name string, parent ksuid.KSUID) (*branches.Config, error)

func (*Root) CreatePool 

func (r *Root) CreatePool(ctx context.Context, name string, sortKey order.SortKey, seekStride int, thresh int64) (*Pool, error)

func (*Root) ListPools 

func (r *Root) ListPools(ctx context.Context) ([]pools.Config, error)

func (*Root) MergeBranch 

func (r *Root) MergeBranch(ctx context.Context, poolID ksuid.KSUID, childBranch, parentBranch, author, message string) (ksuid.KSUID, error)

MergeBranch merges the indicated branch into its parent returning the commit tag of the new commit into the parent branch.

func (*Root) Open 

func (r *Root) Open(context.Context, *zed.Context, string, string, zbuf.Filter) (zbuf.Puller, error)

func (*Root) OpenPool 

func (r *Root) OpenPool(ctx context.Context, id ksuid.KSUID) (*Pool, error)

func (*Root) PoolID 

func (r *Root) PoolID(ctx context.Context, poolName string) (ksuid.KSUID, error)

func (*Root) RemoveBranch 

func (r *Root) RemoveBranch(ctx context.Context, poolID ksuid.KSUID, name string) error

func (*Root) RemovePool 

func (r *Root) RemovePool(ctx context.Context, id ksuid.KSUID) error

RemovePool deletes a pool from the configuration journal and deletes all data associated with the pool.

func (*Root) RenamePool 

func (r *Root) RenamePool(ctx context.Context, id ksuid.KSUID, newName string) error

func (*Root) Revert 

func (r *Root) Revert(ctx context.Context, poolID ksuid.KSUID, branchName string, commitID ksuid.KSUID, author, message string) (ksuid.KSUID, error)

func (*Root) SortKey added in v1.8.0 

func (r *Root) SortKey(ctx context.Context, src dag.Op) order.SortKey

func (*Root) VectorCache added in v1.12.0 

func (r *Root) VectorCache() *vcache.Cache

type SortedWriter added in v1.2.0 

type SortedWriter struct {
	// contains filtered or unexported fields
}

func NewSortedWriter added in v1.2.0 

func NewSortedWriter(ctx context.Context, zctx *zed.Context, pool *Pool, vectorEnabled bool) *SortedWriter

func (*SortedWriter) Abort added in v1.2.0 

func (w *SortedWriter) Abort()

func (*SortedWriter) Close added in v1.2.0 

func (w *SortedWriter) Close() error

func (*SortedWriter) Objects added in v1.2.0 

func (w *SortedWriter) Objects() []*data.Object

func (*SortedWriter) Vectors added in v1.10.0 

func (w *SortedWriter) Vectors() []ksuid.KSUID

func (*SortedWriter) Write added in v1.2.0 

func (w *SortedWriter) Write(val zed.Value) error

type Writer 

type Writer struct {
	// contains filtered or unexported fields
}

Writer is a zio.Writer that consumes records into memory according to the pools data object threshold, sorts each resulting buffer, and writes it as an immutable object to the storage system. The presumption is that each buffer's worth of data fits into memory.

func NewWriter 

func NewWriter(ctx context.Context, zctx *zed.Context, pool *Pool) (*Writer, error)

NewWriter creates a zio.Writer compliant writer for writing data to an a data pool presuming the input is not guaranteed to be sorted. XXX we should make another writer that takes sorted input and is a bit more efficient. This other writer could have different commit triggers to do useful things like paritioning given the context is a rollup.

func (*Writer) Close 

func (w *Writer) Close() error

func (*Writer) Objects 

func (w *Writer) Objects() []data.Object

func (*Writer) Stats 

func (w *Writer) Stats() ImportStats

func (*Writer) Write 

func (w *Writer) Write(rec zed.Value) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.