fragment

type AzureStoreConfig ¶ added in v0.89.0

type AzureStoreConfig struct {
	RewriterConfig
	// contains filtered or unexported fields
}

AzureStoreConfig configures a Fragment store of the "azure://" scheme. It is initialized from parsed URL parametrs of the pb.FragmentStore

type CoverSet ¶

type CoverSet []Fragment

CoverSet maintains Fragments ordered on |Begin| and |End|, with the invariant that no Fragment is fully overlapped by another Fragment in the set (though it may be overlapped by a combination of other Fragments). Intuitively, CoverSet represents the set of offsets which are "covered" by a collection of Fragments, and is able to map, for each byte offset, a "best" covering Fragment. It employs a heuristic of preferring larger fragments (and will replace spans of overlapped smaller fragments). An implication of its invariant is that no two Fragments have the same |Begin| or |End| (as that would imply an overlap). Both are monotonically increasing in the set: set[0].Begin represents the minimum offset, and set[len(set)-1].End represents the maximum offset.

func CoverSetDifference(a, b CoverSet) CoverSet

func WalkAllStores(ctx context.Context, name pb.Journal, stores []pb.FragmentStore) (CoverSet, error)

func (s CoverSet) Add(fragment Fragment) (CoverSet, bool)

func (s CoverSet) BeginOffset() int64

func (s CoverSet) EndOffset() int64

func (s CoverSet) LongestOverlappingFragment(offset int64) (ind int, found bool)

type File ¶

type File interface {
	io.ReaderAt
	io.Seeker
	io.WriterAt
	io.Writer
	io.Closer
}

File is the subset of os.File used in backing Fragments with local files.

type FileStoreConfig ¶ added in v0.84.1

type FileStoreConfig struct {
	RewriterConfig
}

FileStoreConfig configures a Fragment store of the "file://" scheme. It is initialized from parsed URL parameters of the pb.FragmentStore.

type Fragment ¶

type Fragment struct {
	protocol.Fragment
	// Local uncompressed file of the Fragment, or nil iff the Fragment is remote.
	File File
}

Fragment wraps the protocol.Fragment type with a nil-able backing local File.

type GSStoreConfig ¶ added in v0.84.1

type GSStoreConfig struct {
	RewriterConfig
	// contains filtered or unexported fields
}

GSStoreConfig configures a Fragment store of the "gs://" scheme. It is initialized from parsed URL parameters of the pb.FragmentStore.

type Index ¶

type Index struct {
	// contains filtered or unexported fields
}

Index maintains a queryable index of local and remote journal Fragments.

func NewIndex(ctx context.Context) *Index

func (fi *Index) EndOffset() int64

func (fi *Index) FirstRefreshCh() <-chan struct{}

func (fi *Index) Inspect(ctx context.Context, callback func(CoverSet) error) error

func (fi *Index) Query(ctx context.Context, req *pb.ReadRequest) (*pb.ReadResponse, File, error)

func (fi *Index) ReplaceRemote(set CoverSet)

func (fi *Index) SpoolCommit(frag Fragment)

type Persister ¶

type Persister struct {
	// contains filtered or unexported fields
}

Persister asynchronously persists completed Fragments to their backing pb.FragmentStore.

func NewPersister(ks *keyspace.KeySpace) *Persister

func (p *Persister) Finish()

func (p *Persister) Serve()

func (p *Persister) SpoolComplete(spool Spool, primary bool)

type RewriterConfig ¶ added in v0.84.1

type RewriterConfig struct {
	// Find is the string to replace in the unmodified journal name.
	Find string
	// Replace is the string with which Find is replaced in the constructed store path.
	Replace string
}

RewriterConfig rewrites the path under which journal fragments are stored by finding and replacing a portion of the journal's name in the final constructed path. Its use is uncommon and not recommended, but it can help in the implementation of new journal naming taxonomies which don't disrupt journal fragments that are already written.

var cfg = RewriterConfig{
    Replace: "/old-path/page-views/
    Find:    "/bar/v1/page-views/",
}
// Remaps journal name => fragment store URL:
//  "/foo/bar/v1/page-views/part-000" => "s3://my-bucket/foo/old-path/page-views/part-000" // Matched.
//  "/foo/bar/v2/page-views/part-000" => "s3://my-bucket/foo/bar/v2/page-views/part-000"   // Not matched.

type S3StoreConfig ¶ added in v0.84.1

type S3StoreConfig struct {
	RewriterConfig
	// AWS Profile to extract credentials from the shared credentials file.
	// For details, see:
	//   https://aws.amazon.com/blogs/security/a-new-and-standardized-way-to-manage-credentials-in-the-aws-sdks/
	// If empty, the default credentials are used.
	Profile string
	// Endpoint to connect to S3. If empty, the default S3 service is used.
	Endpoint string
	// ACL applied when persisting new fragments. By default, this is
	// s3.ObjectCannedACLBucketOwnerFullControl.
	ACL string
	// Storage class applied when persisting new fragments. By default,
	// this is s3.ObjectStorageClassStandard.
	StorageClass string
	// SSE is the server-side encryption type to be applied (eg, "AES256").
	// By default, encryption is not used.
	SSE string
	// contains filtered or unexported fields
}

S3StoreConfig configures a Fragment store of the "s3://" scheme. It is initialized from parsed URL parameters of the pb.FragmentStore.

type Spool ¶

type Spool struct {
	// Fragment at time of last commit.
	Fragment
	// FirstAppendTime is the UTC Time of the first commit of the Spool Fragment.
	FirstAppendTime time.Time
	// Registers of the journal.
	Registers pb.LabelSet
	// contains filtered or unexported fields
}

Spool is a Fragment which is in the process of being created, backed by a local *os.File. As commits occur and the file extent is updated, the Spool Fragment is also updated to reflect the new committed extent. At all times, the Spool Fragment is a consistent, valid Fragment.

func NewSpool(journal pb.Journal, observer SpoolObserver) Spool

func (s *Spool) Apply(r *pb.ReplicateRequest, primary bool) (pb.ReplicateResponse, error)

func (s *Spool) MustApply(r *pb.ReplicateRequest)

func (s *Spool) Next() pb.Fragment

func (s Spool) String() string

type SpoolObserver ¶

type SpoolObserver interface {
	// SpoolCommit is called when the Spool Fragment is extended.
	SpoolCommit(Fragment)
	// SpoolComplete is called when the Spool has been completed.
	SpoolComplete(_ Spool, primary bool)
}

SpoolObserver is notified of important events in the Spool lifecycle.