Documentation
¶
Index ¶
- func EditProviderFromDisk(nbf *types.NomsBinFormat, vrw types.ValueReadWriter, path string) (types.EditProvider, error)
- func FlushEditsToDisk(ctx context.Context, directory string, ea types.EditAccumulator) (string, error)
- func NewAsyncSortedEditsWithDefaults(nbf *types.NomsBinFormat) types.EditAccumulator
- type AsyncSortedEdits
- func (ase *AsyncSortedEdits) AddEdit(k types.LesserValuable, v types.Valuable)
- func (ase *AsyncSortedEdits) Close(ctx context.Context) error
- func (ase *AsyncSortedEdits) EditsAdded() int
- func (ase *AsyncSortedEdits) FinishedEditing() (types.EditProvider, error)
- func (ase *AsyncSortedEdits) Size() int64
- type DiskBackedEditAcc
- type DiskEditFlusher
- type EPMerger
- type FlushResult
- type FlushResults
- type KVPCollBuilder
- type KVPCollItr
- type KVPCollection
- type SortedEditItr
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func EditProviderFromDisk ¶
func EditProviderFromDisk(nbf *types.NomsBinFormat, vrw types.ValueReadWriter, path string) (types.EditProvider, error)
EditProviderFromDisk returns a types.EditProvider instance which reads data from the specified file
func FlushEditsToDisk ¶
func FlushEditsToDisk(ctx context.Context, directory string, ea types.EditAccumulator) (string, error)
FlushEditsToDisk writes the contents of a types.EditAccumulator to disk and returns the path where the associated file exists.
func NewAsyncSortedEditsWithDefaults ¶
func NewAsyncSortedEditsWithDefaults(nbf *types.NomsBinFormat) types.EditAccumulator
NewAsyncSortedEditsWithDefaults creates a new AsyncSortedEdit instance with default concurrency and buffer size values
Types ¶
type AsyncSortedEdits ¶
type AsyncSortedEdits struct {
// contains filtered or unexported fields
}
AsyncSortedEdits is a data structure that can have edits added to it, and as they are added it will send them in batches to be sorted. Once all edits have been added the batches of edits can then be merge sorted together.
func NewAsyncSortedEdits ¶
func NewAsyncSortedEdits(nbf *types.NomsBinFormat, sliceSize, asyncConcurrency, sortConcurrency int) *AsyncSortedEdits
NewAsyncSortedEdits creates an AsyncSortedEdits object that creates batches of size 'sliceSize' and kicks off 'asyncConcurrency' go routines for background sorting of batches. The final Sort call is processed with 'sortConcurrency' go routines
func (*AsyncSortedEdits) AddEdit ¶
func (ase *AsyncSortedEdits) AddEdit(k types.LesserValuable, v types.Valuable)
AddEdit adds an edit. Not thread safe
func (*AsyncSortedEdits) Close ¶
func (ase *AsyncSortedEdits) Close(ctx context.Context) error
Close ensures that the accumulator is closed. Repeat calls are allowed. This and FinishedEditing are not thread safe, and thus external synchronization is required.
func (*AsyncSortedEdits) EditsAdded ¶
func (ase *AsyncSortedEdits) EditsAdded() int
EditsAdded returns the number of edits that have been added to this EditAccumulator
func (*AsyncSortedEdits) FinishedEditing ¶
func (ase *AsyncSortedEdits) FinishedEditing() (types.EditProvider, error)
FinishedEditing should be called once all edits have been added. Once FinishedEditing is called adding more edits will have undefined behavior.
func (*AsyncSortedEdits) Size ¶
func (ase *AsyncSortedEdits) Size() int64
Size returns the number of edits
type DiskBackedEditAcc ¶
type DiskBackedEditAcc struct {
// contains filtered or unexported fields
}
DiskBackedEditAcc is an EditAccumulator implementation that flushes the edits to disk at regular intervals
func NewDiskBackedEditAcc ¶
func NewDiskBackedEditAcc(ctx context.Context, nbf *types.NomsBinFormat, vrw types.ValueReadWriter, flushInterval int64, directory string, newEditAcc func() types.EditAccumulator) *DiskBackedEditAcc
NewDiskBackedEditAcc returns a new DiskBackedEditAccumulator instance
func (*DiskBackedEditAcc) AddEdit ¶
func (dbea *DiskBackedEditAcc) AddEdit(key types.LesserValuable, val types.Valuable)
AddEdit adds an edit. Not thread safe
func (*DiskBackedEditAcc) Close ¶
func (dbea *DiskBackedEditAcc) Close(ctx context.Context) error
Close ensures that the accumulator is closed. Repeat calls are allowed. Not guaranteed to be thread-safe, thus requires external synchronization.
func (*DiskBackedEditAcc) EditsAdded ¶
func (dbea *DiskBackedEditAcc) EditsAdded() int
EditsAdded returns the number of edits that have been added to this EditAccumulator
func (*DiskBackedEditAcc) FinishedEditing ¶
func (dbea *DiskBackedEditAcc) FinishedEditing() (types.EditProvider, error)
FinishedEditing should be called when all edits have been added to get an EditProvider which provides the edits in sorted order. Adding more edits after calling FinishedEditing is an error.
type DiskEditFlusher ¶
type DiskEditFlusher struct {
// contains filtered or unexported fields
}
DiskEditFlusher is a class that handles asynchronously flushing types.EditAccumulators to disk then allows getting an associated types.EditProvider for each flushed accumulator at a later time.
func NewDiskEditFlusher ¶
func NewDiskEditFlusher(ctx context.Context, directory string, nbf *types.NomsBinFormat, vrw types.ValueReadWriter) *DiskEditFlusher
NewDiskEditFlusher returns a new DiskEditFlusher instance
func (*DiskEditFlusher) Flush ¶
func (ef *DiskEditFlusher) Flush(accumulator types.EditAccumulator, id uint64)
Flush kicks off a new go routine to write the edits from the types.EditAccumulator to disk. An id is provided along with the accumulator to allow for differentiating which results came from which flush.
func (*DiskEditFlusher) Wait ¶
func (ef *DiskEditFlusher) Wait(ctx context.Context) (FlushResults, error)
Wait waits for asynchronous flushing tasks to complete and then returns their results. The FlushResult.Edits needs to be closed by the caller for each result. FlushResults will be sorted by ID
func (*DiskEditFlusher) WaitForIDs ¶
func (ef *DiskEditFlusher) WaitForIDs(ctx context.Context, idFilter *set.Uint64Set) (FlushResults, error)
WaitForIDs waits for asynchronous flushing tasks to complete and then returns the results of flushing the specified ids. The FlushResult.Edits needs to be closed by the caller for each result. FlushResults will be sorted by ID
type EPMerger ¶
type EPMerger struct {
// contains filtered or unexported fields
}
func NewEPMerger ¶
func NewEPMerger(ctx context.Context, nbf *types.NomsBinFormat, eps []types.EditProvider) (*EPMerger, error)
NewEPMerger takes a slice of TupleReaders, whose contents should be key sorted key value tuple pairs, and return a *EPMerger
func (*EPMerger) Next ¶
Next returns the next KVP representing the next edit to be applied. Next will always return KVPs in key sorted order. Once all KVPs have been read io.EOF will be returned.
func (*EPMerger) ReachedEOF ¶
ReachedEOF returns true once all data is exhausted. If ReachedEOF returns false that does not mean that there is more data, only that io.EOF has not been returned previously. If ReachedEOF returns true then all edits have been read
type FlushResult ¶
type FlushResult struct {
Edits types.EditProvider
ID uint64
}
FlushResult contains the results of flushing a types.EditAccumulator to disk and the ID associated with it
type FlushResults ¶
type FlushResults []*FlushResult
FlushResults is a sortable slice of FlushResult instances
func (FlushResults) Sort ¶
func (res FlushResults) Sort()
type KVPCollBuilder ¶
type KVPCollBuilder struct {
// contains filtered or unexported fields
}
KVPCollBuilder is used to build a KVPCollection. It creates two buffers which it fills with KVPs. When a buffer is filled the target buffer is changed for subsequent adds. New buffers can be added to the builder so that buffers of other KVPCollections can be reused.
func NewKVPCollBuilder ¶
func NewKVPCollBuilder(buffSize int, nbf *types.NomsBinFormat) *KVPCollBuilder
NewKVPCollBuilder creates a builder which can be used to
func (*KVPCollBuilder) AddBuffer ¶
func (cb *KVPCollBuilder) AddBuffer(buff types.KVPSlice)
AddBuffer adds a buffer of KVPs that can be filled.
func (*KVPCollBuilder) AddKVP ¶
func (cb *KVPCollBuilder) AddKVP(kvp types.KVP)
AddKVP adds a KVP to the current buffer
func (*KVPCollBuilder) Build ¶
func (cb *KVPCollBuilder) Build() *KVPCollection
Build takes all the filled and partially filled buffers and creates a new KVPCollection from them.
func (*KVPCollBuilder) MoveRemaining ¶
func (cb *KVPCollBuilder) MoveRemaining(itr *KVPCollItr)
MoveRemaining takes a KVPCollItr and moves all the KVPs that still need to be iterated over and moves them into the internal KVP buffers.
type KVPCollItr ¶
type KVPCollItr struct {
// contains filtered or unexported fields
}
KVPCollItr is a KVPIterator implementation for iterating over a KVPCollection
func NewItr ¶
func NewItr(nbf *types.NomsBinFormat, coll *KVPCollection) *KVPCollItr
NewItr creates a new KVPCollItr from a KVPCollection
func (*KVPCollItr) Less ¶
func (itr *KVPCollItr) Less(other *KVPCollItr) (bool, error)
Less returns whether the current key this iterator is less than the current key for another iterator
func (*KVPCollItr) Next ¶
func (itr *KVPCollItr) Next() (*types.KVP, error)
Next returns the next KVP representing the next edit to be applied. Next will always return KVPs in key sorted order. Once all KVPs have been read io.EOF will be returned.
func (*KVPCollItr) Peek ¶
func (itr *KVPCollItr) Peek() *types.KVP
Peek returns the next KVP without advancing
func (*KVPCollItr) ReachedEOF ¶
func (itr *KVPCollItr) ReachedEOF() bool
ReachedEOF returns true once all data is exhausted. If ReachedEOF returns false that does not mean that there is more data, only that io.EOF has not been returned previously. If ReachedEOF returns true then all edits have been read
func (*KVPCollItr) Reset ¶
func (itr *KVPCollItr) Reset()
Reset sets the iterator back to the beginning of the collection so it can be iterated over again.
type KVPCollection ¶
type KVPCollection struct {
// contains filtered or unexported fields
}
KVPCollection is a collection of sorted KVPs
func NewKVPCollection ¶
func NewKVPCollection(nbf *types.NomsBinFormat, sl types.KVPSlice) *KVPCollection
NewKVPCollection creates a new KVPCollection from a sorted KVPSlice
func (*KVPCollection) DestructiveMerge ¶
func (left *KVPCollection) DestructiveMerge(right *KVPCollection) (*KVPCollection, error)
DestructiveMerge merges two KVPCollections into a new collection. This KVPCollection and the collection it is being merged with will no longer be valid once this method is called. A new KVPCollection will be returned which holds the merged collections.
func (*KVPCollection) Iterator ¶
func (coll *KVPCollection) Iterator() *KVPCollItr
Iterator returns an iterator that will iterate over the KVPs in the collection in order.
func (*KVPCollection) Size ¶
func (coll *KVPCollection) Size() int64
Size returns the total number of elements in the collection
type SortedEditItr ¶
type SortedEditItr struct {
// contains filtered or unexported fields
}
SortedEditItr is a KVPIterator implementation that takes two KVPCollItr and merges them as it iterates
func NewSortedEditItr ¶
func NewSortedEditItr(nbf *types.NomsBinFormat, left, right *KVPCollection) *SortedEditItr
NewSortedEditItr creates an iterator from two KVPCollection references. As the iterator iterates it merges the collections and iterates in order
func (*SortedEditItr) Next ¶
func (itr *SortedEditItr) Next() (*types.KVP, error)
Next returns the next KVP representing the next edit to be applied. Next will always return KVPs in key sorted order. Once all KVPs have been read io.EOF will be returned.
func (*SortedEditItr) Peek ¶
func (itr *SortedEditItr) Peek() (*types.KVP, error)
Peek returns the next KVP without advancing
func (*SortedEditItr) ReachedEOF ¶
func (itr *SortedEditItr) ReachedEOF() bool
ReachedEOF returns true once all data is exhausted. If ReachedEOF returns false that does not mean that there is more data, only that io.EOF has not been returned previously. If ReachedEOF returns true then all edits have been read
func (*SortedEditItr) Size ¶
func (itr *SortedEditItr) Size() int64
Size returns the total number of elements this iterator will iterate over.
Source Files