Documentation ¶
Index ¶
- Constants
- Variables
- func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, filename string) (err error)
- type BinaryTOC
- type Config
- type FileWriter
- func (fw *FileWriter) AddPadding(size int) error
- func (fw *FileWriter) Close() error
- func (fw *FileWriter) Flush() error
- func (fw *FileWriter) Pos() uint64
- func (fw *FileWriter) Remove() error
- func (fw *FileWriter) Write(bufs ...[]byte) error
- func (fw *FileWriter) WriteAt(buf []byte, pos uint64) error
- type LazyBinaryReader
- func (r *LazyBinaryReader) Close() error
- func (r *LazyBinaryReader) EagerLoad()
- func (r *LazyBinaryReader) IndexVersion() (int, error)
- func (r *LazyBinaryReader) LabelNames() ([]string, error)
- func (r *LazyBinaryReader) LabelValuesOffsets(name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)
- func (r *LazyBinaryReader) LookupSymbol(o uint32) (string, error)
- func (r *LazyBinaryReader) PostingsOffset(name, value string) (index.Range, error)
- func (r *LazyBinaryReader) SymbolsReader() (streamindex.SymbolsReader, error)
- type LazyBinaryReaderMetrics
- type LazyLoadedHeadersSnapshotConfig
- type Reader
- type ReaderPool
- type ReaderPoolMetrics
- type StreamBinaryReader
- func (r *StreamBinaryReader) Close() error
- func (r *StreamBinaryReader) IndexVersion() (int, error)
- func (r *StreamBinaryReader) LabelNames() ([]string, error)
- func (r *StreamBinaryReader) LabelValuesOffsets(name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)
- func (r *StreamBinaryReader) LookupSymbol(o uint32) (string, error)
- func (r *StreamBinaryReader) PostingsOffset(name, value string) (index.Range, error)
- func (r *StreamBinaryReader) SymbolsReader() (streamindex.SymbolsReader, error)
- type StreamBinaryReaderMetrics
Constants ¶
const ( // BinaryFormatV1 represents first version of index-header file. BinaryFormatV1 = 1 // MagicIndex are 4 bytes at the head of an index-header file. MagicIndex = 0xBAAAD792 )
const ( DefaultIndexHeaderLazyLoadingEnabled = true DefaultIndexHeaderLazyLoadingIdleTimeout = 60 * time.Minute )
Variables ¶
var NotFoundRangeErr = errors.New("range not found") //nolint:revive
NotFoundRangeErr is an error returned by PostingsOffset when there is no posting for given name and value pairs.
Functions ¶
func WriteBinary ¶
func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, filename string) (err error)
WriteBinary build index-header file from the pieces of index in object storage.
Types ¶
type BinaryTOC ¶
type BinaryTOC struct { // Symbols holds start to the same symbols section as index related to this index header. Symbols uint64 // PostingsOffsetTable holds start to the same Postings Offset Table section as index related to this index header. PostingsOffsetTable uint64 }
BinaryTOC is a table of content for index-header file.
type Config ¶
type Config struct { MaxIdleFileHandles uint `yaml:"max_idle_file_handles" category:"advanced"` EagerLoadingStartupEnabled bool `yaml:"eager_loading_startup_enabled" category:"experimental"` // Controls whether index-header lazy loading is enabled. LazyLoadingEnabled bool `yaml:"lazy_loading_enabled" category:"advanced"` LazyLoadingIdleTimeout time.Duration `yaml:"lazy_loading_idle_timeout" category:"advanced"` // Maximum index-headers loaded into store-gateway concurrently LazyLoadingConcurrency int `yaml:"lazy_loading_concurrency" category:"advanced"` VerifyOnLoad bool `yaml:"verify_on_load" category:"advanced"` }
func (*Config) RegisterFlagsWithPrefix ¶
type FileWriter ¶
type FileWriter struct {
// contains filtered or unexported fields
}
func NewFileWriter ¶
func NewFileWriter(name string, size int) (*FileWriter, error)
TODO(bwplotka): Added size to method, upstream this.
func (*FileWriter) AddPadding ¶
func (fw *FileWriter) AddPadding(size int) error
AddPadding adds zero byte padding until the file size is a multiple size.
func (*FileWriter) Close ¶
func (fw *FileWriter) Close() error
func (*FileWriter) Flush ¶
func (fw *FileWriter) Flush() error
func (*FileWriter) Pos ¶
func (fw *FileWriter) Pos() uint64
func (*FileWriter) Remove ¶
func (fw *FileWriter) Remove() error
func (*FileWriter) Write ¶
func (fw *FileWriter) Write(bufs ...[]byte) error
type LazyBinaryReader ¶
type LazyBinaryReader struct {
// contains filtered or unexported fields
}
LazyBinaryReader wraps BinaryReader and loads (mmap or streaming read) the index-header only upon the first Reader function is called.
func NewLazyBinaryReader ¶
func NewLazyBinaryReader( ctx context.Context, readerFactory func() (Reader, error), logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, metrics *LazyBinaryReaderMetrics, onClosed func(*LazyBinaryReader), lazyLoadingGate gate.Gate, ) (*LazyBinaryReader, error)
NewLazyBinaryReader makes a new LazyBinaryReader. If the index-header does not exist on the local disk at dir location, this function will build it downloading required sections from the full index stored in the bucket. However, this function doesn't load (mmap or streaming read) the index-header; it will be loaded at first Reader function call.
func (*LazyBinaryReader) Close ¶
func (r *LazyBinaryReader) Close() error
Close implements Reader. It unloads the index-header from memory (releasing the mmap area), but a subsequent call to any other Reader function will automatically reload it.
func (*LazyBinaryReader) EagerLoad ¶
func (r *LazyBinaryReader) EagerLoad()
EagerLoad attempts to eagerly load this index header.
func (*LazyBinaryReader) IndexVersion ¶
func (r *LazyBinaryReader) IndexVersion() (int, error)
IndexVersion implements Reader.
func (*LazyBinaryReader) LabelNames ¶
func (r *LazyBinaryReader) LabelNames() ([]string, error)
LabelNames implements Reader.
func (*LazyBinaryReader) LabelValuesOffsets ¶
func (r *LazyBinaryReader) LabelValuesOffsets(name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)
LabelValuesOffsets implements Reader.
func (*LazyBinaryReader) LookupSymbol ¶
func (r *LazyBinaryReader) LookupSymbol(o uint32) (string, error)
LookupSymbol implements Reader.
func (*LazyBinaryReader) PostingsOffset ¶
func (r *LazyBinaryReader) PostingsOffset(name, value string) (index.Range, error)
PostingsOffset implements Reader.
func (*LazyBinaryReader) SymbolsReader ¶
func (r *LazyBinaryReader) SymbolsReader() (streamindex.SymbolsReader, error)
SymbolsReader implements Reader.
type LazyBinaryReaderMetrics ¶
type LazyBinaryReaderMetrics struct {
// contains filtered or unexported fields
}
LazyBinaryReaderMetrics holds metrics tracked by LazyBinaryReader.
func NewLazyBinaryReaderMetrics ¶
func NewLazyBinaryReaderMetrics(reg prometheus.Registerer) *LazyBinaryReaderMetrics
NewLazyBinaryReaderMetrics makes new LazyBinaryReaderMetrics.
type LazyLoadedHeadersSnapshotConfig ¶
type LazyLoadedHeadersSnapshotConfig struct { // Path stores where lazy loaded blocks will be tracked in a single file per tenant Path string UserID string }
LazyLoadedHeadersSnapshotConfig stores information needed to track lazy loaded index headers.
type Reader ¶
type Reader interface { io.Closer // IndexVersion returns version of index. IndexVersion() (int, error) // PostingsOffset returns start and end offsets of postings for given name and value. // The Start is inclusive and is the byte offset of the number_of_entries field of a posting list. // The End is exclusive and is typically the byte offset of the CRC32 field. // The End might be bigger than the actual posting ending, but not larger than the whole index file. // NotFoundRangeErr is returned when no index can be found for given name and value. PostingsOffset(name string, value string) (index.Range, error) // LookupSymbol returns string based on given reference. // Error is return if the symbol can't be found. LookupSymbol(o uint32) (string, error) SymbolsReader() (streamindex.SymbolsReader, error) // LabelValuesOffsets returns all label values and the offsets for their posting lists for given label name or error. // The returned label values are sorted lexicographically (which the same as sorted by posting offset). // The ranges of each posting list are the same as returned by PostingsOffset. // If no values are found for label name, or label name does not exists, // then empty slice is returned and no error. // If non-empty prefix is provided, only posting lists starting with the prefix are returned. // If non-nil filter is provided, then only posting lists for which filter returns true are returned. LabelValuesOffsets(name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error) // LabelNames returns all label names in sorted order. LabelNames() ([]string, error) }
Reader is an interface allowing to read essential, minimal number of index fields from the small portion of index file called header.
type ReaderPool ¶
type ReaderPool struct {
// contains filtered or unexported fields
}
ReaderPool is used to istantiate new index-header readers and keep track of them. When the lazy reader is enabled, the pool keeps track of all instantiated readers and automatically close them once the idle timeout is reached. A closed lazy reader will be automatically re-opened upon next usage.
func NewReaderPool ¶
func NewReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics, lazyLoadedSnapshotConfig LazyLoadedHeadersSnapshotConfig) *ReaderPool
NewReaderPool makes a new ReaderPool. If lazy-loading is enabled, NewReaderPool also starts a background task for unloading idle Readers and persisting a list of loaded Readers to disk.
func (*ReaderPool) Close ¶
func (p *ReaderPool) Close()
Close the pool and stop checking for idle readers. No reader tracked by this pool will be closed. It's the caller responsibility to close readers.
func (*ReaderPool) LoadedBlocks ¶
func (p *ReaderPool) LoadedBlocks() map[ulid.ULID]int64
LoadedBlocks returns a new map of lazy-loaded block IDs and the last time they were used in milliseconds.
func (*ReaderPool) NewBinaryReader ¶
func (p *ReaderPool) NewBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int, cfg Config, initialSync bool) (Reader, error)
NewBinaryReader creates and returns a new binary reader. If the pool has been configured with lazy reader enabled, this function will return a lazy reader. The returned lazy reader is tracked by the pool and automatically closed once the idle timeout expires.
type ReaderPoolMetrics ¶
type ReaderPoolMetrics struct {
// contains filtered or unexported fields
}
ReaderPoolMetrics holds metrics tracked by ReaderPool.
func NewReaderPoolMetrics ¶
func NewReaderPoolMetrics(reg prometheus.Registerer) *ReaderPoolMetrics
NewReaderPoolMetrics makes new ReaderPoolMetrics.
type StreamBinaryReader ¶
type StreamBinaryReader struct {
// contains filtered or unexported fields
}
func NewStreamBinaryReader ¶
func NewStreamBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int, metrics *StreamBinaryReaderMetrics, cfg Config) (*StreamBinaryReader, error)
NewStreamBinaryReader loads or builds new index-header if not present on disk.
func (*StreamBinaryReader) Close ¶
func (r *StreamBinaryReader) Close() error
func (*StreamBinaryReader) IndexVersion ¶
func (r *StreamBinaryReader) IndexVersion() (int, error)
func (*StreamBinaryReader) LabelNames ¶
func (r *StreamBinaryReader) LabelNames() ([]string, error)
func (*StreamBinaryReader) LabelValuesOffsets ¶
func (r *StreamBinaryReader) LabelValuesOffsets(name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)
func (*StreamBinaryReader) LookupSymbol ¶
func (r *StreamBinaryReader) LookupSymbol(o uint32) (string, error)
func (*StreamBinaryReader) PostingsOffset ¶
func (r *StreamBinaryReader) PostingsOffset(name, value string) (index.Range, error)
func (*StreamBinaryReader) SymbolsReader ¶
func (r *StreamBinaryReader) SymbolsReader() (streamindex.SymbolsReader, error)
type StreamBinaryReaderMetrics ¶
type StreamBinaryReaderMetrics struct {
// contains filtered or unexported fields
}
func NewStreamBinaryReaderMetrics ¶
func NewStreamBinaryReaderMetrics(reg prometheus.Registerer) *StreamBinaryReaderMetrics