config

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 8, 2024 License: AGPL-3.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (

	// ObjectStorageIndexRequiredPeriod defines the required index period for object storage based index stores like boltdb-shipper and tsdb
	ObjectStorageIndexRequiredPeriod = 24 * time.Hour
)

Variables

This section is empty.

Functions

func ActivePeriodConfig

func ActivePeriodConfig(configs []PeriodConfig) int

ActivePeriodConfig returns index of active PeriodicConfig which would be applicable to logs that would be pushed starting now. Note: Another PeriodicConfig might be applicable for future logs which can change index type.

func ExtractTableNumberFromName

func ExtractTableNumberFromName(tableName string) (int64, error)

ExtractTableNumberFromName extracts the table number from a given tableName. returns -1 on error.

func IsObjectStorageIndex

func IsObjectStorageIndex(indexType string) bool

IsObjectStorageIndex returns true if the index type is either boltdb-shipper or tsdb.

func UsingObjectStorageIndex

func UsingObjectStorageIndex(configs []PeriodConfig) bool

UsingObjectStorageIndex returns true if the current or any of the upcoming periods use an object store index.

func ValidatePathPrefix

func ValidatePathPrefix(prefix string) error

Types

type ActiveTableProvisionConfig

type ActiveTableProvisionConfig struct {
	ProvisionedThroughputOnDemandMode bool  `yaml:"enable_ondemand_throughput_mode"`
	ProvisionedWriteThroughput        int64 `yaml:"provisioned_write_throughput"`
	ProvisionedReadThroughput         int64 `yaml:"provisioned_read_throughput"`

	WriteScale AutoScalingConfig `yaml:"write_scale"`
	ReadScale  AutoScalingConfig `yaml:"read_scale"`
}

func (ActiveTableProvisionConfig) BuildTableDesc

func (cfg ActiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags) TableDesc

func (*ActiveTableProvisionConfig) RegisterFlags

func (cfg *ActiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet. Make sure defaults are set in the respective fields before calling RegisterFlags.

type AutoScalingConfig

type AutoScalingConfig struct {
	Enabled     bool    `yaml:"enabled"`
	RoleARN     string  `yaml:"role_arn"`
	MinCapacity int64   `yaml:"min_capacity"`
	MaxCapacity int64   `yaml:"max_capacity"`
	OutCooldown int64   `yaml:"out_cooldown"`
	InCooldown  int64   `yaml:"in_cooldown"`
	TargetValue float64 `yaml:"target"`
}

AutoScalingConfig for DynamoDB tables.

func (*AutoScalingConfig) RegisterFlags

func (cfg *AutoScalingConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type ChunkStoreConfig

type ChunkStoreConfig struct {
	ChunkCacheConfig       cache.Config `yaml:"chunk_cache_config"`
	ChunkCacheConfigL2     cache.Config `yaml:"chunk_cache_config_l2"`
	WriteDedupeCacheConfig cache.Config `` /* 274-byte string literal not displayed */

	L2ChunkCacheHandoff   time.Duration  `yaml:"l2_chunk_cache_handoff"`
	CacheLookupsOlderThan model.Duration `yaml:"cache_lookups_older_than"`

	// When DisableIndexDeduplication is true and chunk is already there in cache, only index would be written to the store and not chunk.
	DisableIndexDeduplication bool `yaml:"-"`
	// contains filtered or unexported fields
}

func (*ChunkStoreConfig) ChunkCacheStubs

func (cfg *ChunkStoreConfig) ChunkCacheStubs() bool

func (*ChunkStoreConfig) RegisterFlags

func (cfg *ChunkStoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*ChunkStoreConfig) Validate

func (cfg *ChunkStoreConfig) Validate() error

type DayTable

type DayTable struct {
	DayTime
	Prefix string
}

func NewDayTable

func NewDayTable(d DayTime, prefix string) DayTable

func (DayTable) Addr

func (d DayTable) Addr() string

Addr returns the prefix (if any) and the unix day offset as a string, which is used as the address for the index table in storage.

func (DayTable) String

func (d DayTable) String() string

type DayTime

type DayTime struct {
	model.Time
}

DayTime is a model.Time what holds day-aligned values, and marshals to/from YAML in YYYY-MM-DD format.

func NewDayTime

func NewDayTime(d model.Time) DayTime

func (DayTime) After

func (d DayTime) After(other DayTime) bool

func (DayTime) Before

func (d DayTime) Before(other DayTime) bool

func (DayTime) Bounds

func (d DayTime) Bounds() (model.Time, model.Time)

func (DayTime) Dec

func (d DayTime) Dec() DayTime

func (DayTime) Inc

func (d DayTime) Inc() DayTime

func (DayTime) MarshalYAML

func (d DayTime) MarshalYAML() (interface{}, error)

MarshalYAML implements yaml.Marshaller.

func (DayTime) ModelTime

func (d DayTime) ModelTime() model.Time

func (DayTime) String

func (d DayTime) String() string

func (*DayTime) UnmarshalYAML

func (d *DayTime) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaller.

type InactiveTableProvisionConfig

type InactiveTableProvisionConfig struct {
	InactiveThroughputOnDemandMode bool  `yaml:"enable_inactive_throughput_on_demand_mode"`
	InactiveWriteThroughput        int64 `yaml:"inactive_write_throughput"`
	InactiveReadThroughput         int64 `yaml:"inactive_read_throughput"`

	InactiveWriteScale AutoScalingConfig `yaml:"inactive_write_scale"`
	InactiveReadScale  AutoScalingConfig `yaml:"inactive_read_scale"`

	InactiveWriteScaleLastN int64 `yaml:"inactive_write_scale_lastn"`
	InactiveReadScaleLastN  int64 `yaml:"inactive_read_scale_lastn"`
}

func (InactiveTableProvisionConfig) BuildTableDesc

func (cfg InactiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags, disableAutoscale bool) TableDesc

func (*InactiveTableProvisionConfig) RegisterFlags

func (cfg *InactiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type IndexPeriodicTableConfig

type IndexPeriodicTableConfig struct {
	PathPrefix          string `` /* 168-byte string literal not displayed */
	PeriodicTableConfig `yaml:",inline"`
}

func (IndexPeriodicTableConfig) MarshalYAML

func (cfg IndexPeriodicTableConfig) MarshalYAML() (interface{}, error)

MarshalYAML implements the yaml.Marshaler interface.

func (*IndexPeriodicTableConfig) UnmarshalYAML

func (cfg *IndexPeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*IndexPeriodicTableConfig) Validate

func (cfg *IndexPeriodicTableConfig) Validate() error

type PeriodConfig

type PeriodConfig struct {
	// used when working with config
	From DayTime `` /* 261-byte string literal not displayed */
	// type of index client to use.
	IndexType string `` /* 267-byte string literal not displayed */
	// type of object client to use.
	ObjectType  string                   `` /* 308-byte string literal not displayed */
	Schema      string                   `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v13."`
	IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."`
	ChunkTables PeriodicTableConfig      `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."`
	RowShards   uint32                   `yaml:"row_shards" doc:"default=16|description=How many shards will be created. Only used if schema is v10 or greater."`
	// contains filtered or unexported fields
}

PeriodConfig defines the schema and tables to use for a period of time

func (*PeriodConfig) ChunkFormat

func (cfg *PeriodConfig) ChunkFormat() (byte, chunkenc.HeadBlockFmt, error)

ChunkFormat returns chunk format including it's headBlockFormat corresponding to the `schema` version in the given `PeriodConfig`.

func (*PeriodConfig) GetIndexTableNumberRange

func (cfg *PeriodConfig) GetIndexTableNumberRange(schemaEndDate DayTime) TableRange

GetIndexTableNumberRange returns the table number range calculated based on the configured schema start date, index table period and the given schemaEndDate

func (*PeriodConfig) TSDBFormat

func (cfg *PeriodConfig) TSDBFormat() (int, error)

TSDBFormat returns index format corresponding to the `schema` version in the given `PeriodConfig`.

func (*PeriodConfig) UnmarshalYAML

func (cfg *PeriodConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaller.

func (*PeriodConfig) VersionAsInt

func (cfg *PeriodConfig) VersionAsInt() (int, error)

type PeriodicTableConfig

type PeriodicTableConfig struct {
	Prefix string        `yaml:"prefix" doc:"description=Table prefix for all period tables."`
	Period time.Duration `yaml:"period" doc:"description=Table period."`
	Tags   Tags          `yaml:"tags" doc:"description=A map to be added to all managed tables."`
}

PeriodicTableConfig is configuration for a set of time-sharded tables.

func (PeriodicTableConfig) MarshalYAML

func (cfg PeriodicTableConfig) MarshalYAML() (interface{}, error)

MarshalYAML implements the yaml.Marshaler interface.

func (*PeriodicTableConfig) PeriodicTables

func (cfg *PeriodicTableConfig) PeriodicTables(from, through model.Time, pCfg ProvisionConfig, beginGrace, endGrace time.Duration, retention time.Duration) []TableDesc

func (*PeriodicTableConfig) TableFor

func (cfg *PeriodicTableConfig) TableFor(t model.Time) string

TableFor calculates the table shard for a given point in time.

func (*PeriodicTableConfig) UnmarshalYAML

func (cfg *PeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (PeriodicTableConfig) Validate

func (cfg PeriodicTableConfig) Validate() error

type ProvisionConfig

type ProvisionConfig struct {
	ActiveTableProvisionConfig   `yaml:",inline"`
	InactiveTableProvisionConfig `yaml:",inline"`
}

ProvisionConfig holds config for provisioning capacity for index and chunk tables (on DynamoDB for now)

func (*ProvisionConfig) RegisterFlags

func (cfg *ProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type SchemaConfig

type SchemaConfig struct {
	Configs []PeriodConfig `yaml:"configs"`
	// contains filtered or unexported fields
}

SchemaConfig contains the config for our chunk index schemas

func (SchemaConfig) ChunkTableFor

func (cfg SchemaConfig) ChunkTableFor(t model.Time) (string, error)

ChunkTableFor calculates the chunk table shard for a given point in time.

func (SchemaConfig) ExternalKey

func (cfg SchemaConfig) ExternalKey(ref logproto.ChunkRef) string

Generate the appropriate external key based on cfg.Schema, chunk.Checksum, and chunk.From

func (*SchemaConfig) ForEachAfter

func (cfg *SchemaConfig) ForEachAfter(t model.Time, f func(config *PeriodConfig))

ForEachAfter will call f() on every entry after t, splitting entries if necessary so there is an entry starting at t

func (*SchemaConfig) Load

func (cfg *SchemaConfig) Load() error

Load the yaml file, or build the config from legacy command-line flags

func (*SchemaConfig) RegisterFlags

func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

func (SchemaConfig) SchemaForTime

func (cfg SchemaConfig) SchemaForTime(t model.Time) (PeriodConfig, error)

SchemaForTime returns the Schema PeriodConfig to use for a given point in time.

func (*SchemaConfig) Validate

func (cfg *SchemaConfig) Validate() error

Validate the schema config and returns an error if the validation doesn't pass

func (SchemaConfig) VersionForChunk

func (cfg SchemaConfig) VersionForChunk(ref logproto.ChunkRef) int

VersionForChunk will return the schema version associated with the `From` timestamp of a chunk. The schema and chunk must be valid+compatible as the errors are not checked.

type TableDesc

type TableDesc struct {
	Name              string
	UseOnDemandIOMode bool
	ProvisionedRead   int64
	ProvisionedWrite  int64
	Tags              Tags
	WriteScale        AutoScalingConfig
	ReadScale         AutoScalingConfig
}

TableDesc describes a table.

func (TableDesc) Equals

func (desc TableDesc) Equals(other TableDesc) bool

Equals returns true if other matches desc.

type TableRange

type TableRange struct {
	Start, End   int64
	PeriodConfig *PeriodConfig
}

TableRange represents a range of table numbers built based on the configured schema start/end date and the table period. Both Start and End are inclusive.

func (TableRange) ConfigForTableNumber

func (t TableRange) ConfigForTableNumber(tableNumber int64) *PeriodConfig

func (TableRange) TableInRange

func (t TableRange) TableInRange(tableName string) (bool, error)

TableInRange tells whether given table falls in the range and the tableName has the right prefix based on the schema config.

type TableRanges

type TableRanges []TableRange

TableRanges represents a list of table ranges for multiple schemas.

func GetIndexStoreTableRanges

func GetIndexStoreTableRanges(indexType string, periodicConfigs []PeriodConfig) TableRanges

func (TableRanges) ConfigForTableNumber

func (t TableRanges) ConfigForTableNumber(tableNumber int64) *PeriodConfig

func (TableRanges) TableInRange

func (t TableRanges) TableInRange(tableName string) (bool, error)

TableInRange tells whether given table falls in any of the ranges and the tableName has the right prefix based on the schema config.

func (TableRanges) TableNameFor

func (t TableRanges) TableNameFor(table int64) (string, bool)

type Tags

type Tags map[string]string

Tags is a string-string map that implements flag.Value.

func (Tags) Equals

func (ts Tags) Equals(other Tags) bool

Equals returns true is other matches ts.

func (*Tags) Set

func (ts *Tags) Set(s string) error

Set implements flag.Value

func (Tags) String

func (ts Tags) String() string

String implements flag.Value

func (*Tags) UnmarshalYAML

func (ts *Tags) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaler.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL