logql

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 8, 2024 License: AGPL-3.0 Imports: 59 Imported by: 0

Documentation

Overview

Fork of https://raw.githubusercontent.com/cockroachdb/cockroach/065aa74206c9ec9bcd8b9ea2a6c62ddde8aab0a6/pkg/util/treeprinter/tree_printer.go

Index

Constants

View Source 
const (
	StreamsKey = "streams"
	MetricsKey = "metrics"
)

expression type used in metrics

View Source 
const (
	SuccessKey = "success"
	FailureKey = "failure"
	NoopKey    = "noop"
)

parsing evaluation result used in metrics

View Source 
const (
	QueryTypeMetric  = "metric"
	QueryTypeFilter  = "filter"
	QueryTypeLimited = "limited"
	QueryTypeLabels  = "labels"
	QueryTypeSeries  = "series"
	QueryTypeStats   = "stats"
	QueryTypeShards  = "shards"
	QueryTypeVolume  = "volume"
)
View Source 
const (
	DefaultBlockedQueryMessage = "blocked by policy"
)
View Source 
const MaxChildrenDisplay = 3

MaxChildrenDisplay defines the maximum number of children that should be shown by explain.

View Source 
const (
	QuantileSketchMatrixType = "QuantileSketchMatrix"
)
View Source 
const (
	ShardQuantileOverTime = "quantile_over_time"
)

Variables

View Source 
var (
	QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "logql",
		Name:      "query_duration_seconds",
		Help:      "LogQL query timings",
		Buckets:   prometheus.DefBuckets,
	}, []string{"query_type"})

	QueriesBlocked = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "blocked_queries",
		Help:      "Count of queries blocked by per-tenant policy",
	}, []string{"user"})
)
View Source 
var (
	NoLimits = &fakeLimits{maxSeries: math.MaxInt32}
)

Functions

func EvaluatorUnsupportedType 

func EvaluatorUnsupportedType(expr syntax.Expr, ev EvaluatorFactory) error

EvaluatorUnsupportedType is a helper for signaling that an evaluator does not support an Expr type

func MatchForSeriesRequest 

func MatchForSeriesRequest(xs []string) ([][]*labels.Matcher, error)

MatchForSeriesRequest extracts and parses multiple matcher groups from a slice of strings. Does not perform validation as it's used for series queries which allow empty matchers

func MergeQuantileSketchVector 

func MergeQuantileSketchVector(next bool, r StepResult, stepEvaluator StepEvaluator, params Params) (promql_parser.Value, error)

MergeQuantileSketchVector joins the results from stepEvaluator into a ProbabilisticQuantileMatrix.

func ParseShard 

func ParseShard(s string) (Shard, ShardVersion, error)

func ParseShards 

func ParseShards(strs []string) (Shards, ShardVersion, error)

ParseShards parses a list of string encoded shards

func PopulateMatrixFromScalar 

func PopulateMatrixFromScalar(data promql.Scalar, params Params) promql.Matrix

func PrintMatches 

func PrintMatches(matches []string) string

func Quantile 

func Quantile(q float64, values vector.HeapByMaxValue) float64

Quantile calculates the given Quantile of a vector of samples.

The Vector will be sorted. If 'values' has zero elements, NaN is returned. If q<0, -Inf is returned. If q>1, +Inf is returned.

func QueryType 

func QueryType(expr syntax.Expr) (string, error)

func RecordDetectedFieldsQueryMetrics 

func RecordDetectedFieldsQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, stats logql_stats.Result)

func RecordDetectedLabelsQueryMetrics 

func RecordDetectedLabelsQueryMetrics(_ context.Context, _ log.Logger, _ time.Time, _ time.Time, _ string, _ string, _ logql_stats.Result)

func RecordLabelQueryMetrics 

func RecordLabelQueryMetrics(
	ctx context.Context,
	log log.Logger,
	start, end time.Time,
	label, query, status string,
	stats logql_stats.Result,
)

func RecordRangeAndInstantQueryMetrics 

func RecordRangeAndInstantQueryMetrics(
	ctx context.Context,
	log log.Logger,
	p Params,
	status string,
	stats logql_stats.Result,
	result promql_parser.Value,
)

func RecordSeriesQueryMetrics 

func RecordSeriesQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, match []string, status string, shards []string, stats logql_stats.Result)

func RecordShardsQueryMetrics 

func RecordShardsQueryMetrics(
	ctx context.Context,
	log log.Logger,
	start,
	end time.Time,
	query string,
	targetBytesPerShard uint64,
	status string,
	shards int,
	stats logql_stats.Result,
)

func RecordStatsQueryMetrics 

func RecordStatsQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, stats logql_stats.Result)

func RecordVolumeQueryMetrics 

func RecordVolumeQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, limit uint32, step time.Duration, status string, stats logql_stats.Result)

func ResultIterator 

func ResultIterator(res logqlmodel.Result, direction logproto.Direction) (iter.EntryIterator, error)

ResultIterator coerces a downstream streams result into an iter.EntryIterator

func Sortable 

func Sortable(q Params) (bool, error)

Sortable logql contain sort or sort_desc.

Types

type AbsentRangeVectorEvaluator 

type AbsentRangeVectorEvaluator struct {
	// contains filtered or unexported fields
}

func (AbsentRangeVectorEvaluator) Close 

func (r AbsentRangeVectorEvaluator) Close() error

func (AbsentRangeVectorEvaluator) Error 

func (r AbsentRangeVectorEvaluator) Error() error

func (*AbsentRangeVectorEvaluator) Explain 

func (e *AbsentRangeVectorEvaluator) Explain(parent Node)

func (*AbsentRangeVectorEvaluator) Next 

func (r *AbsentRangeVectorEvaluator) Next() (bool, int64, StepResult)

type AccumulatedStreams 

type AccumulatedStreams struct {
	// contains filtered or unexported fields
}

heap impl for keeping only the top n results across m streams importantly, AccumulatedStreams is _bounded_, so it will only store the top `limit` results across all streams. To implement this, we use a min-heap when looking for the max values (logproto.FORWARD) and vice versa for logproto.BACKWARD. This allows us to easily find the 'worst' value and replace it with a better one. Once we've fully processed all log lines, we return the heap in opposite order and then reverse it to get the correct order. Heap implements container/heap.Interface solely to use heap.Interface as a library. It is not intended for the heap pkg functions to otherwise call this type.

func NewStreamAccumulator 

func NewStreamAccumulator(params Params) *AccumulatedStreams

NewStreamAccumulator returns an accumulator for limited log queries. Log queries, sharded thousands of times and each returning <limit> results, can be _considerably_ larger. In this case, we eagerly accumulate the results into a logsAccumulator, discarding values over the limit to keep memory pressure down while other subqueries are executing.

func (*AccumulatedStreams) Accumulate 

func (acc *AccumulatedStreams) Accumulate(_ context.Context, x logqlmodel.Result, _ int) error

func (*AccumulatedStreams) Find 

func (acc *AccumulatedStreams) Find(labels string) (int, bool)

func (*AccumulatedStreams) Len 

func (acc *AccumulatedStreams) Len() int

number of streams

func (*AccumulatedStreams) Less 

func (acc *AccumulatedStreams) Less(i, j int) bool

first order by timestamp, then by labels

func (*AccumulatedStreams) Pop 

func (acc *AccumulatedStreams) Pop() any

Pop returns a stream with one entry. It pops the first entry of the first stream

func (*AccumulatedStreams) Push 

func (acc *AccumulatedStreams) Push(x any)

func (*AccumulatedStreams) Result 

func (acc *AccumulatedStreams) Result() []logqlmodel.Result

Note: can only be called once as it will alter stream ordreing.

func (*AccumulatedStreams) Swap 

func (acc *AccumulatedStreams) Swap(i, j int)

type Accumulator 

type Accumulator interface {
	Accumulate(context.Context, logqlmodel.Result, int) error
	Result() []logqlmodel.Result
}

Accumulator is an interface for accumulating query results.

type AvgOverTime 

type AvgOverTime struct {
	// contains filtered or unexported fields
}

type BatchRangeVectorAggregator 

type BatchRangeVectorAggregator func([]promql.FPoint) float64

BatchRangeVectorAggregator aggregates samples for a given range of samples. It receives the current milliseconds timestamp and the list of point within the range.

type BinOpStepEvaluator 

type BinOpStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*BinOpStepEvaluator) Close 

func (e *BinOpStepEvaluator) Close() (lastError error)

func (*BinOpStepEvaluator) Error 

func (e *BinOpStepEvaluator) Error() error

func (*BinOpStepEvaluator) Explain 

func (e *BinOpStepEvaluator) Explain(parent Node)

func (*BinOpStepEvaluator) Next 

func (e *BinOpStepEvaluator) Next() (bool, int64, StepResult)

type BufferedAccumulator 

type BufferedAccumulator struct {
	// contains filtered or unexported fields
}

func NewBufferedAccumulator 

func NewBufferedAccumulator(n int) *BufferedAccumulator

NewBufferedAccumulator returns an accumulator which aggregates all query results in a slice. This is useful for metric queries, which are generally small payloads and the memory overhead for buffering is negligible.

func (*BufferedAccumulator) Accumulate 

func (a *BufferedAccumulator) Accumulate(_ context.Context, acc logqlmodel.Result, i int) error

func (*BufferedAccumulator) Result 

func (a *BufferedAccumulator) Result() []logqlmodel.Result

type ConcatLogSelectorExpr 

type ConcatLogSelectorExpr struct {
	DownstreamLogSelectorExpr
	// contains filtered or unexported fields
}

ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr

func (*ConcatLogSelectorExpr) Pretty 

func (c *ConcatLogSelectorExpr) Pretty(level int) string

ConcatLogSelectorExpr has no representation in LogQL. Its prettified version is e.g. `concat(downstream<{foo="bar"} |= "error", shard=1_of_3>)`

func (*ConcatLogSelectorExpr) String 

func (c *ConcatLogSelectorExpr) String() string

type ConcatSampleExpr 

type ConcatSampleExpr struct {
	DownstreamSampleExpr
	// contains filtered or unexported fields
}

ConcatSampleExpr is an expr for concatenating multiple SampleExpr Contract: The embedded SampleExprs within a linked list of ConcatSampleExprs must be of the same structure. This makes special implementations of SampleExpr.Associative() unnecessary.

func (*ConcatSampleExpr) Pretty 

func (c *ConcatSampleExpr) Pretty(level int) string

ConcatSampleExpr has no LogQL repretenstation. It is expressed in in the prettified version as e.g. `concat(downstream<count_over_time({foo="bar"}), shard=...> ++ )`

func (*ConcatSampleExpr) String 

func (c *ConcatSampleExpr) String() string

func (*ConcatSampleExpr) Walk 

func (c *ConcatSampleExpr) Walk(f syntax.WalkFn)

type ConcatStepEvaluator 

type ConcatStepEvaluator struct {
	// contains filtered or unexported fields
}

func NewConcatStepEvaluator 

func NewConcatStepEvaluator(evaluators []StepEvaluator) *ConcatStepEvaluator

NewConcatStepEvaluator joins multiple StepEvaluators. Contract: They must be of identical start, end, and step values.

func (*ConcatStepEvaluator) Close 

func (e *ConcatStepEvaluator) Close() (lastErr error)

func (*ConcatStepEvaluator) Error 

func (e *ConcatStepEvaluator) Error() error

func (*ConcatStepEvaluator) Explain 

func (e *ConcatStepEvaluator) Explain(parent Node)

func (*ConcatStepEvaluator) Next 

func (e *ConcatStepEvaluator) Next() (bool, int64, StepResult)

type ConstantShards 

type ConstantShards int

func (ConstantShards) GetStats 

func (s ConstantShards) GetStats(_ syntax.Expr) (stats.Stats, error)

func (ConstantShards) ShardingRanges 

func (s ConstantShards) ShardingRanges(_ syntax.Expr, _ uint64) ([]logproto.Shard, error)

func (ConstantShards) Shards 

func (s ConstantShards) Shards(_ syntax.Expr) (int, uint64, error)

type CountOverTime 

type CountOverTime struct {
	// contains filtered or unexported fields
}

type DefaultEvaluator 

type DefaultEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultEvaluator 

func NewDefaultEvaluator(querier Querier, maxLookBackPeriod time.Duration) *DefaultEvaluator

NewDefaultEvaluator constructs a DefaultEvaluator

func (*DefaultEvaluator) NewIterator 

func (ev *DefaultEvaluator) NewIterator(ctx context.Context, expr syntax.LogSelectorExpr, q Params) (iter.EntryIterator, error)

func (*DefaultEvaluator) NewStepEvaluator 

func (ev *DefaultEvaluator) NewStepEvaluator(
	ctx context.Context,
	nextEvFactory SampleEvaluatorFactory,
	expr syntax.SampleExpr,
	q Params,
) (StepEvaluator, error)

type DownstreamEngine 

type DownstreamEngine struct {
	// contains filtered or unexported fields
}

DownstreamEngine is an Engine implementation that can split queries into more parallelizable forms via querying the underlying backend shards individually and re-aggregating them.

func NewDownstreamEngine 

func NewDownstreamEngine(opts EngineOpts, downstreamable Downstreamable, limits Limits, logger log.Logger) *DownstreamEngine

NewDownstreamEngine constructs a *DownstreamEngine

func (*DownstreamEngine) Opts 

func (ng *DownstreamEngine) Opts() EngineOpts

func (*DownstreamEngine) Query 

func (ng *DownstreamEngine) Query(ctx context.Context, p Params) Query

Query constructs a Query

type DownstreamEvaluator 

type DownstreamEvaluator struct {
	Downstreamer
	// contains filtered or unexported fields
}

DownstreamEvaluator is an evaluator which handles shard aware AST nodes

func NewDownstreamEvaluator 

func NewDownstreamEvaluator(downstreamer Downstreamer) *DownstreamEvaluator

func (DownstreamEvaluator) Downstream 

func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery, acc Accumulator) ([]logqlmodel.Result, error)

Downstream runs queries and collects stats from the embedded Downstreamer

func (*DownstreamEvaluator) NewIterator 

func (ev *DownstreamEvaluator) NewIterator(
	ctx context.Context,
	expr syntax.LogSelectorExpr,
	params Params,
) (iter.EntryIterator, error)

NewIterator returns the iter.EntryIterator for a given LogSelectorExpr

func (*DownstreamEvaluator) NewStepEvaluator 

func (ev *DownstreamEvaluator) NewStepEvaluator(
	ctx context.Context,
	nextEvFactory SampleEvaluatorFactory,
	expr syntax.SampleExpr,
	params Params,
) (StepEvaluator, error)

NewStepEvaluator returns a NewStepEvaluator for a given SampleExpr

type DownstreamLogSelectorExpr 

type DownstreamLogSelectorExpr struct {
	syntax.LogSelectorExpr
	// contains filtered or unexported fields
}

DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation

func (DownstreamLogSelectorExpr) Pretty 

func (d DownstreamLogSelectorExpr) Pretty(level int) string

The DownstreamLogSelectorExpr is not part of LogQL. In the prettified version it's represented as e.g. `downstream<{foo="bar"} |= "error", shard=1_of_3>`

func (DownstreamLogSelectorExpr) String 

func (d DownstreamLogSelectorExpr) String() string

type DownstreamQuery 

type DownstreamQuery struct {
	Params Params
}

type DownstreamSampleExpr 

type DownstreamSampleExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

DownstreamSampleExpr is a SampleExpr which signals downstream computation

func (DownstreamSampleExpr) Pretty 

func (d DownstreamSampleExpr) Pretty(level int) string

The DownstreamSampleExpr is not part of LogQL. In the prettified version it's represented as e.g. `downstream<count_over_time({foo="bar"} |= "error"), shard=1_of_3>`

func (DownstreamSampleExpr) String 

func (d DownstreamSampleExpr) String() string

func (DownstreamSampleExpr) Walk 

func (d DownstreamSampleExpr) Walk(f syntax.WalkFn)

type Downstreamable 

type Downstreamable interface {
	Downstreamer(context.Context) Downstreamer
}

type Downstreamer 

type Downstreamer interface {
	Downstream(context.Context, []DownstreamQuery, Accumulator) ([]logqlmodel.Result, error)
}

Downstreamer is an interface for deferring responsibility for query execution. It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs.

type DynamicBoundsStrategy 

type DynamicBoundsStrategy struct {
	// contains filtered or unexported fields
}

func NewDynamicBoundsStrategy 

func NewDynamicBoundsStrategy(resolver ShardResolver, targetBytesPerShard uint64) DynamicBoundsStrategy

func (DynamicBoundsStrategy) Resolver 

func (s DynamicBoundsStrategy) Resolver() ShardResolver

func (DynamicBoundsStrategy) Shards 

func (s DynamicBoundsStrategy) Shards(expr syntax.Expr) (Shards, uint64, error)

type Engine 

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the LogQL engine.

func NewEngine 

func NewEngine(opts EngineOpts, q Querier, l Limits, logger log.Logger) *Engine

NewEngine creates a new LogQL Engine.

func (*Engine) Query 

func (ng *Engine) Query(params Params) Query

Query creates a new LogQL query. Instant/Range type is derived from the parameters.

type EngineOpts 

type EngineOpts struct {
	// MaxLookBackPeriod is the maximum amount of time to look back for log lines.
	// only used for instant log queries.
	MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"`

	// LogExecutingQuery will control if we log the query when Exec is called.
	LogExecutingQuery bool `yaml:"-"`
}

EngineOpts is the list of options to use with the LogQL query engine.

func (*EngineOpts) RegisterFlagsWithPrefix 

func (opts *EngineOpts) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type EntryEvaluatorFactory 

type EntryEvaluatorFactory interface {
	// NewIterator returns the iter.EntryIterator for a given LogSelectorExpr
	NewIterator(context.Context, syntax.LogSelectorExpr, Params) (iter.EntryIterator, error)
}

type EvaluatorFactory 

type EvaluatorFactory interface {
	SampleEvaluatorFactory
	EntryEvaluatorFactory
}

EvaluatorFactory is an interface for iterating over data at different nodes in the AST

type FirstOverTime 

type FirstOverTime struct {
	// contains filtered or unexported fields
}

type LabelReplaceEvaluator 

type LabelReplaceEvaluator struct {
	// contains filtered or unexported fields
}

func (*LabelReplaceEvaluator) Close 

func (e *LabelReplaceEvaluator) Close() error

func (*LabelReplaceEvaluator) Error 

func (e *LabelReplaceEvaluator) Error() error

func (*LabelReplaceEvaluator) Explain 

func (e *LabelReplaceEvaluator) Explain(parent Node)

func (*LabelReplaceEvaluator) Next 

func (e *LabelReplaceEvaluator) Next() (bool, int64, StepResult)

type LastOverTime 

type LastOverTime struct {
	// contains filtered or unexported fields
}

type Limits 

type Limits interface {
	MaxQuerySeries(context.Context, string) int
	MaxQueryRange(ctx context.Context, userID string) time.Duration
	QueryTimeout(context.Context, string) time.Duration
	BlockedQueries(context.Context, string) []*validation.BlockedQuery
}

Limits allow the engine to fetch limits for a given users.

type LiteralParams 

type LiteralParams struct {
	// contains filtered or unexported fields
}

LiteralParams impls Params

func NewLiteralParams 

func NewLiteralParams(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
) (LiteralParams, error)

func (LiteralParams) Copy 

func (p LiteralParams) Copy() LiteralParams

func (LiteralParams) Direction 

func (p LiteralParams) Direction() logproto.Direction

Direction impls Params

func (LiteralParams) End 

func (p LiteralParams) End() time.Time

End impls Params

func (LiteralParams) GetExpression 

func (p LiteralParams) GetExpression() syntax.Expr

GetExpression impls Params

func (LiteralParams) Interval 

func (p LiteralParams) Interval() time.Duration

Interval impls Params

func (LiteralParams) Limit 

func (p LiteralParams) Limit() uint32

Limit impls Params

func (LiteralParams) QueryString 

func (p LiteralParams) QueryString() string

String impls Params

func (LiteralParams) Shards 

func (p LiteralParams) Shards() []string

Shards impls Params

func (LiteralParams) Start 

func (p LiteralParams) Start() time.Time

Start impls Params

func (LiteralParams) Step 

func (p LiteralParams) Step() time.Duration

Step impls Params

type LiteralStepEvaluator 

type LiteralStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*LiteralStepEvaluator) Close 

func (e *LiteralStepEvaluator) Close() error

func (*LiteralStepEvaluator) Error 

func (e *LiteralStepEvaluator) Error() error

func (*LiteralStepEvaluator) Explain 

func (e *LiteralStepEvaluator) Explain(parent Node)

func (*LiteralStepEvaluator) Next 

func (e *LiteralStepEvaluator) Next() (bool, int64, StepResult)

type MapperMetrics 

type MapperMetrics struct {
	DownstreamQueries *prometheus.CounterVec // downstream queries total, partitioned by streams/metrics
	ParsedQueries     *prometheus.CounterVec // parsed ASTs total, partitioned by success/failure/noop
	DownstreamFactor  prometheus.Histogram   // per request downstream factor
}

MapperMetrics is the metrics wrapper used in logql mapping (shard and range)

func NewRangeMapperMetrics 

func NewRangeMapperMetrics(registerer prometheus.Registerer) *MapperMetrics

func NewShardMapperMetrics 

func NewShardMapperMetrics(registerer prometheus.Registerer) *MapperMetrics

type MapperStats 

type MapperStats struct {
	// contains filtered or unexported fields
}

func NewMapperStats 

func NewMapperStats() *MapperStats

func (*MapperStats) AddSplitQueries 

func (s *MapperStats) AddSplitQueries(num int)

AddSplitQueries add num split queries to the counter

func (*MapperStats) GetSplitQueries 

func (s *MapperStats) GetSplitQueries() int

GetSplitQueries returns the number of split queries

type MatrixStepEvaluator 

type MatrixStepEvaluator struct {
	// contains filtered or unexported fields
}

MatrixStepEvaluator exposes a promql.Matrix as a StepEvaluator. Ensure that the resulting StepEvaluator maintains the same shape that the parameters expect. For example, it's possible that a downstream query returns matches no log streams and thus returns an empty matrix. However, we still need to ensure that it can be merged effectively with another leg that may match series. Therefore, we determine our steps from the parameters and not the underlying Matrix.

func NewMatrixStepEvaluator 

func NewMatrixStepEvaluator(start, end time.Time, step time.Duration, m promql.Matrix) *MatrixStepEvaluator

func (*MatrixStepEvaluator) Close 

func (m *MatrixStepEvaluator) Close() error

func (*MatrixStepEvaluator) Error 

func (m *MatrixStepEvaluator) Error() error

func (*MatrixStepEvaluator) Explain 

func (m *MatrixStepEvaluator) Explain(parent Node)

func (*MatrixStepEvaluator) Next 

func (m *MatrixStepEvaluator) Next() (bool, int64, StepResult)

type MaxOverTime 

type MaxOverTime struct {
	// contains filtered or unexported fields
}

type MinOverTime 

type MinOverTime struct {
	// contains filtered or unexported fields
}

type MockDownstreamer 

type MockDownstreamer struct {
	*Engine
}

func (MockDownstreamer) Downstream 

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery, _ Accumulator) ([]logqlmodel.Result, error)

func (MockDownstreamer) Downstreamer 

func (m MockDownstreamer) Downstreamer(_ context.Context) Downstreamer

type MockQuerier 

type MockQuerier struct {
	// contains filtered or unexported fields
}

Shard aware mock querier

func NewMockQuerier 

func NewMockQuerier(shards int, streams []logproto.Stream) MockQuerier

func (MockQuerier) SelectLogs 

func (q MockQuerier) SelectLogs(_ context.Context, req SelectLogParams) (iter.EntryIterator, error)

func (MockQuerier) SelectSamples 

func (q MockQuerier) SelectSamples(_ context.Context, req SelectSampleParams) (iter.SampleIterator, error)

type Node 

type Node struct {
	// contains filtered or unexported fields
}

Node is a handle associated with a specific depth in a tree. See below for sample usage.

func NewTree 

func NewTree() Node

NewTree creates a tree printer and returns a sentinel node reference which should be used to add the root. Sample usage: 

tp := NewTree()
root := n.Child("root")
root.Child("child-1")
root.Child("child-2").Child("grandchild\ngrandchild-more-info")
root.Child("child-3")

fmt.Print(tp.String())

Output: 

root
 ├── child-1
 ├── child-2
 │    └── grandchild
 │        grandchild-more-info
 └── child-3

Note that the Child calls can't be rearranged arbitrarily; they have to be in the order they need to be displayed (depth-first pre-order).

func (Node) AddEmptyLine 

func (n Node) AddEmptyLine()

AddEmptyLine adds an empty line to the output; used to introduce vertical spacing as needed.

func (Node) AddLine 

func (n Node) AddLine(v string)

AddLine adds a new line to a child node without an edge.

func (Node) Child 

func (n Node) Child(text string) Node

Child adds a node as a child of the given node. Multi-line strings are supported with appropriate indentation.

func (Node) Childf 

func (n Node) Childf(format string, args ...interface{}) Node

Childf adds a node as a child of the given node.

func (Node) FormattedRows 

func (n Node) FormattedRows() []string

FormattedRows returns the formatted rows. Can only be called on the result of treeprinter.New.

func (Node) String 

func (n Node) String() string

type OneOverTime 

type OneOverTime struct {
}

type Params 

type Params interface {
	QueryString() string
	Start() time.Time
	End() time.Time
	Step() time.Duration
	Interval() time.Duration
	Limit() uint32
	Direction() logproto.Direction
	Shards() []string
	GetExpression() syntax.Expr
}

Params details the parameters associated with a loki request

type ParamsWithExpressionOverride 

type ParamsWithExpressionOverride struct {
	Params
	ExpressionOverride syntax.Expr
}

ParamsWithExpressionOverride overrides the query expression so that the query string and the expression can differ. This is useful for for query planning when plan my not match externally available logql syntax

func (ParamsWithExpressionOverride) GetExpression 

func (p ParamsWithExpressionOverride) GetExpression() syntax.Expr

GetExpression returns the parsed expression of the query.

type ParamsWithShardsOverride 

type ParamsWithShardsOverride struct {
	Params
	ShardsOverride []string
}

ParamsWithExpressionOverride overrides the shards. Since the backing implementation of the Params interface is unknown they are embedded and the original shards are shadowed.

func (ParamsWithShardsOverride) Shards 

func (p ParamsWithShardsOverride) Shards() []string

Shards returns this overwriting shards.

type PowerOfTwoStrategy 

type PowerOfTwoStrategy struct {
	// contains filtered or unexported fields
}

func NewPowerOfTwoStrategy 

func NewPowerOfTwoStrategy(resolver ShardResolver) PowerOfTwoStrategy

func (PowerOfTwoStrategy) Resolver 

func (s PowerOfTwoStrategy) Resolver() ShardResolver

func (PowerOfTwoStrategy) Shards 

func (s PowerOfTwoStrategy) Shards(expr syntax.Expr) (Shards, uint64, error)

type ProbabilisticQuantileMatrix 

type ProbabilisticQuantileMatrix []ProbabilisticQuantileVector

func ProbabilisticQuantileMatrixFromProto 

func ProbabilisticQuantileMatrixFromProto(proto *logproto.QuantileSketchMatrix) (ProbabilisticQuantileMatrix, error)

func (ProbabilisticQuantileMatrix) Merge 

func (m ProbabilisticQuantileMatrix) Merge(right ProbabilisticQuantileMatrix) (ProbabilisticQuantileMatrix, error)

func (ProbabilisticQuantileMatrix) Release 

func (m ProbabilisticQuantileMatrix) Release()

func (ProbabilisticQuantileMatrix) String 

func (ProbabilisticQuantileMatrix) String() string

func (ProbabilisticQuantileMatrix) ToProto 

func (m ProbabilisticQuantileMatrix) ToProto() *logproto.QuantileSketchMatrix

func (ProbabilisticQuantileMatrix) Type 

func (ProbabilisticQuantileMatrix) Type() promql_parser.ValueType

type ProbabilisticQuantileSample 

type ProbabilisticQuantileSample struct {
	T int64
	F sketch.QuantileSketch

	Metric labels.Labels
}

func (ProbabilisticQuantileSample) ToProto 

func (q ProbabilisticQuantileSample) ToProto() *logproto.QuantileSketchSample

type ProbabilisticQuantileVector 

type ProbabilisticQuantileVector []ProbabilisticQuantileSample

func ProbabilisticQuantileVectorFromProto 

func ProbabilisticQuantileVectorFromProto(proto *logproto.QuantileSketchVector) (ProbabilisticQuantileVector, error)

func (ProbabilisticQuantileVector) Merge 

func (q ProbabilisticQuantileVector) Merge(right ProbabilisticQuantileVector) (ProbabilisticQuantileVector, error)

func (ProbabilisticQuantileVector) QuantileSketchVec 

func (q ProbabilisticQuantileVector) QuantileSketchVec() ProbabilisticQuantileVector

func (ProbabilisticQuantileVector) Release 

func (q ProbabilisticQuantileVector) Release()

func (ProbabilisticQuantileVector) SampleVector 

func (ProbabilisticQuantileVector) SampleVector() promql.Vector

func (ProbabilisticQuantileVector) ToProto 

func (q ProbabilisticQuantileVector) ToProto() *logproto.QuantileSketchVector

type QuantileOverTime 

type QuantileOverTime struct {
	// contains filtered or unexported fields
}

type QuantileSketchAccumulator 

type QuantileSketchAccumulator struct {
	// contains filtered or unexported fields
}

func (*QuantileSketchAccumulator) Accumulate 

func (a *QuantileSketchAccumulator) Accumulate(_ context.Context, res logqlmodel.Result, _ int) error

func (*QuantileSketchAccumulator) Result 

func (a *QuantileSketchAccumulator) Result() []logqlmodel.Result

type QuantileSketchEvalExpr 

type QuantileSketchEvalExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

QuantileSketchEvalExpr evaluates a quantile sketch to the actual quantile.

func (QuantileSketchEvalExpr) String 

func (e QuantileSketchEvalExpr) String() string

func (*QuantileSketchEvalExpr) Walk 

func (e *QuantileSketchEvalExpr) Walk(f syntax.WalkFn)

type QuantileSketchMatrixStepEvaluator 

type QuantileSketchMatrixStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchMatrixStepEvaluator steps through a matrix of quantile sketch vectors, ie t-digest or DDSketch structures per time step.

func NewQuantileSketchMatrixStepEvaluator 

func NewQuantileSketchMatrixStepEvaluator(m ProbabilisticQuantileMatrix, params Params) *QuantileSketchMatrixStepEvaluator

func (*QuantileSketchMatrixStepEvaluator) Close 

func (*QuantileSketchMatrixStepEvaluator) Close() error

func (*QuantileSketchMatrixStepEvaluator) Error 

func (*QuantileSketchMatrixStepEvaluator) Error() error

func (*QuantileSketchMatrixStepEvaluator) Explain 

func (*QuantileSketchMatrixStepEvaluator) Explain(parent Node)

func (*QuantileSketchMatrixStepEvaluator) Next 

func (m *QuantileSketchMatrixStepEvaluator) Next() (bool, int64, StepResult)

type QuantileSketchMergeExpr 

type QuantileSketchMergeExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

func (QuantileSketchMergeExpr) String 

func (e QuantileSketchMergeExpr) String() string

func (*QuantileSketchMergeExpr) Walk 

func (e *QuantileSketchMergeExpr) Walk(f syntax.WalkFn)

type QuantileSketchMergeStepEvaluator 

type QuantileSketchMergeStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchMergeStepEvaluator merges multiple quantile sketches into one for each step.

func NewQuantileSketchMergeStepEvaluator 

func NewQuantileSketchMergeStepEvaluator(evaluators []StepEvaluator) *QuantileSketchMergeStepEvaluator

func (*QuantileSketchMergeStepEvaluator) Close 

func (*QuantileSketchMergeStepEvaluator) Close() error

func (*QuantileSketchMergeStepEvaluator) Error 

func (e *QuantileSketchMergeStepEvaluator) Error() error

func (*QuantileSketchMergeStepEvaluator) Explain 

func (e *QuantileSketchMergeStepEvaluator) Explain(parent Node)

func (*QuantileSketchMergeStepEvaluator) Next 

func (e *QuantileSketchMergeStepEvaluator) Next() (bool, int64, StepResult)

type QuantileSketchStepEvaluator 

type QuantileSketchStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*QuantileSketchStepEvaluator) Close 

func (e *QuantileSketchStepEvaluator) Close() error

func (*QuantileSketchStepEvaluator) Error 

func (e *QuantileSketchStepEvaluator) Error() error

func (*QuantileSketchStepEvaluator) Explain 

func (e *QuantileSketchStepEvaluator) Explain(parent Node)

func (*QuantileSketchStepEvaluator) Next 

func (e *QuantileSketchStepEvaluator) Next() (bool, int64, StepResult)

type QuantileSketchVectorStepEvaluator 

type QuantileSketchVectorStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchVectorStepEvaluator evaluates a quantile sketch into a promql.Vector.

func NewQuantileSketchVectorStepEvaluator 

func NewQuantileSketchVectorStepEvaluator(inner StepEvaluator, quantile float64) *QuantileSketchVectorStepEvaluator

func (*QuantileSketchVectorStepEvaluator) Close 

func (*QuantileSketchVectorStepEvaluator) Close() error

func (*QuantileSketchVectorStepEvaluator) Error 

func (*QuantileSketchVectorStepEvaluator) Error() error

func (*QuantileSketchVectorStepEvaluator) Explain 

func (e *QuantileSketchVectorStepEvaluator) Explain(parent Node)

func (*QuantileSketchVectorStepEvaluator) Next 

func (e *QuantileSketchVectorStepEvaluator) Next() (bool, int64, StepResult)

type Querier 

type Querier interface {
	SelectLogs(context.Context, SelectLogParams) (iter.EntryIterator, error)
	SelectSamples(context.Context, SelectSampleParams) (iter.SampleIterator, error)
}

Querier allows a LogQL expression to fetch an EntryIterator for a set of matchers and filters

type Query 

type Query interface {
	// Exec processes the query.
	Exec(ctx context.Context) (logqlmodel.Result, error)
}

Query is a LogQL query to be executed.

type QueryParams 

type QueryParams interface {
	LogSelector() (syntax.LogSelectorExpr, error)
	GetStart() time.Time
	GetEnd() time.Time
	GetShards() []string
}

type QueryRangeType 

type QueryRangeType string
var (
	InstantType QueryRangeType = "instant"
	RangeType   QueryRangeType = "range"
)

func GetRangeType 

func GetRangeType(q Params) QueryRangeType

GetRangeType returns whether a query is an instant query or range query

type RangeMapper 

type RangeMapper struct {
	// contains filtered or unexported fields
}

RangeMapper is used to rewrite LogQL sample expressions into multiple downstream sample expressions with a smaller time range that can be executed using the downstream engine.

A rewrite is performed using the following rules:

  1. Check if query is splittable based on the range.
  2. Check if the query is splittable based on the query AST
  3. Range aggregations are split into multiple downstream range aggregation expressions that are concatenated with an appropriate vector aggregator with a grouping operator. If the range aggregation has a grouping, the grouping is also applied to the resultant vector aggregator expression. If the range aggregation has no grouping, a grouping operator using "without" is applied to the resultant vector aggregator expression to preserve the stream labels.
  4. Vector aggregations are split into multiple downstream vector aggregations that are merged with vector aggregation using "without" and then aggregated using the vector aggregation with the same operator, either with or without grouping.
  5. Left and right-hand side of binary operations are split individually using the same rules as above.

func NewRangeMapper 

func NewRangeMapper(interval time.Duration, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error)

NewRangeMapper creates a new RangeMapper instance with the given duration as split interval. The interval must be greater than 0.

func NewRangeMapperWithSplitAlign 

func NewRangeMapperWithSplitAlign(interval time.Duration, splitAlign time.Time, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error)

NewRangeMapperWithSplitAlign is similar to `NewRangeMapper` except it accepts additional `splitAlign` argument and used to align the subqueries generated according to that. Look at `rangeSplitAlign` method for more information.

func (RangeMapper) Map 

func (m RangeMapper) Map(expr syntax.SampleExpr, vectorAggrPushdown *syntax.VectorAggregationExpr, recorder *downstreamRecorder) (syntax.SampleExpr, error)

Map rewrites sample expression expr and returns the resultant sample expression to be executed by the downstream engine It is called recursively on the expression tree. The function takes an optional vector aggregation as second argument, that is pushed down to the downstream expression.

func (RangeMapper) Parse 

func (m RangeMapper) Parse(expr syntax.Expr) (bool, syntax.Expr, error)

Parse parses the given LogQL query string into a sample expression and applies the rewrite rules for splitting it into a sample expression that can be executed by the downstream engine. It returns a boolean indicating whether a rewrite was possible, the rewritten sample expression, and an error in case the rewrite failed.

type RangeStreamingAgg 

type RangeStreamingAgg interface {
	// contains filtered or unexported methods
}

RangeStreamingAgg streaming aggregates sample for each sample

type RangeVectorEvaluator 

type RangeVectorEvaluator struct {
	// contains filtered or unexported fields
}

func (*RangeVectorEvaluator) Close 

func (r *RangeVectorEvaluator) Close() error

func (*RangeVectorEvaluator) Error 

func (r *RangeVectorEvaluator) Error() error

func (*RangeVectorEvaluator) Explain 

func (r *RangeVectorEvaluator) Explain(parent Node)

func (*RangeVectorEvaluator) Next 

func (r *RangeVectorEvaluator) Next() (bool, int64, StepResult)

type RangeVectorIterator 

type RangeVectorIterator interface {
	Next() bool
	At() (int64, StepResult)
	Close() error
	Error() error
}

RangeVectorIterator iterates through a range of samples. To fetch the current vector use `At` with a `BatchRangeVectorAggregator` or `RangeStreamingAgg`.

type RateCounterOverTime 

type RateCounterOverTime struct {
	// contains filtered or unexported fields
}

rateCounter calculates the per-second rate of values extracted from log lines and treat them like a "counter" metric.

type RateLogBytesOverTime 

type RateLogBytesOverTime struct {
	// contains filtered or unexported fields
}

rateLogBytes calculates the per-second rate of log bytes.

type RateLogsOverTime 

type RateLogsOverTime struct {
	// contains filtered or unexported fields
}

rateLogs calculates the per-second rate of log lines or values extracted from log lines

type Resp 

type Resp struct {
	I   int
	Res logqlmodel.Result
	Err error
}

type SampleEvaluatorFactory 

type SampleEvaluatorFactory interface {
	// NewStepEvaluator returns a NewStepEvaluator for a given SampleExpr. It's explicitly passed another NewStepEvaluator// in order to enable arbitrary computation of embedded expressions. This allows more modular & extensible
	// NewStepEvaluator implementations which can be composed.
	NewStepEvaluator(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)
}

type SampleEvaluatorFunc 

type SampleEvaluatorFunc func(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

func (SampleEvaluatorFunc) NewStepEvaluator 

func (s SampleEvaluatorFunc) NewStepEvaluator(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

type SampleVector 

type SampleVector promql.Vector

func (SampleVector) QuantileSketchVec 

func (p SampleVector) QuantileSketchVec() ProbabilisticQuantileVector

func (SampleVector) SampleVector 

func (p SampleVector) SampleVector() promql.Vector

type SelectLogParams 

type SelectLogParams struct {
	*logproto.QueryRequest
}

SelectParams specifies parameters passed to data selections.

func (SelectLogParams) LogSelector 

func (s SelectLogParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectLogParams) String 

func (s SelectLogParams) String() string

type SelectSampleParams 

type SelectSampleParams struct {
	*logproto.SampleQueryRequest
}

func (SelectSampleParams) Expr 

func (s SelectSampleParams) Expr() (syntax.SampleExpr, error)

Expr returns the SampleExpr from the SelectSampleParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) LogSelector 

func (s SelectSampleParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type Shard 

type Shard struct {
	PowerOfTwo *index.ShardAnnotation
	Bounded    *logproto.Shard
}

Shard represents a shard annotation It holds either a power of two shard (legacy) or a bounded shard

func NewBoundedShard 

func NewBoundedShard(shard logproto.Shard) Shard

func NewPowerOfTwoShard 

func NewPowerOfTwoShard(shard index.ShardAnnotation) Shard

func (*Shard) GetFromThrough 

func (s *Shard) GetFromThrough() (model.Fingerprint, model.Fingerprint)

func (*Shard) Match 

func (s *Shard) Match(fp model.Fingerprint) bool

implement FingerprintFilter

func (Shard) Ptr 

func (s Shard) Ptr() *Shard

convenience method for unaddressability concerns using constructors in literals (tests)

func (Shard) String 

func (s Shard) String() string

func (*Shard) Variant 

func (s *Shard) Variant() ShardVersion

type ShardMapper 

type ShardMapper struct {
	// contains filtered or unexported fields
}

func NewShardMapper 

func NewShardMapper(strategy ShardingStrategy, metrics *MapperMetrics, shardAggregation []string) ShardMapper

func (ShardMapper) Map 

func (m ShardMapper) Map(expr syntax.Expr, r *downstreamRecorder, topLevel bool) (syntax.Expr, uint64, error)

func (ShardMapper) Parse 

func (m ShardMapper) Parse(parsed syntax.Expr) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error)

type ShardResolver 

type ShardResolver interface {
	Shards(expr syntax.Expr) (int, uint64, error)
	ShardingRanges(expr syntax.Expr, targetBytesPerShard uint64) ([]logproto.Shard, error)
	GetStats(e syntax.Expr) (stats.Stats, error)
}

type ShardVersion 

type ShardVersion uint8
const (
	PowerOfTwoVersion ShardVersion = iota
	BoundedVersion
)

func ParseShardVersion 

func ParseShardVersion(s string) (ShardVersion, error)

func (ShardVersion) Strategy 

func (v ShardVersion) Strategy(resolver ShardResolver, defaultTargetShardBytes uint64) ShardingStrategy

func (ShardVersion) String 

func (v ShardVersion) String() string

type ShardingStrategy 

type ShardingStrategy interface {
	Shards(expr syntax.Expr) (shards Shards, maxBytesPerShard uint64, err error)
	Resolver() ShardResolver
}

type Shards 

type Shards []Shard

func (Shards) Encode 

func (xs Shards) Encode() (encoded []string)

type StddevOverTime 

type StddevOverTime struct {
	// contains filtered or unexported fields
}

type StdvarOverTime 

type StdvarOverTime struct {
	// contains filtered or unexported fields
}

type StepEvaluator 

type StepEvaluator interface {
	// while Next returns a promql.Value, the only acceptable types are Scalar and Vector.
	Next() (ok bool, ts int64, r StepResult)
	// Close all resources used.
	Close() error
	// Reports any error
	Error() error
	// Explain returns a print of the step evaluation tree
	Explain(Node)
}

StepEvaluator evaluate a single step of a query.

func NewResultStepEvaluator 

func NewResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator, error)

NewResultStepEvaluator coerces a downstream vector or matrix into a StepEvaluator

type StepResult 

type StepResult interface {
	SampleVector() promql.Vector
	QuantileSketchVec() ProbabilisticQuantileVector
}

type SumOverTime 

type SumOverTime struct {
	// contains filtered or unexported fields
}

type VectorAggEvaluator 

type VectorAggEvaluator struct {
	// contains filtered or unexported fields
}

func (*VectorAggEvaluator) Close 

func (e *VectorAggEvaluator) Close() error

func (*VectorAggEvaluator) Error 

func (e *VectorAggEvaluator) Error() error

func (*VectorAggEvaluator) Explain 

func (e *VectorAggEvaluator) Explain(parent Node)

func (*VectorAggEvaluator) Next 

func (e *VectorAggEvaluator) Next() (bool, int64, StepResult)

type VectorIterator 

type VectorIterator struct {
	// contains filtered or unexported fields
}

VectorIterator return simple vector like (1).

func (*VectorIterator) Close 

func (r *VectorIterator) Close() error

func (*VectorIterator) Error 

func (r *VectorIterator) Error() error

func (*VectorIterator) Explain 

func (i *VectorIterator) Explain(parent Node)

func (*VectorIterator) Next 

func (r *VectorIterator) Next() (bool, int64, StepResult)

type VectorStepEvaluator 

type VectorStepEvaluator struct {
	// contains filtered or unexported fields
}

func NewVectorStepEvaluator 

func NewVectorStepEvaluator(start time.Time, data promql.Vector) *VectorStepEvaluator

func (*VectorStepEvaluator) Close 

func (e *VectorStepEvaluator) Close() error

func (*VectorStepEvaluator) Error 

func (e *VectorStepEvaluator) Error() error

func (*VectorStepEvaluator) Explain 

func (e *VectorStepEvaluator) Explain(parent Node)

func (*VectorStepEvaluator) Next 

func (e *VectorStepEvaluator) Next() (bool, int64, StepResult)

Source Files

View all Source files

Directories

Path Synopsis
log
jsonexpr
logfmt
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
 Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
pattern
This file is taken from the golang text/scanner package so `bufLen` can be set to `maxInputSize`.
 This file is taken from the golang text/scanner package so `bufLen` can be set to `maxInputSize`.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.