logql

package
v2.5.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 27, 2022 License: AGPL-3.0 Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source
const (
	QueryTypeMetric  = "metric"
	QueryTypeFilter  = "filter"
	QueryTypeLimited = "limited"
)
View Source
const (
	StreamsKey = "streams"
	MetricsKey = "metrics"
	SuccessKey = "success"
	FailureKey = "failure"
	NoopKey    = "noop"
)

keys used in metrics

Variables

View Source
var (
	NoLimits = &fakeLimits{maxSeries: math.MaxInt32}
)

Functions

func EvaluatorUnsupportedType

func EvaluatorUnsupportedType(expr syntax.Expr, ev Evaluator) error

EvaluatorUnsupportedType is a helper for signaling that an evaluator does not support an Expr type

func Match

func Match(xs []string) ([][]*labels.Matcher, error)

Match extracts and parses multiple matcher groups from a slice of strings

func PopulateMatrixFromScalar

func PopulateMatrixFromScalar(data promql.Scalar, params Params) promql.Matrix

func QueryType

func QueryType(query string) (string, error)

func RecordMetrics

func RecordMetrics(ctx context.Context, log log.Logger, p Params, status string, stats logql_stats.Result, result promql_parser.Value)

func ResultIterator

func ResultIterator(res logqlmodel.Result, params Params) (iter.EntryIterator, error)

ResultIterator coerces a downstream streams result into an iter.EntryIterator

Types

type ConcatLogSelectorExpr

type ConcatLogSelectorExpr struct {
	DownstreamLogSelectorExpr
	// contains filtered or unexported fields
}

ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr

func (ConcatLogSelectorExpr) String

func (c ConcatLogSelectorExpr) String() string

type ConcatSampleExpr

type ConcatSampleExpr struct {
	DownstreamSampleExpr
	// contains filtered or unexported fields
}

ConcatSampleExpr is an expr for concatenating multiple SampleExpr Contract: The embedded SampleExprs within a linked list of ConcatSampleExprs must be of the same structure. This makes special implementations of SampleExpr.Associative() unnecessary.

func (ConcatSampleExpr) String

func (c ConcatSampleExpr) String() string

func (ConcatSampleExpr) Walk

func (c ConcatSampleExpr) Walk(f syntax.WalkFn)

type DefaultEvaluator

type DefaultEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultEvaluator

func NewDefaultEvaluator(querier Querier, maxLookBackPeriod time.Duration) *DefaultEvaluator

NewDefaultEvaluator constructs a DefaultEvaluator

func (*DefaultEvaluator) Iterator

func (*DefaultEvaluator) StepEvaluator

func (ev *DefaultEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv SampleEvaluator,
	expr syntax.SampleExpr,
	q Params,
) (StepEvaluator, error)

type DownstreamEngine

type DownstreamEngine struct {
	// contains filtered or unexported fields
}

DownstreamEngine is an Engine implementation that can split queries into more parallelizable forms via querying the underlying backend shards individually and re-aggregating them.

func NewDownstreamEngine

func NewDownstreamEngine(opts EngineOpts, downstreamable Downstreamable, metrics *ShardingMetrics, limits Limits, logger log.Logger) *DownstreamEngine

NewDownstreamEngine constructs a *DownstreamEngine

func (*DownstreamEngine) Query

func (ng *DownstreamEngine) Query(p Params, mapped syntax.Expr) Query

Query constructs a Query

type DownstreamEvaluator

type DownstreamEvaluator struct {
	Downstreamer
	// contains filtered or unexported fields
}

DownstreamEvaluator is an evaluator which handles shard aware AST nodes

func NewDownstreamEvaluator

func NewDownstreamEvaluator(downstreamer Downstreamer) *DownstreamEvaluator

func (DownstreamEvaluator) Downstream

func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery) ([]logqlmodel.Result, error)

Downstream runs queries and collects stats from the embedded Downstreamer

func (*DownstreamEvaluator) Iterator

func (ev *DownstreamEvaluator) Iterator(
	ctx context.Context,
	expr syntax.LogSelectorExpr,
	params Params,
) (iter.EntryIterator, error)

Iterator returns the iter.EntryIterator for a given LogSelectorExpr

func (*DownstreamEvaluator) StepEvaluator

func (ev *DownstreamEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv SampleEvaluator,
	expr syntax.SampleExpr,
	params Params,
) (StepEvaluator, error)

StepEvaluator returns a StepEvaluator for a given SampleExpr

type DownstreamLogSelectorExpr

type DownstreamLogSelectorExpr struct {
	syntax.LogSelectorExpr
	// contains filtered or unexported fields
}

DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation

func (DownstreamLogSelectorExpr) String

func (d DownstreamLogSelectorExpr) String() string

type DownstreamQuery

type DownstreamQuery struct {
	Expr   syntax.Expr
	Params Params
	Shards Shards
}

type DownstreamSampleExpr

type DownstreamSampleExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

DownstreamSampleExpr is a SampleExpr which signals downstream computation

func (DownstreamSampleExpr) String

func (d DownstreamSampleExpr) String() string

func (DownstreamSampleExpr) Walk

type Downstreamable

type Downstreamable interface {
	Downstreamer() Downstreamer
}

type Downstreamer

type Downstreamer interface {
	Downstream(context.Context, []DownstreamQuery) ([]logqlmodel.Result, error)
}

Downstreamer is an interface for deferring responsibility for query execution. It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs.

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the LogQL engine.

func NewEngine

func NewEngine(opts EngineOpts, q Querier, l Limits, logger log.Logger) *Engine

NewEngine creates a new LogQL Engine.

func (*Engine) Query

func (ng *Engine) Query(params Params) Query

Query creates a new LogQL query. Instant/Range type is derived from the parameters.

type EngineOpts

type EngineOpts struct {
	// Timeout for queries execution
	Timeout time.Duration `yaml:"timeout"`
	// MaxLookBackPeriod is the maximum amount of time to look back for log lines.
	// only used for instant log queries.
	MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"`
}

EngineOpts is the list of options to use with the LogQL query engine.

func (*EngineOpts) RegisterFlagsWithPrefix

func (opts *EngineOpts) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type EntryEvaluator

type EntryEvaluator interface {
	// Iterator returns the iter.EntryIterator for a given LogSelectorExpr
	Iterator(context.Context, syntax.LogSelectorExpr, Params) (iter.EntryIterator, error)
}

type Evaluator

type Evaluator interface {
	SampleEvaluator
	EntryEvaluator
}

Evaluator is an interface for iterating over data at different nodes in the AST

type Limits

type Limits interface {
	MaxQuerySeries(userID string) int
}

Limits allow the engine to fetch limits for a given users.

type LiteralParams

type LiteralParams struct {
	// contains filtered or unexported fields
}

LiteralParams impls Params

func NewLiteralParams

func NewLiteralParams(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
) LiteralParams

func (LiteralParams) Copy

func (p LiteralParams) Copy() LiteralParams

func (LiteralParams) Direction

func (p LiteralParams) Direction() logproto.Direction

Direction impls Params

func (LiteralParams) End

func (p LiteralParams) End() time.Time

End impls Params

func (LiteralParams) Interval

func (p LiteralParams) Interval() time.Duration

Interval impls Params

func (LiteralParams) Limit

func (p LiteralParams) Limit() uint32

Limit impls Params

func (LiteralParams) Query

func (p LiteralParams) Query() string

String impls Params

func (LiteralParams) Shards

func (p LiteralParams) Shards() []string

Shards impls Params

func (LiteralParams) Start

func (p LiteralParams) Start() time.Time

Start impls Params

func (LiteralParams) Step

func (p LiteralParams) Step() time.Duration

Step impls Params

type MatrixStepper

type MatrixStepper struct {
	// contains filtered or unexported fields
}

MatrixStepper exposes a promql.Matrix as a StepEvaluator. Ensure that the resulting StepEvaluator maintains the same shape that the parameters expect. For example, it's possible that a downstream query returns matches no log streams and thus returns an empty matrix. However, we still need to ensure that it can be merged effectively with another leg that may match series. Therefore, we determine our steps from the parameters and not the underlying Matrix.

func NewMatrixStepper

func NewMatrixStepper(start, end time.Time, step time.Duration, m promql.Matrix) *MatrixStepper

func (*MatrixStepper) Close

func (m *MatrixStepper) Close() error

func (*MatrixStepper) Error

func (m *MatrixStepper) Error() error

func (*MatrixStepper) Next

func (m *MatrixStepper) Next() (bool, int64, promql.Vector)

type MockDownstreamer

type MockDownstreamer struct {
	*Engine
}

func (MockDownstreamer) Downstream

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery) ([]logqlmodel.Result, error)

func (MockDownstreamer) Downstreamer

func (m MockDownstreamer) Downstreamer() Downstreamer

type MockQuerier

type MockQuerier struct {
	// contains filtered or unexported fields
}

Shard aware mock querier

func NewMockQuerier

func NewMockQuerier(shards int, streams []logproto.Stream) MockQuerier

func (MockQuerier) SelectLogs

func (MockQuerier) SelectSamples

func (q MockQuerier) SelectSamples(ctx context.Context, req SelectSampleParams) (iter.SampleIterator, error)

type Params

type Params interface {
	Query() string
	Start() time.Time
	End() time.Time
	Step() time.Duration
	Interval() time.Duration
	Limit() uint32
	Direction() logproto.Direction
	Shards() []string
}

Params details the parameters associated with a loki request

type Querier

type Querier interface {
	SelectLogs(context.Context, SelectLogParams) (iter.EntryIterator, error)
	SelectSamples(context.Context, SelectSampleParams) (iter.SampleIterator, error)
}

Querier allows a LogQL expression to fetch an EntryIterator for a set of matchers and filters

type Query

type Query interface {
	// Exec processes the query.
	Exec(ctx context.Context) (logqlmodel.Result, error)
}

Query is a LogQL query to be executed.

type QueryParams

type QueryParams interface {
	LogSelector() (syntax.LogSelectorExpr, error)
	GetStart() time.Time
	GetEnd() time.Time
	GetShards() []string
}

type QueryRangeType

type QueryRangeType string
var (
	InstantType QueryRangeType = "instant"
	RangeType   QueryRangeType = "range"
)

func GetRangeType

func GetRangeType(q Params) QueryRangeType

GetRangeType returns whether a query is an instant query or range query

type RangeMapper

type RangeMapper struct {
	// contains filtered or unexported fields
}

RangeMapper is used to rewrite LogQL sample expressions into multiple downstream sample expressions with a smaller time range that can be executed using the downstream engine.

A rewrite is performed using the following rules:

  1. Check if query is splittable based on the range.
  2. Check if the query is splittable based on the query AST
  3. Range aggregations are split into multiple downstream range aggregation expressions that are concatenated with an appropriate vector aggregator with a grouping operator. If the range aggregation has a grouping, the grouping is also applied to the resultant vector aggregator expression. If the range aggregation has no grouping, a grouping operator using "without" is applied to the resultant vector aggregator expression to preserve the stream labels.
  4. Vector aggregations are split into multiple downstream vector aggregations that are merged with vector aggregation using "without" and then aggregated using the vector aggregation with the same operator, either with or without grouping.
  5. Left and right-hand side of binary operations are split individually using the same rules as above.

func NewRangeMapper

func NewRangeMapper(interval time.Duration) (RangeMapper, error)

NewRangeMapper creates a new RangeMapper instance with the given duration as split interval. The interval must be greater than 0.

func (RangeMapper) Map

func (m RangeMapper) Map(expr syntax.SampleExpr, vectorAggrPushdown *syntax.VectorAggregationExpr) (syntax.SampleExpr, error)

Map rewrites sample expression expr and returns the resultant sample expression to be executed by the downstream engine It is called recursively on the expression tree. The function takes an optional vector aggregation as second argument, that is pushed down to the downstream expression.

func (RangeMapper) Parse

func (m RangeMapper) Parse(query string) (bool, syntax.Expr, error)

Parse parses the given LogQL query string into a sample expression and applies the rewrite rules for splitting it into a sample expression that can be executed by the downstream engine. It returns a boolean indicating whether a rewrite was possible, the rewritten sample expression, and an error in case the rewrite failed.

type RangeVectorAggregator

type RangeVectorAggregator func([]promql.Point) float64

RangeVectorAggregator aggregates samples for a given range of samples. It receives the current milliseconds timestamp and the list of point within the range.

type RangeVectorIterator

type RangeVectorIterator interface {
	Next() bool
	At(aggregator RangeVectorAggregator) (int64, promql.Vector)
	Close() error
	Error() error
}

RangeVectorIterator iterates through a range of samples. To fetch the current vector use `At` with a `RangeVectorAggregator`.

type SampleEvaluator

type SampleEvaluator interface {
	// StepEvaluator returns a StepEvaluator for a given SampleExpr. It's explicitly passed another StepEvaluator// in order to enable arbitrary computation of embedded expressions. This allows more modular & extensible
	// StepEvaluator implementations which can be composed.
	StepEvaluator(ctx context.Context, nextEvaluator SampleEvaluator, expr syntax.SampleExpr, p Params) (StepEvaluator, error)
}

type SampleEvaluatorFunc

type SampleEvaluatorFunc func(ctx context.Context, nextEvaluator SampleEvaluator, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

func (SampleEvaluatorFunc) StepEvaluator

func (s SampleEvaluatorFunc) StepEvaluator(ctx context.Context, nextEvaluator SampleEvaluator, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

type SelectLogParams

type SelectLogParams struct {
	*logproto.QueryRequest
}

SelectParams specifies parameters passed to data selections.

func (SelectLogParams) LogSelector

func (s SelectLogParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectLogParams) String

func (s SelectLogParams) String() string

type SelectSampleParams

type SelectSampleParams struct {
	*logproto.SampleQueryRequest
}

func (SelectSampleParams) Expr

Expr returns the SampleExpr from the SelectSampleParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) LogSelector

func (s SelectSampleParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type ShardMapper

type ShardMapper struct {
	// contains filtered or unexported fields
}

func NewShardMapper

func NewShardMapper(shards int, metrics *ShardingMetrics) (ShardMapper, error)

func (ShardMapper) Map

func (m ShardMapper) Map(expr syntax.Expr, r *shardRecorder) (syntax.Expr, error)

func (ShardMapper) Parse

func (m ShardMapper) Parse(query string) (noop bool, expr syntax.Expr, err error)

type ShardingMetrics

type ShardingMetrics struct {
	Shards      *prometheus.CounterVec // sharded queries total, partitioned by (streams/metric)
	ShardFactor prometheus.Histogram   // per request shard factor
	// contains filtered or unexported fields
}

ShardingMetrics is the metrics wrapper used in shard mapping

func NewShardingMetrics

func NewShardingMetrics(registerer prometheus.Registerer) *ShardingMetrics

type Shards

type Shards []astmapper.ShardAnnotation

func ParseShards

func ParseShards(strs []string) (Shards, error)

ParseShards parses a list of string encoded shards

func (Shards) Encode

func (xs Shards) Encode() (encoded []string)

type StepEvaluator

type StepEvaluator interface {
	// while Next returns a promql.Value, the only acceptable types are Scalar and Vector.
	Next() (ok bool, ts int64, vec promql.Vector)
	// Close all resources used.
	Close() error
	// Reports any error
	Error() error
}

StepEvaluator evaluate a single step of a query.

func ConcatEvaluator

func ConcatEvaluator(evaluators []StepEvaluator) (StepEvaluator, error)

ConcatEvaluator joins multiple StepEvaluators. Contract: They must be of identical start, end, and step values.

func ResultStepEvaluator

func ResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator, error)

ResultStepEvaluator coerces a downstream vector or matrix into a StepEvaluator

Directories

Path Synopsis
log
logfmt
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL