logql

package
v0.0.0-...-360277a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 25, 2022 License: Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// vector ops
	OpTypeSum     = "sum"
	OpTypeAvg     = "avg"
	OpTypeMax     = "max"
	OpTypeMin     = "min"
	OpTypeCount   = "count"
	OpTypeStddev  = "stddev"
	OpTypeStdvar  = "stdvar"
	OpTypeBottomK = "bottomk"
	OpTypeTopK    = "topk"

	// range vector ops
	OpRangeTypeCount     = "count_over_time"
	OpRangeTypeRate      = "rate"
	OpRangeTypeBytes     = "bytes_over_time"
	OpRangeTypeBytesRate = "bytes_rate"
	OpRangeTypeAvg       = "avg_over_time"
	OpRangeTypeSum       = "sum_over_time"
	OpRangeTypeMin       = "min_over_time"
	OpRangeTypeMax       = "max_over_time"
	OpRangeTypeStdvar    = "stdvar_over_time"
	OpRangeTypeStddev    = "stddev_over_time"
	OpRangeTypeQuantile  = "quantile_over_time"
	OpRangeTypeAbsent    = "absent_over_time"

	// binops - logical/set
	OpTypeOr     = "or"
	OpTypeAnd    = "and"
	OpTypeUnless = "unless"

	// binops - operations
	OpTypeAdd = "+"
	OpTypeSub = "-"
	OpTypeMul = "*"
	OpTypeDiv = "/"
	OpTypeMod = "%"
	OpTypePow = "^"

	// binops - comparison
	OpTypeCmpEQ = "=="
	OpTypeNEQ   = "!="
	OpTypeGT    = ">"
	OpTypeGTE   = ">="
	OpTypeLT    = "<"
	OpTypeLTE   = "<="

	// parsers
	OpParserTypeJSON   = "json"
	OpParserTypeLogfmt = "logfmt"
	OpParserTypeRegexp = "regexp"
	OpParserTypeUnpack = "unpack"

	OpFmtLine  = "line_format"
	OpFmtLabel = "label_format"

	OpPipe   = "|"
	OpUnwrap = "unwrap"

	// conversion Op
	OpConvBytes           = "bytes"
	OpConvDuration        = "duration"
	OpConvDurationSeconds = "duration_seconds"

	OpLabelReplace = "label_replace"
)
View Source
const (
	QueryTypeMetric  = "metric"
	QueryTypeFilter  = "filter"
	QueryTypeLimited = "limited"
)
View Source
const (
	StreamsKey = "streams"
	MetricsKey = "metrics"
	SuccessKey = "success"
	FailureKey = "failure"
	NoopKey    = "noop"
)

keys used in metrics

View Source
const ABSENT_OVER_TIME = 57400
View Source
const ADD = 57412
View Source
const AND = 57404
View Source
const AVG = 57372
View Source
const AVG_OVER_TIME = 57390
View Source
const BOOL = 57382
View Source
const BOTTOMK = 57378
View Source
const BY = 57367
View Source
const BYTES = 57346
View Source
const BYTES_CONV = 57397
View Source
const BYTES_OVER_TIME = 57380
View Source
const BYTES_RATE = 57381
View Source
const CLOSE_BRACE = 57358
View Source
const CLOSE_BRACKET = 57360
View Source
const CLOSE_PARENTHESIS = 57366
View Source
const CMP_EQ = 57406
View Source
const COMMA = 57361
View Source
const COUNT = 57375
View Source
const COUNT_OVER_TIME = 57369
View Source
const DIV = 57415
View Source
const DOT = 57362
View Source
const DURATION = 57350
View Source
const DURATION_CONV = 57398
View Source
const DURATION_SECONDS_CONV = 57399
View Source
const EQ = 57354
View Source
const GT = 57410
View Source
const GTE = 57411
View Source
const IDENTIFIER = 57347
View Source
const JSON = 57383
View Source
const LABELS = 57353
View Source
const LABEL_FMT = 57388
View Source
const LABEL_REPLACE = 57401
View Source
const LINE_FMT = 57387
View Source
const LOGFMT = 57385
View Source
const LT = 57408
View Source
const LTE = 57409
View Source
const MATCHERS = 57352
View Source
const MAX = 57373
View Source
const MAX_OVER_TIME = 57393
View Source
const MIN = 57374
View Source
const MIN_OVER_TIME = 57392
View Source
const MOD = 57416
View Source
const MUL = 57414
View Source
const NEQ = 57407
View Source
const NRE = 57356
View Source
const NUMBER = 57349
View Source
const OPEN_BRACE = 57357
View Source
const OPEN_BRACKET = 57359
View Source
const OPEN_PARENTHESIS = 57365
View Source
const OR = 57403
View Source
const PIPE = 57386
View Source
const PIPE_EXACT = 57364
View Source
const PIPE_MATCH = 57363
View Source
const POW = 57417
View Source
const QUANTILE_OVER_TIME = 57396
View Source
const RANGE = 57351
View Source
const RATE = 57370
View Source
const RE = 57355
View Source
const REGEXP = 57384
View Source
const STDDEV = 57376
View Source
const STDDEV_OVER_TIME = 57395
View Source
const STDVAR = 57377
View Source
const STDVAR_OVER_TIME = 57394
View Source
const STRING = 57348
View Source
const SUB = 57413
View Source
const SUM = 57371
View Source
const SUM_OVER_TIME = 57391
View Source
const TOPK = 57379
View Source
const UNLESS = 57405
View Source
const UNPACK = 57402
View Source
const UNWRAP = 57389
View Source
const ValueTypeStreams = "streams"

ValueTypeStreams promql.ValueType for log streams

View Source
const WITHOUT = 57368

Variables

View Source
var (
	ErrParse    = errors.New("failed to parse the log query")
	ErrPipeline = errors.New("failed execute pipeline")
	ErrLimit    = errors.New("limit reached while evaluating the query")
)

Those errors are useful for comparing error returned by the engine. e.g. errors.Is(err,logql.ErrParse) let you know if this is a ast parsing error.

View Source
var (
	NoLimits = &fakeLimits{maxSeries: math.MaxInt32}
)

Functions

func EvaluatorUnsupportedType

func EvaluatorUnsupportedType(expr Expr, ev Evaluator) error

EvaluatorUnsupportedType is a helper for signaling that an evaluator does not support an Expr type

func IsComparisonOperator

func IsComparisonOperator(op string) bool

func IsLogicalBinOp

func IsLogicalBinOp(op string) bool

IsLogicalBinOp tests whether an operation is a logical/set binary operation

func ParseLabels

func ParseLabels(lbs string) (labels.Labels, error)

ParseLabels parses labels from a string using logql parser.

func ParseMatchers

func ParseMatchers(input string) ([]*labels.Matcher, error)

ParseMatchers parses a string and returns labels matchers, if the expression contains anything else it will return an error.

func PopulateMatrixFromScalar

func PopulateMatrixFromScalar(data promql.Scalar, params Params) promql.Matrix

func QueryType

func QueryType(query string) (string, error)

func RecordMetrics

func RecordMetrics(ctx context.Context, p Params, status string, stats stats.Result, result promql_parser.Value)

func ResultIterator

func ResultIterator(res Result, params Params) (iter.EntryIterator, error)

ResultIterator coerces a downstream streams result into an iter.EntryIterator

Types

type BinOpOptions

type BinOpOptions struct {
	ReturnBool bool
}

type ConcatLogSelectorExpr

type ConcatLogSelectorExpr struct {
	DownstreamLogSelectorExpr
	// contains filtered or unexported fields
}

ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr

func (ConcatLogSelectorExpr) String

func (c ConcatLogSelectorExpr) String() string

type ConcatSampleExpr

type ConcatSampleExpr struct {
	DownstreamSampleExpr
	// contains filtered or unexported fields
}

ConcatSampleExpr is an expr for concatenating multiple SampleExpr Contract: The embedded SampleExprs within a linked list of ConcatSampleExprs must be of the same structure. This makes special implementations of SampleExpr.Associative() unnecessary.

func (ConcatSampleExpr) String

func (c ConcatSampleExpr) String() string

type DefaultEvaluator

type DefaultEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultEvaluator

func NewDefaultEvaluator(querier Querier, maxLookBackPeriod time.Duration) *DefaultEvaluator

NewDefaultEvaluator constructs a DefaultEvaluator

func (*DefaultEvaluator) Iterator

func (*DefaultEvaluator) StepEvaluator

func (ev *DefaultEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv SampleEvaluator,
	expr SampleExpr,
	q Params,
) (StepEvaluator, error)

type DownstreamEvaluator

type DownstreamEvaluator struct {
	Downstreamer
	// contains filtered or unexported fields
}

DownstreamEvaluator is an evaluator which handles shard aware AST nodes

func NewDownstreamEvaluator

func NewDownstreamEvaluator(downstreamer Downstreamer) *DownstreamEvaluator

func (DownstreamEvaluator) Downstream

func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery) ([]Result, error)

Downstream runs queries and collects stats from the embedded Downstreamer

func (*DownstreamEvaluator) Iterator

func (ev *DownstreamEvaluator) Iterator(
	ctx context.Context,
	expr LogSelectorExpr,
	params Params,
) (iter.EntryIterator, error)

Iterator returns the iter.EntryIterator for a given LogSelectorExpr

func (*DownstreamEvaluator) StepEvaluator

func (ev *DownstreamEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv SampleEvaluator,
	expr SampleExpr,
	params Params,
) (StepEvaluator, error)

Evaluator returns a StepEvaluator for a given SampleExpr

type DownstreamLogSelectorExpr

type DownstreamLogSelectorExpr struct {
	LogSelectorExpr
	// contains filtered or unexported fields
}

DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation

func (DownstreamLogSelectorExpr) String

func (d DownstreamLogSelectorExpr) String() string

type DownstreamQuery

type DownstreamQuery struct {
	Expr   Expr
	Params Params
	Shards Shards
}

type DownstreamSampleExpr

type DownstreamSampleExpr struct {
	SampleExpr
	// contains filtered or unexported fields
}

DownstreamSampleExpr is a SampleExpr which signals downstream computation

func (DownstreamSampleExpr) String

func (d DownstreamSampleExpr) String() string

type Downstreamable

type Downstreamable interface {
	Downstreamer() Downstreamer
}

type Downstreamer

type Downstreamer interface {
	Downstream(context.Context, []DownstreamQuery) ([]Result, error)
}

Downstreamer is an interface for deferring responsibility for query execution. It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs.

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the LogQL engine.

func NewEngine

func NewEngine(opts EngineOpts, q Querier, l Limits) *Engine

NewEngine creates a new LogQL Engine.

func (*Engine) Query

func (ng *Engine) Query(params Params) Query

Query creates a new LogQL query. Instant/Range type is derived from the parameters.

type EngineOpts

type EngineOpts struct {
	// Timeout for queries execution
	Timeout time.Duration `yaml:"timeout"`
	// MaxLookBackPeriod is the maximum amount of time to look back for log lines.
	// only used for instant log queries.
	MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"`
}

EngineOpts is the list of options to use with the LogQL query engine.

func (*EngineOpts) RegisterFlagsWithPrefix

func (opts *EngineOpts) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type EntryEvaluator

type EntryEvaluator interface {
	// Iterator returns the iter.EntryIterator for a given LogSelectorExpr
	Iterator(context.Context, LogSelectorExpr, Params) (iter.EntryIterator, error)
}

type Evaluator

type Evaluator interface {
	SampleEvaluator
	EntryEvaluator
}

Evaluator is an interface for iterating over data at different nodes in the AST

type Expr

type Expr interface {
	Shardable() bool // A recursive check on the AST to see if it's shardable.
	fmt.Stringer
	// contains filtered or unexported methods
}

Expr is the root expression which can be a SampleExpr or LogSelectorExpr

func ParseExpr

func ParseExpr(input string) (expr Expr, err error)

ParseExpr parses a string and returns an Expr.

type Limits

type Limits interface {
	MaxQuerySeries(userID string) int
}

Limits allow the engine to fetch limits for a given users.

type LiteralParams

type LiteralParams struct {
	// contains filtered or unexported fields
}

LiteralParams impls Params

func NewLiteralParams

func NewLiteralParams(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
) LiteralParams

func (LiteralParams) Copy

func (p LiteralParams) Copy() LiteralParams

func (LiteralParams) Direction

func (p LiteralParams) Direction() logproto.Direction

Direction impls Params

func (LiteralParams) End

func (p LiteralParams) End() time.Time

End impls Params

func (LiteralParams) Interval

func (p LiteralParams) Interval() time.Duration

Interval impls Params

func (LiteralParams) Limit

func (p LiteralParams) Limit() uint32

Limit impls Params

func (LiteralParams) Query

func (p LiteralParams) Query() string

String impls Params

func (LiteralParams) Shards

func (p LiteralParams) Shards() []string

Shards impls Params

func (LiteralParams) Start

func (p LiteralParams) Start() time.Time

Start impls Params

func (LiteralParams) Step

func (p LiteralParams) Step() time.Duration

Step impls Params

type LogSelectorExpr

type LogSelectorExpr interface {
	Matchers() []*labels.Matcher
	PipelineExpr
	HasFilter() bool
	Expr
}

LogSelectorExpr is a LogQL expression filtering and returning logs.

func AddFilterExpr

func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, match string) (LogSelectorExpr, error)

AddFilterExpr adds a filter expression to a logselector expression.

func ParseLogSelector

func ParseLogSelector(input string) (LogSelectorExpr, error)

ParseLogSelector parses a log selector expression `{app="foo"} |= "filter"`

type MatrixStepper

type MatrixStepper struct {
	// contains filtered or unexported fields
}

MatrixStepper exposes a promql.Matrix as a StepEvaluator. Ensure that the resulting StepEvaluator maintains the same shape that the parameters expect. For example, it's possible that a downstream query returns matches no log streams and thus returns an empty matrix. However, we still need to ensure that it can be merged effectively with another leg that may match series. Therefore, we determine our steps from the parameters and not the underlying Matrix.

func NewMatrixStepper

func NewMatrixStepper(start, end time.Time, step time.Duration, m promql.Matrix) *MatrixStepper

func (*MatrixStepper) Close

func (m *MatrixStepper) Close() error

func (*MatrixStepper) Error

func (m *MatrixStepper) Error() error

func (*MatrixStepper) Next

func (m *MatrixStepper) Next() (bool, int64, promql.Vector)

type MockDownstreamer

type MockDownstreamer struct {
	*Engine
}

func (MockDownstreamer) Downstream

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery) ([]Result, error)

func (MockDownstreamer) Downstreamer

func (m MockDownstreamer) Downstreamer() Downstreamer

type MockQuerier

type MockQuerier struct {
	// contains filtered or unexported fields
}

Shard aware mock querier

func NewMockQuerier

func NewMockQuerier(shards int, streams []logproto.Stream) MockQuerier

func (MockQuerier) SelectLogs

func (MockQuerier) SelectSamples

func (q MockQuerier) SelectSamples(ctx context.Context, req SelectSampleParams) (iter.SampleIterator, error)

type MultiStageExpr

type MultiStageExpr []StageExpr

MultiStageExpr is multiple stages which implement a PipelineExpr.

func (MultiStageExpr) Pipeline

func (m MultiStageExpr) Pipeline() (log.Pipeline, error)

func (MultiStageExpr) String

func (m MultiStageExpr) String() string

type Params

type Params interface {
	Query() string
	Start() time.Time
	End() time.Time
	Step() time.Duration
	Interval() time.Duration
	Limit() uint32
	Direction() logproto.Direction
	Shards() []string
}

Params details the parameters associated with a loki request

type ParseError

type ParseError struct {
	// contains filtered or unexported fields
}

ParseError is what is returned when we failed to parse.

func (ParseError) Error

func (p ParseError) Error() string

func (ParseError) Is

func (p ParseError) Is(target error) bool

Is allows to use errors.Is(err,ErrParse) on this error.

type Pipeline

type Pipeline = log.Pipeline

Type alias for backward compatibility

type PipelineExpr

type PipelineExpr interface {
	Pipeline() (Pipeline, error)
	Expr
}

PipelineExpr is an expression defining a log pipeline.

type Querier

type Querier interface {
	SelectLogs(context.Context, SelectLogParams) (iter.EntryIterator, error)
	SelectSamples(context.Context, SelectSampleParams) (iter.SampleIterator, error)
}

Querier allows a LogQL expression to fetch an EntryIterator for a set of matchers and filters

type Query

type Query interface {
	// Exec processes the query.
	Exec(ctx context.Context) (Result, error)
}

Query is a LogQL query to be executed.

type QueryParams

type QueryParams interface {
	LogSelector() (LogSelectorExpr, error)
	GetStart() time.Time
	GetEnd() time.Time
	GetShards() []string
}

type QueryRangeType

type QueryRangeType string
var (
	InstantType QueryRangeType = "instant"
	RangeType   QueryRangeType = "range"
)

func GetRangeType

func GetRangeType(q Params) QueryRangeType

GetRangeType returns whether a query is an instant query or range query

type RangeVectorAggregator

type RangeVectorAggregator func([]promql.Point) float64

RangeVectorAggregator aggregates samples for a given range of samples. It receives the current milliseconds timestamp and the list of point within the range.

type RangeVectorIterator

type RangeVectorIterator interface {
	Next() bool
	At(aggregator RangeVectorAggregator) (int64, promql.Vector)
	Close() error
	Error() error
}

RangeVectorIterator iterates through a range of samples. To fetch the current vector use `At` with a `RangeVectorAggregator`.

type Result

type Result struct {
	Data       promql_parser.Value
	Statistics stats.Result
}

Result is the result of a query execution.

type SampleEvaluator

type SampleEvaluator interface {
	// StepEvaluator returns a StepEvaluator for a given SampleExpr. It's explicitly passed another StepEvaluator// in order to enable arbitrary computation of embedded expressions. This allows more modular & extensible
	// StepEvaluator implementations which can be composed.
	StepEvaluator(ctx context.Context, nextEvaluator SampleEvaluator, expr SampleExpr, p Params) (StepEvaluator, error)
}

type SampleEvaluatorFunc

type SampleEvaluatorFunc func(ctx context.Context, nextEvaluator SampleEvaluator, expr SampleExpr, p Params) (StepEvaluator, error)

func (SampleEvaluatorFunc) StepEvaluator

func (s SampleEvaluatorFunc) StepEvaluator(ctx context.Context, nextEvaluator SampleEvaluator, expr SampleExpr, p Params) (StepEvaluator, error)

type SampleExpr

type SampleExpr interface {
	// Selector is the LogQL selector to apply when retrieving logs.
	Selector() LogSelectorExpr
	Extractor() (SampleExtractor, error)
	Expr
}

SampleExpr is a LogQL expression filtering logs and returning metric samples.

func ParseSampleExpr

func ParseSampleExpr(input string) (SampleExpr, error)

ParseSampleExpr parses a string and returns the sampleExpr

type SampleExtractor

type SampleExtractor = log.SampleExtractor

Type alias for backward compatibility

type SelectLogParams

type SelectLogParams struct {
	*logproto.QueryRequest
}

SelectParams specifies parameters passed to data selections.

func (SelectLogParams) LogSelector

func (s SelectLogParams) LogSelector() (LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type SelectSampleParams

type SelectSampleParams struct {
	*logproto.SampleQueryRequest
}

func (SelectSampleParams) Expr

func (s SelectSampleParams) Expr() (SampleExpr, error)

Expr returns the SampleExpr from the SelectSampleParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) LogSelector

func (s SelectSampleParams) LogSelector() (LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type ShardMapper

type ShardMapper struct {
	// contains filtered or unexported fields
}

func NewShardMapper

func NewShardMapper(shards int, metrics *ShardingMetrics) (ShardMapper, error)

func (ShardMapper) Map

func (m ShardMapper) Map(expr Expr, r *shardRecorder) (Expr, error)

func (ShardMapper) Parse

func (m ShardMapper) Parse(query string) (noop bool, expr Expr, err error)

type ShardedEngine

type ShardedEngine struct {
	// contains filtered or unexported fields
}

ShardedEngine is an Engine implementation that can split queries into more parallelizable forms via querying the underlying backend shards individually and reaggregating them.

func NewShardedEngine

func NewShardedEngine(opts EngineOpts, downstreamable Downstreamable, metrics *ShardingMetrics, limits Limits) *ShardedEngine

NewShardedEngine constructs a *ShardedEngine

func (*ShardedEngine) Query

func (ng *ShardedEngine) Query(p Params, mapped Expr) Query

Query constructs a Query

type ShardingMetrics

type ShardingMetrics struct {
	// contains filtered or unexported fields
}

ShardingMetrics is the metrics wrapper used in shard mapping

func NewShardingMetrics

func NewShardingMetrics(registerer prometheus.Registerer) *ShardingMetrics

type Shards

type Shards []astmapper.ShardAnnotation

func ParseShards

func ParseShards(strs []string) (Shards, error)

ParseShards parses a list of string encoded shards

func (Shards) Encode

func (xs Shards) Encode() (encoded []string)

type StageExpr

type StageExpr interface {
	Stage() (log.Stage, error)
	Expr
}

StageExpr is an expression defining a single step into a log pipeline

type StepEvaluator

type StepEvaluator interface {
	// while Next returns a promql.Value, the only acceptable types are Scalar and Vector.
	Next() (bool, int64, promql.Vector)
	// Close all resources used.
	Close() error
	// Reports any error
	Error() error
}

StepEvaluator evaluate a single step of a query.

func ConcatEvaluator

func ConcatEvaluator(evaluators []StepEvaluator) (StepEvaluator, error)

ConcatEvaluator joins multiple StepEvaluators. Contract: They must be of identical start, end, and step values.

func ResultStepEvaluator

func ResultStepEvaluator(res Result, params Params) (StepEvaluator, error)

ResultStepEvaluator coerces a downstream vector or matrix into a StepEvaluator

type Streams

type Streams []logproto.Stream

Streams is promql.Value

func (Streams) Len

func (streams Streams) Len() int

func (Streams) Less

func (streams Streams) Less(i, j int) bool

func (Streams) String

func (Streams) String() string

String implements `promql.Value`

func (Streams) Swap

func (streams Streams) Swap(i, j int)

func (Streams) Type

Type implements `promql.Value`

Directories

Path Synopsis
log
logfmt
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
Package marshal converts internal objects to loghttp model objects.
Package marshal converts internal objects to loghttp model objects.
legacy
Package marshal converts internal objects to loghttp model objects.
Package marshal converts internal objects to loghttp model objects.
Package stats provides primitives for recording metrics across the query path.
Package stats provides primitives for recording metrics across the query path.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL