filter

package
v0.1.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 19, 2021 License: MIT Imports: 22 Imported by: 0

Documentation

Overview

Package filter provides filter components.

Index

Constants

This section is empty.

Variables

All is the list of all baker filters.

View Source
var ClauseFilterDesc = baker.FilterDesc{
	Name:   "ClauseFilter",
	New:    NewClauseFilter,
	Config: &ClauseFilterConfig{},
	Help:   clauseFilterHelpMsg,
}

ClauseFilterDesc describes the ClauseFilter filter

View Source
var ClearFieldsDesc = baker.FilterDesc{
	Name:   "ClearFields",
	New:    NewClearFields,
	Config: &ClearFieldsConfig{},
	Help:   "Reset a set of fields of all records passing through",
}

ClearFieldsDesc describes the ClearFields filter

View Source
var ConcatenateDesc = baker.FilterDesc{
	Name:   "Concatenate",
	New:    NewConcatenate,
	Config: &ConcatenateConfig{},
	Help:   `Concatenate up to 10 fields' values to a single field`,
}

ConcatenateDesc describes the Concatenate filter

View Source
var DedupDesc = baker.FilterDesc{
	Name:   "Dedup",
	New:    NewDedup,
	Config: &DedupConfig{},
	Help:   dedupHelp,
}
View Source
var ExpandJSONDesc = baker.FilterDesc{
	Name:   "ExpandJSON",
	New:    NewExpandJSON,
	Config: &ExpandJSONConfig{},
	Help:   expandJSONhelp,
}
View Source
var ExpandListDesc = baker.FilterDesc{
	Name:   "ExpandList",
	New:    NewExpandList,
	Config: &ExpandListConfig{},
	Help:   expandListhelp,
}
View Source
var FormatTimeDesc = baker.FilterDesc{
	Name:   "FormatTime",
	New:    NewFormatTime,
	Config: &FormatTimeConfig{},
	Help:   formatTimeHelp,
}
View Source
var HashDesc = baker.FilterDesc{
	Name:   "Hash",
	New:    NewHash,
	Config: &HashConfig{},
	Help: `This filter hashes a field using a specified hash function and writes the value 
to another (or the same) field. In order to have control over the set of characters
present, the hashed value can optionally be encoded.
	
	
Supported hash functions:
 - md5
 - sha256

Supported encodings:
- hex (hexadecimal encoding)
`,
}

HashDesc describes the Hash filter

View Source
var MetadataLastModifiedDesc = baker.FilterDesc{
	Name:   "MetadataLastModified",
	New:    NewMetadataLastModified,
	Config: &MetadataLastModifiedConfig{},
	Help:   `Extract the "last modified" timestamp from the record Metadata and write it to the selected field.`,
}
View Source
var MetadataUrlDesc = baker.FilterDesc{
	Name:   "MetadataUrl",
	New:    NewMetadataUrl,
	Config: &MetadataUrlConfig{},
	Help:   metadataUrlHelp,
}
View Source
var NotNullDesc = baker.FilterDesc{
	Name:   "NotNull",
	New:    NewNotNull,
	Config: &NotNullConfig{},
	Help:   "Discard the records having null (i.e empty) fields.\n",
}

NotNullDesc describes the NotNull filter.

View Source
var PartialCloneDesc = baker.FilterDesc{
	Name:   "PartialClone",
	New:    NewPartialClone,
	Config: &PartialCloneConfig{},
	Help:   "Copy a list of fields to a new record and process this new record, discarding the original one",
}
View Source
var RegexMatchDesc = baker.FilterDesc{
	Name:   "RegexMatch",
	New:    NewRegexMatch,
	Config: &RegexMatchConfig{},
	Help:   "Discard a record if one or more fields don't match the corresponding regular expressions",
}

RegexMatchDesc describes the RegexMatch filter

View Source
var ReplaceFieldsDesc = baker.FilterDesc{
	Name:   "ReplaceFields",
	New:    NewReplaceFields,
	Config: &ReplaceFieldsConfig{},
	Help:   `Copy a field value or a fixed value to another field. Can copy multiple fields.`,
}

ReplaceFieldsDesc describes the ReplaceFields filter

View Source
var SetStringFromURLDesc = baker.FilterDesc{
	Name:   "SetStringFromURL",
	New:    NewSetStringFromURL,
	Config: &SetStringFromURLConfig{},
	Help:   helpSetStringFromURL,
}

SetStringFromURLDesc describes the SetStringFromURL filter.

View Source
var StringMatchDesc = baker.FilterDesc{
	Name:   "StringMatch",
	New:    NewStringMatch,
	Config: &StringMatchConfig{},
	Help:   "Discard records if a field matches any of the provided strings",
}

StringMatchDesc describes the StringMatch filter

View Source
var TimestampDesc = baker.FilterDesc{
	Name:   "Timestamp",
	New:    NewTimestamp,
	Config: &TimestampConfig{},
	Help:   "Sets a field to the Unix Epoch timestamp at which the record is processed",
}

TimestampDesc describes the Timestamp filter.

View Source
var TimestampRangeDesc = baker.FilterDesc{
	Name:   "TimestampRange",
	New:    NewTimestampRange,
	Config: &TimestampRangeConfig{},
	Help:   "Discard records if the value of a field containing a timestamp is out of the given time range (i.e StartDateTime <= value < EndDateTime)",
}

TimestampRangeDesc describes the TimestampRange filter.

Functions

func NewClauseFilter

func NewClauseFilter(cfg baker.FilterParams) (baker.Filter, error)

func NewClearFields

func NewClearFields(cfg baker.FilterParams) (baker.Filter, error)

NewClearFields returns a ClearFields filter.

func NewConcatenate

func NewConcatenate(cfg baker.FilterParams) (baker.Filter, error)

func NewDedup

func NewDedup(cfg baker.FilterParams) (baker.Filter, error)

func NewExpandJSON

func NewExpandJSON(cfg baker.FilterParams) (baker.Filter, error)

func NewExpandList

func NewExpandList(cfg baker.FilterParams) (baker.Filter, error)

func NewFormatTime

func NewFormatTime(cfg baker.FilterParams) (baker.Filter, error)

func NewHash

func NewHash(cfg baker.FilterParams) (baker.Filter, error)

func NewMetadataLastModified

func NewMetadataLastModified(cfg baker.FilterParams) (baker.Filter, error)

func NewMetadataUrl

func NewMetadataUrl(cfg baker.FilterParams) (baker.Filter, error)

func NewNotNull

func NewNotNull(cfg baker.FilterParams) (baker.Filter, error)

NewNotNull creates and configures a new NotNull filter.

func NewPartialClone

func NewPartialClone(cfg baker.FilterParams) (baker.Filter, error)

func NewRegexMatch

func NewRegexMatch(cfg baker.FilterParams) (baker.Filter, error)

NewRegexMatch returns a RegexMatch filter.

func NewReplaceFields

func NewReplaceFields(cfg baker.FilterParams) (baker.Filter, error)

func NewSetStringFromURL

func NewSetStringFromURL(cfg baker.FilterParams) (baker.Filter, error)

func NewStringMatch

func NewStringMatch(cfg baker.FilterParams) (baker.Filter, error)

NewStringMatch returns a StringMatch filter.

func NewTimestamp

func NewTimestamp(cfg baker.FilterParams) (baker.Filter, error)

NewTimestamp creates and configures a Timestamp filter.

func NewTimestampRange

func NewTimestampRange(cfg baker.FilterParams) (baker.Filter, error)

NewTimestampRange creates and configures a TimestampRange filter.

Types

type Clause

type Clause struct {
	// contains filtered or unexported fields
}

Will real sum types ever make it to go?

Validness of the fields in this structure depend on clauseType

type ClauseFilter

type ClauseFilter struct {
	// contains filtered or unexported fields
}

func (*ClauseFilter) Process

func (f *ClauseFilter) Process(l baker.Record, next func(baker.Record))

func (*ClauseFilter) Stats

func (f *ClauseFilter) Stats() baker.FilterStats

type ClauseFilterConfig

type ClauseFilterConfig struct {
	Clause string `help:"Boolean formula describing which events to let through. If empty, let everything through."`
}

ClauseFilterConfig describes the ClauseFilter filter config

type ClearFields

type ClearFields struct {
	// contains filtered or unexported fields
}

ClearFields filter clears (i.e set to the empty string) a set of fields.

func (*ClearFields) Process

func (f *ClearFields) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*ClearFields) Stats

func (f *ClearFields) Stats() baker.FilterStats

Stats returns filter statistics.

type ClearFieldsConfig

type ClearFieldsConfig struct {
	Fields []string `help:"set of fields to clear" required:"true"`
}

ClearFieldsConfig holds config parameters of the ClearFields filter.

type Concatenate

type Concatenate struct {
	// contains filtered or unexported fields
}

func (*Concatenate) Process

func (c *Concatenate) Process(l baker.Record, next func(baker.Record))

func (*Concatenate) Stats

func (c *Concatenate) Stats() baker.FilterStats

type ConcatenateConfig

type ConcatenateConfig struct {
	Fields    []string `help:"The field names to concatenate, in order"`
	Target    string   `help:"The field name to save the concatenated value to"`
	Separator string   `help:"Separator to concatenate the values. Must either be empty or a single ASCII, non-nil char"`
}

type Dedup

type Dedup struct {
	// contains filtered or unexported fields
}

func (*Dedup) Process

func (f *Dedup) Process(l baker.Record, next func(baker.Record))

func (*Dedup) Stats

func (f *Dedup) Stats() baker.FilterStats

type DedupConfig

type DedupConfig struct {
	Fields       []string `help:"fields to consider when comparing records" required:"true"`
	KeySeparator string   `help:"character separator used to build a key from the fields" default:"\\x1e"`
}

type ExpandJSON

type ExpandJSON struct {
	// contains filtered or unexported fields
}

func (*ExpandJSON) Process

func (f *ExpandJSON) Process(l baker.Record, next func(baker.Record))

func (*ExpandJSON) Stats

func (f *ExpandJSON) Stats() baker.FilterStats

type ExpandJSONConfig

type ExpandJSONConfig struct {
	Source          string            `help:"record field that contains the json" required:"true"`
	Fields          map[string]string `help:"<JMESPath -> record field> map, the rest will be ignored" required:"true"`
	TrueFalseValues []string          `help:"bind the json boolean values to correstponding strings" default:"[\"true\", \"false\"]"`
}

type ExpandList

type ExpandList struct {
	// contains filtered or unexported fields
}

func (*ExpandList) Process

func (f *ExpandList) Process(l baker.Record, next func(baker.Record))

func (*ExpandList) Stats

func (f *ExpandList) Stats() baker.FilterStats

type ExpandListConfig

type ExpandListConfig struct {
	Source    string            `help:"record field that contains the list" required:"true"`
	Fields    map[string]string `help:"<list index -> record field> map, the rest will be ignored" required:"true"`
	Separator string            `help:"character separator of the list" default:";"`
}

type FormatTime

type FormatTime struct {
	// contains filtered or unexported fields
}

func (*FormatTime) Process

func (f *FormatTime) Process(l baker.Record, next func(baker.Record))

func (*FormatTime) Stats

func (f *FormatTime) Stats() baker.FilterStats

type FormatTimeConfig

type FormatTimeConfig struct {
	SrcField  string `help:"Field name of the input time" required:"true"`
	DstField  string `help:"Field name of the output time" required:"true"`
	SrcFormat string `help:"Format of the input time" required:"false" default:"UnixDate"`
	DstFormat string `help:"Format of the output time" required:"false" default:"unixms"`
}

type Hash

type Hash struct {
	// contains filtered or unexported fields
}

func (*Hash) Process

func (h *Hash) Process(r baker.Record, next func(baker.Record))

func (*Hash) Stats

func (h *Hash) Stats() baker.FilterStats

type HashConfig

type HashConfig struct {
	SrcField string `help:"Name of the field to hash" required:"true"`
	DstField string `help:"Name of the field to write the result to" required:"true"`
	Function string `help:"Name of the hash function to use" required:"true"`
	Encoding string `help:"Name of the encoding function to use" required:"false"`
}

type MetadataLastModified

type MetadataLastModified struct {
	// contains filtered or unexported fields
}

func (*MetadataLastModified) Process

func (f *MetadataLastModified) Process(l baker.Record, next func(baker.Record))

func (*MetadataLastModified) Stats

type MetadataLastModifiedConfig

type MetadataLastModifiedConfig struct {
	DstField string `help:"Name of the field into which write the timestamp to" required:"true"`
}

type MetadataUrl

type MetadataUrl struct {
	metadata.Cache // type: map[*url.URL]string
	// contains filtered or unexported fields
}

func (*MetadataUrl) Process

func (f *MetadataUrl) Process(l baker.Record, next func(baker.Record))

func (*MetadataUrl) Stats

func (f *MetadataUrl) Stats() baker.FilterStats

type MetadataUrlConfig

type MetadataUrlConfig struct {
	DstField string `help:"Name of the field into to write the url to (or to clear if there's no url)" required:"true"`
}

type NotNull

type NotNull struct {
	// contains filtered or unexported fields
}

NotNull is a baker filter that discards records having null fields.

func (*NotNull) Process

func (v *NotNull) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*NotNull) Stats

func (v *NotNull) Stats() baker.FilterStats

Stats implements baker.Filter.

type NotNullConfig

type NotNullConfig struct {
	Fields []string `help:"Fields is the list of fields to check for null/empty values" required:"true"`
}

NotNullConfig holds configuration parameters for the NotNull filter.

type PartialClone

type PartialClone struct {
	// contains filtered or unexported fields
}

func (*PartialClone) Process

func (s *PartialClone) Process(r baker.Record, next func(baker.Record))

func (*PartialClone) Stats

func (s *PartialClone) Stats() baker.FilterStats

type PartialCloneConfig

type PartialCloneConfig struct {
	Fields []string `help:"Fields that must be copied to the new line" required:"true"`
}

type RegexMatch

type RegexMatch struct {
	// contains filtered or unexported fields
}

RegexMatch filter clears (i.e set to the empty string) a set of fields.

func (*RegexMatch) Process

func (f *RegexMatch) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*RegexMatch) Stats

func (f *RegexMatch) Stats() baker.FilterStats

Stats returns filter statistics.

type RegexMatchConfig

type RegexMatchConfig struct {
	Fields []string `help:"list of fields to match with the corresponding regular expression in Regexs" default:"[]"`
	Regexs []string `help:"list of regular expression to match. Fields[0] must match Regexs[0], Fields[1] Regexs[1] and so on" default:"[]"`
}

RegexMatchConfig holds config parameters of the RegexMatch filter.

type ReplaceFields

type ReplaceFields struct {
	// contains filtered or unexported fields
}

func (*ReplaceFields) Process

func (f *ReplaceFields) Process(l baker.Record, next func(baker.Record))

func (*ReplaceFields) Stats

func (c *ReplaceFields) Stats() baker.FilterStats

type ReplaceFieldsConfig

type ReplaceFieldsConfig struct {
	CopyFields    []string `help:"List of src, dst field pairs, for example [\"srcField1\", \"dstField1\", \"srcField2\", \"dstField2\"]"`
	ReplaceFields []string `help:"List of field, value pairs, for example: [\"Foo\", \"dstField1\", \"Bar\", \"dstField2\"]"`
}

type SetStringFromURL

type SetStringFromURL struct {
	// contains filtered or unexported fields
}

func (*SetStringFromURL) Process

func (f *SetStringFromURL) Process(l baker.Record, next func(baker.Record))

func (*SetStringFromURL) Stats

func (f *SetStringFromURL) Stats() baker.FilterStats

type SetStringFromURLConfig

type SetStringFromURLConfig struct {
	Field   string   `help:"Name of the field to set to" required:"true"`
	Strings []string `help:"Strings to look for in the URL. Discard records not containing any of them." required:"true"`
}

type StringMatch

type StringMatch struct {
	// contains filtered or unexported fields
}

StringMatch filter clears (i.e set to the empty string) a set of fields.

func (*StringMatch) Process

func (f *StringMatch) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*StringMatch) Stats

func (f *StringMatch) Stats() baker.FilterStats

Stats returns filter statistics.

type StringMatchConfig

type StringMatchConfig struct {
	Field       string   `help:"name of the field which value is used for string comparison" required:"true"`
	Strings     []string `help:"list of strings to match." required:"true"`
	InvertMatch bool     `help:"Invert the match outcome, so that records are discarded if they don't match any of the strings" default:"false"`
}

StringMatchConfig holds config parameters of the StringMatch filter.

type Timestamp

type Timestamp struct {
	// contains filtered or unexported fields
}

Timestamp is a baker filter that sets the value of a certain field to the Unix Epoch timestamp (in seconds, UTC) for all the records it processes.

func (*Timestamp) Process

func (f *Timestamp) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*Timestamp) Stats

func (f *Timestamp) Stats() baker.FilterStats

Stats implements baker.Filter.

type TimestampConfig

type TimestampConfig struct {
	Field string `help:"field to set to the unix Epoch timestamp" required:"true"`
}

TimestampConfig holds configuration paramters for the Timestamp filter.

type TimestampRange

type TimestampRange struct {
	// contains filtered or unexported fields
}

TimestampRange is a baker filter that discards records depending on the value of a field representing a Unix timestamp.

func (*TimestampRange) Process

func (f *TimestampRange) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*TimestampRange) Stats

func (f *TimestampRange) Stats() baker.FilterStats

Stats implements baker.Filter.

type TimestampRangeConfig

type TimestampRangeConfig struct {
	StartDatetime string `` /* 149-byte string literal not displayed */
	EndDatetime   string `` /* 149-byte string literal not displayed */
	Field         string `help:"Name of the field containing the Unix EPOCH timestamp" required:"true"`
}

TimestampRangeConfig holds configuration paramters for the TimestampRange filter.

Directories

Path Synopsis
Package metadata provides a metadata-specific (concurrent and size-limited) cache.
Package metadata provides a metadata-specific (concurrent and size-limited) cache.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL