output

package
v0.1.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 19, 2021 License: MIT Imports: 32 Imported by: 0

Documentation

Overview

Package output provides output components

Index

Constants

This section is empty.

Variables

All is the list of all baker outputs.

View Source
var BakerTransport = http.DefaultTransport.(*http.Transport).Clone()
View Source
var DynamoDBDesc = baker.OutputDesc{
	Name:   "DynamoDB",
	New:    NewDynamoDB,
	Config: &DynamoDBConfig{},
	Raw:    false,
	Help: "This output writes the filtered log lines to DynamoDB. It must be\n" +
		"configured specifying the region, the table name, and the columns\n" +
		"to write.\nColumns are specified using the syntax \"t:name\" where \"t\"\n" +
		"is the type of the data, and \"name\" is the name of column. Supported\n" +
		"types are: \"n\" - integers; \"s\" - strings.\n" +
		"The first column (and field) must be the primary key.\n",
}
View Source
var DynamoGlobals dynamoGlobals
View Source
var FileWriterDesc = baker.OutputDesc{
	Name:   "FileWriter",
	New:    NewFileWriter,
	Config: &FileWriterConfig{},
	Raw:    true,
	Help:   helpMsg,
}
View Source
var NopDesc = baker.OutputDesc{
	Name:   "Nop",
	New:    NewNop,
	Config: &NopConfig{},
	Help:   "No-operation output. This output simply drops all lines and does not write them anywhere.",
}
View Source
var OpLogDesc = baker.OutputDesc{
	Name:   "OpLog",
	New:    NewOpLog,
	Config: &OpLogConfig{},
	Raw:    false,
	Help:   "This output writes the filtered log lines into the current baker log, purely for development purpose.\n",
}
View Source
var SQLiteDesc = baker.OutputDesc{
	Name:   "SQLite",
	New:    NewSQLite(false),
	Config: &SQLiteConfig{},
	Help:   "Writes a chosen set of fields as table columns into a local SQLite database file",
	Raw:    false,
}

SQLiteDesc declares the standard (non-raw) SQLite output.

View Source
var SQLiteRawDesc = baker.OutputDesc{
	Name:   "SQLiteRaw",
	New:    NewSQLite(true),
	Config: &SQLiteRawWriterConfig{},
	Help:   "Writes a chosen set of fields, plus the raw record, as table columns into a local SQLite database file",
	Raw:    true,
}

SQLiteRawDesc declares the raw SQLite output.

View Source
var StatsDesc = baker.OutputDesc{
	Name:   "Stats",
	New:    NewStats,
	Config: &StatsConfig{},
	Raw:    true,
	Help: "Compute various distributions of the records it " +
		"receives and dumps that to CSV. It computes the " +
		"distribution of record by size and the distribution " +
		"of the values of certain fields\n",
}
View Source
var WebSocketDesc = baker.OutputDesc{
	Name:   "WebSocket",
	New:    NewWebSocket,
	Config: &WebSocketConfig{},
	Raw:    false,
	Help:   "This output writes the filtered log lines into any connected WebSocket client.\n",
}

Functions

func NewDynamoDB

func NewDynamoDB(cfg baker.OutputParams) (baker.Output, error)

NewDynamoDB create a new DynamoDB output.

TableName is the name of the DynamoDB table to be written. Columns is a slice listing the columns that will be written; the first item in the slice *MUST* be the primary key of the table.

func NewFileWriter

func NewFileWriter(cfg baker.OutputParams) (baker.Output, error)

func NewNop

func NewNop(cfg baker.OutputParams) (baker.Output, error)

func NewOpLog

func NewOpLog(cfg baker.OutputParams) (baker.Output, error)

func NewSQLite

func NewSQLite(isRaw bool) func(baker.OutputParams) (baker.Output, error)

func NewStats

func NewStats(cfg baker.OutputParams) (baker.Output, error)

NewStats returns a new Stats Baker output.

func NewWebSocket

func NewWebSocket(cfg baker.OutputParams) (baker.Output, error)

Types

type DynamoDB

type DynamoDB struct {
	TableName string
	Fields    []baker.FieldIndex
	Columns   []string
	Cfg       *DynamoDBConfig
	// contains filtered or unexported fields
}

DynamoDB is a class to do optimized batched writes to a single DynamoDB table with a fixed schema (same number of columns for all records).

func (*DynamoDB) CanShard

func (b *DynamoDB) CanShard() bool

func (*DynamoDB) Flush

func (b *DynamoDB) Flush()

func (*DynamoDB) NumProcessedRecords

func (b *DynamoDB) NumProcessedRecords() int64

func (*DynamoDB) Run

func (b *DynamoDB) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*DynamoDB) Stats

func (b *DynamoDB) Stats() baker.OutputStats

type DynamoDBConfig

type DynamoDBConfig struct {
	Regions         []string      `help:"DynamoDB regions to connect to" default:"us-west-2"`
	Table           string        `help:"Name of the table to modify" required:"true"`
	Columns         []string      `help:"Table columns that correspond to each of the fields being written"`
	FlushInterval   time.Duration `help:"Interval at which flush the data to DynamoDB even if we have not reached 25 records" default:"1s"`
	MaxWritesPerSec int64         `help:"Maximum number of writes per second that DynamoDB can accept (0 for unlimited)" default:"0"`
	MaxBackoff      time.Duration `help:"Maximum retry/backoff time in case of errors before giving up" default:"2m"`
	// contains filtered or unexported fields
}

type FileWriter

type FileWriter struct {
	Cfg *FileWriterConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*FileWriter) CanShard

func (w *FileWriter) CanShard() bool

func (*FileWriter) Run

func (w *FileWriter) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*FileWriter) Stats

func (w *FileWriter) Stats() baker.OutputStats

type FileWriterConfig

type FileWriterConfig struct {
	PathString           string        `` /* 182-byte string literal not displayed */
	RotateInterval       time.Duration `help:"Time after which data will be rotated. If -1, it will not rotate until the end." default:"60s"`
	ZstdCompressionLevel int           `help:"zstd compression level, ranging from 1 (best speed) to 19 (best compression)." default:"3"`
	ZstdWindowLog        int           `` /* 183-byte string literal not displayed */
}

type Nop

type Nop struct {
	// contains filtered or unexported fields
}

func (*Nop) CanShard

func (b *Nop) CanShard() bool

func (*Nop) Run

func (nop *Nop) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*Nop) Stats

func (nop *Nop) Stats() baker.OutputStats

type NopConfig

type NopConfig struct{}

type OpLog

type OpLog struct {
	Cfg *OpLogConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*OpLog) CanShard

func (b *OpLog) CanShard() bool

func (*OpLog) Run

func (w *OpLog) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*OpLog) Stats

func (w *OpLog) Stats() baker.OutputStats

type OpLogConfig

type OpLogConfig struct{}

type SQLite

type SQLite struct {
	// contains filtered or unexported fields
}

func (*SQLite) CanShard

func (c *SQLite) CanShard() bool

func (*SQLite) Run

func (c *SQLite) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*SQLite) Stats

func (c *SQLite) Stats() baker.OutputStats

type SQLiteConfig

type SQLiteConfig struct {
	PathString string   `` /* 167-byte string literal not displayed */
	TableName  string   `help:"Table name to which to write the records to." required:"true"`
	PreRun     []string `help:"List of SQL statements to run at startup (before table creation)."`
	PostRun    []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
	Clear      bool     `` /* 233-byte string literal not displayed */
	Vacuum     bool     `` /* 259-byte string literal not displayed */
	Wal        bool     `` /* 167-byte string literal not displayed */
	PageSize   int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
}

SQLiteConfig holds the configuration parameters for the standard SQLite baker output.

type SQLiteRawWriterConfig

type SQLiteRawWriterConfig struct {
	PathString     string   `` /* 167-byte string literal not displayed */
	TableName      string   `help:"Table name to which to write the records to." required:"true"`
	PreRun         []string `help:"List of SQL statements to run at startup (before table creation)."`
	PostRun        []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
	Clear          bool     `` /* 233-byte string literal not displayed */
	Vacuum         bool     `` /* 259-byte string literal not displayed */
	Wal            bool     `` /* 167-byte string literal not displayed */
	PageSize       int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
	RecordBlobName string   `help:"Name of the column in which the whole raw record should be put." required:"true"`
}

SQLiteRawWriterConfig holds the configuration parameters for the raw SQLite baker output.

type Stats

type Stats struct {
	// contains filtered or unexported fields
}

The Stats output gathers statistics about the sizes of log lines it sees.

func (*Stats) CanShard

func (s *Stats) CanShard() bool

CanShard implements baker.Output

func (*Stats) Run

func (s *Stats) Run(input <-chan baker.OutputRecord, _ chan<- string) error

Run implements baker.Output

func (*Stats) Stats

func (s *Stats) Stats() baker.OutputStats

Stats implements baker.Output

type StatsConfig

type StatsConfig struct {
	CountEmptyFields bool   `help:"Whether fields with empty values are counted or not" default:"false"`
	CSVPath          string `help:"Path of the CSV file to create" default:"stats.csv"`
	TimestampField   string `help:"Name of a field containing a POSIX timestamp (in seconds) used to build the times stats" required:"true"`
}

type WebSocket

type WebSocket struct {
	Cfg *WebSocketConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*WebSocket) CanShard

func (b *WebSocket) CanShard() bool

func (*WebSocket) Run

func (w *WebSocket) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*WebSocket) Stats

func (w *WebSocket) Stats() baker.OutputStats

type WebSocketConfig

type WebSocketConfig struct{}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL