flow

package module
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 12, 2021 License: MIT Imports: 8 Imported by: 0

README

flow

Documentation

Index

Constants

View Source
const HeartbeatInterval = time.Second

Variables

View Source
var ErrTerminate = errors.New("source terminated")

ErrTerminate should be returned from Run to stop running the source.

Functions

func Batch

func Batch(topic Topic, size int, flushEvery time.Duration) *batchTransform

func BatchByTopic

func BatchByTopic(maxBatchSize int, flushEvery time.Duration) *batchByTopicTransform

func NewBigquerySink

func NewBigquerySink(ctx context.Context, projectID, datasetID string) (*bqSink, error)

func Run

func Run(ctx context.Context, runnables ...Runnable)

Types

type Fact

type Fact struct {
	FactID        string    `bigquery:"_factId"`
	TransactionID string    `bigquery:"_transactionId"`
	Data          string    `bigquery:"_data"`
	Timestamp     time.Time `bigquery:"_timestamp"`
}

type Runnable

type Runnable interface {
	Heartbeat(ctx context.Context)
	Run(ctx context.Context) (nextInterval time.Duration, err error)
}

type Sink

type Sink interface {
	Step
}

type Source

type Source struct {
	Next Step
}

func (*Source) Heartbeat

func (s *Source) Heartbeat(ctx context.Context)

func (*Source) Into

func (s *Source) Into(next Sink)

func (*Source) Run

func (s *Source) Run(ctx context.Context) (time.Duration, error)

func (*Source) Transform

func (s *Source) Transform(next Transform) Transform

type Step

type Step interface {
	Heartbeat(ctx context.Context)
	Write(ctx context.Context, topic Topic, facts ...Fact) error
}

type Topic

type Topic = string

type Transform

type Transform interface {
	Step
	Transform(next Transform) Transform
	Into(next Sink)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL