gbigtable

package
v0.5.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 24, 2024 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source
const (
	PreDefinedRowKeyTimestampIso      = "timestampIso"
	PreDefinedRowKeyUuid              = "uuid"
	PreDefinedRowKeyInvertedTimestamp = "invertedTimestamp"
	PreDefinedRowKeyKeysInMap         = "keysInMap"

	GarbageCollectionPolicyMaxVersions = "maxVersions"
	GarbageCollectionPolicyMaxAge      = "maxAge"
)

Variables

View Source
var ErrNotApplicable = errors.New("not applicable")

Functions

func NewLoaderFactory

func NewLoaderFactory(ctx context.Context, config Config) (entity.LoaderFactory, error)

Types

type BigTableAdminClient

type BigTableAdminClient interface {
	Tables(ctx context.Context) ([]string, error)
	CreateTable(ctx context.Context, table string) error
	TableInfo(ctx context.Context, table string) (*bigtable.TableInfo, error)
	CreateColumnFamily(ctx context.Context, table, family string) error
	SetGCPolicy(ctx context.Context, table, family string, policy bigtable.GCPolicy) error
	Close() error
}

type BigTableClient

type BigTableClient interface {
	Open(table string) *bigtable.Table
	Close() error
}

type BigTableTable

type BigTableTable interface {
	Apply(ctx context.Context, row string, m *bigtable.Mutation, opts ...bigtable.ApplyOption) (err error)

	// A missing row will return a zero-length map and a nil error.
	ReadRow(ctx context.Context, row string, opts ...bigtable.ReadOption) (bigtable.Row, error)
}

type ColumnFamily added in v0.4.0

type ColumnFamily struct {
	Name                    string                   `json:"name"`
	GarbageCollectionPolicy *GarbageCollectionPolicy `json:"garbageCollectionPolicy"`
	ColumnQualifiers        []ColumnQualifier        `json:"columnQualifiers"`
}

type ColumnQualifier added in v0.4.0

type ColumnQualifier struct {
	Id         string      `json:"id"`
	Name       string      `json:"name,omitempty"`
	NameFromId *NameFromId `json:"nameFromId,omitempty"`
}

The Id field can be used directly in the Transformed map to fetch the value to be inserted The Name field is the actual CQ name to be used in the table. Either Name or NameFromId must be present, not both.

type Config

type Config struct {

	// ProjectId (required) specifies GCP project ID for this deployment.
	ProjectId string

	// InstanceId (required) specifies BigTable instance ID
	InstanceId string
}

type GarbageCollectionPolicy added in v0.4.0

type GarbageCollectionPolicy struct {
	Type  string `json:"type"`
	Value int    `json:"value"`
}

TODO: Add support for Intersection and Union policies The following types are supported: - MaxVersions: where Value takes an integer of number of old versions to keep (-1) - MaxAge: where Value takes an integer of number of hours before deleting the data.

type NameFromId added in v0.4.0

type NameFromId struct {
	Prefix       string `json:"prefix"`
	SuffixFromId string `json:"suffixFromId"`
}

Creates a Column/CQ name from id outputs in transloaded event map

type Query

type Query struct {
	Type    QueryType
	Table   string
	RowKey  string
	LatestN int
}

type QueryType

type QueryType int
const (
	InvalidQueryType QueryType = iota
	KeyValue
	LatestN
	All
)

type RowKey added in v0.4.0

type RowKey struct {
	Predefined string   `json:"predefined,omitempty"`
	Keys       []string `json:"keys,omitempty"`
	Delimiter  string   `json:"delimiter,omitempty"`

	// Only required when using the Predefined rowkey option "keysInMap". This id should map to the transformed
	// output map item specified in ExtractItemsFromArray.Id
	MapId string `json:"mapId,omitempty"`
}

RowKey specifies how the row-key should be generated for BigTable sinks. If one of the Predefined options are set, that will be used. Currently available Predefined options are:

"timestampIso"
"invertedTimestamp"
"uuid"
"keysInMap"

If Predefined is not set, the Keys array should be used to specify which extracted fields from the event should be used. TODO: Add padding config

type RowKeyValue

type RowKeyValue struct {
	RowKey string
	Value  []byte
}

type SinkConfig added in v0.4.0

type SinkConfig struct {
	Tables []Table `json:"tables,omitempty"`
}

SinkConfig specifies the schema for the "customConfig" field in the "sink" section of the stream spec. It enables arbitrary connector specific fields to be present in the stream spec.

func NewSinkConfig added in v0.4.0

func NewSinkConfig(spec *entity.Spec) (sc SinkConfig, err error)

type Table added in v0.4.0

type Table struct {
	Name string `json:"name"`

	// Table spec for BigTable are built up by RowKey and ColumnFamilies
	RowKey         RowKey         `json:"rowKey"`
	ColumnFamilies []ColumnFamily `json:"columnFamilies"`

	// Only input transformations satisfying the whitelist key/value filter will be
	// processed by the sink (mostly needed in multi-table Sink specs)
	Whitelist *Whitelist `json:"whitelist,omitempty"`
}

type Whitelist added in v0.4.0

type Whitelist struct {
	Id     string   `json:"id"`
	Type   string   `json:"type"`
	Values []string `json:"values"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL