file_storage

package
v0.0.0-...-c6cbf5f Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 7, 2023 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const FilenameDate = "2006_01_02T15_04_05"
View Source
const GCSAutocommitUnsupported = "Stream mode is not supported for GCS. Please use 'batch' mode"
View Source
const GCSBulkerTypeId = "gcs"
View Source
const S3AutocommitUnsupported = "Stream mode is not supported for GCS. Please use 'batch' mode"
View Source
const S3BulkerTypeId = "s3"

Variables

This section is empty.

Functions

func NewGCSBulker

func NewGCSBulker(bulkerConfig bulker.Config) (bulker.Bulker, error)

func NewReplacePartitionStream

func NewReplacePartitionStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error)

func NewReplaceTableStream

func NewReplaceTableStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error)

func NewS3Bulker

func NewS3Bulker(bulkerConfig bulker.Config) (bulker.Bulker, error)

func NewTransactionalStream

func NewTransactionalStream(id string, p implementations.FileAdapter, tableName string, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error)

Types

type AbstractFileStorageStream

type AbstractFileStorageStream struct {
	// contains filtered or unexported fields
}

func (*AbstractFileStorageStream) Abort

func (ps *AbstractFileStorageStream) Abort(ctx context.Context) (state bulker.State, err error)

func (*AbstractFileStorageStream) Complete

func (ps *AbstractFileStorageStream) Complete(ctx context.Context) (state bulker.State, err error)

func (*AbstractFileStorageStream) Consume

func (ps *AbstractFileStorageStream) Consume(ctx context.Context, object types.Object) (state bulker.State, processedObjects []types.Object, err error)

type GCSBulker

type GCSBulker struct {
	implementations.GoogleCloudStorage
}

func (*GCSBulker) CreateStream

func (gcs *GCSBulker) CreateStream(id, tableName string, mode bulker.BulkMode, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error)

type GCSConfig

type GCSConfig struct {
	implementations.FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
	Bucket                     string `mapstructure:"bucket,omitempty" json:"bucket,omitempty" yaml:"bucket,omitempty"`
	AccessKey                  any    `mapstructure:"accessKey,omitempty" json:"accessKey,omitempty" yaml:"accessKey,omitempty"`
}

type ReplacePartitionStream

type ReplacePartitionStream struct {
	AbstractFileStorageStream
}

func (*ReplacePartitionStream) Complete

func (ps *ReplacePartitionStream) Complete(ctx context.Context) (state bulker.State, err error)

type ReplaceTableStream

type ReplaceTableStream struct {
	AbstractFileStorageStream
}

func (*ReplaceTableStream) Complete

func (ps *ReplaceTableStream) Complete(ctx context.Context) (state bulker.State, err error)

type S3Bulker

type S3Bulker struct {
	implementations.S3
}

func (*S3Bulker) CreateStream

func (s3 *S3Bulker) CreateStream(id, tableName string, mode bulker.BulkMode, streamOptions ...bulker.StreamOption) (bulker.BulkerStream, error)

type TransactionalStream

type TransactionalStream struct {
	AbstractFileStorageStream
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL