storage

package
v0.0.16 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 24, 2023 License: MIT Imports: 7 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type GCSConfig

type GCSConfig struct {
	BucketName string `env:"GCS_BUCKET_NAME,required"`
	ProjectID  string `env:"GCP_PROJECT_ID,required"`
	RangeSize  uint64 `env:"GCS_DIR_RANGE_SIZE" envDefault:"10000"`
}

type GCSConnector

type GCSConnector struct {
	// contains filtered or unexported fields
}

func MustNewGCSConnector

func MustNewGCSConnector(ctx context.Context, cfg *GCSConfig, logger framework.Logger) *GCSConnector

func NewGCSConnector

func NewGCSConnector(ctx context.Context, cfg *GCSConfig) (*GCSConnector, error)

func (*GCSConnector) Bucket

func (g *GCSConnector) Bucket() string

func (*GCSConnector) ProjectID

func (g *GCSConnector) ProjectID() string

func (*GCSConnector) RangeSize

func (g *GCSConnector) RangeSize() uint64

func (*GCSConnector) WriteMany

func (g *GCSConnector) WriteMany(ctx context.Context, input []interface{}, mapToStruct interface{}, filename string) error

Write writes a mutliple parquets to GCS storage

func (*GCSConnector) WriteOne

func (g *GCSConnector) WriteOne(ctx context.Context, input interface{}, mapToStruct interface{}, filename string) error

Write writes a single parquet to GCS storage

type Store

type Store interface {
	WriteOne(ctx context.Context, input interface{}, mapToStruct interface{}, filename string) error
	WriteMany(ctx context.Context, input []interface{}, mapToStruct interface{}, filename string) error
	ProjectID() string
	Bucket() string
	RangeSize() uint64
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL