bigquery

package
v0.12.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 2, 2024 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ConcurrentTicketPerSec = 5
	ConcurrentLimit        = 20
)
View Source
const (
	EntityDataset = "dataset"

	DatesetNameSections = 2
	TableNameSections   = 3
)
View Source
const (
	ModeNullable = "nullable"
	ModeRequired = "required"
	ModeRepeated = "repeated"

	EntityResourceSchema = "bigquery_schema"
)
View Source
const (
	KindDataset       string = "dataset"
	KindTable         string = "table"
	KindView          string = "view"
	KindExternalTable string = "external_table"
)
View Source
const (
	EntityExternalTable = "resource_external_table"
)
View Source
const (
	EntityTable = "resource_table"
)
View Source
const (
	EntityView = "resource_view"
)

Variables

This section is empty.

Functions

func BackupResources

func BackupResources(ctx context.Context, backup *resource.Backup, resources []*resource.Resource, client Client) (*resource.BackupResult, error)

func BackupTable

func BackupTable(ctx context.Context, backup *resource.Backup, source *resource.Resource, client Client) (string, error)

func BatchesFrom

func BatchesFrom(resources []*resource.Resource, provider ClientProvider) (map[string]*Batch, error)

func ConfigAs

func ConfigAs[T any](mapping map[string]any, key string) T

func ConvertSpecTo

func ConvertSpecTo[T DatasetDetails | Table | View | ExternalTable](res *resource.Resource) (*T, error)

func CopyTable

func CopyTable(ctx context.Context, source, destination TableResourceHandle) error

func CreateIfDatasetDoesNotExist

func CreateIfDatasetDoesNotExist(ctx context.Context, client Client, dataset Dataset) error

func DestinationExpiry

func DestinationExpiry(backup *resource.Backup) (time.Time, error)

func DestinationName

func DestinationName(sourceDatasetName, sourceName string, backup *resource.Backup) string

func ResourceNameFor

func ResourceNameFor(name resource.Name, kind string) (string, error)

func URNFor

func URNFor(res *resource.Resource) (resource.URN, error)

func ValidateName

func ValidateName(res *resource.Resource) error

Types

type BQCopier

type BQCopier interface {
	Run(context.Context) (*bigquery.Job, error)
}

type BQJob

type BQJob interface {
	Wait(context.Context) (*bigquery.JobStatus, error)
}

type Batch

type Batch struct {
	Dataset        Dataset
	DatasetDetails *resource.Resource

	Tables         []*resource.Resource
	ExternalTables []*resource.Resource
	Views          []*resource.Resource
	// contains filtered or unexported fields
}

func (*Batch) QueueJobs

func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel.Runner) error

type BqClient

type BqClient struct {
	*bigquery.Client
}

func NewClient

func NewClient(ctx context.Context, svcAccount string) (*BqClient, error)

func (*BqClient) BulkGetDDLView added in v0.10.1

func (c *BqClient) BulkGetDDLView(ctx context.Context, pd ProjectDataset, names []string) (map[ResourceURN]string, error)

func (*BqClient) DatasetHandleFrom

func (c *BqClient) DatasetHandleFrom(ds Dataset) ResourceHandle

func (*BqClient) ExternalTableHandleFrom

func (c *BqClient) ExternalTableHandleFrom(ds Dataset, name string) ResourceHandle

func (*BqClient) TableHandleFrom

func (c *BqClient) TableHandleFrom(ds Dataset, name string) TableResourceHandle

func (*BqClient) ViewHandleFrom

func (c *BqClient) ViewHandleFrom(ds Dataset, name string) ResourceHandle

type BqClientProvider

type BqClientProvider struct{}

func NewClientProvider

func NewClientProvider() *BqClientProvider

func (BqClientProvider) Get

func (BqClientProvider) Get(ctx context.Context, account string) (Client, error)

type Client

type Client interface {
	DatasetHandleFrom(dataset Dataset) ResourceHandle
	TableHandleFrom(dataset Dataset, name string) TableResourceHandle
	ExternalTableHandleFrom(dataset Dataset, name string) ResourceHandle
	ViewHandleFrom(dataset Dataset, name string) ResourceHandle
	BulkGetDDLView(ctx context.Context, dataset ProjectDataset, names []string) (map[ResourceURN]string, error)
	Close() error
}

type ClientProvider

type ClientProvider interface {
	Get(ctx context.Context, account string) (Client, error)
}

type Cluster

type Cluster struct {
	Using []string `mapstructure:"using,omitempty"`
}

func (Cluster) Validate

func (c Cluster) Validate() error

type Copier

type Copier struct {
	// contains filtered or unexported fields
}

func NewCopier

func NewCopier(bqCopier BQCopier) *Copier

func (Copier) Run

func (c Copier) Run(ctx context.Context) (CopyJob, error)

type CopyJob

type CopyJob interface {
	Wait(ctx context.Context) error
}

type Dataset

type Dataset struct {
	Project     string
	DatasetName string
}

func DataSetFor

func DataSetFor(name resource.Name) (Dataset, error)

func DataSetFrom

func DataSetFrom(project, datasetName string) (Dataset, error)

func DestinationDataset

func DestinationDataset(project string, backup *resource.Backup) (Dataset, error)

func (Dataset) FullName

func (d Dataset) FullName() string

type DatasetDetails

type DatasetDetails struct {
	Name resource.Name

	Description string                 `mapstructure:"description,omitempty"`
	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (DatasetDetails) FullName

func (d DatasetDetails) FullName() string

func (DatasetDetails) Validate

func (DatasetDetails) Validate() error

type DatasetHandle

type DatasetHandle struct {
	// contains filtered or unexported fields
}

func NewDatasetHandle

func NewDatasetHandle(ds BqDataset) *DatasetHandle

func (DatasetHandle) Create

func (d DatasetHandle) Create(ctx context.Context, res *resource.Resource) error

func (DatasetHandle) Exists

func (d DatasetHandle) Exists(ctx context.Context) bool

func (DatasetHandle) Update

func (d DatasetHandle) Update(ctx context.Context, res *resource.Resource) error

type ExternalSource

type ExternalSource struct {
	SourceType string   `mapstructure:"type,omitempty"`
	SourceURIs []string `mapstructure:"uris,omitempty"`

	// Additional configs for CSV, GoogleSheets, Bigtable, and Parquet formats.
	Config map[string]interface{} `mapstructure:"config"`
}

func (ExternalSource) Validate

func (e ExternalSource) Validate() error

type ExternalTable

type ExternalTable struct {
	Name resource.Name

	Description string          `mapstructure:"description,omitempty"`
	Schema      Schema          `mapstructure:"schema,omitempty"`
	Source      *ExternalSource `mapstructure:"source,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*ExternalTable) FullName

func (e *ExternalTable) FullName() string

func (*ExternalTable) Validate

func (e *ExternalTable) Validate() error

type ExternalTableHandle

type ExternalTableHandle struct {
	// contains filtered or unexported fields
}

func NewExternalTableHandle

func NewExternalTableHandle(bq BqTable) *ExternalTableHandle

func (ExternalTableHandle) Create

func (ExternalTableHandle) Exists

func (et ExternalTableHandle) Exists(ctx context.Context) bool

func (ExternalTableHandle) Update

type Field

type Field struct {
	Name        string `mapstructure:"name,omitempty"`
	Type        string `mapstructure:"type,omitempty"`
	Description string `mapstructure:"description,omitempty"`
	Mode        string `mapstructure:"mode,omitempty"`

	// optional sub-schema, when record type
	Schema Schema `mapstructure:"schema,omitempty"`
}

func (Field) Validate

func (f Field) Validate() error

type JobHandle

type JobHandle struct {
	// contains filtered or unexported fields
}

func NewJob

func NewJob(job BQJob) *JobHandle

func (JobHandle) Wait

func (j JobHandle) Wait(ctx context.Context) error

type Partition

type Partition struct {
	Field string `mapstructure:"field,omitempty"`

	Type       string `mapstructure:"type,omitempty"`
	Expiration int64  `mapstructure:"expiration,omitempty"`

	Range *Range `mapstructure:"range,omitempty"`
}

func (Partition) Validate

func (p Partition) Validate() error

type ProjectDataset added in v0.10.1

type ProjectDataset struct {
	Project string
	Dataset string
}

type Range

type Range struct {
	Start    int64 `mapstructure:"start,omitempty"`
	End      int64 `mapstructure:"end,omitempty"`
	Interval int64 `mapstructure:"interval,omitempty"`
}

type ResourceHandle

type ResourceHandle interface {
	Create(ctx context.Context, res *resource.Resource) error
	Update(ctx context.Context, res *resource.Resource) error
	Exists(ctx context.Context) bool
}

type ResourceURN added in v0.10.1

type ResourceURN struct {
	// contains filtered or unexported fields
}

func NewResourceURN added in v0.10.1

func NewResourceURN(project, dataset, name string) (ResourceURN, error)

func NewResourceURNFromString added in v0.10.1

func NewResourceURNFromString(urn string) (ResourceURN, error)

func (ResourceURN) Dataset added in v0.10.1

func (n ResourceURN) Dataset() string

func (ResourceURN) Name added in v0.10.1

func (n ResourceURN) Name() string

func (ResourceURN) Project added in v0.10.1

func (n ResourceURN) Project() string

func (ResourceURN) URN added in v0.10.1

func (n ResourceURN) URN() string

type ResourceURNWithUpstreams added in v0.10.2

type ResourceURNWithUpstreams struct {
	ResourceURN ResourceURN
	Upstreams   []*ResourceURNWithUpstreams
}

type ResourceURNWithUpstreamsList added in v0.10.2

type ResourceURNWithUpstreamsList []*ResourceURNWithUpstreams

func (ResourceURNWithUpstreamsList) FlattenUnique added in v0.10.2

type ResourceURNs added in v0.10.1

type ResourceURNs []ResourceURN

func (ResourceURNs) GroupByProjectDataset added in v0.10.1

func (n ResourceURNs) GroupByProjectDataset() map[ProjectDataset][]string

type Schema

type Schema []Field

func (Schema) Validate

func (s Schema) Validate() error

type SecretProvider

type SecretProvider interface {
	GetSecret(ctx context.Context, tnnt tenant.Tenant, key string) (*tenant.PlainTextSecret, error)
}

type Store

type Store struct {
	// contains filtered or unexported fields
}

func NewBigqueryDataStore

func NewBigqueryDataStore(secretProvider SecretProvider, clientProvider ClientProvider) *Store

func (Store) Backup

func (s Store) Backup(ctx context.Context, backup *resource.Backup, resources []*resource.Resource) (*resource.BackupResult, error)

func (Store) BatchUpdate

func (s Store) BatchUpdate(ctx context.Context, resources []*resource.Resource) error

func (Store) Create

func (s Store) Create(ctx context.Context, res *resource.Resource) error

func (Store) Exist added in v0.12.0

func (s Store) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error)

func (Store) GetURN

func (Store) GetURN(res *resource.Resource) (resource.URN, error)

func (Store) Update

func (s Store) Update(ctx context.Context, res *resource.Resource) error

func (Store) Validate

func (Store) Validate(r *resource.Resource) error

type Table

type Table struct {
	Name resource.Name

	Description string     `mapstructure:"description,omitempty"`
	Schema      Schema     `mapstructure:"schema,omitempty"`
	Cluster     *Cluster   `mapstructure:"cluster,omitempty"`
	Partition   *Partition `mapstructure:"partition,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*Table) FullName

func (t *Table) FullName() string

func (*Table) Validate

func (t *Table) Validate() error

type TableCopier

type TableCopier interface {
	Run(ctx context.Context) (CopyJob, error)
}

type TableHandle

type TableHandle struct {
	// contains filtered or unexported fields
}

func NewTableHandle

func NewTableHandle(bq BqTable) *TableHandle

func (TableHandle) CopierFrom

func (t TableHandle) CopierFrom(source TableResourceHandle) (TableCopier, error)

func (TableHandle) Create

func (t TableHandle) Create(ctx context.Context, res *resource.Resource) error

func (TableHandle) Exists

func (t TableHandle) Exists(ctx context.Context) bool

func (TableHandle) GetBQTable

func (t TableHandle) GetBQTable() (*bigquery.Table, error)

func (TableHandle) Update

func (t TableHandle) Update(ctx context.Context, res *resource.Resource) error

func (TableHandle) UpdateExpiry

func (t TableHandle) UpdateExpiry(ctx context.Context, name string, expiry time.Time) error

type TableResourceHandle

type TableResourceHandle interface {
	ResourceHandle
	GetBQTable() (*bq.Table, error)
	CopierFrom(source TableResourceHandle) (TableCopier, error)
	UpdateExpiry(ctx context.Context, name string, expiry time.Time) error
}

type View

type View struct {
	Name resource.Name

	Description string `mapstructure:"description,omitempty"`
	ViewQuery   string `mapstructure:"view_query,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*View) Validate

func (v *View) Validate() error

type ViewHandle

type ViewHandle struct {
	// contains filtered or unexported fields
}

func NewViewHandle

func NewViewHandle(bq BqTable) *ViewHandle

func (ViewHandle) Create

func (v ViewHandle) Create(ctx context.Context, res *resource.Resource) error

func (ViewHandle) Exists

func (v ViewHandle) Exists(ctx context.Context) bool

func (ViewHandle) Update

func (v ViewHandle) Update(ctx context.Context, res *resource.Resource) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL