dataflowdatav1beta3

package
v0.8.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 29, 2024 License: Apache-2.0 Imports: 6 Imported by: 0

Documentation

Overview

Package dataflowdatav1beta3 provides Dataflow type definitions for CloudEvent data payloads.

Supported CloudEvent Types

  • google.cloud.dataflow.job.v1beta3.statusChanged

Index

Constants

This section is empty.

Variables

View Source
var (
	JobType_name = map[int32]string{
		0: "JOB_TYPE_UNKNOWN",
		1: "JOB_TYPE_BATCH",
		2: "JOB_TYPE_STREAMING",
	}
	JobType_value = map[string]int32{
		"JOB_TYPE_UNKNOWN":   0,
		"JOB_TYPE_BATCH":     1,
		"JOB_TYPE_STREAMING": 2,
	}
)

Enum value maps for JobType.

View Source
var (
	FlexResourceSchedulingGoal_name = map[int32]string{
		0: "FLEXRS_UNSPECIFIED",
		1: "FLEXRS_SPEED_OPTIMIZED",
		2: "FLEXRS_COST_OPTIMIZED",
	}
	FlexResourceSchedulingGoal_value = map[string]int32{
		"FLEXRS_UNSPECIFIED":     0,
		"FLEXRS_SPEED_OPTIMIZED": 1,
		"FLEXRS_COST_OPTIMIZED":  2,
	}
)

Enum value maps for FlexResourceSchedulingGoal.

View Source
var (
	TeardownPolicy_name = map[int32]string{
		0: "TEARDOWN_POLICY_UNKNOWN",
		1: "TEARDOWN_ALWAYS",
		2: "TEARDOWN_ON_SUCCESS",
		3: "TEARDOWN_NEVER",
	}
	TeardownPolicy_value = map[string]int32{
		"TEARDOWN_POLICY_UNKNOWN": 0,
		"TEARDOWN_ALWAYS":         1,
		"TEARDOWN_ON_SUCCESS":     2,
		"TEARDOWN_NEVER":          3,
	}
)

Enum value maps for TeardownPolicy.

View Source
var (
	DefaultPackageSet_name = map[int32]string{
		0: "DEFAULT_PACKAGE_SET_UNKNOWN",
		1: "DEFAULT_PACKAGE_SET_NONE",
		2: "DEFAULT_PACKAGE_SET_JAVA",
		3: "DEFAULT_PACKAGE_SET_PYTHON",
	}
	DefaultPackageSet_value = map[string]int32{
		"DEFAULT_PACKAGE_SET_UNKNOWN": 0,
		"DEFAULT_PACKAGE_SET_NONE":    1,
		"DEFAULT_PACKAGE_SET_JAVA":    2,
		"DEFAULT_PACKAGE_SET_PYTHON":  3,
	}
)

Enum value maps for DefaultPackageSet.

View Source
var (
	AutoscalingAlgorithm_name = map[int32]string{
		0: "AUTOSCALING_ALGORITHM_UNKNOWN",
		1: "AUTOSCALING_ALGORITHM_NONE",
		2: "AUTOSCALING_ALGORITHM_BASIC",
	}
	AutoscalingAlgorithm_value = map[string]int32{
		"AUTOSCALING_ALGORITHM_UNKNOWN": 0,
		"AUTOSCALING_ALGORITHM_NONE":    1,
		"AUTOSCALING_ALGORITHM_BASIC":   2,
	}
)

Enum value maps for AutoscalingAlgorithm.

View Source
var (
	WorkerIPAddressConfiguration_name = map[int32]string{
		0: "WORKER_IP_UNSPECIFIED",
		1: "WORKER_IP_PUBLIC",
		2: "WORKER_IP_PRIVATE",
	}
	WorkerIPAddressConfiguration_value = map[string]int32{
		"WORKER_IP_UNSPECIFIED": 0,
		"WORKER_IP_PUBLIC":      1,
		"WORKER_IP_PRIVATE":     2,
	}
)

Enum value maps for WorkerIPAddressConfiguration.

View Source
var (
	ShuffleMode_name = map[int32]string{
		0: "SHUFFLE_MODE_UNSPECIFIED",
		1: "VM_BASED",
		2: "SERVICE_BASED",
	}
	ShuffleMode_value = map[string]int32{
		"SHUFFLE_MODE_UNSPECIFIED": 0,
		"VM_BASED":                 1,
		"SERVICE_BASED":            2,
	}
)

Enum value maps for ShuffleMode.

View Source
var (
	JobState_name = map[int32]string{
		0:  "JOB_STATE_UNKNOWN",
		1:  "JOB_STATE_STOPPED",
		2:  "JOB_STATE_RUNNING",
		3:  "JOB_STATE_DONE",
		4:  "JOB_STATE_FAILED",
		5:  "JOB_STATE_CANCELLED",
		6:  "JOB_STATE_UPDATED",
		7:  "JOB_STATE_DRAINING",
		8:  "JOB_STATE_DRAINED",
		9:  "JOB_STATE_PENDING",
		10: "JOB_STATE_CANCELLING",
		11: "JOB_STATE_QUEUED",
		12: "JOB_STATE_RESOURCE_CLEANING_UP",
	}
	JobState_value = map[string]int32{
		"JOB_STATE_UNKNOWN":              0,
		"JOB_STATE_STOPPED":              1,
		"JOB_STATE_RUNNING":              2,
		"JOB_STATE_DONE":                 3,
		"JOB_STATE_FAILED":               4,
		"JOB_STATE_CANCELLED":            5,
		"JOB_STATE_UPDATED":              6,
		"JOB_STATE_DRAINING":             7,
		"JOB_STATE_DRAINED":              8,
		"JOB_STATE_PENDING":              9,
		"JOB_STATE_CANCELLING":           10,
		"JOB_STATE_QUEUED":               11,
		"JOB_STATE_RESOURCE_CLEANING_UP": 12,
	}
)

Enum value maps for JobState.

View Source
var (
	SdkVersion_SdkSupportStatus_name = map[int32]string{
		0: "UNKNOWN",
		1: "SUPPORTED",
		2: "STALE",
		3: "DEPRECATED",
		4: "UNSUPPORTED",
	}
	SdkVersion_SdkSupportStatus_value = map[string]int32{
		"UNKNOWN":     0,
		"SUPPORTED":   1,
		"STALE":       2,
		"DEPRECATED":  3,
		"UNSUPPORTED": 4,
	}
)

Enum value maps for SdkVersion_SdkSupportStatus.

View Source
var File_cloud_dataflow_v1beta3_data_proto protoreflect.FileDescriptor

Functions

This section is empty.

Types

type AutoscalingAlgorithm

type AutoscalingAlgorithm int32

Specifies the algorithm used to determine the number of worker processes to run at any given point in time, based on the amount of data left to process, the number of workers, and how quickly existing workers are processing data.

const (
	// The algorithm is unknown, or unspecified.
	AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_UNKNOWN AutoscalingAlgorithm = 0
	// Disable autoscaling.
	AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_NONE AutoscalingAlgorithm = 1
	// Increase worker count over time to reduce job execution time.
	AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_BASIC AutoscalingAlgorithm = 2
)

func (AutoscalingAlgorithm) Descriptor

func (AutoscalingAlgorithm) Enum

func (AutoscalingAlgorithm) EnumDescriptor deprecated

func (AutoscalingAlgorithm) EnumDescriptor() ([]byte, []int)

Deprecated: Use AutoscalingAlgorithm.Descriptor instead.

func (AutoscalingAlgorithm) Number

func (AutoscalingAlgorithm) String

func (x AutoscalingAlgorithm) String() string

func (AutoscalingAlgorithm) Type

type AutoscalingSettings

type AutoscalingSettings struct {

	// The algorithm to use for autoscaling.
	Algorithm AutoscalingAlgorithm `` /* 135-byte string literal not displayed */
	// The maximum number of workers to cap scaling at.
	MaxNumWorkers int32 `protobuf:"varint,2,opt,name=max_num_workers,json=maxNumWorkers,proto3" json:"max_num_workers,omitempty"`
	// contains filtered or unexported fields
}

Settings for WorkerPool autoscaling.

func (*AutoscalingSettings) Descriptor deprecated

func (*AutoscalingSettings) Descriptor() ([]byte, []int)

Deprecated: Use AutoscalingSettings.ProtoReflect.Descriptor instead.

func (*AutoscalingSettings) GetAlgorithm

func (x *AutoscalingSettings) GetAlgorithm() AutoscalingAlgorithm

func (*AutoscalingSettings) GetMaxNumWorkers

func (x *AutoscalingSettings) GetMaxNumWorkers() int32

func (*AutoscalingSettings) ProtoMessage

func (*AutoscalingSettings) ProtoMessage()

func (*AutoscalingSettings) ProtoReflect

func (x *AutoscalingSettings) ProtoReflect() protoreflect.Message

func (*AutoscalingSettings) Reset

func (x *AutoscalingSettings) Reset()

func (*AutoscalingSettings) String

func (x *AutoscalingSettings) String() string

type BigQueryIODetails

type BigQueryIODetails struct {

	// Table accessed in the connection.
	Table string `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"`
	// Dataset accessed in the connection.
	Dataset string `protobuf:"bytes,2,opt,name=dataset,proto3" json:"dataset,omitempty"`
	// Project accessed in the connection.
	ProjectId string `protobuf:"bytes,3,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
	// Query used to access data in the connection.
	Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a BigQuery connector used by the job.

func (*BigQueryIODetails) Descriptor deprecated

func (*BigQueryIODetails) Descriptor() ([]byte, []int)

Deprecated: Use BigQueryIODetails.ProtoReflect.Descriptor instead.

func (*BigQueryIODetails) GetDataset

func (x *BigQueryIODetails) GetDataset() string

func (*BigQueryIODetails) GetProjectId

func (x *BigQueryIODetails) GetProjectId() string

func (*BigQueryIODetails) GetQuery

func (x *BigQueryIODetails) GetQuery() string

func (*BigQueryIODetails) GetTable

func (x *BigQueryIODetails) GetTable() string

func (*BigQueryIODetails) ProtoMessage

func (*BigQueryIODetails) ProtoMessage()

func (*BigQueryIODetails) ProtoReflect

func (x *BigQueryIODetails) ProtoReflect() protoreflect.Message

func (*BigQueryIODetails) Reset

func (x *BigQueryIODetails) Reset()

func (*BigQueryIODetails) String

func (x *BigQueryIODetails) String() string

type BigTableIODetails

type BigTableIODetails struct {

	// ProjectId accessed in the connection.
	ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
	// InstanceId accessed in the connection.
	InstanceId string `protobuf:"bytes,2,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"`
	// TableId accessed in the connection.
	TableId string `protobuf:"bytes,3,opt,name=table_id,json=tableId,proto3" json:"table_id,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a Cloud Bigtable connector used by the job.

func (*BigTableIODetails) Descriptor deprecated

func (*BigTableIODetails) Descriptor() ([]byte, []int)

Deprecated: Use BigTableIODetails.ProtoReflect.Descriptor instead.

func (*BigTableIODetails) GetInstanceId

func (x *BigTableIODetails) GetInstanceId() string

func (*BigTableIODetails) GetProjectId

func (x *BigTableIODetails) GetProjectId() string

func (*BigTableIODetails) GetTableId

func (x *BigTableIODetails) GetTableId() string

func (*BigTableIODetails) ProtoMessage

func (*BigTableIODetails) ProtoMessage()

func (*BigTableIODetails) ProtoReflect

func (x *BigTableIODetails) ProtoReflect() protoreflect.Message

func (*BigTableIODetails) Reset

func (x *BigTableIODetails) Reset()

func (*BigTableIODetails) String

func (x *BigTableIODetails) String() string

type DatastoreIODetails

type DatastoreIODetails struct {

	// Namespace used in the connection.
	Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"`
	// ProjectId accessed in the connection.
	ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a Datastore connector used by the job.

func (*DatastoreIODetails) Descriptor deprecated

func (*DatastoreIODetails) Descriptor() ([]byte, []int)

Deprecated: Use DatastoreIODetails.ProtoReflect.Descriptor instead.

func (*DatastoreIODetails) GetNamespace

func (x *DatastoreIODetails) GetNamespace() string

func (*DatastoreIODetails) GetProjectId

func (x *DatastoreIODetails) GetProjectId() string

func (*DatastoreIODetails) ProtoMessage

func (*DatastoreIODetails) ProtoMessage()

func (*DatastoreIODetails) ProtoReflect

func (x *DatastoreIODetails) ProtoReflect() protoreflect.Message

func (*DatastoreIODetails) Reset

func (x *DatastoreIODetails) Reset()

func (*DatastoreIODetails) String

func (x *DatastoreIODetails) String() string

type DebugOptions

type DebugOptions struct {

	// When true, enables the logging of the literal hot key to the user's Cloud
	// Logging.
	EnableHotKeyLogging bool `protobuf:"varint,1,opt,name=enable_hot_key_logging,json=enableHotKeyLogging,proto3" json:"enable_hot_key_logging,omitempty"`
	// contains filtered or unexported fields
}

Describes any options that have an effect on the debugging of pipelines.

func (*DebugOptions) Descriptor deprecated

func (*DebugOptions) Descriptor() ([]byte, []int)

Deprecated: Use DebugOptions.ProtoReflect.Descriptor instead.

func (*DebugOptions) GetEnableHotKeyLogging

func (x *DebugOptions) GetEnableHotKeyLogging() bool

func (*DebugOptions) ProtoMessage

func (*DebugOptions) ProtoMessage()

func (*DebugOptions) ProtoReflect

func (x *DebugOptions) ProtoReflect() protoreflect.Message

func (*DebugOptions) Reset

func (x *DebugOptions) Reset()

func (*DebugOptions) String

func (x *DebugOptions) String() string

type DefaultPackageSet

type DefaultPackageSet int32

The default set of packages to be staged on a pool of workers.

const (
	// The default set of packages to stage is unknown, or unspecified.
	DefaultPackageSet_DEFAULT_PACKAGE_SET_UNKNOWN DefaultPackageSet = 0
	// Indicates that no packages should be staged at the worker unless
	// explicitly specified by the job.
	DefaultPackageSet_DEFAULT_PACKAGE_SET_NONE DefaultPackageSet = 1
	// Stage packages typically useful to workers written in Java.
	DefaultPackageSet_DEFAULT_PACKAGE_SET_JAVA DefaultPackageSet = 2
	// Stage packages typically useful to workers written in Python.
	DefaultPackageSet_DEFAULT_PACKAGE_SET_PYTHON DefaultPackageSet = 3
)

func (DefaultPackageSet) Descriptor

func (DefaultPackageSet) Enum

func (DefaultPackageSet) EnumDescriptor deprecated

func (DefaultPackageSet) EnumDescriptor() ([]byte, []int)

Deprecated: Use DefaultPackageSet.Descriptor instead.

func (DefaultPackageSet) Number

func (DefaultPackageSet) String

func (x DefaultPackageSet) String() string

func (DefaultPackageSet) Type

type Environment

type Environment struct {

	// The prefix of the resources the system should use for temporary
	// storage.  The system will append the suffix "/temp-{JOBNAME} to
	// this resource prefix, where {JOBNAME} is the value of the
	// job_name field.  The resulting bucket and object prefix is used
	// as the prefix of the resources used to store temporary data
	// needed during the job execution.  NOTE: This will override the
	// value in taskrunner_settings.
	// The supported resource type is:
	//
	// Google Cloud Storage:
	//
	//	storage.googleapis.com/{bucket}/{object}
	//	bucket.storage.googleapis.com/{object}
	TempStoragePrefix string `protobuf:"bytes,1,opt,name=temp_storage_prefix,json=tempStoragePrefix,proto3" json:"temp_storage_prefix,omitempty"`
	// The type of cluster manager API to use.  If unknown or
	// unspecified, the service will attempt to choose a reasonable
	// default.  This should be in the form of the API service name,
	// e.g. "compute.googleapis.com".
	ClusterManagerApiService string `` /* 137-byte string literal not displayed */
	// The list of experiments to enable. This field should be used for SDK
	// related experiments and not for service related experiments. The proper
	// field for service related experiments is service_options.
	Experiments []string `protobuf:"bytes,3,rep,name=experiments,proto3" json:"experiments,omitempty"`
	// The list of service options to enable. This field should be used for
	// service related experiments only. These experiments, when graduating to GA,
	// should be replaced by dedicated fields or become default (i.e. always on).
	ServiceOptions []string `protobuf:"bytes,16,rep,name=service_options,json=serviceOptions,proto3" json:"service_options,omitempty"`
	// If set, contains the Cloud KMS key identifier used to encrypt data
	// at rest, AKA a Customer Managed Encryption Key (CMEK).
	//
	// Format:
	//
	//	projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
	ServiceKmsKeyName string `protobuf:"bytes,12,opt,name=service_kms_key_name,json=serviceKmsKeyName,proto3" json:"service_kms_key_name,omitempty"`
	// The worker pools. At least one "harness" worker pool must be
	// specified in order for the job to have workers.
	WorkerPools []*WorkerPool `protobuf:"bytes,4,rep,name=worker_pools,json=workerPools,proto3" json:"worker_pools,omitempty"`
	// A description of the process that generated the request.
	UserAgent *structpb.Struct `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
	// A structure describing which components and their versions of the service
	// are required in order to run the job.
	Version *structpb.Struct `protobuf:"bytes,6,opt,name=version,proto3" json:"version,omitempty"`
	// The dataset for the current project where various workflow
	// related tables are stored.
	//
	// The supported resource type is:
	//
	// Google BigQuery:
	//
	//	bigquery.googleapis.com/{dataset}
	Dataset string `protobuf:"bytes,7,opt,name=dataset,proto3" json:"dataset,omitempty"`
	// The Cloud Dataflow SDK pipeline options specified by the user. These
	// options are passed through the service and are used to recreate the
	// SDK pipeline options on the worker in a language agnostic and platform
	// independent way.
	SdkPipelineOptions *structpb.Struct `protobuf:"bytes,8,opt,name=sdk_pipeline_options,json=sdkPipelineOptions,proto3" json:"sdk_pipeline_options,omitempty"`
	// Identity to run virtual machines as. Defaults to the default account.
	ServiceAccountEmail string `protobuf:"bytes,10,opt,name=service_account_email,json=serviceAccountEmail,proto3" json:"service_account_email,omitempty"`
	// Which Flexible Resource Scheduling mode to run in.
	FlexResourceSchedulingGoal FlexResourceSchedulingGoal `` /* 214-byte string literal not displayed */
	// The Compute Engine region
	// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
	// which worker processing should occur, e.g. "us-west1". Mutually exclusive
	// with worker_zone. If neither worker_region nor worker_zone is specified,
	// default to the control plane's region.
	WorkerRegion string `protobuf:"bytes,13,opt,name=worker_region,json=workerRegion,proto3" json:"worker_region,omitempty"`
	// The Compute Engine zone
	// (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
	// which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
	// with worker_region. If neither worker_region nor worker_zone is specified,
	// a zone in the control plane's region is chosen based on available capacity.
	WorkerZone string `protobuf:"bytes,14,opt,name=worker_zone,json=workerZone,proto3" json:"worker_zone,omitempty"`
	// Output only. The shuffle mode used for the job.
	ShuffleMode ShuffleMode `` /* 150-byte string literal not displayed */
	// Any debugging options to be supplied to the job.
	DebugOptions *DebugOptions `protobuf:"bytes,17,opt,name=debug_options,json=debugOptions,proto3" json:"debug_options,omitempty"`
	// contains filtered or unexported fields
}

Describes the environment in which a Dataflow Job runs.

func (*Environment) Descriptor deprecated

func (*Environment) Descriptor() ([]byte, []int)

Deprecated: Use Environment.ProtoReflect.Descriptor instead.

func (*Environment) GetClusterManagerApiService

func (x *Environment) GetClusterManagerApiService() string

func (*Environment) GetDataset

func (x *Environment) GetDataset() string

func (*Environment) GetDebugOptions

func (x *Environment) GetDebugOptions() *DebugOptions

func (*Environment) GetExperiments

func (x *Environment) GetExperiments() []string

func (*Environment) GetFlexResourceSchedulingGoal

func (x *Environment) GetFlexResourceSchedulingGoal() FlexResourceSchedulingGoal

func (*Environment) GetSdkPipelineOptions

func (x *Environment) GetSdkPipelineOptions() *structpb.Struct

func (*Environment) GetServiceAccountEmail

func (x *Environment) GetServiceAccountEmail() string

func (*Environment) GetServiceKmsKeyName

func (x *Environment) GetServiceKmsKeyName() string

func (*Environment) GetServiceOptions

func (x *Environment) GetServiceOptions() []string

func (*Environment) GetShuffleMode

func (x *Environment) GetShuffleMode() ShuffleMode

func (*Environment) GetTempStoragePrefix

func (x *Environment) GetTempStoragePrefix() string

func (*Environment) GetUserAgent

func (x *Environment) GetUserAgent() *structpb.Struct

func (*Environment) GetVersion

func (x *Environment) GetVersion() *structpb.Struct

func (*Environment) GetWorkerPools

func (x *Environment) GetWorkerPools() []*WorkerPool

func (*Environment) GetWorkerRegion

func (x *Environment) GetWorkerRegion() string

func (*Environment) GetWorkerZone

func (x *Environment) GetWorkerZone() string

func (*Environment) ProtoMessage

func (*Environment) ProtoMessage()

func (*Environment) ProtoReflect

func (x *Environment) ProtoReflect() protoreflect.Message

func (*Environment) Reset

func (x *Environment) Reset()

func (*Environment) String

func (x *Environment) String() string

type ExecutionStageState

type ExecutionStageState struct {

	// The name of the execution stage.
	ExecutionStageName string `protobuf:"bytes,1,opt,name=execution_stage_name,json=executionStageName,proto3" json:"execution_stage_name,omitempty"`
	// Executions stage states allow the same set of values as JobState.
	ExecutionStageState JobState `` /* 172-byte string literal not displayed */
	// The time at which the stage transitioned to this state.
	CurrentStateTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=current_state_time,json=currentStateTime,proto3" json:"current_state_time,omitempty"`
	// contains filtered or unexported fields
}

A message describing the state of a particular execution stage.

func (*ExecutionStageState) Descriptor deprecated

func (*ExecutionStageState) Descriptor() ([]byte, []int)

Deprecated: Use ExecutionStageState.ProtoReflect.Descriptor instead.

func (*ExecutionStageState) GetCurrentStateTime

func (x *ExecutionStageState) GetCurrentStateTime() *timestamppb.Timestamp

func (*ExecutionStageState) GetExecutionStageName

func (x *ExecutionStageState) GetExecutionStageName() string

func (*ExecutionStageState) GetExecutionStageState

func (x *ExecutionStageState) GetExecutionStageState() JobState

func (*ExecutionStageState) ProtoMessage

func (*ExecutionStageState) ProtoMessage()

func (*ExecutionStageState) ProtoReflect

func (x *ExecutionStageState) ProtoReflect() protoreflect.Message

func (*ExecutionStageState) Reset

func (x *ExecutionStageState) Reset()

func (*ExecutionStageState) String

func (x *ExecutionStageState) String() string

type FileIODetails

type FileIODetails struct {

	// File Pattern used to access files by the connector.
	FilePattern string `protobuf:"bytes,1,opt,name=file_pattern,json=filePattern,proto3" json:"file_pattern,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a File connector used by the job.

func (*FileIODetails) Descriptor deprecated

func (*FileIODetails) Descriptor() ([]byte, []int)

Deprecated: Use FileIODetails.ProtoReflect.Descriptor instead.

func (*FileIODetails) GetFilePattern

func (x *FileIODetails) GetFilePattern() string

func (*FileIODetails) ProtoMessage

func (*FileIODetails) ProtoMessage()

func (*FileIODetails) ProtoReflect

func (x *FileIODetails) ProtoReflect() protoreflect.Message

func (*FileIODetails) Reset

func (x *FileIODetails) Reset()

func (*FileIODetails) String

func (x *FileIODetails) String() string

type FlexResourceSchedulingGoal

type FlexResourceSchedulingGoal int32

Specifies the resource to optimize for in Flexible Resource Scheduling.

const (
	// Run in the default mode.
	FlexResourceSchedulingGoal_FLEXRS_UNSPECIFIED FlexResourceSchedulingGoal = 0
	// Optimize for lower execution time.
	FlexResourceSchedulingGoal_FLEXRS_SPEED_OPTIMIZED FlexResourceSchedulingGoal = 1
	// Optimize for lower cost.
	FlexResourceSchedulingGoal_FLEXRS_COST_OPTIMIZED FlexResourceSchedulingGoal = 2
)

func (FlexResourceSchedulingGoal) Descriptor

func (FlexResourceSchedulingGoal) Enum

func (FlexResourceSchedulingGoal) EnumDescriptor deprecated

func (FlexResourceSchedulingGoal) EnumDescriptor() ([]byte, []int)

Deprecated: Use FlexResourceSchedulingGoal.Descriptor instead.

func (FlexResourceSchedulingGoal) Number

func (FlexResourceSchedulingGoal) String

func (FlexResourceSchedulingGoal) Type

type Job

type Job struct {

	// The unique ID of this job.
	//
	// This field is set by the Cloud Dataflow service when the Job is
	// created, and is immutable for the life of the job.
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// The ID of the Cloud Platform project that the job belongs to.
	ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
	// The user-specified Cloud Dataflow job name.
	//
	// Only one Job with a given name can exist in a project within one region at
	// any given time. Jobs in different regions can have the same name.
	// If a caller attempts to create a Job with the same
	// name as an already-existing Job, the attempt returns the
	// existing Job.
	//
	// The name must match the regular expression
	// `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?`
	Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
	// The type of Cloud Dataflow job.
	Type JobType `protobuf:"varint,4,opt,name=type,proto3,enum=google.events.cloud.dataflow.v1beta3.JobType" json:"type,omitempty"`
	// The environment for the job.
	Environment *Environment `protobuf:"bytes,5,opt,name=environment,proto3" json:"environment,omitempty"`
	// The Cloud Storage location where the steps are stored.
	StepsLocation string `protobuf:"bytes,24,opt,name=steps_location,json=stepsLocation,proto3" json:"steps_location,omitempty"`
	// The current state of the job.
	//
	// Jobs are created in the `JOB_STATE_STOPPED` state unless otherwise
	// specified.
	//
	// A job in the `JOB_STATE_RUNNING` state may asynchronously enter a
	// terminal state. After a job has reached a terminal state, no
	// further state updates may be made.
	//
	// This field may be mutated by the Cloud Dataflow service;
	// callers cannot mutate it.
	CurrentState JobState `` /* 149-byte string literal not displayed */
	// The timestamp associated with the current state.
	CurrentStateTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=current_state_time,json=currentStateTime,proto3" json:"current_state_time,omitempty"`
	// The job's requested state.
	//
	// `UpdateJob` may be used to switch between the `JOB_STATE_STOPPED` and
	// `JOB_STATE_RUNNING` states, by setting requested_state.  `UpdateJob` may
	// also be used to directly set a job's requested state to
	// `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the
	// job if it has not already reached a terminal state.
	RequestedState JobState `` /* 155-byte string literal not displayed */
	// Deprecated.
	ExecutionInfo *JobExecutionInfo `protobuf:"bytes,10,opt,name=execution_info,json=executionInfo,proto3" json:"execution_info,omitempty"`
	// The timestamp when the job was initially created. Immutable and set by the
	// Cloud Dataflow service.
	CreateTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"`
	// If this job is an update of an existing job, this field is the job ID
	// of the job it replaced.
	//
	// When sending a `CreateJobRequest`, you can update a job by specifying it
	// here. The job named here is stopped, and its intermediate state is
	// transferred to this job.
	ReplaceJobId string `protobuf:"bytes,12,opt,name=replace_job_id,json=replaceJobId,proto3" json:"replace_job_id,omitempty"`
	// The client's unique identifier of the job, re-used across retried attempts.
	// If this field is set, the service will ensure its uniqueness.
	// The request to create a job will fail if the service has knowledge of a
	// previously submitted job with the same client's ID and job name.
	// The caller may use this field to ensure idempotence of job
	// creation across retried attempts to create a job.
	// By default, the field is empty and, in that case, the service ignores it.
	ClientRequestId string `protobuf:"bytes,14,opt,name=client_request_id,json=clientRequestId,proto3" json:"client_request_id,omitempty"`
	// If another job is an update of this job (and thus, this job is in
	// `JOB_STATE_UPDATED`), this field contains the ID of that job.
	ReplacedByJobId string `protobuf:"bytes,15,opt,name=replaced_by_job_id,json=replacedByJobId,proto3" json:"replaced_by_job_id,omitempty"`
	// A set of files the system should be aware of that are used
	// for temporary storage. These temporary files will be
	// removed on job completion.
	// No duplicates are allowed.
	// No file patterns are supported.
	//
	// The supported files are:
	//
	// Google Cloud Storage:
	//
	//	storage.googleapis.com/{bucket}/{object}
	//	bucket.storage.googleapis.com/{object}
	TempFiles []string `protobuf:"bytes,16,rep,name=temp_files,json=tempFiles,proto3" json:"temp_files,omitempty"`
	// User-defined labels for this job.
	//
	// The labels map can contain no more than 64 entries.  Entries of the labels
	// map are UTF8 strings that comply with the following restrictions:
	//
	// * Keys must conform to regexp:  [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
	// * Values must conform to regexp:  [\p{Ll}\p{Lo}\p{N}_-]{0,63}
	// * Both keys and values are additionally constrained to be <= 128 bytes in
	// size.
	Labels map[string]string `` /* 154-byte string literal not displayed */
	// The [regional endpoint]
	// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
	// contains this job.
	Location string `protobuf:"bytes,18,opt,name=location,proto3" json:"location,omitempty"`
	// This field may be mutated by the Cloud Dataflow service;
	// callers cannot mutate it.
	StageStates []*ExecutionStageState `protobuf:"bytes,20,rep,name=stage_states,json=stageStates,proto3" json:"stage_states,omitempty"`
	// This field is populated by the Dataflow service to support filtering jobs
	// by the metadata values provided here. Populated for ListJobs and all GetJob
	// views SUMMARY and higher.
	JobMetadata *JobMetadata `protobuf:"bytes,21,opt,name=job_metadata,json=jobMetadata,proto3" json:"job_metadata,omitempty"`
	// The timestamp when the job was started (transitioned to JOB_STATE_PENDING).
	// Flexible resource scheduling jobs are started with some delay after job
	// creation, so start_time is unset before start and is updated when the
	// job is started by the Cloud Dataflow service. For other jobs, start_time
	// always equals to create_time and is immutable and set by the Cloud Dataflow
	// service.
	StartTime *timestamppb.Timestamp `protobuf:"bytes,22,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
	// If this is specified, the job's initial state is populated from the given
	// snapshot.
	CreatedFromSnapshotId string `` /* 129-byte string literal not displayed */
	// Reserved for future use. This field is set only in responses from the
	// server; it is ignored if it is set in any requests.
	SatisfiesPzs bool `protobuf:"varint,25,opt,name=satisfies_pzs,json=satisfiesPzs,proto3" json:"satisfies_pzs,omitempty"`
	// contains filtered or unexported fields
}

Defines a job to be run by the Cloud Dataflow service. Do not enter confidential information when you supply string values using the API. Fields stripped from source Job proto: - steps - pipeline_description - transform_name_mapping

func (*Job) Descriptor deprecated

func (*Job) Descriptor() ([]byte, []int)

Deprecated: Use Job.ProtoReflect.Descriptor instead.

func (*Job) GetClientRequestId

func (x *Job) GetClientRequestId() string

func (*Job) GetCreateTime

func (x *Job) GetCreateTime() *timestamppb.Timestamp

func (*Job) GetCreatedFromSnapshotId

func (x *Job) GetCreatedFromSnapshotId() string

func (*Job) GetCurrentState

func (x *Job) GetCurrentState() JobState

func (*Job) GetCurrentStateTime

func (x *Job) GetCurrentStateTime() *timestamppb.Timestamp

func (*Job) GetEnvironment

func (x *Job) GetEnvironment() *Environment

func (*Job) GetExecutionInfo

func (x *Job) GetExecutionInfo() *JobExecutionInfo

func (*Job) GetId

func (x *Job) GetId() string

func (*Job) GetJobMetadata

func (x *Job) GetJobMetadata() *JobMetadata

func (*Job) GetLabels

func (x *Job) GetLabels() map[string]string

func (*Job) GetLocation

func (x *Job) GetLocation() string

func (*Job) GetName

func (x *Job) GetName() string

func (*Job) GetProjectId

func (x *Job) GetProjectId() string

func (*Job) GetReplaceJobId

func (x *Job) GetReplaceJobId() string

func (*Job) GetReplacedByJobId

func (x *Job) GetReplacedByJobId() string

func (*Job) GetRequestedState

func (x *Job) GetRequestedState() JobState

func (*Job) GetSatisfiesPzs

func (x *Job) GetSatisfiesPzs() bool

func (*Job) GetStageStates

func (x *Job) GetStageStates() []*ExecutionStageState

func (*Job) GetStartTime

func (x *Job) GetStartTime() *timestamppb.Timestamp

func (*Job) GetStepsLocation

func (x *Job) GetStepsLocation() string

func (*Job) GetTempFiles

func (x *Job) GetTempFiles() []string

func (*Job) GetType

func (x *Job) GetType() JobType

func (*Job) ProtoMessage

func (*Job) ProtoMessage()

func (*Job) ProtoReflect

func (x *Job) ProtoReflect() protoreflect.Message

func (*Job) Reset

func (x *Job) Reset()

func (*Job) String

func (x *Job) String() string

type JobEventData

type JobEventData struct {

	// The Job event payload.
	Payload *Job `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"`
	// contains filtered or unexported fields
}

The data within all Job events.

func (*JobEventData) Descriptor deprecated

func (*JobEventData) Descriptor() ([]byte, []int)

Deprecated: Use JobEventData.ProtoReflect.Descriptor instead.

func (*JobEventData) GetPayload

func (x *JobEventData) GetPayload() *Job

func (*JobEventData) ProtoMessage

func (*JobEventData) ProtoMessage()

func (*JobEventData) ProtoReflect

func (x *JobEventData) ProtoReflect() protoreflect.Message

func (*JobEventData) Reset

func (x *JobEventData) Reset()

func (*JobEventData) String

func (x *JobEventData) String() string

type JobExecutionInfo

type JobExecutionInfo struct {

	// A mapping from each stage to the information about that stage.
	Stages map[string]*JobExecutionStageInfo `` /* 153-byte string literal not displayed */
	// contains filtered or unexported fields
}

Additional information about how a Cloud Dataflow job will be executed that isn't contained in the submitted job.

func (*JobExecutionInfo) Descriptor deprecated

func (*JobExecutionInfo) Descriptor() ([]byte, []int)

Deprecated: Use JobExecutionInfo.ProtoReflect.Descriptor instead.

func (*JobExecutionInfo) GetStages

func (x *JobExecutionInfo) GetStages() map[string]*JobExecutionStageInfo

func (*JobExecutionInfo) ProtoMessage

func (*JobExecutionInfo) ProtoMessage()

func (*JobExecutionInfo) ProtoReflect

func (x *JobExecutionInfo) ProtoReflect() protoreflect.Message

func (*JobExecutionInfo) Reset

func (x *JobExecutionInfo) Reset()

func (*JobExecutionInfo) String

func (x *JobExecutionInfo) String() string

type JobExecutionStageInfo

type JobExecutionStageInfo struct {

	// The steps associated with the execution stage.
	// Note that stages may have several steps, and that a given step
	// might be run by more than one stage.
	StepName []string `protobuf:"bytes,1,rep,name=step_name,json=stepName,proto3" json:"step_name,omitempty"`
	// contains filtered or unexported fields
}

Contains information about how a particular [google.dataflow.v1beta3.Step][google.dataflow.v1beta3.Step] will be executed.

func (*JobExecutionStageInfo) Descriptor deprecated

func (*JobExecutionStageInfo) Descriptor() ([]byte, []int)

Deprecated: Use JobExecutionStageInfo.ProtoReflect.Descriptor instead.

func (*JobExecutionStageInfo) GetStepName

func (x *JobExecutionStageInfo) GetStepName() []string

func (*JobExecutionStageInfo) ProtoMessage

func (*JobExecutionStageInfo) ProtoMessage()

func (*JobExecutionStageInfo) ProtoReflect

func (x *JobExecutionStageInfo) ProtoReflect() protoreflect.Message

func (*JobExecutionStageInfo) Reset

func (x *JobExecutionStageInfo) Reset()

func (*JobExecutionStageInfo) String

func (x *JobExecutionStageInfo) String() string

type JobMetadata

type JobMetadata struct {

	// The SDK version used to run the job.
	SdkVersion *SdkVersion `protobuf:"bytes,1,opt,name=sdk_version,json=sdkVersion,proto3" json:"sdk_version,omitempty"`
	// Identification of a Spanner source used in the Dataflow job.
	SpannerDetails []*SpannerIODetails `protobuf:"bytes,2,rep,name=spanner_details,json=spannerDetails,proto3" json:"spanner_details,omitempty"`
	// Identification of a BigQuery source used in the Dataflow job.
	BigqueryDetails []*BigQueryIODetails `protobuf:"bytes,3,rep,name=bigquery_details,json=bigqueryDetails,proto3" json:"bigquery_details,omitempty"`
	// Identification of a Cloud Bigtable source used in the Dataflow job.
	BigTableDetails []*BigTableIODetails `protobuf:"bytes,4,rep,name=big_table_details,json=bigTableDetails,proto3" json:"big_table_details,omitempty"`
	// Identification of a Pub/Sub source used in the Dataflow job.
	PubsubDetails []*PubSubIODetails `protobuf:"bytes,5,rep,name=pubsub_details,json=pubsubDetails,proto3" json:"pubsub_details,omitempty"`
	// Identification of a File source used in the Dataflow job.
	FileDetails []*FileIODetails `protobuf:"bytes,6,rep,name=file_details,json=fileDetails,proto3" json:"file_details,omitempty"`
	// Identification of a Datastore source used in the Dataflow job.
	DatastoreDetails []*DatastoreIODetails `protobuf:"bytes,7,rep,name=datastore_details,json=datastoreDetails,proto3" json:"datastore_details,omitempty"`
	// contains filtered or unexported fields
}

Metadata available primarily for filtering jobs. Will be included in the ListJob response and Job SUMMARY view.

func (*JobMetadata) Descriptor deprecated

func (*JobMetadata) Descriptor() ([]byte, []int)

Deprecated: Use JobMetadata.ProtoReflect.Descriptor instead.

func (*JobMetadata) GetBigTableDetails

func (x *JobMetadata) GetBigTableDetails() []*BigTableIODetails

func (*JobMetadata) GetBigqueryDetails

func (x *JobMetadata) GetBigqueryDetails() []*BigQueryIODetails

func (*JobMetadata) GetDatastoreDetails

func (x *JobMetadata) GetDatastoreDetails() []*DatastoreIODetails

func (*JobMetadata) GetFileDetails

func (x *JobMetadata) GetFileDetails() []*FileIODetails

func (*JobMetadata) GetPubsubDetails

func (x *JobMetadata) GetPubsubDetails() []*PubSubIODetails

func (*JobMetadata) GetSdkVersion

func (x *JobMetadata) GetSdkVersion() *SdkVersion

func (*JobMetadata) GetSpannerDetails

func (x *JobMetadata) GetSpannerDetails() []*SpannerIODetails

func (*JobMetadata) ProtoMessage

func (*JobMetadata) ProtoMessage()

func (*JobMetadata) ProtoReflect

func (x *JobMetadata) ProtoReflect() protoreflect.Message

func (*JobMetadata) Reset

func (x *JobMetadata) Reset()

func (*JobMetadata) String

func (x *JobMetadata) String() string

type JobState

type JobState int32

Describes the overall state of a [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job].

const (
	// The job's run state isn't specified.
	JobState_JOB_STATE_UNKNOWN JobState = 0
	// `JOB_STATE_STOPPED` indicates that the job has not
	// yet started to run.
	JobState_JOB_STATE_STOPPED JobState = 1
	// `JOB_STATE_RUNNING` indicates that the job is currently running.
	JobState_JOB_STATE_RUNNING JobState = 2
	// `JOB_STATE_DONE` indicates that the job has successfully completed.
	// This is a terminal job state.  This state may be set by the Cloud Dataflow
	// service, as a transition from `JOB_STATE_RUNNING`. It may also be set via a
	// Cloud Dataflow `UpdateJob` call, if the job has not yet reached a terminal
	// state.
	JobState_JOB_STATE_DONE JobState = 3
	// `JOB_STATE_FAILED` indicates that the job has failed.  This is a
	// terminal job state.  This state may only be set by the Cloud Dataflow
	// service, and only as a transition from `JOB_STATE_RUNNING`.
	JobState_JOB_STATE_FAILED JobState = 4
	// `JOB_STATE_CANCELLED` indicates that the job has been explicitly
	// cancelled. This is a terminal job state. This state may only be
	// set via a Cloud Dataflow `UpdateJob` call, and only if the job has not
	// yet reached another terminal state.
	JobState_JOB_STATE_CANCELLED JobState = 5
	// `JOB_STATE_UPDATED` indicates that the job was successfully updated,
	// meaning that this job was stopped and another job was started, inheriting
	// state from this one. This is a terminal job state. This state may only be
	// set by the Cloud Dataflow service, and only as a transition from
	// `JOB_STATE_RUNNING`.
	JobState_JOB_STATE_UPDATED JobState = 6
	// `JOB_STATE_DRAINING` indicates that the job is in the process of draining.
	// A draining job has stopped pulling from its input sources and is processing
	// any data that remains in-flight. This state may be set via a Cloud Dataflow
	// `UpdateJob` call, but only as a transition from `JOB_STATE_RUNNING`. Jobs
	// that are draining may only transition to `JOB_STATE_DRAINED`,
	// `JOB_STATE_CANCELLED`, or `JOB_STATE_FAILED`.
	JobState_JOB_STATE_DRAINING JobState = 7
	// `JOB_STATE_DRAINED` indicates that the job has been drained.
	// A drained job terminated by stopping pulling from its input sources and
	// processing any data that remained in-flight when draining was requested.
	// This state is a terminal state, may only be set by the Cloud Dataflow
	// service, and only as a transition from `JOB_STATE_DRAINING`.
	JobState_JOB_STATE_DRAINED JobState = 8
	// `JOB_STATE_PENDING` indicates that the job has been created but is not yet
	// running.  Jobs that are pending may only transition to `JOB_STATE_RUNNING`,
	// or `JOB_STATE_FAILED`.
	JobState_JOB_STATE_PENDING JobState = 9
	// `JOB_STATE_CANCELLING` indicates that the job has been explicitly cancelled
	// and is in the process of stopping.  Jobs that are cancelling may only
	// transition to `JOB_STATE_CANCELLED` or `JOB_STATE_FAILED`.
	JobState_JOB_STATE_CANCELLING JobState = 10
	// `JOB_STATE_QUEUED` indicates that the job has been created but is being
	// delayed until launch. Jobs that are queued may only transition to
	// `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`.
	JobState_JOB_STATE_QUEUED JobState = 11
	// `JOB_STATE_RESOURCE_CLEANING_UP` indicates that the batch job's associated
	// resources are currently being cleaned up after a successful run.
	// Currently, this is an opt-in feature, please reach out to Cloud support
	// team if you are interested.
	JobState_JOB_STATE_RESOURCE_CLEANING_UP JobState = 12
)

func (JobState) Descriptor

func (JobState) Descriptor() protoreflect.EnumDescriptor

func (JobState) Enum

func (x JobState) Enum() *JobState

func (JobState) EnumDescriptor deprecated

func (JobState) EnumDescriptor() ([]byte, []int)

Deprecated: Use JobState.Descriptor instead.

func (JobState) Number

func (x JobState) Number() protoreflect.EnumNumber

func (JobState) String

func (x JobState) String() string

func (JobState) Type

type JobType

type JobType int32

Specifies the processing model used by a [google.dataflow.v1beta3.Job], which determines the way the Job is managed by the Cloud Dataflow service (how workers are scheduled, how inputs are sharded, etc).

const (
	// The type of the job is unspecified, or unknown.
	JobType_JOB_TYPE_UNKNOWN JobType = 0
	// A batch job with a well-defined end point: data is read, data is
	// processed, data is written, and the job is done.
	JobType_JOB_TYPE_BATCH JobType = 1
	// A continuously streaming job with no end: data is read,
	// processed, and written continuously.
	JobType_JOB_TYPE_STREAMING JobType = 2
)

func (JobType) Descriptor

func (JobType) Descriptor() protoreflect.EnumDescriptor

func (JobType) Enum

func (x JobType) Enum() *JobType

func (JobType) EnumDescriptor deprecated

func (JobType) EnumDescriptor() ([]byte, []int)

Deprecated: Use JobType.Descriptor instead.

func (JobType) Number

func (x JobType) Number() protoreflect.EnumNumber

func (JobType) String

func (x JobType) String() string

func (JobType) Type

func (JobType) Type() protoreflect.EnumType

type Package

type Package struct {

	// The name of the package.
	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
	// The resource to read the package from. The supported resource type is:
	//
	// Google Cloud Storage:
	//
	//	storage.googleapis.com/{bucket}
	//	bucket.storage.googleapis.com/
	Location string `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"`
	// contains filtered or unexported fields
}

The packages that must be installed in order for a worker to run the steps of the Cloud Dataflow job that will be assigned to its worker pool.

This is the mechanism by which the Cloud Dataflow SDK causes code to be loaded onto the workers. For example, the Cloud Dataflow Java SDK might use this to install jars containing the user's code and all of the various dependencies (libraries, data files, etc.) required in order for that code to run.

func (*Package) Descriptor deprecated

func (*Package) Descriptor() ([]byte, []int)

Deprecated: Use Package.ProtoReflect.Descriptor instead.

func (*Package) GetLocation

func (x *Package) GetLocation() string

func (*Package) GetName

func (x *Package) GetName() string

func (*Package) ProtoMessage

func (*Package) ProtoMessage()

func (*Package) ProtoReflect

func (x *Package) ProtoReflect() protoreflect.Message

func (*Package) Reset

func (x *Package) Reset()

func (*Package) String

func (x *Package) String() string

type PubSubIODetails

type PubSubIODetails struct {

	// Topic accessed in the connection.
	Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
	// Subscription used in the connection.
	Subscription string `protobuf:"bytes,2,opt,name=subscription,proto3" json:"subscription,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a Pub/Sub connector used by the job.

func (*PubSubIODetails) Descriptor deprecated

func (*PubSubIODetails) Descriptor() ([]byte, []int)

Deprecated: Use PubSubIODetails.ProtoReflect.Descriptor instead.

func (*PubSubIODetails) GetSubscription

func (x *PubSubIODetails) GetSubscription() string

func (*PubSubIODetails) GetTopic

func (x *PubSubIODetails) GetTopic() string

func (*PubSubIODetails) ProtoMessage

func (*PubSubIODetails) ProtoMessage()

func (*PubSubIODetails) ProtoReflect

func (x *PubSubIODetails) ProtoReflect() protoreflect.Message

func (*PubSubIODetails) Reset

func (x *PubSubIODetails) Reset()

func (*PubSubIODetails) String

func (x *PubSubIODetails) String() string

type SdkHarnessContainerImage

type SdkHarnessContainerImage struct {

	// A docker container image that resides in Google Container Registry.
	ContainerImage string `protobuf:"bytes,1,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"`
	// If true, recommends the Dataflow service to use only one core per SDK
	// container instance with this image. If false (or unset) recommends using
	// more than one core per SDK container instance with this image for
	// efficiency. Note that Dataflow service may choose to override this property
	// if needed.
	UseSingleCorePerContainer bool `` /* 143-byte string literal not displayed */
	// Environment ID for the Beam runner API proto Environment that corresponds
	// to the current SDK Harness.
	EnvironmentId string `protobuf:"bytes,3,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"`
	// The set of capabilities enumerated in the above Environment proto. See also
	// [beam_runner_api.proto](https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto)
	Capabilities []string `protobuf:"bytes,4,rep,name=capabilities,proto3" json:"capabilities,omitempty"`
	// contains filtered or unexported fields
}

Defines an SDK harness container for executing Dataflow pipelines.

func (*SdkHarnessContainerImage) Descriptor deprecated

func (*SdkHarnessContainerImage) Descriptor() ([]byte, []int)

Deprecated: Use SdkHarnessContainerImage.ProtoReflect.Descriptor instead.

func (*SdkHarnessContainerImage) GetCapabilities

func (x *SdkHarnessContainerImage) GetCapabilities() []string

func (*SdkHarnessContainerImage) GetContainerImage

func (x *SdkHarnessContainerImage) GetContainerImage() string

func (*SdkHarnessContainerImage) GetEnvironmentId

func (x *SdkHarnessContainerImage) GetEnvironmentId() string

func (*SdkHarnessContainerImage) GetUseSingleCorePerContainer

func (x *SdkHarnessContainerImage) GetUseSingleCorePerContainer() bool

func (*SdkHarnessContainerImage) ProtoMessage

func (*SdkHarnessContainerImage) ProtoMessage()

func (*SdkHarnessContainerImage) ProtoReflect

func (x *SdkHarnessContainerImage) ProtoReflect() protoreflect.Message

func (*SdkHarnessContainerImage) Reset

func (x *SdkHarnessContainerImage) Reset()

func (*SdkHarnessContainerImage) String

func (x *SdkHarnessContainerImage) String() string

type SdkVersion

type SdkVersion struct {

	// The version of the SDK used to run the job.
	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
	// A readable string describing the version of the SDK.
	VersionDisplayName string `protobuf:"bytes,2,opt,name=version_display_name,json=versionDisplayName,proto3" json:"version_display_name,omitempty"`
	// The support status for this SDK version.
	SdkSupportStatus SdkVersion_SdkSupportStatus `` /* 182-byte string literal not displayed */
	// contains filtered or unexported fields
}

The version of the SDK used to run the job.

func (*SdkVersion) Descriptor deprecated

func (*SdkVersion) Descriptor() ([]byte, []int)

Deprecated: Use SdkVersion.ProtoReflect.Descriptor instead.

func (*SdkVersion) GetSdkSupportStatus

func (x *SdkVersion) GetSdkSupportStatus() SdkVersion_SdkSupportStatus

func (*SdkVersion) GetVersion

func (x *SdkVersion) GetVersion() string

func (*SdkVersion) GetVersionDisplayName

func (x *SdkVersion) GetVersionDisplayName() string

func (*SdkVersion) ProtoMessage

func (*SdkVersion) ProtoMessage()

func (*SdkVersion) ProtoReflect

func (x *SdkVersion) ProtoReflect() protoreflect.Message

func (*SdkVersion) Reset

func (x *SdkVersion) Reset()

func (*SdkVersion) String

func (x *SdkVersion) String() string

type SdkVersion_SdkSupportStatus

type SdkVersion_SdkSupportStatus int32

The support status of the SDK used to run the job.

const (
	// Cloud Dataflow is unaware of this version.
	SdkVersion_UNKNOWN SdkVersion_SdkSupportStatus = 0
	// This is a known version of an SDK, and is supported.
	SdkVersion_SUPPORTED SdkVersion_SdkSupportStatus = 1
	// A newer version of the SDK family exists, and an update is recommended.
	SdkVersion_STALE SdkVersion_SdkSupportStatus = 2
	// This version of the SDK is deprecated and will eventually be
	// unsupported.
	SdkVersion_DEPRECATED SdkVersion_SdkSupportStatus = 3
	// Support for this SDK version has ended and it should no longer be used.
	SdkVersion_UNSUPPORTED SdkVersion_SdkSupportStatus = 4
)

func (SdkVersion_SdkSupportStatus) Descriptor

func (SdkVersion_SdkSupportStatus) Enum

func (SdkVersion_SdkSupportStatus) EnumDescriptor deprecated

func (SdkVersion_SdkSupportStatus) EnumDescriptor() ([]byte, []int)

Deprecated: Use SdkVersion_SdkSupportStatus.Descriptor instead.

func (SdkVersion_SdkSupportStatus) Number

func (SdkVersion_SdkSupportStatus) String

func (SdkVersion_SdkSupportStatus) Type

type ShuffleMode

type ShuffleMode int32

Specifies the shuffle mode used by a [google.dataflow.v1beta3.Job], which determines the approach data is shuffled during processing. More details in: https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-shuffle

const (
	// Shuffle mode information is not available.
	ShuffleMode_SHUFFLE_MODE_UNSPECIFIED ShuffleMode = 0
	// Shuffle is done on the worker VMs.
	ShuffleMode_VM_BASED ShuffleMode = 1
	// Shuffle is done on the service side.
	ShuffleMode_SERVICE_BASED ShuffleMode = 2
)

func (ShuffleMode) Descriptor

func (ShuffleMode) Enum

func (x ShuffleMode) Enum() *ShuffleMode

func (ShuffleMode) EnumDescriptor deprecated

func (ShuffleMode) EnumDescriptor() ([]byte, []int)

Deprecated: Use ShuffleMode.Descriptor instead.

func (ShuffleMode) Number

func (x ShuffleMode) Number() protoreflect.EnumNumber

func (ShuffleMode) String

func (x ShuffleMode) String() string

func (ShuffleMode) Type

type SpannerIODetails

type SpannerIODetails struct {

	// ProjectId accessed in the connection.
	ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
	// InstanceId accessed in the connection.
	InstanceId string `protobuf:"bytes,2,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"`
	// DatabaseId accessed in the connection.
	DatabaseId string `protobuf:"bytes,3,opt,name=database_id,json=databaseId,proto3" json:"database_id,omitempty"`
	// contains filtered or unexported fields
}

Metadata for a Spanner connector used by the job.

func (*SpannerIODetails) Descriptor deprecated

func (*SpannerIODetails) Descriptor() ([]byte, []int)

Deprecated: Use SpannerIODetails.ProtoReflect.Descriptor instead.

func (*SpannerIODetails) GetDatabaseId

func (x *SpannerIODetails) GetDatabaseId() string

func (*SpannerIODetails) GetInstanceId

func (x *SpannerIODetails) GetInstanceId() string

func (*SpannerIODetails) GetProjectId

func (x *SpannerIODetails) GetProjectId() string

func (*SpannerIODetails) ProtoMessage

func (*SpannerIODetails) ProtoMessage()

func (*SpannerIODetails) ProtoReflect

func (x *SpannerIODetails) ProtoReflect() protoreflect.Message

func (*SpannerIODetails) Reset

func (x *SpannerIODetails) Reset()

func (*SpannerIODetails) String

func (x *SpannerIODetails) String() string

type TeardownPolicy

type TeardownPolicy int32

Specifies what happens to a resource when a Cloud Dataflow [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job] has completed.

const (
	// The teardown policy isn't specified, or is unknown.
	TeardownPolicy_TEARDOWN_POLICY_UNKNOWN TeardownPolicy = 0
	// Always teardown the resource.
	TeardownPolicy_TEARDOWN_ALWAYS TeardownPolicy = 1
	// Teardown the resource on success. This is useful for debugging
	// failures.
	TeardownPolicy_TEARDOWN_ON_SUCCESS TeardownPolicy = 2
	// Never teardown the resource. This is useful for debugging and
	// development.
	TeardownPolicy_TEARDOWN_NEVER TeardownPolicy = 3
)

func (TeardownPolicy) Descriptor

func (TeardownPolicy) Enum

func (x TeardownPolicy) Enum() *TeardownPolicy

func (TeardownPolicy) EnumDescriptor deprecated

func (TeardownPolicy) EnumDescriptor() ([]byte, []int)

Deprecated: Use TeardownPolicy.Descriptor instead.

func (TeardownPolicy) Number

func (TeardownPolicy) String

func (x TeardownPolicy) String() string

func (TeardownPolicy) Type

type WorkerIPAddressConfiguration

type WorkerIPAddressConfiguration int32

Specifies how IP addresses should be allocated to the worker machines.

const (
	// The configuration is unknown, or unspecified.
	WorkerIPAddressConfiguration_WORKER_IP_UNSPECIFIED WorkerIPAddressConfiguration = 0
	// Workers should have public IP addresses.
	WorkerIPAddressConfiguration_WORKER_IP_PUBLIC WorkerIPAddressConfiguration = 1
	// Workers should have private IP addresses.
	WorkerIPAddressConfiguration_WORKER_IP_PRIVATE WorkerIPAddressConfiguration = 2
)

func (WorkerIPAddressConfiguration) Descriptor

func (WorkerIPAddressConfiguration) Enum

func (WorkerIPAddressConfiguration) EnumDescriptor deprecated

func (WorkerIPAddressConfiguration) EnumDescriptor() ([]byte, []int)

Deprecated: Use WorkerIPAddressConfiguration.Descriptor instead.

func (WorkerIPAddressConfiguration) Number

func (WorkerIPAddressConfiguration) String

func (WorkerIPAddressConfiguration) Type

type WorkerPool

type WorkerPool struct {

	// The kind of the worker pool; currently only `harness` and `shuffle`
	// are supported.
	Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"`
	// Number of Google Compute Engine workers in this pool needed to
	// execute the job.  If zero or unspecified, the service will
	// attempt to choose a reasonable default.
	NumWorkers int32 `protobuf:"varint,2,opt,name=num_workers,json=numWorkers,proto3" json:"num_workers,omitempty"`
	// Packages to be installed on workers.
	Packages []*Package `protobuf:"bytes,3,rep,name=packages,proto3" json:"packages,omitempty"`
	// The default package set to install.  This allows the service to
	// select a default set of packages which are useful to worker
	// harnesses written in a particular language.
	DefaultPackageSet DefaultPackageSet `` /* 175-byte string literal not displayed */
	// Machine type (e.g. "n1-standard-1").  If empty or unspecified, the
	// service will attempt to choose a reasonable default.
	MachineType string `protobuf:"bytes,5,opt,name=machine_type,json=machineType,proto3" json:"machine_type,omitempty"`
	// Sets the policy for determining when to turndown worker pool.
	// Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and
	// `TEARDOWN_NEVER`.
	// `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether
	// the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down
	// if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn
	// down.
	//
	// If the workers are not torn down by the service, they will
	// continue to run and use Google Compute Engine VM resources in the
	// user's project until they are explicitly terminated by the user.
	// Because of this, Google recommends using the `TEARDOWN_ALWAYS`
	// policy except for small, manually supervised test jobs.
	//
	// If unknown or unspecified, the service will attempt to choose a reasonable
	// default.
	TeardownPolicy TeardownPolicy `` /* 161-byte string literal not displayed */
	// Size of root disk for VMs, in GB.  If zero or unspecified, the service will
	// attempt to choose a reasonable default.
	DiskSizeGb int32 `protobuf:"varint,7,opt,name=disk_size_gb,json=diskSizeGb,proto3" json:"disk_size_gb,omitempty"`
	// Type of root disk for VMs.  If empty or unspecified, the service will
	// attempt to choose a reasonable default.
	DiskType string `protobuf:"bytes,16,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
	// Fully qualified source image for disks.
	DiskSourceImage string `protobuf:"bytes,8,opt,name=disk_source_image,json=diskSourceImage,proto3" json:"disk_source_image,omitempty"`
	// Zone to run the worker pools in.  If empty or unspecified, the service
	// will attempt to choose a reasonable default.
	Zone string `protobuf:"bytes,9,opt,name=zone,proto3" json:"zone,omitempty"`
	// The action to take on host maintenance, as defined by the Google
	// Compute Engine API.
	OnHostMaintenance string `protobuf:"bytes,11,opt,name=on_host_maintenance,json=onHostMaintenance,proto3" json:"on_host_maintenance,omitempty"`
	// Metadata to set on the Google Compute Engine VMs.
	Metadata map[string]string `` /* 158-byte string literal not displayed */
	// Settings for autoscaling of this WorkerPool.
	AutoscalingSettings *AutoscalingSettings `protobuf:"bytes,14,opt,name=autoscaling_settings,json=autoscalingSettings,proto3" json:"autoscaling_settings,omitempty"`
	// Network to which VMs will be assigned.  If empty or unspecified,
	// the service will use the network "default".
	Network string `protobuf:"bytes,17,opt,name=network,proto3" json:"network,omitempty"`
	// Subnetwork to which VMs will be assigned, if desired.  Expected to be of
	// the form "regions/REGION/subnetworks/SUBNETWORK".
	Subnetwork string `protobuf:"bytes,19,opt,name=subnetwork,proto3" json:"subnetwork,omitempty"`
	// Required. Docker container image that executes the Cloud Dataflow worker
	// harness, residing in Google Container Registry.
	//
	// Deprecated for the Fn API path. Use sdk_harness_container_images instead.
	WorkerHarnessContainerImage string `` /* 147-byte string literal not displayed */
	// The number of threads per worker harness. If empty or unspecified, the
	// service will choose a number of threads (according to the number of cores
	// on the selected machine type for batch, or 1 by convention for streaming).
	NumThreadsPerWorker int32 `protobuf:"varint,20,opt,name=num_threads_per_worker,json=numThreadsPerWorker,proto3" json:"num_threads_per_worker,omitempty"`
	// Configuration for VM IPs.
	IpConfiguration WorkerIPAddressConfiguration `` /* 179-byte string literal not displayed */
	// Set of SDK harness containers needed to execute this pipeline. This will
	// only be set in the Fn API path. For non-cross-language pipelines this
	// should have only one entry. Cross-language pipelines will have two or more
	// entries.
	SdkHarnessContainerImages []*SdkHarnessContainerImage `` /* 141-byte string literal not displayed */
	// contains filtered or unexported fields
}

Describes one particular pool of Cloud Dataflow workers to be instantiated by the Cloud Dataflow service in order to perform the computations required by a job. Note that a workflow job may use multiple pools, in order to match the various computational requirements of the various stages of the job.

func (*WorkerPool) Descriptor deprecated

func (*WorkerPool) Descriptor() ([]byte, []int)

Deprecated: Use WorkerPool.ProtoReflect.Descriptor instead.

func (*WorkerPool) GetAutoscalingSettings

func (x *WorkerPool) GetAutoscalingSettings() *AutoscalingSettings

func (*WorkerPool) GetDefaultPackageSet

func (x *WorkerPool) GetDefaultPackageSet() DefaultPackageSet

func (*WorkerPool) GetDiskSizeGb

func (x *WorkerPool) GetDiskSizeGb() int32

func (*WorkerPool) GetDiskSourceImage

func (x *WorkerPool) GetDiskSourceImage() string

func (*WorkerPool) GetDiskType

func (x *WorkerPool) GetDiskType() string

func (*WorkerPool) GetIpConfiguration

func (x *WorkerPool) GetIpConfiguration() WorkerIPAddressConfiguration

func (*WorkerPool) GetKind

func (x *WorkerPool) GetKind() string

func (*WorkerPool) GetMachineType

func (x *WorkerPool) GetMachineType() string

func (*WorkerPool) GetMetadata

func (x *WorkerPool) GetMetadata() map[string]string

func (*WorkerPool) GetNetwork

func (x *WorkerPool) GetNetwork() string

func (*WorkerPool) GetNumThreadsPerWorker

func (x *WorkerPool) GetNumThreadsPerWorker() int32

func (*WorkerPool) GetNumWorkers

func (x *WorkerPool) GetNumWorkers() int32

func (*WorkerPool) GetOnHostMaintenance

func (x *WorkerPool) GetOnHostMaintenance() string

func (*WorkerPool) GetPackages

func (x *WorkerPool) GetPackages() []*Package

func (*WorkerPool) GetSdkHarnessContainerImages

func (x *WorkerPool) GetSdkHarnessContainerImages() []*SdkHarnessContainerImage

func (*WorkerPool) GetSubnetwork

func (x *WorkerPool) GetSubnetwork() string

func (*WorkerPool) GetTeardownPolicy

func (x *WorkerPool) GetTeardownPolicy() TeardownPolicy

func (*WorkerPool) GetWorkerHarnessContainerImage

func (x *WorkerPool) GetWorkerHarnessContainerImage() string

func (*WorkerPool) GetZone

func (x *WorkerPool) GetZone() string

func (*WorkerPool) ProtoMessage

func (*WorkerPool) ProtoMessage()

func (*WorkerPool) ProtoReflect

func (x *WorkerPool) ProtoReflect() protoreflect.Message

func (*WorkerPool) Reset

func (x *WorkerPool) Reset()

func (*WorkerPool) String

func (x *WorkerPool) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL