plugins

package
v0.16.6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 10, 2020 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var SparkApplication_Type_name = map[int32]string{
	0: "PYTHON",
	1: "JAVA",
	2: "SCALA",
	3: "R",
}
View Source
var SparkApplication_Type_value = map[string]int32{
	"PYTHON": 0,
	"JAVA":   1,
	"SCALA":  2,
	"R":      3,
}

Functions

This section is empty.

Types

type ArrayJob

type ArrayJob struct {
	// Defines the minimum number of instances to bring up concurrently at any given point. Note that this is an
	// optimistic restriction and that, due to network partitioning or other failures, the actual number of currently
	// running instances might be more. This has to be a positive number if assigned. Default value is size.
	Parallelism int64 `protobuf:"varint,1,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
	// Defines the number of instances to launch at most. This number should match the size of the input if the job
	// requires processing of all input data. This has to be a positive number.
	// +required.
	Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
	// An absolute number of the minimum number of successful completions of subtasks. As soon as this criteria is met,
	// the array job will be marked as successful and outputs will be computed. This has to be a non-negative number if
	// assigned. Default value is size.
	MinSuccesses         int64    `protobuf:"varint,3,opt,name=min_successes,json=minSuccesses,proto3" json:"min_successes,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

Describes a job that can process independent pieces of data concurrently. Multiple copies of the runnable component will be executed concurrently.

func (*ArrayJob) Descriptor

func (*ArrayJob) Descriptor() ([]byte, []int)

func (*ArrayJob) GetMinSuccesses

func (m *ArrayJob) GetMinSuccesses() int64

func (*ArrayJob) GetParallelism

func (m *ArrayJob) GetParallelism() int64

func (*ArrayJob) GetSize

func (m *ArrayJob) GetSize() int64

func (*ArrayJob) ProtoMessage

func (*ArrayJob) ProtoMessage()

func (*ArrayJob) Reset

func (m *ArrayJob) Reset()

func (*ArrayJob) String

func (m *ArrayJob) String() string

func (*ArrayJob) Validate

func (m *ArrayJob) Validate() error

Validate checks the field values on ArrayJob with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*ArrayJob) XXX_DiscardUnknown

func (m *ArrayJob) XXX_DiscardUnknown()

func (*ArrayJob) XXX_Marshal

func (m *ArrayJob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ArrayJob) XXX_Merge

func (m *ArrayJob) XXX_Merge(src proto.Message)

func (*ArrayJob) XXX_Size

func (m *ArrayJob) XXX_Size() int

func (*ArrayJob) XXX_Unmarshal

func (m *ArrayJob) XXX_Unmarshal(b []byte) error

type ArrayJobValidationError

type ArrayJobValidationError struct {
	// contains filtered or unexported fields
}

ArrayJobValidationError is the validation error returned by ArrayJob.Validate if the designated constraints aren't met.

func (ArrayJobValidationError) Cause

func (e ArrayJobValidationError) Cause() error

Cause function returns cause value.

func (ArrayJobValidationError) Error

func (e ArrayJobValidationError) Error() string

Error satisfies the builtin error interface

func (ArrayJobValidationError) ErrorName

func (e ArrayJobValidationError) ErrorName() string

ErrorName returns error name.

func (ArrayJobValidationError) Field

func (e ArrayJobValidationError) Field() string

Field function returns field value.

func (ArrayJobValidationError) Key

func (e ArrayJobValidationError) Key() bool

Key function returns key value.

func (ArrayJobValidationError) Reason

func (e ArrayJobValidationError) Reason() string

Reason function returns reason value.

type HiveQuery

type HiveQuery struct {
	Query                string   `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
	TimeoutSec           uint32   `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
	RetryCount           uint32   `protobuf:"varint,3,opt,name=retryCount,proto3" json:"retryCount,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

Defines a query to execute on a hive cluster.

func (*HiveQuery) Descriptor

func (*HiveQuery) Descriptor() ([]byte, []int)

func (*HiveQuery) GetQuery

func (m *HiveQuery) GetQuery() string

func (*HiveQuery) GetRetryCount

func (m *HiveQuery) GetRetryCount() uint32

func (*HiveQuery) GetTimeoutSec

func (m *HiveQuery) GetTimeoutSec() uint32

func (*HiveQuery) ProtoMessage

func (*HiveQuery) ProtoMessage()

func (*HiveQuery) Reset

func (m *HiveQuery) Reset()

func (*HiveQuery) String

func (m *HiveQuery) String() string

func (*HiveQuery) Validate

func (m *HiveQuery) Validate() error

Validate checks the field values on HiveQuery with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*HiveQuery) XXX_DiscardUnknown

func (m *HiveQuery) XXX_DiscardUnknown()

func (*HiveQuery) XXX_Marshal

func (m *HiveQuery) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*HiveQuery) XXX_Merge

func (m *HiveQuery) XXX_Merge(src proto.Message)

func (*HiveQuery) XXX_Size

func (m *HiveQuery) XXX_Size() int

func (*HiveQuery) XXX_Unmarshal

func (m *HiveQuery) XXX_Unmarshal(b []byte) error

type HiveQueryCollection

type HiveQueryCollection struct {
	Queries              []*HiveQuery `protobuf:"bytes,2,rep,name=queries,proto3" json:"queries,omitempty"`
	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
	XXX_unrecognized     []byte       `json:"-"`
	XXX_sizecache        int32        `json:"-"`
}

Defines a collection of hive queries.

func (*HiveQueryCollection) Descriptor

func (*HiveQueryCollection) Descriptor() ([]byte, []int)

func (*HiveQueryCollection) GetQueries

func (m *HiveQueryCollection) GetQueries() []*HiveQuery

func (*HiveQueryCollection) ProtoMessage

func (*HiveQueryCollection) ProtoMessage()

func (*HiveQueryCollection) Reset

func (m *HiveQueryCollection) Reset()

func (*HiveQueryCollection) String

func (m *HiveQueryCollection) String() string

func (*HiveQueryCollection) Validate

func (m *HiveQueryCollection) Validate() error

Validate checks the field values on HiveQueryCollection with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*HiveQueryCollection) XXX_DiscardUnknown

func (m *HiveQueryCollection) XXX_DiscardUnknown()

func (*HiveQueryCollection) XXX_Marshal

func (m *HiveQueryCollection) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*HiveQueryCollection) XXX_Merge

func (m *HiveQueryCollection) XXX_Merge(src proto.Message)

func (*HiveQueryCollection) XXX_Size

func (m *HiveQueryCollection) XXX_Size() int

func (*HiveQueryCollection) XXX_Unmarshal

func (m *HiveQueryCollection) XXX_Unmarshal(b []byte) error

type HiveQueryCollectionValidationError

type HiveQueryCollectionValidationError struct {
	// contains filtered or unexported fields
}

HiveQueryCollectionValidationError is the validation error returned by HiveQueryCollection.Validate if the designated constraints aren't met.

func (HiveQueryCollectionValidationError) Cause

Cause function returns cause value.

func (HiveQueryCollectionValidationError) Error

Error satisfies the builtin error interface

func (HiveQueryCollectionValidationError) ErrorName

ErrorName returns error name.

func (HiveQueryCollectionValidationError) Field

Field function returns field value.

func (HiveQueryCollectionValidationError) Key

Key function returns key value.

func (HiveQueryCollectionValidationError) Reason

Reason function returns reason value.

type HiveQueryValidationError

type HiveQueryValidationError struct {
	// contains filtered or unexported fields
}

HiveQueryValidationError is the validation error returned by HiveQuery.Validate if the designated constraints aren't met.

func (HiveQueryValidationError) Cause

func (e HiveQueryValidationError) Cause() error

Cause function returns cause value.

func (HiveQueryValidationError) Error

func (e HiveQueryValidationError) Error() string

Error satisfies the builtin error interface

func (HiveQueryValidationError) ErrorName

func (e HiveQueryValidationError) ErrorName() string

ErrorName returns error name.

func (HiveQueryValidationError) Field

func (e HiveQueryValidationError) Field() string

Field function returns field value.

func (HiveQueryValidationError) Key

Key function returns key value.

func (HiveQueryValidationError) Reason

func (e HiveQueryValidationError) Reason() string

Reason function returns reason value.

type QuboleHiveJob

type QuboleHiveJob struct {
	ClusterLabel         string               `protobuf:"bytes,1,opt,name=cluster_label,json=clusterLabel,proto3" json:"cluster_label,omitempty"`
	QueryCollection      *HiveQueryCollection `protobuf:"bytes,2,opt,name=query_collection,json=queryCollection,proto3" json:"query_collection,omitempty"` // Deprecated: Do not use.
	Tags                 []string             `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"`
	Query                *HiveQuery           `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"`
	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
	XXX_unrecognized     []byte               `json:"-"`
	XXX_sizecache        int32                `json:"-"`
}

This message works with the 'hive' task type in the SDK and is the object that will be in the 'custom' field of a hive task's TaskTemplate

func (*QuboleHiveJob) Descriptor

func (*QuboleHiveJob) Descriptor() ([]byte, []int)

func (*QuboleHiveJob) GetClusterLabel

func (m *QuboleHiveJob) GetClusterLabel() string

func (*QuboleHiveJob) GetQuery added in v0.1.1

func (m *QuboleHiveJob) GetQuery() *HiveQuery

func (*QuboleHiveJob) GetQueryCollection deprecated

func (m *QuboleHiveJob) GetQueryCollection() *HiveQueryCollection

Deprecated: Do not use.

func (*QuboleHiveJob) GetTags

func (m *QuboleHiveJob) GetTags() []string

func (*QuboleHiveJob) ProtoMessage

func (*QuboleHiveJob) ProtoMessage()

func (*QuboleHiveJob) Reset

func (m *QuboleHiveJob) Reset()

func (*QuboleHiveJob) String

func (m *QuboleHiveJob) String() string

func (*QuboleHiveJob) Validate

func (m *QuboleHiveJob) Validate() error

Validate checks the field values on QuboleHiveJob with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*QuboleHiveJob) XXX_DiscardUnknown

func (m *QuboleHiveJob) XXX_DiscardUnknown()

func (*QuboleHiveJob) XXX_Marshal

func (m *QuboleHiveJob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*QuboleHiveJob) XXX_Merge

func (m *QuboleHiveJob) XXX_Merge(src proto.Message)

func (*QuboleHiveJob) XXX_Size

func (m *QuboleHiveJob) XXX_Size() int

func (*QuboleHiveJob) XXX_Unmarshal

func (m *QuboleHiveJob) XXX_Unmarshal(b []byte) error

type QuboleHiveJobValidationError

type QuboleHiveJobValidationError struct {
	// contains filtered or unexported fields
}

QuboleHiveJobValidationError is the validation error returned by QuboleHiveJob.Validate if the designated constraints aren't met.

func (QuboleHiveJobValidationError) Cause

Cause function returns cause value.

func (QuboleHiveJobValidationError) Error

Error satisfies the builtin error interface

func (QuboleHiveJobValidationError) ErrorName

func (e QuboleHiveJobValidationError) ErrorName() string

ErrorName returns error name.

func (QuboleHiveJobValidationError) Field

Field function returns field value.

func (QuboleHiveJobValidationError) Key

Key function returns key value.

func (QuboleHiveJobValidationError) Reason

Reason function returns reason value.

type SidecarJob

type SidecarJob struct {
	PodSpec              *v1.PodSpec `protobuf:"bytes,1,opt,name=pod_spec,json=podSpec,proto3" json:"pod_spec,omitempty"`
	PrimaryContainerName string      `protobuf:"bytes,2,opt,name=primary_container_name,json=primaryContainerName,proto3" json:"primary_container_name,omitempty"`
	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
	XXX_unrecognized     []byte      `json:"-"`
	XXX_sizecache        int32       `json:"-"`
}

A sidecar job brings up the desired pod_spec. The plugin executor is responsible for keeping the pod alive until the primary container terminates or the task itself times out.

func (*SidecarJob) Descriptor

func (*SidecarJob) Descriptor() ([]byte, []int)

func (*SidecarJob) GetPodSpec

func (m *SidecarJob) GetPodSpec() *v1.PodSpec

func (*SidecarJob) GetPrimaryContainerName

func (m *SidecarJob) GetPrimaryContainerName() string

func (*SidecarJob) ProtoMessage

func (*SidecarJob) ProtoMessage()

func (*SidecarJob) Reset

func (m *SidecarJob) Reset()

func (*SidecarJob) String

func (m *SidecarJob) String() string

func (*SidecarJob) Validate

func (m *SidecarJob) Validate() error

Validate checks the field values on SidecarJob with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*SidecarJob) XXX_DiscardUnknown

func (m *SidecarJob) XXX_DiscardUnknown()

func (*SidecarJob) XXX_Marshal

func (m *SidecarJob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*SidecarJob) XXX_Merge

func (m *SidecarJob) XXX_Merge(src proto.Message)

func (*SidecarJob) XXX_Size

func (m *SidecarJob) XXX_Size() int

func (*SidecarJob) XXX_Unmarshal

func (m *SidecarJob) XXX_Unmarshal(b []byte) error

type SidecarJobValidationError

type SidecarJobValidationError struct {
	// contains filtered or unexported fields
}

SidecarJobValidationError is the validation error returned by SidecarJob.Validate if the designated constraints aren't met.

func (SidecarJobValidationError) Cause

func (e SidecarJobValidationError) Cause() error

Cause function returns cause value.

func (SidecarJobValidationError) Error

Error satisfies the builtin error interface

func (SidecarJobValidationError) ErrorName

func (e SidecarJobValidationError) ErrorName() string

ErrorName returns error name.

func (SidecarJobValidationError) Field

Field function returns field value.

func (SidecarJobValidationError) Key

Key function returns key value.

func (SidecarJobValidationError) Reason

func (e SidecarJobValidationError) Reason() string

Reason function returns reason value.

type SparkApplication

type SparkApplication struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*SparkApplication) Descriptor

func (*SparkApplication) Descriptor() ([]byte, []int)

func (*SparkApplication) ProtoMessage

func (*SparkApplication) ProtoMessage()

func (*SparkApplication) Reset

func (m *SparkApplication) Reset()

func (*SparkApplication) String

func (m *SparkApplication) String() string

func (*SparkApplication) Validate

func (m *SparkApplication) Validate() error

Validate checks the field values on SparkApplication with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*SparkApplication) XXX_DiscardUnknown

func (m *SparkApplication) XXX_DiscardUnknown()

func (*SparkApplication) XXX_Marshal

func (m *SparkApplication) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*SparkApplication) XXX_Merge

func (m *SparkApplication) XXX_Merge(src proto.Message)

func (*SparkApplication) XXX_Size

func (m *SparkApplication) XXX_Size() int

func (*SparkApplication) XXX_Unmarshal

func (m *SparkApplication) XXX_Unmarshal(b []byte) error

type SparkApplicationValidationError

type SparkApplicationValidationError struct {
	// contains filtered or unexported fields
}

SparkApplicationValidationError is the validation error returned by SparkApplication.Validate if the designated constraints aren't met.

func (SparkApplicationValidationError) Cause

Cause function returns cause value.

func (SparkApplicationValidationError) Error

Error satisfies the builtin error interface

func (SparkApplicationValidationError) ErrorName

ErrorName returns error name.

func (SparkApplicationValidationError) Field

Field function returns field value.

func (SparkApplicationValidationError) Key

Key function returns key value.

func (SparkApplicationValidationError) Reason

Reason function returns reason value.

type SparkApplication_Type

type SparkApplication_Type int32
const (
	SparkApplication_PYTHON SparkApplication_Type = 0
	SparkApplication_JAVA   SparkApplication_Type = 1
	SparkApplication_SCALA  SparkApplication_Type = 2
	SparkApplication_R      SparkApplication_Type = 3
)

func (SparkApplication_Type) EnumDescriptor

func (SparkApplication_Type) EnumDescriptor() ([]byte, []int)

func (SparkApplication_Type) String

func (x SparkApplication_Type) String() string

type SparkJob

type SparkJob struct {
	ApplicationType      SparkApplication_Type `` /* 128-byte string literal not displayed */
	MainApplicationFile  string                `protobuf:"bytes,2,opt,name=mainApplicationFile,proto3" json:"mainApplicationFile,omitempty"`
	MainClass            string                `protobuf:"bytes,3,opt,name=mainClass,proto3" json:"mainClass,omitempty"`
	SparkConf            map[string]string     `` /* 159-byte string literal not displayed */
	HadoopConf           map[string]string     `` /* 161-byte string literal not displayed */
	ExecutorPath         string                `protobuf:"bytes,6,opt,name=executorPath,proto3" json:"executorPath,omitempty"`
	XXX_NoUnkeyedLiteral struct{}              `json:"-"`
	XXX_unrecognized     []byte                `json:"-"`
	XXX_sizecache        int32                 `json:"-"`
}

Custom Proto for Spark Plugin.

func (*SparkJob) Descriptor

func (*SparkJob) Descriptor() ([]byte, []int)

func (*SparkJob) GetApplicationType

func (m *SparkJob) GetApplicationType() SparkApplication_Type

func (*SparkJob) GetExecutorPath

func (m *SparkJob) GetExecutorPath() string

func (*SparkJob) GetHadoopConf

func (m *SparkJob) GetHadoopConf() map[string]string

func (*SparkJob) GetMainApplicationFile

func (m *SparkJob) GetMainApplicationFile() string

func (*SparkJob) GetMainClass

func (m *SparkJob) GetMainClass() string

func (*SparkJob) GetSparkConf

func (m *SparkJob) GetSparkConf() map[string]string

func (*SparkJob) ProtoMessage

func (*SparkJob) ProtoMessage()

func (*SparkJob) Reset

func (m *SparkJob) Reset()

func (*SparkJob) String

func (m *SparkJob) String() string

func (*SparkJob) Validate

func (m *SparkJob) Validate() error

Validate checks the field values on SparkJob with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*SparkJob) XXX_DiscardUnknown

func (m *SparkJob) XXX_DiscardUnknown()

func (*SparkJob) XXX_Marshal

func (m *SparkJob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*SparkJob) XXX_Merge

func (m *SparkJob) XXX_Merge(src proto.Message)

func (*SparkJob) XXX_Size

func (m *SparkJob) XXX_Size() int

func (*SparkJob) XXX_Unmarshal

func (m *SparkJob) XXX_Unmarshal(b []byte) error

type SparkJobValidationError

type SparkJobValidationError struct {
	// contains filtered or unexported fields
}

SparkJobValidationError is the validation error returned by SparkJob.Validate if the designated constraints aren't met.

func (SparkJobValidationError) Cause

func (e SparkJobValidationError) Cause() error

Cause function returns cause value.

func (SparkJobValidationError) Error

func (e SparkJobValidationError) Error() string

Error satisfies the builtin error interface

func (SparkJobValidationError) ErrorName

func (e SparkJobValidationError) ErrorName() string

ErrorName returns error name.

func (SparkJobValidationError) Field

func (e SparkJobValidationError) Field() string

Field function returns field value.

func (SparkJobValidationError) Key

func (e SparkJobValidationError) Key() bool

Key function returns key value.

func (SparkJobValidationError) Reason

func (e SparkJobValidationError) Reason() string

Reason function returns reason value.

type Waitable

type Waitable struct {
	WfExecId             *core.WorkflowExecutionIdentifier `protobuf:"bytes,1,opt,name=wf_exec_id,json=wfExecId,proto3" json:"wf_exec_id,omitempty"`
	Phase                core.WorkflowExecution_Phase      `protobuf:"varint,2,opt,name=phase,proto3,enum=flyteidl.core.WorkflowExecution_Phase" json:"phase,omitempty"`
	WorkflowId           string                            `protobuf:"bytes,3,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{}                          `json:"-"`
	XXX_unrecognized     []byte                            `json:"-"`
	XXX_sizecache        int32                             `json:"-"`
}

Represents an Execution that was launched and could be waited on.

func (*Waitable) Descriptor

func (*Waitable) Descriptor() ([]byte, []int)

func (*Waitable) GetPhase

func (m *Waitable) GetPhase() core.WorkflowExecution_Phase

func (*Waitable) GetWfExecId

func (m *Waitable) GetWfExecId() *core.WorkflowExecutionIdentifier

func (*Waitable) GetWorkflowId

func (m *Waitable) GetWorkflowId() string

func (*Waitable) ProtoMessage

func (*Waitable) ProtoMessage()

func (*Waitable) Reset

func (m *Waitable) Reset()

func (*Waitable) String

func (m *Waitable) String() string

func (*Waitable) Validate

func (m *Waitable) Validate() error

Validate checks the field values on Waitable with the rules defined in the proto definition for this message. If any rules are violated, an error is returned.

func (*Waitable) XXX_DiscardUnknown

func (m *Waitable) XXX_DiscardUnknown()

func (*Waitable) XXX_Marshal

func (m *Waitable) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Waitable) XXX_Merge

func (m *Waitable) XXX_Merge(src proto.Message)

func (*Waitable) XXX_Size

func (m *Waitable) XXX_Size() int

func (*Waitable) XXX_Unmarshal

func (m *Waitable) XXX_Unmarshal(b []byte) error

type WaitableValidationError

type WaitableValidationError struct {
	// contains filtered or unexported fields
}

WaitableValidationError is the validation error returned by Waitable.Validate if the designated constraints aren't met.

func (WaitableValidationError) Cause

func (e WaitableValidationError) Cause() error

Cause function returns cause value.

func (WaitableValidationError) Error

func (e WaitableValidationError) Error() string

Error satisfies the builtin error interface

func (WaitableValidationError) ErrorName

func (e WaitableValidationError) ErrorName() string

ErrorName returns error name.

func (WaitableValidationError) Field

func (e WaitableValidationError) Field() string

Field function returns field value.

func (WaitableValidationError) Key

func (e WaitableValidationError) Key() bool

Key function returns key value.

func (WaitableValidationError) Reason

func (e WaitableValidationError) Reason() string

Reason function returns reason value.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL