plugins

type ArrayJob struct {

	// Defines the maximum number of instances to bring up concurrently at any given point. Note that this is an
	// optimistic restriction and that, due to network partitioning or other failures, the actual number of currently
	// running instances might be more. This has to be a positive number if assigned. Default value is size.
	Parallelism int64 `protobuf:"varint,1,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
	// Defines the number of instances to launch at most. This number should match the size of the input if the job
	// requires processing of all input data. This has to be a positive number.
	// In the case this is not defined, the back-end will determine the size at run-time by reading the inputs.
	Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
	// Types that are assignable to SuccessCriteria:
	//
	//	*ArrayJob_MinSuccesses
	//	*ArrayJob_MinSuccessRatio
	SuccessCriteria isArrayJob_SuccessCriteria `protobuf_oneof:"success_criteria"`
	// contains filtered or unexported fields
}

type ArrayJob_MinSuccessRatio struct {
	// If the array job size is not known beforehand, the min_success_ratio can instead be used to determine when an array
	// job can be marked successful.
	MinSuccessRatio float32 `protobuf:"fixed32,4,opt,name=min_success_ratio,json=minSuccessRatio,proto3,oneof"`
}

type ArrayJob_MinSuccesses struct {
	// An absolute number of the minimum number of successful completions of subtasks. As soon as this criteria is met,
	// the array job will be marked as successful and outputs will be computed. This has to be a non-negative number if
	// assigned. Default value is size (if specified).
	MinSuccesses int64 `protobuf:"varint,3,opt,name=min_successes,json=minSuccesses,proto3,oneof"`
}

type DaskJob struct {

	// Spec for the scheduler pod.
	Scheduler *DaskScheduler `protobuf:"bytes,1,opt,name=scheduler,proto3" json:"scheduler,omitempty"`
	// Spec of the default worker group.
	Workers *DaskWorkerGroup `protobuf:"bytes,2,opt,name=workers,proto3" json:"workers,omitempty"`
	// contains filtered or unexported fields
}

type DaskScheduler struct {

	// Optional image to use. If unset, will use the default image.
	Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"`
	// Resources assigned to the scheduler pod.
	Resources *core.Resources `protobuf:"bytes,2,opt,name=resources,proto3" json:"resources,omitempty"`
	// contains filtered or unexported fields
}

type DaskWorkerGroup struct {

	// Number of workers in the group.
	NumberOfWorkers uint32 `protobuf:"varint,1,opt,name=number_of_workers,json=numberOfWorkers,proto3" json:"number_of_workers,omitempty"`
	// Optional image to use for the pods of the worker group. If unset, will use the default image.
	Image string `protobuf:"bytes,2,opt,name=image,proto3" json:"image,omitempty"`
	// Resources assigned to the all pods of the worker group.
	// As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices
	// it is advised to only set limits. If requests are not explicitly set, the plugin will make
	// sure to set requests==limits.
	// The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit.
	Resources *core.Resources `protobuf:"bytes,3,opt,name=resources,proto3" json:"resources,omitempty"`
	// contains filtered or unexported fields
}

type DistributedMPITrainingTask struct {

	// number of worker spawned in the cluster for this job
	NumWorkers int32 `protobuf:"varint,1,opt,name=num_workers,json=numWorkers,proto3" json:"num_workers,omitempty"`
	// number of launcher replicas spawned in the cluster for this job
	// The launcher pod invokes mpirun and communicates with worker pods through MPI.
	NumLauncherReplicas int32 `protobuf:"varint,2,opt,name=num_launcher_replicas,json=numLauncherReplicas,proto3" json:"num_launcher_replicas,omitempty"`
	// number of slots per worker used in hostfile.
	// The available slots (GPUs) in each pod.
	Slots int32 `protobuf:"varint,3,opt,name=slots,proto3" json:"slots,omitempty"`
	// contains filtered or unexported fields
}

type DistributedPyTorchTrainingTask struct {

	// number of worker replicas spawned in the cluster for this job
	Workers int32 `protobuf:"varint,1,opt,name=workers,proto3" json:"workers,omitempty"`
	// config for an elastic pytorch job
	ElasticConfig *ElasticConfig `protobuf:"bytes,2,opt,name=elastic_config,json=elasticConfig,proto3" json:"elastic_config,omitempty"`
	// contains filtered or unexported fields
}

type DistributedTensorflowTrainingTask struct {

	// number of worker replicas spawned in the cluster for this job
	Workers int32 `protobuf:"varint,1,opt,name=workers,proto3" json:"workers,omitempty"`
	// PS -> Parameter server
	// number of ps replicas spawned in the cluster for this job
	PsReplicas int32 `protobuf:"varint,2,opt,name=ps_replicas,json=psReplicas,proto3" json:"ps_replicas,omitempty"`
	// number of chief replicas spawned in the cluster for this job
	ChiefReplicas int32 `protobuf:"varint,3,opt,name=chief_replicas,json=chiefReplicas,proto3" json:"chief_replicas,omitempty"`
	// number of evaluator replicas spawned in the cluster for this job
	EvaluatorReplicas int32 `protobuf:"varint,4,opt,name=evaluator_replicas,json=evaluatorReplicas,proto3" json:"evaluator_replicas,omitempty"`
	// contains filtered or unexported fields
}

type ElasticConfig struct {
	RdzvBackend  string `protobuf:"bytes,1,opt,name=rdzv_backend,json=rdzvBackend,proto3" json:"rdzv_backend,omitempty"`
	MinReplicas  int32  `protobuf:"varint,2,opt,name=min_replicas,json=minReplicas,proto3" json:"min_replicas,omitempty"`
	MaxReplicas  int32  `protobuf:"varint,3,opt,name=max_replicas,json=maxReplicas,proto3" json:"max_replicas,omitempty"`
	NprocPerNode int32  `protobuf:"varint,4,opt,name=nproc_per_node,json=nprocPerNode,proto3" json:"nproc_per_node,omitempty"`
	MaxRestarts  int32  `protobuf:"varint,5,opt,name=max_restarts,json=maxRestarts,proto3" json:"max_restarts,omitempty"`
	// contains filtered or unexported fields
}

type HeadGroupSpec struct {

	// Optional. RayStartParams are the params of the start command: address, object-store-memory.
	// Refer to https://docs.ray.io/en/latest/ray-core/package-ref.html#ray-start
	RayStartParams map[string]string `` /* 193-byte string literal not displayed */
	// contains filtered or unexported fields
}

type HiveQuery struct {
	Query      string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
	TimeoutSec uint32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"`
	RetryCount uint32 `protobuf:"varint,3,opt,name=retryCount,proto3" json:"retryCount,omitempty"`
	// contains filtered or unexported fields
}

type HiveQueryCollection struct {
	Queries []*HiveQuery `protobuf:"bytes,2,rep,name=queries,proto3" json:"queries,omitempty"`
	// contains filtered or unexported fields
}

type PrestoQuery struct {
	RoutingGroup string `protobuf:"bytes,1,opt,name=routing_group,json=routingGroup,proto3" json:"routing_group,omitempty"`
	Catalog      string `protobuf:"bytes,2,opt,name=catalog,proto3" json:"catalog,omitempty"`
	Schema       string `protobuf:"bytes,3,opt,name=schema,proto3" json:"schema,omitempty"`
	Statement    string `protobuf:"bytes,4,opt,name=statement,proto3" json:"statement,omitempty"`
	// contains filtered or unexported fields
}

type QuboleHiveJob struct {
	ClusterLabel string `protobuf:"bytes,1,opt,name=cluster_label,json=clusterLabel,proto3" json:"cluster_label,omitempty"`
	// Deprecated: Marked as deprecated in kozmoidl/plugins/qubole.proto.
	QueryCollection *HiveQueryCollection `protobuf:"bytes,2,opt,name=query_collection,json=queryCollection,proto3" json:"query_collection,omitempty"`
	Tags            []string             `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"`
	Query           *HiveQuery           `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"`
	// contains filtered or unexported fields
}

type RayCluster struct {

	// HeadGroupSpecs are the spec for the head pod
	HeadGroupSpec *HeadGroupSpec `protobuf:"bytes,1,opt,name=head_group_spec,json=headGroupSpec,proto3" json:"head_group_spec,omitempty"`
	// WorkerGroupSpecs are the specs for the worker pods
	WorkerGroupSpec []*WorkerGroupSpec `protobuf:"bytes,2,rep,name=worker_group_spec,json=workerGroupSpec,proto3" json:"worker_group_spec,omitempty"`
	// Whether to enable autoscaling.
	EnableAutoscaling bool `protobuf:"varint,3,opt,name=enable_autoscaling,json=enableAutoscaling,proto3" json:"enable_autoscaling,omitempty"`
	// contains filtered or unexported fields
}

type RayJob struct {

	// RayClusterSpec is the cluster template to run the job
	RayCluster *RayCluster `protobuf:"bytes,1,opt,name=ray_cluster,json=rayCluster,proto3" json:"ray_cluster,omitempty"`
	// runtime_env is base64 encoded.
	// Ray runtime environments: https://docs.ray.io/en/latest/ray-core/handling-dependencies.html#runtime-environments
	RuntimeEnv string `protobuf:"bytes,2,opt,name=runtime_env,json=runtimeEnv,proto3" json:"runtime_env,omitempty"`
	// shutdown_after_job_finishes specifies whether the RayCluster should be deleted after the RayJob finishes.
	ShutdownAfterJobFinishes bool `` /* 138-byte string literal not displayed */
	// ttl_seconds_after_finished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes.
	TtlSecondsAfterFinished int32 `` /* 135-byte string literal not displayed */
	// contains filtered or unexported fields
}

type SparkApplication struct {
	// contains filtered or unexported fields
}

type SparkApplication_Type int32

type SparkJob struct {
	ApplicationType     SparkApplication_Type `` /* 128-byte string literal not displayed */
	MainApplicationFile string                `protobuf:"bytes,2,opt,name=mainApplicationFile,proto3" json:"mainApplicationFile,omitempty"`
	MainClass           string                `protobuf:"bytes,3,opt,name=mainClass,proto3" json:"mainClass,omitempty"`
	SparkConf           map[string]string     ``                                                                              /* 159-byte string literal not displayed */
	HadoopConf          map[string]string     ``                                                                              /* 161-byte string literal not displayed */
	ExecutorPath        string                `protobuf:"bytes,6,opt,name=executorPath,proto3" json:"executorPath,omitempty"` // Executor path for Python jobs.
	// Databricks job configuration.
	// Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure.
	DatabricksConf *structpb.Struct `protobuf:"bytes,7,opt,name=databricksConf,proto3" json:"databricksConf,omitempty"`
	// Databricks access token. https://docs.databricks.com/dev-tools/api/latest/authentication.html
	// This token can be set in either kozmopropeller or kozmokit.
	DatabricksToken string `protobuf:"bytes,8,opt,name=databricksToken,proto3" json:"databricksToken,omitempty"`
	// Domain name of your deployment. Use the form <account>.cloud.databricks.com.
	// This instance name can be set in either kozmopropeller or kozmokit.
	DatabricksInstance string `protobuf:"bytes,9,opt,name=databricksInstance,proto3" json:"databricksInstance,omitempty"`
	// contains filtered or unexported fields
}

type Waitable struct {
	WfExecId   *core.WorkflowExecutionIdentifier `protobuf:"bytes,1,opt,name=wf_exec_id,json=wfExecId,proto3" json:"wf_exec_id,omitempty"`
	Phase      core.WorkflowExecution_Phase      `protobuf:"varint,2,opt,name=phase,proto3,enum=kozmoidl.core.WorkflowExecution_Phase" json:"phase,omitempty"`
	WorkflowId string                            `protobuf:"bytes,3,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"`
	// contains filtered or unexported fields
}

type WorkerGroupSpec struct {

	// Required. RayCluster can have multiple worker groups, and it distinguishes them by name
	GroupName string `protobuf:"bytes,1,opt,name=group_name,json=groupName,proto3" json:"group_name,omitempty"`
	// Required. Desired replicas of the worker group. Defaults to 1.
	Replicas int32 `protobuf:"varint,2,opt,name=replicas,proto3" json:"replicas,omitempty"`
	// Optional. Min replicas of the worker group. MinReplicas defaults to 1.
	MinReplicas int32 `protobuf:"varint,3,opt,name=min_replicas,json=minReplicas,proto3" json:"min_replicas,omitempty"`
	// Optional. Max replicas of the worker group. MaxReplicas defaults to maxInt32
	MaxReplicas int32 `protobuf:"varint,4,opt,name=max_replicas,json=maxReplicas,proto3" json:"max_replicas,omitempty"`
	// Optional. RayStartParams are the params of the start command: address, object-store-memory.
	// Refer to https://docs.ray.io/en/latest/ray-core/package-ref.html#ray-start
	RayStartParams map[string]string `` /* 193-byte string literal not displayed */
	// contains filtered or unexported fields
}