Documentation
¶
Index ¶
- Constants
- Variables
- func DecodeResourceName(resName ResourceName) (string, error)
- func ParseResourceID(rpath ResourceID) (ResourceType, ResourceName, error)
- func ToResourceRequirement(jobID JobID, resourceIDs ...ResourceID) []*pb.ResourceKey
- type Config
- type ExecutorID
- type GCSConfig
- type JobID
- type LocalFileConfig
- type ResourceID
- type ResourceKey
- type ResourceMeta
- type ResourceName
- type ResourceType
- type S3Config
- type WorkerID
Constants ¶
const ( ResourceTypeLocalFile = ResourceType("local") ResourceTypeS3 = ResourceType("s3") ResourceTypeGCS = ResourceType("gs") ResourceTypeNone = ResourceType("none") )
Define all supported resource types.
Variables ¶
var DefaultConfig = Config{ Local: LocalFileConfig{BaseDir: ""}, S3: S3Config{ S3BackendOptions: brStorage.S3BackendOptions{ ForcePathStyle: true, }, Bucket: "", Prefix: "", }, GCS: GCSConfig{ Bucket: "", Prefix: "", }, }
DefaultConfig defines the default configuration for external storage
var ResourceUpdateColumns = []string{
"updated_at",
"project_id",
"tenant_id",
"id",
"job_id",
"worker_id",
"executor_id",
"deleted",
}
ResourceUpdateColumns is used in gorm update
Functions ¶
func DecodeResourceName ¶
func DecodeResourceName(resName ResourceName) (string, error)
DecodeResourceName decodes resource name to raw resource name.
func ParseResourceID ¶
func ParseResourceID(rpath ResourceID) (ResourceType, ResourceName, error)
ParseResourceID returns the ResourceType and the path suffix.
func ToResourceRequirement ¶
func ToResourceRequirement(jobID JobID, resourceIDs ...ResourceID) []*pb.ResourceKey
ToResourceRequirement return the resource requirement of pb
Types ¶
type Config ¶
type Config struct {
Local LocalFileConfig `json:"local" toml:"local"`
S3 S3Config `json:"s3" toml:"s3"`
GCS GCSConfig `json:"gcs" toml:"gcs"`
}
Config defines configurations for an external storage resource
func (*Config) Adjust ¶
func (c *Config) Adjust(executorID ExecutorID)
Adjust adjusts the configuration
func (Config) GCSEnabled ¶
GCSEnabled returns true if the GCS storage is enabled
func (Config) LocalEnabled ¶
LocalEnabled returns true if the local storage is enabled
func (Config) ToBrBackendOptions ¶
func (c Config) ToBrBackendOptions() (opts *brStorage.BackendOptions, bucket, prefix string, tp ResourceType)
ToBrBackendOptions return BackendOptions for brStorage Make sure the Config is a valid config
type GCSConfig ¶
type GCSConfig struct {
brStorage.GCSBackendOptions
Bucket string `json:"bucket" toml:"bucket"`
Prefix string `json:"prefix" toml:"prefix"`
}
GCSConfig defines configurations for gcs based resources
type LocalFileConfig ¶
type LocalFileConfig struct {
BaseDir string `json:"base-dir" toml:"base-dir"`
}
LocalFileConfig defines configurations for a local file based resource
func (*LocalFileConfig) Adjust ¶
func (c *LocalFileConfig) Adjust(executorID ExecutorID)
Adjust adjusts the local file config
type ResourceID ¶
type ResourceID = string
ResourceID should be in the form of `/<type>/<unique-name>`, currently only local type is available.
func BuildResourceID ¶
func BuildResourceID(rtype ResourceType, resName ResourceName) ResourceID
BuildResourceID returns an ResourceID based on given ResourceType and ResourceName.
type ResourceKey ¶
type ResourceKey struct {
JobID JobID
ID ResourceID
}
ResourceKey is the unique identifier for the resource
func ToResourceKeys ¶
func ToResourceKeys(requires []*pb.ResourceKey) []ResourceKey
ToResourceKeys converts resource requirements in pb to resource keys
type ResourceMeta ¶
type ResourceMeta struct {
ormModel.Model
ProjectID tenant.ProjectID `json:"project-id" gorm:"column:project_id;type:varchar(128) not null;"`
TenantID tenant.Tenant `json:"tenant-id" gorm:"column:tenant_id;type:varchar(128) not null;"`
ID ResourceID `json:"id" gorm:"column:id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:2;index:idx_rei,priority:2"`
Job JobID `json:"job" gorm:"column:job_id;type:varchar(128) not null;uniqueIndex:uidx_rid,priority:1"`
Worker WorkerID `json:"worker" gorm:"column:worker_id;type:varchar(128) not null"`
Executor ExecutorID `json:"executor" gorm:"column:executor_id;type:varchar(128) not null;index:idx_rei,priority:1"`
GCPending bool `json:"gc-pending" gorm:"column:gc_pending;type:BOOLEAN"`
// TODO soft delete has not be implemented, because it requires modifying too many
// unit tests in engine/pkg/orm
Deleted bool `json:"deleted" gorm:"column:deleted;type:BOOLEAN"`
}
ResourceMeta is the records stored in the metastore.
func (*ResourceMeta) GetID ¶
func (m *ResourceMeta) GetID() string
GetID implements dataset.DataEntry
func (*ResourceMeta) Map ¶
func (m *ResourceMeta) Map() map[string]interface{}
Map is used in gorm update
func (*ResourceMeta) ToQueryResourceResponse ¶
func (m *ResourceMeta) ToQueryResourceResponse() *pb.QueryResourceResponse
ToQueryResourceResponse converts the ResourceMeta to pb.QueryResourceResponse
type ResourceName ¶
type ResourceName = string
ResourceName is a string encoding raw resource name in hexadecimal. The raw resource name is the ResourceID with its type prefix removed. For example, the raw resource name of `/local/resource-1` is `resource-1`.
func EncodeResourceName ¶
func EncodeResourceName(rawResName string) ResourceName
EncodeResourceName encodes raw resource name to a valid resource name.
type ResourceType ¶
type ResourceType string
ResourceType represents the type of the resource
func (ResourceType) BuildPrefix ¶
func (r ResourceType) BuildPrefix() string
BuildPrefix returns the prefix of the resource type.
Source Files