Documentation
¶
Index ¶
- Constants
- Variables
- func DatumID(inputs []*Input) string
- func HashDatum(pipelineSalt string, inputs []*Input) string
- func MatchDatum(filter []string, inputs []*pps.InputFile) bool
- func MetaFilePath(id string) string
- func Shard(ctx context.Context, c pfs.APIClient, fileSetIDs []string) ([]*pfs.PathRange, error)
- type Input
- func (*Input) Descriptor() ([]byte, []int)deprecated
- func (x *Input) GetBranch() string
- func (x *Input) GetEmptyFiles() bool
- func (x *Input) GetFileInfo() *pfs.FileInfo
- func (x *Input) GetGroupBy() string
- func (x *Input) GetJoinOn() string
- func (x *Input) GetLazy() bool
- func (x *Input) GetName() string
- func (x *Input) GetOuterJoin() bool
- func (x *Input) GetS3() bool
- func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*Input) ProtoMessage()
- func (x *Input) ProtoReflect() protoreflect.Message
- func (x *Input) Reset()
- func (x *Input) String() string
- func (m *Input) Validate() error
- func (m *Input) ValidateAll() error
- type InputMultiError
- type InputValidationError
Constants ¶
const ( // MetaPrefix is the prefix for the meta path. MetaPrefix = "meta" // MetaFileName is the name of the meta file. MetaFileName = "meta" // PFSPrefix is the prefix for the pfs path. PFSPrefix = "pfs" // OutputPrefix is the prefix for the output path. OutputPrefix = "out" // TmpFileName is the name of the tmp file. TmpFileName = "tmp" // EnvFileName is the name of the env file. EnvFileName = ".env" TTL = 15 * time.Minute )
Variables ¶
var File_server_worker_common_common_proto protoreflect.FileDescriptor
Functions ¶
func MatchDatum ¶
MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and inputs is irrelevant.
func MetaFilePath ¶
Types ¶
type Input ¶
type Input struct {
FileInfo *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo,proto3" json:"file_info,omitempty"`
Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
JoinOn string `protobuf:"bytes,4,opt,name=join_on,json=joinOn,proto3" json:"join_on,omitempty"`
OuterJoin bool `protobuf:"varint,5,opt,name=outer_join,json=outerJoin,proto3" json:"outer_join,omitempty"`
GroupBy string `protobuf:"bytes,6,opt,name=group_by,json=groupBy,proto3" json:"group_by,omitempty"`
Lazy bool `protobuf:"varint,7,opt,name=lazy,proto3" json:"lazy,omitempty"`
Branch string `protobuf:"bytes,8,opt,name=branch,proto3" json:"branch,omitempty"`
EmptyFiles bool `protobuf:"varint,10,opt,name=empty_files,json=emptyFiles,proto3" json:"empty_files,omitempty"`
S3 bool `protobuf:"varint,11,opt,name=s3,proto3" json:"s3,omitempty"` // If set, workers won't create an input directory for this input
// contains filtered or unexported fields
}
func (*Input) Descriptor
deprecated
func (*Input) GetEmptyFiles ¶
func (*Input) GetFileInfo ¶
func (*Input) GetGroupBy ¶
func (*Input) GetOuterJoin ¶
func (*Input) MarshalLogObject ¶
func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*Input) ProtoMessage ¶
func (*Input) ProtoMessage()
func (*Input) ProtoReflect ¶ added in v2.7.0
func (x *Input) ProtoReflect() protoreflect.Message
func (*Input) Validate ¶ added in v2.8.0
Validate checks the field values on Input with the rules defined in the proto definition for this message. If any rules are violated, the first error encountered is returned, or nil if there are no violations.
func (*Input) ValidateAll ¶ added in v2.8.0
ValidateAll checks the field values on Input with the rules defined in the proto definition for this message. If any rules are violated, the result is a list of violation errors wrapped in InputMultiError, or nil if none found.
type InputMultiError ¶ added in v2.8.0
type InputMultiError []error
InputMultiError is an error wrapping multiple validation errors returned by Input.ValidateAll() if the designated constraints aren't met.
func (InputMultiError) AllErrors ¶ added in v2.8.0
func (m InputMultiError) AllErrors() []error
AllErrors returns a list of validation violation errors.
func (InputMultiError) Error ¶ added in v2.8.0
func (m InputMultiError) Error() string
Error returns a concatenation of all the error messages it wraps.
type InputValidationError ¶ added in v2.8.0
type InputValidationError struct {
// contains filtered or unexported fields
}
InputValidationError is the validation error returned by Input.Validate if the designated constraints aren't met.
func (InputValidationError) Cause ¶ added in v2.8.0
func (e InputValidationError) Cause() error
Cause function returns cause value.
func (InputValidationError) Error ¶ added in v2.8.0
func (e InputValidationError) Error() string
Error satisfies the builtin error interface
func (InputValidationError) ErrorName ¶ added in v2.8.0
func (e InputValidationError) ErrorName() string
ErrorName returns error name.
func (InputValidationError) Field ¶ added in v2.8.0
func (e InputValidationError) Field() string
Field function returns field value.
func (InputValidationError) Key ¶ added in v2.8.0
func (e InputValidationError) Key() bool
Key function returns key value.
func (InputValidationError) Reason ¶ added in v2.8.0
func (e InputValidationError) Reason() string
Reason function returns reason value.
Source Files