Documentation ¶
Index ¶
- Constants
- Variables
- func DFS(node string, graphNodes map[string]bool, graphEdges map[string][]string, ...) error
- func DownloadToolFile(url string) ([]byte, error)
- func FetchTools(cmd *cobra.Command) (map[string]Tool, error)
- func GetAllToolFile(rootPath string) ([]string, error)
- func GetExecutionName(inputs map[string]Input) string
- func GetExecutionNamespace(inputs map[string]Input) string
- func GetInputType(val interface{}) string
- func GetRangeFuncParam(str string) (string, string, string)
- func GetStringValue(key string, inputs map[string]Input) string
- func GetVariantName(str string) string
- func Inputs2ReplaceData(inputs map[string]Input) map[string]string
- func InstantiateCheckResultFunc(prefix, str string, data map[string]string) common.Var
- func InstantiateCondition(prefix string, condition interface{}, data map[string]string) (common.Var, error)
- func InstantiateGetResultFunc(prefix, str string, data map[string]string) common.Var
- func InstantiateRangeFunc(prefix, str string, data map[string]string) (common.Var, error)
- func InstantiateVars(prefix string, vars []interface{}, data map[string]string) ([]common.Var, error)
- func InstantiateVarsIter(prefix string, vars []interface{}, data map[string]string) ([]common.Var, bool, error)
- func InstantiateWorkflow(workflow *Workflow, inputs map[string]interface{}, tools map[string]Tool) error
- func IsCheckResultFunc(str string) bool
- func IsCommandIterEmpty(commandIter CommandsIter) bool
- func IsGetResultFunc(str string) bool
- func IsPathIterEmpty(pathIter PathsIter) bool
- func IsRangeFunc(str string) bool
- func IsValidInputValue(val interface{}, typeStr string) bool
- func IsValidType(typeStr string, typeList []string) bool
- func IsVariant(str string) bool
- func MergeInputs(inputsDefult map[string]Input, inputs map[string]interface{}) (map[string]Input, error)
- func ReplaceArray(base []string, kv map[string]string) []string
- func SetDefaultInputs(workflow *Workflow)
- func SetDefaultWorkflow(workflow *Workflow)
- func ToArrayIndex(str string) (int, bool)
- func TransCommandIter2ExecCommandIter(commandsIter CommandsIter) *execv1alpha1.CommandsIter
- func TransCond2ExecCond(condition interface{}) *execv1alpha1.Condition
- func TransDepend2ExecDepend(depends []Depend) []execv1alpha1.Dependent
- func TransGenericCond2ExecGenericCond(condition *GenericCondition) *execv1alpha1.GenericCondition
- func TransTools2Map(tools []Tool) map[string]Tool
- func TransVolume2ExecVolume(volumes map[string]Volume) map[string]execv1alpha1.Volume
- func TransWorkflow2Execution(workflow *Workflow) (*execv1alpha1.Execution, error)
- func ValidateCPU(prefix, cpu string) error
- func ValidateDependsCircle(jobs map[string]JobInfo) error
- func ValidateInstantiatedVars(prefix string, varsArray []common.Var) (int, error)
- func ValidateMemory(prefix, memory string) error
- func ValidateRangeFuncParam(prefix, param string, inputs map[string]Input) error
- func ValidateToolAttr(tool Tool) error
- func ValidateVarType(prefix string, varValue interface{}, inputs map[string]Input) error
- func ValidateVariant(prefix, varStr string, types []string, inputs map[string]Input) error
- type CommandsIter
- type Depend
- type ErrorList
- func ValidateCommands(jobName string, commands []string, inputs map[string]Input) ErrorList
- func ValidateCommandsIter(jobName string, commandsIter CommandsIter, inputs map[string]Input, ...) ErrorList
- func ValidateDepend(prefix string, depend Depend, jobs map[string]JobInfo) ErrorList
- func ValidateDepends(jobName string, depends []Depend, jobs map[string]JobInfo) ErrorList
- func ValidateGenericCondition(jobName string, gCondition *GenericCondition, inputs map[string]Input, ...) ErrorList
- func ValidateInputs(inputs map[string]Input) ErrorList
- func ValidateJobName(jobName string) ErrorList
- func ValidatePaths(outputName string, paths []string, inputs map[string]Input) ErrorList
- func ValidatePathsIter(outputName string, pathIter PathsIter, inputs map[string]Input) ErrorList
- func ValidateRangeFunc(prefix, str string, inputs map[string]Input) ErrorList
- func ValidateResources(jobName string, res Resources) ErrorList
- func ValidateTemplate(template, prefix, typeStr string, inputs map[string]Input) (int, ErrorList)
- func ValidateTool(jobName string, toolName string) ErrorList
- func ValidateVarsArray(prefix string, varsArray []interface{}, inputs map[string]Input) ErrorList
- func ValidateVarsIterArray(prefix string, varsArray []interface{}, inputs map[string]Input, ...) ErrorList
- func ValidateVarsIterTypes(prefix string, vars interface{}, inputs map[string]Input, jobName string, ...) ErrorList
- func ValidateVarsTypes(prefix string, vars interface{}, inputs map[string]Input) ErrorList
- func ValidateVolumes(volumes map[string]Volume, inputs map[string]Input) ErrorList
- func ValidateWorkflow(workflow *Workflow) ErrorList
- type GenericCondition
- type Input
- type JobInfo
- type MatchOperator
- type MatchRule
- type OutputDesc
- type PathsIter
- type Resources
- type Tool
- type Volume
- type VolumeSource
- type Workflow
Constants ¶
const ( StringType = "string" NumberType = "number" BoolType = "bool" ArrayType = "array" )
const ( WholeDependType = "whole" IterateDependType = "iterate" )
const CPURegexFmt = `^\d+(\.\d+)?[cC]?$`
const GetRangeFuncRegexFmt = "range\\(([^,]+)\\s*,\\s*([^,]+)(,\\s*([^,]+))?\\)"
const IsCheckResultFuncRegexFmt = `^check_result\(\s*([^,]+)\s*(,\s*("([^,]+)"|'([^,]+)'|\$\{[^,]+\}))?\)$`
const IsGetResultFuncRegexFmt = `^get_result\(\s*([^,]+)\s*(,\s*("([^,]+)"|'([^,]+)'|\$\{[^,]+\}))?\)$`
const IsRangeFuncRegexFmt = "^range\\(([^,]+)\\s*,\\s*([^,]+)(,\\s*([^,]+))?\\)$"
const JobNameRegexFmt = "^[a-z]([-a-z0-9]*[a-z0-9])?$"
const MemoryRegexFmt = `^\d+(\.\d+)?[gG]?$`
Variables ¶
var InputTypeList = []string{StringType, NumberType, BoolType, ArrayType}
Functions ¶
func DownloadToolFile ¶
func GetAllToolFile ¶
func GetExecutionName ¶
func GetExecutionNamespace ¶
func GetInputType ¶
func GetInputType(val interface{}) string
func GetRangeFuncParam ¶
GetRangeFuncParam extract parameter from range function. for example:
Str ---> range(2,${npart},2) result ---> 2, ${npart}, 2 Str ---> range(2, 10, 2) result ---> 2, 10, 2
func GetVariantName ¶
GetVariantName extract variant Name. such as, ${var} --> var
func InstantiateCondition ¶
func InstantiateRangeFunc ¶
func InstantiateVars ¶
func InstantiateVarsIter ¶
func InstantiateWorkflow ¶
func IsCheckResultFunc ¶
IsCheckResultFunc checks a string whether is a check_result function. check_result function in the workflows must follow the format:
check_result(jobName, exp)
The jobName of the Job The exp used to determine if the output of jobName is equal or not:
Can be a single character or a string; Use double quotes to indicate, such as "\n"; Variables such as ${input} can be used.
check_result function example
---- check_result(job-a, "\n") ---- check_result(job-target, ${input})
func IsCommandIterEmpty ¶
func IsCommandIterEmpty(commandIter CommandsIter) bool
func IsGetResultFunc ¶
IsGetResultFunc checks a string whether is a get_result function. get_result function in the workflows must follow the format:
get_result(jobName, sep)
The jobName of the Job The separator used to split the string:
Can be a single character or a string; Use double quotes to indicate, such as "\n"; Variables such as ${input} can be used.
get_result function example
---- get_result(job-a, "\n") ---- get_result(job-target) ---- get_result(job-target, ${input})
func IsPathIterEmpty ¶
func IsRangeFunc ¶
IsRangeFunc checks a string whether is a range function. Range function in the workflows must follow the format:
range(start, end, step)
start, end, and step must be integer type or variant that reference a integer data. If step is not specified, default to 1.
range function example
---- range(2, ${npart}, 2) ---- range(1, 10, 2) ---- range(1, 10)
func IsValidInputValue ¶
func IsValidType ¶
func IsVariant ¶
IsVariant checks whether a string is variant. valid variant format ${.var}, ${var}.
func MergeInputs ¶
func SetDefaultInputs ¶
func SetDefaultInputs(workflow *Workflow)
func ToArrayIndex ¶
func TransCommandIter2ExecCommandIter ¶
func TransCommandIter2ExecCommandIter(commandsIter CommandsIter) *execv1alpha1.CommandsIter
func TransCond2ExecCond ¶
func TransCond2ExecCond(condition interface{}) *execv1alpha1.Condition
func TransDepend2ExecDepend ¶
func TransDepend2ExecDepend(depends []Depend) []execv1alpha1.Dependent
func TransGenericCond2ExecGenericCond ¶
func TransGenericCond2ExecGenericCond(condition *GenericCondition) *execv1alpha1.GenericCondition
func TransTools2Map ¶
func TransVolume2ExecVolume ¶
func TransVolume2ExecVolume(volumes map[string]Volume) map[string]execv1alpha1.Volume
func TransWorkflow2Execution ¶
func TransWorkflow2Execution(workflow *Workflow) (*execv1alpha1.Execution, error)
func ValidateCPU ¶
func ValidateDependsCircle ¶
func ValidateMemory ¶
func ValidateRangeFuncParam ¶
func ValidateToolAttr ¶
func ValidateVarType ¶
Types ¶
type CommandsIter ¶
type CommandsIter struct { // Command is the base command that contains variables. Command string `json:"command" yaml:"command"` // Vars list all the parameters for the command. // // commandsIter example // // commands_iter: // command: sh /tmp/scripts/step1.splitfq.sh ${1} ${2} /tmp/data ${3} // vars: // - ["sample1", 0, 25] # Each member of the array will be the ${1}, ${2}, ${3} // - ["sample2", 0, 25] // - ["sample1", 1, 25] // - ["sample2", 1, 25] // // then the final command will be: // // sh /tmp/scripts/step1.splitfq.sh sample1 0 ${sample-path} 25 // sh /tmp/scripts/step1.splitfq.sh sample2 0 ${sample-path} 25 // sh /tmp/scripts/step1.splitfq.sh sample1 1 ${sample-path} 25 // sh /tmp/scripts/step1.splitfq.sh sample2 1 ${sample-path} 25 Vars []interface{} `json:"vars,omitempty" yaml:"vars,omitempty"` // VarsIter list all the possible parameters for every position in the command line. // And we will use algorithm Of Full Permutation to generate all the permutation and // combinations for these parameter that will be used to replace the ${number} variable. // // commandsIter example // // commands_iter: // command: sh /tmp/scripts/step1.splitfq.sh ${1} ${2} /tmp/data ${3} // vars_iter: // - ["sample1", "sample2"] // - [0, 1] // - [25] // // then the final command will be: // // sh /tmp/scripts/step1.splitfq.sh sample1 0 /tmp/data 25 // sh /tmp/scripts/step1.splitfq.sh sample2 0 /tmp/data 25 // sh /tmp/scripts/step1.splitfq.sh sample1 1 /tmp/data 25 // sh /tmp/scripts/step1.splitfq.sh sample2 1 /tmp/data 25 VarsIter []interface{} `json:"vars_iter,omitempty" yaml:"vars_iter,omitempty"` }
CommandsIter defines command for workflows job. If both Vars and Vars_iter are specified, the generate command will be merged.
type Depend ¶
type Depend struct { // Target is the Name of job this depends on. Target string `json:"target" yaml:"target"` // Type is the depends type. // One of whole, iterate // Default to `whole`. // // Examples: // // "whole" - A-->B": jobs of B depends on all jobs of A execution done. // "iterate" - A[1,2,3]-->B[1,2,3]: jobs of B depends on jobs of A one by one. // That is to say: A[1]->B[1], A[2]->B[2], A[3]->B[3] Type string `json:"type,omitempty" yaml:"type,omitempty"` }
type ErrorList ¶
type ErrorList []error
ErrorList holds a set of Errors.
func ValidateCommands ¶
func ValidateCommandsIter ¶
func ValidateDepend ¶
func ValidateDepends ¶
func ValidateInputs ¶
func ValidateJobName ¶
func ValidatePaths ¶
func ValidatePathsIter ¶
func ValidateRangeFunc ¶
ValidateRangeFunc validate parameter of range function is valid.
func ValidateResources ¶
func ValidateTemplate ¶
func ValidateTool ¶
func ValidateVarsArray ¶
func ValidateVarsIterArray ¶
func ValidateVarsIterTypes ¶
func ValidateVarsTypes ¶
func ValidateVolumes ¶
func ValidateWorkflow ¶
type GenericCondition ¶
type GenericCondition struct { // when we use generic condition this DependJobName should be same as jobName in Depends // and Depends should have only one JobName DependJobName string `json:"depend_job_name" yaml:"depend_job_name"` MatchRules []MatchRule `json:"match_rules" yaml:"match_rules"` }
generic Conditional dynamic handling match rules are ORed.
type Input ¶
type Input struct { // Default is the default value that will be used for an input parameter // if a value was not provided. Default interface{} `json:"default,omitempty" yaml:"default,omitempty"` // Value is the literal value to use for the parameter. Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` // Description is the information about what the parameter is used for. Description string `json:"description,omitempty" yaml:"description,omitempty"` // Type defines the type for the input parameter. // One of string, number, bool, array. // The default type is string if not specified. Type string `json:"type,omitempty" yaml:"type,omitempty"` }
Input defines input parameter that used for gene sequencing.
input example
sample:
default: /sample/data description: the path that stores the sample data. type: string
type JobInfo ¶
type JobInfo struct { // Description describes what this job is to do. Description string `json:"description,omitempty" yaml:"description,omitempty"` // tool Name Tool string `json:"tool" yaml:"tool"` // docker image Name. Image string `json:"image,omitempty" yaml:"image,omitempty"` // Compute Resources required by this job. Resources Resources `json:"resources,omitempty" yaml:"resources,omitempty"` // command to run for gene sequencing. Commands []string `json:"commands,omitempty" yaml:"commands,omitempty"` // CommandsIter defines batch command for workflows job. CommandsIter CommandsIter `json:"commands_iter,omitempty" yaml:"commands_iter,omitempty"` // Depends is the Name of task this depends on. Depends []Depend `json:"depends,omitempty" yaml:"depends,omitempty"` // conditional branch handling Condition interface{} `json:"condition,omitempty" yaml:"condition,omitempty"` // generic conditional handling using the match rules are ORed. GenericCondition *GenericCondition `json:"generic_condition,omitempty" yaml:"generic_condition,omitempty"` }
job information.
type MatchOperator ¶
type MatchOperator string
A match operator is the set of operators that can be used in a MatchRequirement.
const ( MatchOperatorOpIn MatchOperator = "In" MatchOperatorOpNotIn MatchOperator = "NotIn" MatchOperatorOpExists MatchOperator = "Exists" MatchOperatorOpDoesNotExist MatchOperator = "DoesNotExist" MatchOperatorOpGt MatchOperator = "Gt" MatchOperatorOpLt MatchOperator = "Lt" MatchOperatorOpEqual MatchOperator = "=" MatchOperatorOpNotEqual MatchOperator = "!=" MatchOperatorOpDoubleEqual MatchOperator = "==" )
type MatchRule ¶
type MatchRule struct { // The key that the requirement applies to. Key string `json:"key" yaml:"key"` // Represents a key's relationship to a set of values. // Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. Operator MatchOperator `json:"operator" yaml:"operator"` // An array of string values. If the operator is In or NotIn, // the values array must be non-empty. If the operator is Exists or DoesNotExist, // the values array must be empty. If the operator is Gt or Lt, the values // array must have a single element, which will be interpreted as an integer. // +optional Values []string `json:"values,omitempty" yaml:"values,omitempty"` }
A matching rule is a requirement that contains values, a key, and an operator that relates the key and values.
type OutputDesc ¶
type PathsIter ¶
type PathsIter struct { Path string `json:"path" yaml:"path"` Vars []interface{} `json:"vars,omitempty" yaml:"vars,omitempty"` VarsIter []interface{} `json:"vars_iter,omitempty" yaml:"vars_iter,omitempty"` }
PathsIter similar to CommandsIter.
type Resources ¶
type Resources struct { Memory string `json:"memory,omitempty" yaml:"memory,omitempty"` Cpu string `json:"cpu,omitempty" yaml:"cpu,omitempty"` }
Compute Resources required by this container.
type Tool ¶
type Tool struct { // The Name of tool. // Required. Name string `json:"Name" yaml:"Name"` // The version of tool. // Required. Version string `json:"version" yaml:"version"` // Docker image Name. // Required. Image string `json:"image" yaml:"image"` // Command is the task that will be run for gene sequencing. // If set, it will append to workflows commands. Command string `json:"command,omitempty" yaml:"command,omitempty"` // the type of tool. Type string `json:"type" yaml:"type"` // Description describes what the tool is used for. Description string `json:"description,omitempty" yaml:"description,omitempty"` }
Tool is an abstraction of the gene sequencing container. It contains the basic information about a gene sequencing container. such as image, version, description and so on. When we use it in gene sequencing workflows, we can simply specify the tool as Name:version, and the real image and other information will be replaced when running.
tool example
Name: GATK version: 4.0.1 image: 1.0.0.21:/root/GATK:4.0.1 command: gatk hello world type: basic description: software package to analyze next-generation sequencing data
use example
job-GATK:
tool: GATK:4.0.1 resources: memory: 2G cpu: 2C command: - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample1.sam - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample2.sam
the final workflows job
job-GATK:
tool: GATK:4.0.1 image: 1.0.0.21:/root/GATK:4.0.1 resources: memory: 2G cpu: 2C command: - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample1.sam - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample2.sam
type Volume ¶
type Volume struct { MountPath string `json:"mount_path" yaml:"mount_path"` MountFrom VolumeSource `json:"mount_from" yaml:"mount_from"` }
type VolumeSource ¶
type VolumeSource struct {
PVC string `json:"pvc" yaml:"pvc"`
}
type Workflow ¶
type Workflow struct { // Workflow version Version string `json:"version" yaml:"version"` Inputs map[string]Input `json:"inputs,omitempty" yaml:"inputs,omitempty"` Jobs map[string]JobInfo `json:"workflow" yaml:"workflow"` Volumes map[string]Volume `json:"volumes" yaml:"volumes"` Outputs map[string]OutputDesc `json:"outputs,omitempty" yaml:"outputs,omitempty"` Tools map[string]Tool `json:"tools" yaml:"tools"` }