parser

package
v0.0.0-...-9e48b22 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 15, 2019 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const (
	StringType = "string"
	NumberType = "number"
	BoolType   = "bool"
	ArrayType  = "array"
)
View Source
const (
	WholeDependType   = "whole"
	IterateDependType = "iterate"
)
View Source
const CPURegexFmt = `^\d+(\.\d+)?[cC]?$`
View Source
const GetRangeFuncRegexFmt = "range\\(([^,]+)\\s*,\\s*([^,]+)(,\\s*([^,]+))?\\)"
View Source
const IsCheckResultFuncRegexFmt = `^check_result\(\s*([^,]+)\s*(,\s*("([^,]+)"|'([^,]+)'|\$\{[^,]+\}))?\)$`
View Source
const IsGetResultFuncRegexFmt = `^get_result\(\s*([^,]+)\s*(,\s*("([^,]+)"|'([^,]+)'|\$\{[^,]+\}))?\)$`
View Source
const IsRangeFuncRegexFmt = "^range\\(([^,]+)\\s*,\\s*([^,]+)(,\\s*([^,]+))?\\)$"
View Source
const JobNameRegexFmt = "^[a-z]([-a-z0-9]*[a-z0-9])?$"
View Source
const MemoryRegexFmt = `^\d+(\.\d+)?[gG]?$`

Variables

Functions

func DFS

func DFS(node string, graphNodes map[string]bool, graphEdges map[string][]string, nodeStack []string) error

func DownloadToolFile

func DownloadToolFile(url string) ([]byte, error)

func FetchTools

func FetchTools(cmd *cobra.Command) (map[string]Tool, error)

func GetAllToolFile

func GetAllToolFile(rootPath string) ([]string, error)

func GetExecutionName

func GetExecutionName(inputs map[string]Input) string

func GetExecutionNamespace

func GetExecutionNamespace(inputs map[string]Input) string

func GetInputType

func GetInputType(val interface{}) string

func GetRangeFuncParam

func GetRangeFuncParam(str string) (string, string, string)

GetRangeFuncParam extract parameter from range function. for example:

Str ---> range(2,${npart},2)
result ---> 2, ${npart}, 2

Str ---> range(2, 10, 2)
result ---> 2, 10, 2

func GetStringValue

func GetStringValue(key string, inputs map[string]Input) string

func GetVariantName

func GetVariantName(str string) string

GetVariantName extract variant Name. such as, ${var} --> var

func Inputs2ReplaceData

func Inputs2ReplaceData(inputs map[string]Input) map[string]string

func InstantiateCheckResultFunc

func InstantiateCheckResultFunc(prefix, str string, data map[string]string) common.Var

func InstantiateCondition

func InstantiateCondition(prefix string, condition interface{}, data map[string]string) (common.Var, error)

func InstantiateGetResultFunc

func InstantiateGetResultFunc(prefix, str string, data map[string]string) common.Var

func InstantiateRangeFunc

func InstantiateRangeFunc(prefix, str string, data map[string]string) (common.Var, error)

func InstantiateVars

func InstantiateVars(prefix string, vars []interface{}, data map[string]string) ([]common.Var, error)

func InstantiateVarsIter

func InstantiateVarsIter(prefix string, vars []interface{}, data map[string]string) ([]common.Var, bool, error)

func InstantiateWorkflow

func InstantiateWorkflow(workflow *Workflow, inputs map[string]interface{}, tools map[string]Tool) error

func IsCheckResultFunc

func IsCheckResultFunc(str string) bool

IsCheckResultFunc checks a string whether is a check_result function. check_result function in the workflows must follow the format:

check_result(jobName, exp)

The jobName of the Job The exp used to determine if the output of jobName is equal or not:

Can be a single character or a string;
Use double quotes to indicate, such as "\n";
Variables such as ${input} can be used.

check_result function example

---- check_result(job-a, "\n") ---- check_result(job-target, ${input})

func IsCommandIterEmpty

func IsCommandIterEmpty(commandIter CommandsIter) bool

func IsGetResultFunc

func IsGetResultFunc(str string) bool

IsGetResultFunc checks a string whether is a get_result function. get_result function in the workflows must follow the format:

get_result(jobName, sep)

The jobName of the Job The separator used to split the string:

Can be a single character or a string;
Use double quotes to indicate, such as "\n";
Variables such as ${input} can be used.

get_result function example

---- get_result(job-a, "\n") ---- get_result(job-target) ---- get_result(job-target, ${input})

func IsPathIterEmpty

func IsPathIterEmpty(pathIter PathsIter) bool

func IsRangeFunc

func IsRangeFunc(str string) bool

IsRangeFunc checks a string whether is a range function. Range function in the workflows must follow the format:

range(start, end, step)

start, end, and step must be integer type or variant that reference a integer data. If step is not specified, default to 1.

range function example

---- range(2, ${npart}, 2) ---- range(1, 10, 2) ---- range(1, 10)

func IsValidInputValue

func IsValidInputValue(val interface{}, typeStr string) bool

func IsValidType

func IsValidType(typeStr string, typeList []string) bool

func IsVariant

func IsVariant(str string) bool

IsVariant checks whether a string is variant. valid variant format ${.var}, ${var}.

func MergeInputs

func MergeInputs(inputsDefult map[string]Input, inputs map[string]interface{}) (map[string]Input, error)

func ReplaceArray

func ReplaceArray(base []string, kv map[string]string) []string

func SetDefaultInputs

func SetDefaultInputs(workflow *Workflow)

func SetDefaultWorkflow

func SetDefaultWorkflow(workflow *Workflow)

set default for workflows

func ToArrayIndex

func ToArrayIndex(str string) (int, bool)

func TransCommandIter2ExecCommandIter

func TransCommandIter2ExecCommandIter(commandsIter CommandsIter) *execv1alpha1.CommandsIter

func TransCond2ExecCond

func TransCond2ExecCond(condition interface{}) *execv1alpha1.Condition

func TransDepend2ExecDepend

func TransDepend2ExecDepend(depends []Depend) []execv1alpha1.Dependent

func TransGenericCond2ExecGenericCond

func TransGenericCond2ExecGenericCond(condition *GenericCondition) *execv1alpha1.GenericCondition

func TransTools2Map

func TransTools2Map(tools []Tool) map[string]Tool

func TransVolume2ExecVolume

func TransVolume2ExecVolume(volumes map[string]Volume) map[string]execv1alpha1.Volume

func TransWorkflow2Execution

func TransWorkflow2Execution(workflow *Workflow) (*execv1alpha1.Execution, error)

func ValidateCPU

func ValidateCPU(prefix, cpu string) error

func ValidateDependsCircle

func ValidateDependsCircle(jobs map[string]JobInfo) error

func ValidateInstantiatedVars

func ValidateInstantiatedVars(prefix string, varsArray []common.Var) (int, error)

func ValidateMemory

func ValidateMemory(prefix, memory string) error

func ValidateRangeFuncParam

func ValidateRangeFuncParam(prefix, param string, inputs map[string]Input) error

func ValidateToolAttr

func ValidateToolAttr(tool Tool) error

func ValidateVarType

func ValidateVarType(prefix string, varValue interface{}, inputs map[string]Input) error

func ValidateVariant

func ValidateVariant(prefix, varStr string, types []string, inputs map[string]Input) error

Types

type CommandsIter

type CommandsIter struct {
	// Command is the base command that contains variables.
	Command string `json:"command" yaml:"command"`

	// Vars list all the parameters for the command.
	//
	// commandsIter example
	//
	//    commands_iter:
	//      command: sh /tmp/scripts/step1.splitfq.sh ${1} ${2} /tmp/data ${3}
	//      vars:
	//        - ["sample1", 0, 25] # Each member of the array will be the ${1}, ${2}, ${3}
	//        - ["sample2", 0, 25]
	//        - ["sample1", 1, 25]
	//        - ["sample2", 1, 25]
	//
	// then the final command will be:
	//
	// sh /tmp/scripts/step1.splitfq.sh sample1 0 ${sample-path} 25
	// sh /tmp/scripts/step1.splitfq.sh sample2 0 ${sample-path} 25
	// sh /tmp/scripts/step1.splitfq.sh sample1 1 ${sample-path} 25
	// sh /tmp/scripts/step1.splitfq.sh sample2 1 ${sample-path} 25
	Vars []interface{} `json:"vars,omitempty" yaml:"vars,omitempty"`

	// VarsIter list all the possible parameters for every position in the command line.
	// And we will use algorithm Of Full Permutation to generate all the permutation and
	// combinations for these parameter that will be used to replace the ${number} variable.
	//
	// commandsIter example
	//
	//    commands_iter:
	//      command: sh /tmp/scripts/step1.splitfq.sh ${1} ${2} /tmp/data ${3}
	//      vars_iter:
	//        - ["sample1", "sample2"]
	//        - [0, 1]
	//        - [25]
	//
	// then the final command will be:
	//
	// sh /tmp/scripts/step1.splitfq.sh sample1 0 /tmp/data 25
	// sh /tmp/scripts/step1.splitfq.sh sample2 0 /tmp/data 25
	// sh /tmp/scripts/step1.splitfq.sh sample1 1 /tmp/data 25
	// sh /tmp/scripts/step1.splitfq.sh sample2 1 /tmp/data 25
	VarsIter []interface{} `json:"vars_iter,omitempty" yaml:"vars_iter,omitempty"`
}

CommandsIter defines command for workflows job. If both Vars and Vars_iter are specified, the generate command will be merged.

type Depend

type Depend struct {
	// Target is the Name of job this depends on.
	Target string `json:"target" yaml:"target"`

	// Type is the depends type.
	// One of whole, iterate
	// Default to `whole`.
	//
	// Examples:
	//
	// "whole" - A-->B": jobs of B depends on all jobs of A execution done.
	// "iterate" - A[1,2,3]-->B[1,2,3]: jobs of B depends on jobs of A one by one.
	// That is to say: A[1]->B[1], A[2]->B[2], A[3]->B[3]
	Type string `json:"type,omitempty" yaml:"type,omitempty"`
}

type ErrorList

type ErrorList []error

ErrorList holds a set of Errors.

func ValidateCommands

func ValidateCommands(jobName string, commands []string, inputs map[string]Input) ErrorList

func ValidateCommandsIter

func ValidateCommandsIter(jobName string, commandsIter CommandsIter, inputs map[string]Input, workflow *Workflow) ErrorList

func ValidateDepend

func ValidateDepend(prefix string, depend Depend, jobs map[string]JobInfo) ErrorList

func ValidateDepends

func ValidateDepends(jobName string, depends []Depend, jobs map[string]JobInfo) ErrorList

func ValidateGenericCondition

func ValidateGenericCondition(jobName string, gCondition *GenericCondition, inputs map[string]Input, workflow *Workflow) ErrorList

func ValidateInputs

func ValidateInputs(inputs map[string]Input) ErrorList

func ValidateJobName

func ValidateJobName(jobName string) ErrorList

func ValidatePaths

func ValidatePaths(outputName string, paths []string, inputs map[string]Input) ErrorList

func ValidatePathsIter

func ValidatePathsIter(outputName string, pathIter PathsIter, inputs map[string]Input) ErrorList

func ValidateRangeFunc

func ValidateRangeFunc(prefix, str string, inputs map[string]Input) ErrorList

ValidateRangeFunc validate parameter of range function is valid.

func ValidateResources

func ValidateResources(jobName string, res Resources) ErrorList

func ValidateTemplate

func ValidateTemplate(template, prefix, typeStr string, inputs map[string]Input) (int, ErrorList)

func ValidateTool

func ValidateTool(jobName string, toolName string) ErrorList

func ValidateVarsArray

func ValidateVarsArray(prefix string, varsArray []interface{}, inputs map[string]Input) ErrorList

func ValidateVarsIterArray

func ValidateVarsIterArray(prefix string, varsArray []interface{}, inputs map[string]Input, jobName string, workflow *Workflow) ErrorList

func ValidateVarsIterTypes

func ValidateVarsIterTypes(prefix string, vars interface{}, inputs map[string]Input, jobName string, workflow *Workflow) ErrorList

func ValidateVarsTypes

func ValidateVarsTypes(prefix string, vars interface{}, inputs map[string]Input) ErrorList

func ValidateVolumes

func ValidateVolumes(volumes map[string]Volume, inputs map[string]Input) ErrorList

func ValidateWorkflow

func ValidateWorkflow(workflow *Workflow) ErrorList

type GenericCondition

type GenericCondition struct {
	// when we use generic condition this DependJobName should be same as jobName in Depends
	// and Depends should have only one JobName
	DependJobName string      `json:"depend_job_name" yaml:"depend_job_name"`
	MatchRules    []MatchRule `json:"match_rules" yaml:"match_rules"`
}

generic Conditional dynamic handling match rules are ORed.

type Input

type Input struct {
	// Default is the default value that will be used for an input parameter
	// if a value was not provided.
	Default interface{} `json:"default,omitempty" yaml:"default,omitempty"`

	// Value is the literal value to use for the parameter.
	Value interface{} `json:"value,omitempty" yaml:"value,omitempty"`

	// Description is the information about what the parameter is used for.
	Description string `json:"description,omitempty" yaml:"description,omitempty"`

	// Type defines the type for the input parameter.
	// One of string, number, bool, array.
	// The default type is string if not specified.
	Type string `json:"type,omitempty" yaml:"type,omitempty"`
}

Input defines input parameter that used for gene sequencing.

input example

sample:

default: /sample/data
description: the path that stores the sample data.
type: string

type JobInfo

type JobInfo struct {
	// Description describes what this job is to do.
	Description string `json:"description,omitempty" yaml:"description,omitempty"`
	// tool Name
	Tool string `json:"tool" yaml:"tool"`
	// docker image Name.
	Image string `json:"image,omitempty" yaml:"image,omitempty"`
	// Compute Resources required by this job.
	Resources Resources `json:"resources,omitempty" yaml:"resources,omitempty"`
	// command to run for gene sequencing.
	Commands []string `json:"commands,omitempty" yaml:"commands,omitempty"`
	// CommandsIter defines batch command for workflows job.
	CommandsIter CommandsIter `json:"commands_iter,omitempty" yaml:"commands_iter,omitempty"`
	// Depends is the Name of task this depends on.
	Depends []Depend `json:"depends,omitempty" yaml:"depends,omitempty"`
	// conditional branch handling
	Condition interface{} `json:"condition,omitempty" yaml:"condition,omitempty"`

	// generic conditional handling using the match rules are ORed.
	GenericCondition *GenericCondition `json:"generic_condition,omitempty" yaml:"generic_condition,omitempty"`
}

job information.

type MatchOperator

type MatchOperator string

A match operator is the set of operators that can be used in a MatchRequirement.

const (
	MatchOperatorOpIn           MatchOperator = "In"
	MatchOperatorOpNotIn        MatchOperator = "NotIn"
	MatchOperatorOpExists       MatchOperator = "Exists"
	MatchOperatorOpDoesNotExist MatchOperator = "DoesNotExist"
	MatchOperatorOpGt           MatchOperator = "Gt"
	MatchOperatorOpLt           MatchOperator = "Lt"
	MatchOperatorOpEqual        MatchOperator = "="
	MatchOperatorOpNotEqual     MatchOperator = "!="
	MatchOperatorOpDoubleEqual  MatchOperator = "=="
)

type MatchRule

type MatchRule struct {
	// The key that the requirement applies to.
	Key string `json:"key" yaml:"key"`
	// Represents a key's relationship to a set of values.
	// Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
	Operator MatchOperator `json:"operator" yaml:"operator"`
	// An array of string values. If the operator is In or NotIn,
	// the values array must be non-empty. If the operator is Exists or DoesNotExist,
	// the values array must be empty. If the operator is Gt or Lt, the values
	// array must have a single element, which will be interpreted as an integer.
	// +optional
	Values []string `json:"values,omitempty" yaml:"values,omitempty"`
}

A matching rule is a requirement that contains values, a key, and an operator that relates the key and values.

type OutputDesc

type OutputDesc struct {
	Paths     []string  `json:"paths" yaml:"paths"`
	PathsIter PathsIter `json:"paths_iter,omitempty" yaml:"paths_iter,omitempty"`
}

type PathsIter

type PathsIter struct {
	Path     string        `json:"path" yaml:"path"`
	Vars     []interface{} `json:"vars,omitempty" yaml:"vars,omitempty"`
	VarsIter []interface{} `json:"vars_iter,omitempty" yaml:"vars_iter,omitempty"`
}

PathsIter similar to CommandsIter.

type Resources

type Resources struct {
	Memory string `json:"memory,omitempty" yaml:"memory,omitempty"`
	Cpu    string `json:"cpu,omitempty" yaml:"cpu,omitempty"`
}

Compute Resources required by this container.

type Tool

type Tool struct {
	// The Name of tool.
	// Required.
	Name string `json:"Name" yaml:"Name"`
	// The version of tool.
	// Required.
	Version string `json:"version" yaml:"version"`
	// Docker image Name.
	// Required.
	Image string `json:"image" yaml:"image"`
	// Command is the task that will be run for gene sequencing.
	// If set, it will append to workflows commands.
	Command string `json:"command,omitempty" yaml:"command,omitempty"`
	// the type of tool.
	Type string `json:"type" yaml:"type"`
	// Description describes what the tool is used for.
	Description string `json:"description,omitempty" yaml:"description,omitempty"`
}

Tool is an abstraction of the gene sequencing container. It contains the basic information about a gene sequencing container. such as image, version, description and so on. When we use it in gene sequencing workflows, we can simply specify the tool as Name:version, and the real image and other information will be replaced when running.

tool example

Name: GATK version: 4.0.1 image: 1.0.0.21:/root/GATK:4.0.1 command: gatk hello world type: basic description: software package to analyze next-generation sequencing data

use example

job-GATK:

tool: GATK:4.0.1
resources:
  memory: 2G
  cpu: 2C
command:
  - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample1.sam
  - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample2.sam

the final workflows job

job-GATK:

tool: GATK:4.0.1
image: 1.0.0.21:/root/GATK:4.0.1
resources:
  memory: 2G
  cpu: 2C
command:
  - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample1.sam
  - sh ${obs-path}/${jobid}/bwa_mem.sh obs/path/sample1.fastq.gz obs/path/hg19.fa >obs/path/sample2.sam

func ParseTool

func ParseTool(data []byte) ([]Tool, error)

type Volume

type Volume struct {
	MountPath string       `json:"mount_path" yaml:"mount_path"`
	MountFrom VolumeSource `json:"mount_from" yaml:"mount_from"`
}

type VolumeSource

type VolumeSource struct {
	PVC string `json:"pvc" yaml:"pvc"`
}

type Workflow

type Workflow struct {
	// Workflow version
	Version string                `json:"version" yaml:"version"`
	Inputs  map[string]Input      `json:"inputs,omitempty" yaml:"inputs,omitempty"`
	Jobs    map[string]JobInfo    `json:"workflow" yaml:"workflow"`
	Volumes map[string]Volume     `json:"volumes" yaml:"volumes"`
	Outputs map[string]OutputDesc `json:"outputs,omitempty" yaml:"outputs,omitempty"`
	Tools   map[string]Tool       `json:"tools" yaml:"tools"`
}

func UnmarshalWorkflow

func UnmarshalWorkflow(workflowData []byte) (*Workflow, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL