scipipe: github.com/scipipe/scipipe/components Index | Files

package components

import "github.com/scipipe/scipipe/components"

Index

Package Files

cmd_to_param.go concatenator.go file_combinator.go file_globber.go file_source.go file_splitter.go file_to_params_reader.go maptotags.go param_combinator.go param_source.go streamtosubstream.go utils.go

type CommandToParams Uses

type CommandToParams struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

CommandToParams takes a shell command, runs it, and sens each of its files as parameters on its OutParam parameter port

func NewCommandToParams Uses

func NewCommandToParams(wf *scipipe.Workflow, name string, command string) *CommandToParams

NewCommandToParams returns an initialized new CommandToParams

func (*CommandToParams) OutParam Uses

func (p *CommandToParams) OutParam() *scipipe.OutParamPort

OutParam returns an parameter out-port with lines of the files being read

func (*CommandToParams) Run Uses

func (p *CommandToParams) Run()

Run the CommandToParams

type Concatenator Uses

type Concatenator struct {
    scipipe.BaseProcess
    OutPath string
}

Concatenator is a process that concatenates the content of multiple files received in the in-port In, into one file returned on its out-port, Out

func NewConcatenator Uses

func NewConcatenator(wf *scipipe.Workflow, name string, outPath string) *Concatenator

NewConcatenator returns a new, initialized Concatenator process

func (*Concatenator) In Uses

func (p *Concatenator) In() *scipipe.InPort

In returns the (only) in-port for this process

func (*Concatenator) Out Uses

func (p *Concatenator) Out() *scipipe.OutPort

Out returns the (only) out-port for this process

func (*Concatenator) Run Uses

func (p *Concatenator) Run()

Run runs the Concatenator process

type FileCombinator Uses

type FileCombinator struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

FileCombinator takes a set of input streams of FileIPs, and returns the same number of output streams, where the FileIPs are multiplied so as to guarantee that all combinations of the ips in the input streams are created. Input ports and corresponding out-ports (with the same port names) are created on demand, by accessing them with the p.In(PORTNAME) method. The corresponding out-porta can then be accessed with the same port name with p.Out(PORTNAME)

func NewFileCombinator Uses

func NewFileCombinator(wf *scipipe.Workflow, name string) *FileCombinator

NewFileCombinator returns a new initialized FileCombinator process

func (*FileCombinator) In Uses

func (p *FileCombinator) In(pName string) *scipipe.InPort

In returns the in-port with name pName. If it does not exist, it will create that in-port, and a corresponding out-port with the same port name.

func (*FileCombinator) Out Uses

func (p *FileCombinator) Out(pName string) *scipipe.OutPort

Out returns the outport

func (*FileCombinator) Run Uses

func (p *FileCombinator) Run()

Run runs the FileCombinator process

type FileGlobber Uses

type FileGlobber struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

FileGlobber is initiated with a set of glob patterns paths, which it will use to find concrete file paths, for which it will return a stream of corresponding File IPs on its outport Out()

func NewFileGlobber Uses

func NewFileGlobber(wf *scipipe.Workflow, name string, globPatterns ...string) *FileGlobber

NewFileGlobber returns a new initialized FileGlobber process

func NewFileGlobberDependent Uses

func NewFileGlobberDependent(wf *scipipe.Workflow, name string, globPatterns ...string) *FileGlobber

NewFileGlobberDependent returns a new FileGlobber that depends on upstream files to be received on the InPort InDependency() before it starts globbing files.

func (*FileGlobber) InDependency Uses

func (p *FileGlobber) InDependency() *scipipe.InPort

InDependency takes files which it will wait for before it starts to execute.

func (*FileGlobber) Out Uses

func (p *FileGlobber) Out() *scipipe.OutPort

Out returns the out-port, on which file IPs based on the file paths the process was initialized with, will be retrieved.

func (*FileGlobber) Run Uses

func (p *FileGlobber) Run()

Run runs the FileGlobber process

type FileSource Uses

type FileSource struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

FileSource is initiated with a set of file paths, which it will send as a stream of File IPs on its outport Out()

func NewFileSource Uses

func NewFileSource(wf *scipipe.Workflow, name string, filePaths ...string) *FileSource

NewFileSource returns a new initialized FileSource process

func (*FileSource) Out Uses

func (p *FileSource) Out() *scipipe.OutPort

Out returns the out-port, on which file IPs based on the file paths the process was initialized with, will be retrieved.

func (*FileSource) Run Uses

func (p *FileSource) Run()

Run runs the FileSource process

type FileSplitter Uses

type FileSplitter struct {
    scipipe.BaseProcess
    LinesPerSplit int
}

FileSplitter is a process that will split a file into multiple files, each with LinesPerSplit number of lines per file

func NewFileSplitter Uses

func NewFileSplitter(wf *scipipe.Workflow, name string, linesPerSplit int) *FileSplitter

NewFileSplitter returns an initialized FileSplitter process that will split a file into multiple files, each with linesPerSplit number of lines per file

func (*FileSplitter) InFile Uses

func (p *FileSplitter) InFile() *scipipe.InPort

InFile returns the port for the input file

func (*FileSplitter) OutSplitFile Uses

func (p *FileSplitter) OutSplitFile() *scipipe.OutPort

OutSplitFile returns the resulting split (part) files generated0

func (*FileSplitter) Run Uses

func (p *FileSplitter) Run()

Run runs the FileSplitter process

type FileToParamsReader Uses

type FileToParamsReader struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

FileToParamsReader takes a file path on its FilePath in-port, and returns the file content as []byte on its out-port Out

func NewFileToParamsReader Uses

func NewFileToParamsReader(wf *scipipe.Workflow, name string, filePath string) *FileToParamsReader

NewFileToParamsReader returns an initialized new FileToParamsReader

func (*FileToParamsReader) OutLine Uses

func (p *FileToParamsReader) OutLine() *scipipe.OutParamPort

OutLine returns an parameter out-port with lines of the files being read

func (*FileToParamsReader) Run Uses

func (p *FileToParamsReader) Run()

Run the FileToParamsReader

type MapToTags Uses

type MapToTags struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

MapToTags is a process that runs a function provided by the user, upon initialization, that will provide a map of tag:value pairs, based in IPs read on the In-port. The tag:value pairs (maps) are added to the IPs on the out-port, which are identical to the incoming IPs, except for the new tag:value map

func NewMapToTags Uses

func NewMapToTags(wf *scipipe.Workflow, name string, mapFunc func(ip *scipipe.FileIP) map[string]string) *MapToTags

NewMapToTags returns an initialized MapToTags process

func (*MapToTags) In Uses

func (p *MapToTags) In() *scipipe.InPort

In takes input files the content of which the map function will be run, to generate tags

func (*MapToTags) Out Uses

func (p *MapToTags) Out() *scipipe.OutPort

Out outputs files that are supplemented with tags by the map function.

func (*MapToTags) Run Uses

func (p *MapToTags) Run()

Run runs the MapToTags process

type ParamCombinator Uses

type ParamCombinator struct {
    scipipe.BaseProcess
}

ParamCombinator takes a set of input params, and returns the same number of param streams, where the params are multiplied so as to guarantee that all combinations of the params in the streams are created. Input ports and corresponding out-ports (with the same port names) are created on demand, by accessing them with the p.InParam(PORTNAME) method. The corresponding out-porta can then be accessed with the same port name with p.OutParam(PORTNAME)

func NewParamCombinator Uses

func NewParamCombinator(wf *scipipe.Workflow, name string) *ParamCombinator

NewParamCombinator returns a new initialized ParamCombinator process

func (*ParamCombinator) InParam Uses

func (p *ParamCombinator) InParam(pName string) *scipipe.InParamPort

InParam returns the in-port with name pName. If it does not exist, it will create that in-port, and a corresponding out-port with the same port name.

func (*ParamCombinator) OutParam Uses

func (p *ParamCombinator) OutParam(pName string) *scipipe.OutParamPort

OutParam returns the outport

func (*ParamCombinator) Run Uses

func (p *ParamCombinator) Run()

Run runs the ParamCombinator process

type ParamSource Uses

type ParamSource struct {
    scipipe.BaseProcess
    // contains filtered or unexported fields
}

ParamSource will feed parameters on an out-port

func NewParamSource Uses

func NewParamSource(wf *scipipe.Workflow, name string, params ...string) *ParamSource

NewParamSource returns a new ParamSource

func (*ParamSource) Out Uses

func (p *ParamSource) Out() *scipipe.OutParamPort

Out returns the out-port, on which parameters the process was initialized with, will be retrieved.

func (*ParamSource) Run Uses

func (p *ParamSource) Run()

Run runs the process

type StreamToSubStream Uses

type StreamToSubStream struct {
    scipipe.BaseProcess
}

StreamToSubStream takes a normal stream of IP's representing individual files, and returns one IP where the incoming IPs are sent on its substream.

func NewStreamToSubStream Uses

func NewStreamToSubStream(wf *scipipe.Workflow, name string) *StreamToSubStream

NewStreamToSubStream instantiates a new StreamToSubStream process

func (*StreamToSubStream) In Uses

func (p *StreamToSubStream) In() *scipipe.InPort

In returns the in-port

func (*StreamToSubStream) OutSubStream Uses

func (p *StreamToSubStream) OutSubStream() *scipipe.OutPort

OutSubStream returns the out-port

func (*StreamToSubStream) Run Uses

func (p *StreamToSubStream) Run()

Run runs the StreamToSubStream

Package components imports 12 packages (graph). Updated 2019-06-19. Refresh now. Tools for package owners.