Documentation ¶
Index ¶
- Constants
- func TarDirectory(dirPath string, writer io.Writer) error
- func ZipDirectory(dirPath string, writer io.Writer) error
- type Controller
- func (c *Controller) BuildTemplate(template *Template, fetcher Fetcher) error
- func (c *Controller) CleanOldState() error
- func (c *Controller) DockerFilters() map[string][]string
- func (c *Controller) FindTemplate(id string) (*Template, error)
- func (c *Controller) Init(namePrefix string) error
- func (c *Controller) MapperImageName(job *Job) string
- func (c *Controller) NewJob() (*Job, error)
- func (c *Controller) ReducerImageName(job *Job) string
- func (c *Controller) RunJob(job *Job) error
- func (c *Controller) TemplateImageName(template *Template) string
- type DefinitionReader
- type DockerDescriptor
- type Fetcher
- type FileFetcher
- type Job
- type MapperDescriptor
- type ReducerDescriptor
- type StatefulFetcher
- type Template
- type TemplateDescriptor
- type TemplateState
Constants ¶
const ( // Init was not called on this template. InvalidTemplateState = 0 // The template was assigned an ID, and is blank otherwise. TemplateAllocated = 1 // The template's YAML definition was read into the descriptor. TemplateDefined = 1 // The template's base Docker image was created. TemplateBuilt = 2 // The template's base Docker image is ready for use. TemplateReady = 3 // The template's base Docker image is pending deletion. TemplateDeleting = 4 // Errors were encountered while processing the template. TemplateErrored = 5 )
Variables ¶
This section is empty.
Functions ¶
func TarDirectory ¶
TarDirectory compresses a directory into a tar archive.
Types ¶
type Controller ¶
type Controller struct { // The prefix added to Docker container and image names. // Used to prevent clashes with other images. NamePrefix string // The value for the mapreduced.ctl label added to containers and images. // Used to clean up objects left over from previous runs of the controller. LabelValue string // The number assigned to the next created Job. NextJobId int // contains filtered or unexported fields }
func (*Controller) BuildTemplate ¶
func (c *Controller) BuildTemplate(template *Template, fetcher Fetcher) error
BuildTemplate gets a Template from Allocated to Ready.
func (*Controller) CleanOldState ¶
func (c *Controller) CleanOldState() error
CleanOldState removes all the Docker objects left over from a previous run.
func (*Controller) DockerFilters ¶
func (c *Controller) DockerFilters() map[string][]string
DockerFilters returns the filters used to find this controller's objects.
func (*Controller) FindTemplate ¶
func (c *Controller) FindTemplate(id string) (*Template, error)
FindTemplate locates or creates the job template with the given ID.
func (*Controller) Init ¶
func (c *Controller) Init(namePrefix string) error
Init sets up the controller's Docker connection.
func (*Controller) MapperImageName ¶
func (c *Controller) MapperImageName(job *Job) string
MapperImageName computes the name of a job's mapper image.
func (*Controller) NewJob ¶
func (c *Controller) NewJob() (*Job, error)
NewJob creates a job to be run by this controller, from a YAML description.
func (*Controller) ReducerImageName ¶
func (c *Controller) ReducerImageName(job *Job) string
MapperImageName computes the name of a job's mapper image.
func (*Controller) RunJob ¶
func (c *Controller) RunJob(job *Job) error
func (*Controller) TemplateImageName ¶
func (c *Controller) TemplateImageName(template *Template) string
TemplateImageName computes the name of a template's base Docker image.
type DefinitionReader ¶
type DefinitionReader struct {
// contains filtered or unexported fields
}
Manages a template definition in a .zip file.
func (*DefinitionReader) Init ¶
func (r *DefinitionReader) Init(fetcher Fetcher, template *Template) error
Init sets the reader's source .zip and target Template.
func (*DefinitionReader) WriteImageTar ¶
func (r *DefinitionReader) WriteImageTar(writer io.Writer) error
WriteImageTar saves the .tar context for the template's base Docker image.
type DockerDescriptor ¶
type DockerDescriptor struct { // Executable + paramters used to run the Docker image. EntryPoint []string `yaml:"cmd"` // Current directory when the Docker image starts. WorkDir string `yaml:"chdir"` // Path where the job's input will be copied to. InputPath string `yaml:"input"` // Path where the job's output will be copied from. OutputPath string `yaml:"output"` }
func (*DockerDescriptor) GetDockerFile ¶
func (s *DockerDescriptor) GetDockerFile(sourceImage string, copyFrom string) string
GetDockerFile builds the Dockerfile that creates a phase container's image.
type Fetcher ¶
type Fetcher interface { // Activates this fetcher. // The fetcher must not do any work until Open() is called. // This must not be called multiple times. Open() error // Returns the information needed to create a zip.Reader. // This must not be called multiple times. Reader() (io.ReaderAt, int) // Deactivates this fetcher. The reader it returns is no longer usable. Close() error }
Fetcher has the functions used by DefinitionReader to read raw .zip bytes.
type FileFetcher ¶
type FileFetcher struct {
// contains filtered or unexported fields
}
FileFetcher demonstrates how a Fetcher should be implemented.
func (*FileFetcher) Close ¶
func (f *FileFetcher) Close() error
func (*FileFetcher) Closed ¶
func (f *FileFetcher) Closed() bool
func (*FileFetcher) Open ¶
func (f *FileFetcher) Open() error
func (*FileFetcher) Opened ¶
func (f *FileFetcher) Opened() bool
type Job ¶
type MapperDescriptor ¶
type MapperDescriptor struct { DockerDescriptor `yaml:",inline"` // The name of the environment variable holding the current item's index. ItemEnvVar string `yaml:"env"` }
Information about the mapper in a Map-Reduce job.
type ReducerDescriptor ¶
type ReducerDescriptor struct { DockerDescriptor `yaml:",inline"` // The name of the environment variable holding the number of items. ItemCountVar string `yaml:"env"` }
Information about the reducer in a Map-Reduce job.
type StatefulFetcher ¶
type StatefulFetcher interface { Fetcher // Returns true if Open() has been called on this fetcher. Opened() bool // Returns true if Close() has been called on this fetcher. Closed() bool }
Exposes FileFetcher functionality that is only used in testing.
func NewFileFetcher ¶
func NewFileFetcher(filePath string) StatefulFetcher
NewFileFetcher creates a Fetcher that reads from a file.
type Template ¶
type Template struct { // The descriptor fields are only initialzed if State >= TemplateRead. TemplateDescriptor // Identifier assigned when the template is created ID string // The current state of the template State TemplateState // The first error that occurred while processing the template Err error // The output produced by Docker while building the base Docker image BuildOutput []byte // The ID of the template's base Docker image BaseImageID string }
Information common to Map-Reduce jobs.
func (*Template) ReadDefinition ¶
ReadDefinition gets the template to the TemplateRead state.
func (*Template) SetBaseImage ¶
ReadDefinition gets the template to the TemplateReady state.
func (*Template) SetBuildOutput ¶
SetBuildOutput gets the template to the TemplateBuilt state.
type TemplateDescriptor ¶
type TemplateDescriptor struct { // Number of times the mapper will run. ItemCount int `yaml:"items"` // Information about the map phase of the job. Mapper MapperDescriptor `yaml:"mapper"` // Information about the reduce phase of the job. Reducer ReducerDescriptor `yaml:"reducer"` }
The template information provided in the Yaml file.
func (*TemplateDescriptor) Init ¶
func (d *TemplateDescriptor) Init(templateYaml []byte) error
Init reads a YAML template definition into the descriptor's fields.
type TemplateState ¶
type TemplateState int