skyhook

type AnnotateDataset struct {
	ID      int
	Dataset Dataset
	Inputs  []ExecParent
	Tool    string
	Params  string
}

type ArrayDataSpec struct{}

type ArrayMetadata struct {
	Width    int `json:",omitempty"`
	Height   int `json:",omitempty"`
	Channels int `json:",omitempty"`

	// uint8, uint16, uint32, uint64, int8, int16, int32, int64, float32, float64
	Type string `json:",omitempty"`
}

type ArrayReader struct {
	// contains filtered or unexported fields
}

type ArrayStreamHeader struct {
	Length          int
	BytesPerElement int
}

type ArrayWriter struct {
	// contains filtered or unexported fields
}

type ChanReader struct {
	Ch chan Image
}

type ClosingSequenceReader struct {
	ReadCloser io.ReadCloser
	Reader     SequenceReader
}

type ClosingSequenceWriter struct {
	WriteCloser io.WriteCloser
	Writer      SequenceWriter
}

type Cmd struct {
	// contains filtered or unexported fields
}

type CmdError struct {
	ExitError error
	Lines     []string
}

type ColumnSpec struct {
	Label string
	// data type, one of "string", "int", "float64"
	Type string
}

type CommandOptions struct {
	NoStdin       bool
	NoStdout      bool
	NoStderr      bool
	NoPrintStderr bool
	// Function to arbitrary modify the exec.Cmd, e.g., set working directory.
	// This is called just before starting the process.
	F func(*exec.Cmd)
	// Whether to only print stderr if debug mode is on.
	OnlyDebug bool
	// Whether to keep not just the last stderr line, but all lines, in case of error.
	AllStderrLines bool
}

type ContainerRequest struct {
	Node           Runnable
	JobID          *int
	CoordinatorURL string
	InstanceID     string
}

type ContainerResponse struct {
	// request/container UUID
	UUID string
}

type DataMetadata interface {
	// Produce new metadata where fields that are specified by other overwrite
	// fields in the current metadata.
	Update(other DataMetadata) DataMetadata
}

type DataSpec interface {
	// Decode JSON-encoded metadata into a metadata object.
	DecodeMetadata(rawMetadata string) DataMetadata
	// Read data that has been written via WriteStream.
	ReadStream(r io.Reader) (data interface{}, err error)
	// Write data for reading via ReadStream.
	WriteStream(data interface{}, w io.Writer) error

	// Read data from storage.
	// format is the Item.Format which data types can use to describe how the data is stored.
	Read(format string, metadata DataMetadata, r io.Reader) (data interface{}, err error)
	// Write data to storage.
	Write(data interface{}, format string, metadata DataMetadata, w io.Writer) error

	// Given some data, return a suggested file extension and format to store it with.
	GetDefaultExtAndFormat(data interface{}, metadata DataMetadata) (ext string, format string)
}

type DataType string

type Dataset struct {
	ID   int
	Name string

	// data or computed
	Type string

	DataType DataType
	Metadata string

	// nil unless Type=computed
	Hash *string
}

type Detection struct {
	Left   int
	Top    int
	Right  int
	Bottom int

	// Optional metadata
	Category string            `json:",omitempty"`
	TrackID  int               `json:",omitempty"`
	Score    float64           `json:",omitempty"`
	Metadata map[string]string `json:",omitempty"`
}

type DetectionJsonSpec struct{}

type DetectionMetadata struct {
	CanvasDims [2]int   `json:",omitempty"`
	Categories []string `json:",omitempty"`
}

type EndRequest struct {
	UUID string
}

type ErrorSequenceReader struct {
	Error error
}

type ErrorSequenceWriter struct {
	Error error
}

type ExecBeginRequest struct {
	Node           Runnable
	JobID          *int
	CoordinatorURL string
}

type ExecBeginResponse struct {
	Parallelism int
}

type ExecInput struct {
	Name string
	// nil if input can be any type
	DataTypes []DataType
	// true if this node can accept multiple inputs for this name
	Variable bool
}

type ExecNode struct {
	ID     int
	Name   string
	Op     string
	Params string

	// currently configured parents for each input
	Parents map[string][]ExecParent
}

type ExecOp interface {
	Parallelism() int
	Apply(task ExecTask) error
	Close()
}

type ExecOpConfig struct {
	ID          string
	Name        string
	Description string
}

type ExecOpImpl struct {
	Config       ExecOpConfig
	Requirements func(node Runnable) map[string]int
	GetTasks     func(node Runnable, items map[string][][]Item) ([]ExecTask, error)
	Prepare      func(url string, node Runnable) (ExecOp, error)

	// only one should be set (static/dynamic)
	ImageName    string
	GetImageName func(node Runnable) (string, error)

	// static specification of inputs/outputs
	// one of dynamic/static should be set
	Inputs  []ExecInput
	Outputs []ExecOutput

	// dynamic specification of inputs/outputs
	// one of dynamic/static should be set
	GetInputs  func(params string) []ExecInput
	GetOutputs func(params string, inputTypes map[string][]DataType) []ExecOutput

	// optional; if set, op is considered "incremental"
	Incremental     bool
	GetOutputKeys   func(node ExecNode, inputs map[string][][]string) []string
	GetNeededInputs func(node ExecNode, outputs []string) map[string][][]string

	// more various optional functions
	GetJobOp   func(node Runnable) (JobOp, string)
	Virtualize func(node ExecNode) *VirtualNode
	Resolve    func(node *VirtualNode, inputDatasets map[string][]Dataset, items map[string][][]Item) ExecutionGraph
}

type ExecOpImplProvider struct {
	Impl ExecOpImpl
}

type ExecOpProvider interface {
	// Returns config for front-end.
	Config() ExecOpConfig
	// Returns resource requirements.
	Requirements(node Runnable) map[string]int
	// Returns list of tasks.
	// items: is a map: input name -> input dataset index -> items in that dataset
	GetTasks(node Runnable, items map[string][][]Item) ([]ExecTask, error)
	// Prepare the ExecOp for a node.
	Prepare(url string, node Runnable) (ExecOp, error)
	// Determines the input specification of a node.
	GetInputs(params string) []ExecInput
	// Determines the output specification of a node.
	GetOutputs(params string, inputTypes map[string][]DataType) []ExecOutput

	// Incremental ops support partial computation of their outputs.
	// This is only possible for concrete nodes (Resolve must return nil).
	IsIncremental() bool
	// Must be implemented if Incremental.
	// GetOutputKeys returns all output keys that would be produced given a set of input keys.
	// GetNeededInputs returns the input keys that are needed to compute a given subset of output keys.
	GetOutputKeys(node ExecNode, inputs map[string][][]string) []string
	GetNeededInputs(node ExecNode, outputs []string) map[string][][]string

	// Docker image name
	GetImageName(node Runnable) (string, error)

	// Optional system to provide customized state to store in ExecNode jobs.
	// For example, when training a model, we may want to store the loss history.
	// Can return nil to use defaults.
	// Second return value is the view of the JobOp, empty string to use default view.
	GetJobOp(node Runnable) (JobOp, string)

	// Virtualize is called when constructing an initial ExecutionGraph.
	// For example, if(A) { input B } else { input C } can be implemented by:
	// - Virtualize should return VirtualNode requiring only A
	// - Resolve can load A, and output a new graph that includes B or C depending on A
	Virtualize(node ExecNode) *VirtualNode

	// Optional system for pre-processing steps, dynamic execution graphs, etc.
	// Given a VirtualNode, returns a subgraph of new VirtualNodes that implement it.
	// Or nil if the VirtualNode is already OK.
	// Resolve is called just before executing the node.
	Resolve(node *VirtualNode, inputDatasets map[string][]Dataset, items map[string][][]Item) ExecutionGraph
}

type ExecOutput struct {
	Name     string
	DataType DataType
}

type ExecParent struct {
	// "n" for ExecNode, "d" for Dataset
	Type string
	ID   int

	// name of ExecNode output that is being input
	Name string

	// the data type of this parent
	DataType DataType
}

type ExecTask struct {
	// For incremental operations, this must be the output key that will be created by this task.
	// TODO: operation may need to produce multiple output keys at some task
	// For other operations, I think this can be arbitrary, but usually it's still related to the output key
	Key string

	// Generally maps from input name to list of items in each dataset at that input
	Items map[string][][]Item

	Metadata string
}

type ExecTaskRequest struct {
	Task ExecTask
}

type ExecutionGraph map[GraphID]Node

type ExtFromFormatDataSpec interface {
	DataSpec
	// Given a format, return the standard extension corresponding to the format.
	// If a DataSpec doesn't implement this function, callers should use the format
	// as the file extension.
	GetExtFromFormat(format string) (ext string)
}

type FfmpegReader struct {
	Cmd    *Cmd
	Stdout io.ReadCloser
	Width  int
	Height int
	Buf    []byte
}

type FileDataSpec interface {
	DataSpec
	// Read data directly from a file.
	ReadFile(format string, metadata DataMetadata, fname string) (data interface{}, err error)
	// Write data directly to a file.
	WriteFile(data interface{}, format string, metadata DataMetadata, fname string) error
}

type FileMetadata struct {
	Filename string `json:",omitempty"`
}

type FileSequenceDataSpec interface {
	SequenceDataSpec
	FileReader(format string, metadata DataMetadata, fname string) SequenceReader
	FileWriter(format string, metadata DataMetadata, fname string) SequenceWriter
}

type FileStreamHeader struct {
	Length int
}

type FileTypeDataSpec struct{}

type FloatJsonSpec struct{}

type GeoBbox [4]float64

type GeoImageDataSpec struct{}

type GeoImageItemSource struct {
	Item Item
	// Copy the source image to this offset in our image.
	// Coordinates can be negative. For example, (-50, -50) means that (50, 50) in
	// the source item is placed at (0, 0) in our image.
	Offset [2]int
}

type GeoImageMetadata struct {
	// either "webmercator" or "custom"
	ReferenceType string `json:",omitempty"`

	// For "webmercator" georeference type.
	// Zoom, X, and Y specify the cell that the image spans.
	Zoom int `json:",omitempty"`
	X    int `json:",omitempty"`
	Y    int `json:",omitempty"`
	// Scale specifies resolution of each cell. Usually it is 256.
	Scale int `json:",omitempty"`
	// Offset specifies offset from the X,Y tile.
	Offset [2]int `json:",omitempty"`

	// Width and height corresponding to the specified zoom and scale.
	// This does not necessarily match the image width and height, but if image
	// is resized from this width and height then the x / y axis must still be proportional.
	Width  int `json:",omitempty"`
	Height int `json:",omitempty"`

	// For custom formats, optionally store the longitude-latitude of bottom-left and top-right corners.
	// If set, we assume that the projection is like Mercator where compass direction matches image axes.
	// If not set, we cannot transform between longitude-latitude and pixel coordinates.
	Bbox [4]float64 `json:",omitempty"`

	// image source type
	// "local": image is stored in JPEG file
	// "url": image comes from a tile server
	// "dataset": image comes from another dataset
	SourceType string `json:",omitempty"`

	// For URL type, the tile server URL.
	URL string `json:",omitempty"`

	// For dataset type, define the source items.
	Items []GeoImageItemSource `json:",omitempty"`
}

type GeoJsonData struct {
	Collection *geojson.FeatureCollection
}

type GeoJsonDataSpec struct{}

type GraphID struct {
	// Either "exec" or "dataset"
	Type string
	// ID of ExecNode or Dataset
	ID int
	// For virtual nodes, some unique key for it
	VirtualKey string
}

type Image struct {
	Width  int
	Height int
	Bytes  []byte
}

type ImageDataSpec struct{}

type ImageStreamHeader struct {
	Width           int
	Height          int
	Channels        int
	Length          int
	BytesPerElement int
}

type IntJsonSpec struct{}

type IntMetadata struct {
	Categories []string `json:",omitempty"`
}

type Item struct {
	Dataset  Dataset
	Key      string
	Ext      string
	Format   string
	Metadata string

	// nil to use default storage provider for LoadData / UpdateData
	Provider     *string
	ProviderInfo *string
}

type ItemProvider struct {
	LoadData func(item Item) (interface{}, DataMetadata, error)
	// optional: we panic if UpdateData is called without being supported
	UpdateData func(item Item, data interface{}, metadata DataMetadata) error
	// optional: we return empty string if Fname is called without being supported
	// caller then needs to fallback to loading the data
	Fname func(item Item) string
}

type Job struct {
	ID        int
	Name      string
	Type      string
	Op        string
	Metadata  string
	StartTime time.Time
	State     string

	// If the job succeeds, Done=true and Error="".
	// If it fails, then Done=true and Error is set.
	// If Done=false it implies the job is still running.
	Done  bool
	Error string
}

type JobOp interface {
	// Update the job given the newly received lines from the job output.
	Update(lines []string)
	// Encode the current job state
	Encode() string
	// Stop this job.
	Stop() error
}

type JobUpdate struct {
	JobID int
	Lines []string
}

type MakeVideoOptions struct {
	// The destination of the encoded video data.
	// One of Fname or Writer must be set.
	Fname  string
	Writer io.Writer
}

type MetadataFromFileDataSpec interface {
	DataSpec
	// Given a filename, which should correspond to an actual file stored on disk,
	// returns a suitable format and metadata for reading that file.
	GetMetadataFromFile(fname string) (format string, metadata DataMetadata, err error)
}

type ModelJobState struct {
	TrainLoss []float64
	ValLoss   []float64
}

type NoMetadata struct{}

type Node interface {
	// Parents of this node.
	// These are other nodes that must be executed before this node can be executed.
	// Key defines what kind of parent it is.
	GraphParents() map[string]GraphID
	// The local hash at this node.
	// The hash of a node is computed by merging hash of its parents with its local hash.
	LocalHash() []byte
	// unique identifier
	GraphID() GraphID
}

type PDDImageOptions struct {
	Mode         string
	MaxDimension int
	Width        int
	Height       int
	Multiple     int
}

type PDDParams struct {
	InputOptions []interface{}
	ValPercent   int
}

type PIInputOption struct {
	Idx   int
	Value string
}

type PIOutputDataset struct {
	ComponentIdx int
	Layer        string
	DataType     DataType
}

type PRestore struct {
	SrcPrefix    string
	DstPrefix    string
	SkipPrefixes string
}

type PTDParams struct {
	LearningRate  float64
	Optimizer     string
	BatchSize     int
	AutoBatchSize bool
	StopCondition struct {
		MaxEpochs      int
		ScoreEpsilon   float64
		ScoreMaxEpochs int
	}
	ModelSaver struct {
		Mode string
	}
	RateDecay struct {
		Op               string
		StepSize         int
		StepGamma        float64
		PlateauFactor    float64
		PlateauPatience  int
		PlateauThreshold float64
		PlateauMin       float64
	}
}

type PytorchArch struct {
	ID     string
	Params PytorchArchParams
}

type PytorchArchComponent struct {
	// PytorchComponent ID
	ID string
	// arbitrary JSON parameters
	Params string
	// where should component.Inputs come from
	// these must be layer or input dataset (not target dataset)
	Inputs []PytorchArchInput
	// where should component.Targets come from
	// these could be layer, input dataset, or target dataset
	Targets []PytorchArchInput
}

type PytorchArchInput struct {
	// dataset or layer
	Type string

	ComponentIdx int
	Layer        string

	DatasetIdx int
}

type PytorchArchLoss struct {
	ComponentIdx int
	Layer        string
	Weight       float64
}

type PytorchArchParams struct {
	// datasets during training are numbered starting from inputs, then continuing with targets
	// DatasetIdx refer to this unified numbering scheme
	NumInputs  int
	NumTargets int

	Components []PytorchArchComponent
	Losses     []PytorchArchLoss
	Scores     []PytorchArchLoss
}

type PytorchComponent struct {
	ID     string
	Params PytorchComponentParams
}

type PytorchComponentParams struct {
	// the module can be defined one of three ways:
	// - a built-in module in exec_ops/pytorch/models/X.py
	// - a module X in a git repository Y
	// - hardcoded
	// only one of BuiltInModule, RepositoryModule, and Code should be set
	// if RepositoryModule is set, Repository must be as well
	Module struct {
		BuiltInModule string

		Repository       PytorchRepository
		RepositoryModule string

		Code string
	}

	// inputs/targets are provided as arguments to forward pass
	NumInputs  int
	NumTargets int

	// produces these recommended skyhook outputs
	Outputs map[string]DataType

	// forward pass output dict also includes these layers and losses
	Layers []string
	Losses []string

	Repositories []PytorchRepository
}

type PytorchInferParams struct {
	ArchID         string
	InputOptions   []PIInputOption
	OutputDatasets []PIOutputDataset
	// See PytorchTrainParams.Components
	Components map[int]string
}

type PytorchOpParams struct {
	Op     string
	Params string
}

type PytorchRepository struct {
	URL string

	// Optional (empty string for latest commit in default branch)
	Commit string
}

type PytorchTrainParams struct {
	ArchID string
	// dataset options
	Dataset PytorchOpParams
	// data augmentation
	Augment []PytorchOpParams
	// restoring models, corresponding to the "models" input
	Restore []PRestore
	// training loop parameters
	Train PytorchOpParams
	// Configuring component parameters.
	// This is like PytorchArchComponent.Params, but just overrides or extends
	// those parameters with more options.
	// It is map from component index to JSON-encoded parameters.
	Components map[int]string
}

type RandomAccessDataSpec interface {
	SequenceDataSpec
	// Initialize a SequenceReader that starts reading at index i, and reads up to index j.
	ReadSlice(format string, metadata DataMetadata, fname string, i, j int) SequenceReader
}

type ReadFfmpegOptions struct {
	// The source of the encoded video data.
	// One of Fname or Reader must be set.
	Fname  string
	Reader io.Reader
	// If non-zero, the offset in the file to seek to before reading.
	Start int
	// If non-zero, read only this many frames.
	Length int
}

type RichText struct {
	Text string
	X    int
	Y    int
}

type Runnable struct {
	Name           string
	Op             string
	Params         string
	InputDatasets  map[string][]Dataset
	OutputDatasets map[string]Dataset
}

type SequenceDataSpec interface {
	DataSpec

	// Initialize a SequenceReader for reading data from storage.
	// The SequenceReader should read the data chunk by chunk.
	Reader(format string, metadata DataMetadata, r io.Reader) SequenceReader
	// Initialize a SequenceWriter to write chunk by chunk to storage.
	Writer(format string, metadata DataMetadata, w io.Writer) SequenceWriter

	// Slice operations on the sequence data.
	Length(data interface{}) int
	Append(data interface{}, more interface{}) interface{}
	Slice(data interface{}, i int, j int) interface{}
}

type SequenceJsonDataImpl struct {
	Spec SequenceJsonSpec
}

type SequenceJsonSpec interface {
	DecodeMetadata(rawMetadata string) DataMetadata
	DecodeData(bytes []byte) (interface{}, error)
	GetEmptyMetadata() (metadata DataMetadata)

	Length(data interface{}) int
	Append(data interface{}, more interface{}) interface{}
	Slice(data interface{}, i int, j int) interface{}
}

type SequenceReader interface {
	Read(n int) (interface{}, error)
	Close()
}

type SequenceWriter interface {
	Write(data interface{}) error
	Close() error
}

type Shape struct {
	Type   TypeOfShape
	Points [][2]int

	// Optional metadata
	Category string            `json:",omitempty"`
	TrackID  int               `json:",omitempty"`
	Metadata map[string]string `json:",omitempty"`
}

type ShapeJsonSpec struct{}

type ShapeMetadata struct {
	CanvasDims [2]int   `json:",omitempty"`
	Categories []string `json:",omitempty"`
}

type SimpleExecOp struct {
	ApplyFunc func(ExecTask) error
	P         int
}

type SliceReader struct {
	Data interface{}
	Spec SequenceDataSpec
	// contains filtered or unexported fields
}

type SliceWriter struct {
	Spec     SequenceDataSpec
	Format   string
	Metadata DataMetadata
	Writer   io.Writer
	// contains filtered or unexported fields
}

type StatusRequest struct {
	UUID string
}

type StatusResponse struct {
	// whether the container has been provisioned
	Ready bool
	// if not Ready, some message e.g. position in queue
	Message string
	// if Ready, forwarded ExecBeginResponse
	ExecBeginResponse ExecBeginResponse
	// if Ready, base URL where container can be accessed
	BaseURL string
}

type StringJsonSpec struct{}

type TableData [][]string

type TableDataSpec struct{}

type TableMetadata struct {
	Columns []ColumnSpec `json:",omitempty"`
}

type TailJobOp struct {
	Lines []string
	// contains filtered or unexported fields
}

type TypeOfShape string

type VideoBuilder struct {
	Metadata VideoMetadata
	Fname    string
	Writer   io.Writer
	// contains filtered or unexported fields
}

type VideoDataSpec struct{}

type VideoIterator struct {
	Metadata VideoMetadata
	Fname    string
	Reader   io.Reader
	// contains filtered or unexported fields
}

type VideoMetadata struct {
	// view settings that can be adjusted
	Dims      [2]int `json:",omitempty"`
	Framerate [2]int `json:",omitempty"`

	// cached properties that don't make sense to adjust
	Duration float64 `json:",omitempty"`
}

type VideoReader interface {
	// Error should be io.EOF if there are no more images.
	// If an image is returned, error must NOT be io.EOF.
	// (So no error should be returned on the last image, only after the last image.)
	Read() (Image, error)

	Close()
}

type VideoStreamHeader struct {
	Width           int
	Height          int
	Channels        int
	Length          int
	BytesPerElement int
}

type VirtualNode struct {
	// from ExecNode
	Name   string
	Op     string
	Params string

	// Parents of this node.
	Parents map[string][]VirtualParent
	// the concrete node that this ExecNode was created from
	// if ExecOpImpl.Resolve is not set, then Node == OrigNode
	OrigNode ExecNode
	// A unique identifier for this node if it is actually virtual (Node != OrigNode).
	VirtualKey string
}

type VirtualParent struct {
	GraphID GraphID
	// if GraphID.Type is "exec", then this is name of output that we want to input
	Name string

	DataType DataType
}