core

type ArgumentMap map[string]interface{}

type Binding struct {
	// contains filtered or unexported fields
}

type BindingInfo struct {
	Id          string      `json:"id"`
	Type        string      `json:"type"`
	ValExp      string      `json:"valexp"`
	Mode        string      `json:"mode"`
	Output      string      `json:"output"`
	Sweep       bool        `json:"sweep"`
	SweepRootId string      `json:"sweepRootId"`
	Node        interface{} `json:"node"`
	MatchedFork interface{} `json:"matchedFork"`
	Value       interface{} `json:"value"`
	Waiting     bool        `json:"waiting"`
}

type Chunk struct {
	// contains filtered or unexported fields
}

type ChunkDef struct {
	Resources *JobResources
	Args      ArgumentMap
}

type ChunkInfo struct {
	Index    int           `json:"index"`
	ChunkDef *ChunkDef     `json:"chunkDef"`
	State    MetadataState `json:"state"`
	Metadata *MetadataInfo `json:"metadata"`
}

type ChunkPerfInfo struct {
	Index      int       `json:"index"`
	ChunkStats *PerfInfo `json:"chunk_stats"`
}

type DiskSpaceError struct {
	Bytes   uint64
	Inodes  uint64
	Message string
}

type EdgeInfo struct {
	From string `json:"from"`
	To   string `json:"to"`
}

type Fork struct {
	// contains filtered or unexported fields
}

type ForkBindingsInfo struct {
	Argument []*BindingInfo `json:"Argument"`
	Return   []*BindingInfo `json:"Return"`
}

type ForkInfo struct {
	Index         int                    `json:"index"`
	ArgPermute    map[string]interface{} `json:"argPermute"`
	JoinDef       *JobResources          `json:"joinDef"`
	State         MetadataState          `json:"state"`
	Metadata      *MetadataInfo          `json:"metadata"`
	SplitMetadata *MetadataInfo          `json:"split_metadata"`
	JoinMetadata  *MetadataInfo          `json:"join_metadata"`
	Chunks        []*ChunkInfo           `json:"chunks"`
	Bindings      *ForkBindingsInfo      `json:"bindings"`
}

type ForkPerfCache struct {
	// contains filtered or unexported fields
}

type ForkPerfInfo struct {
	Stages     []*StagePerfInfo `json:"stages"`
	Index      int              `json:"index"`
	Chunks     []*ChunkPerfInfo `json:"chunks"`
	SplitStats *PerfInfo        `json:"split_stats"`
	JoinStats  *PerfInfo        `json:"join_stats"`
	ForkStats  *PerfInfo        `json:"fork_stats"`
}

type ForkStorageEvent struct {
	Name          string
	ChildNames    []string
	TotalBytes    uint64
	ChunkBytes    uint64
	ForkBytes     uint64
	TotalVDRBytes uint64
	ForkVDRBytes  uint64
	Timestamp     time.Time
	VDRTimestamp  time.Time
}

type InvocationData struct {
	Call         string                 `json:"call"`
	Args         map[string]interface{} `json:"args"`
	SweepArgs    []string               `json:"sweepargs"`
	IncludePaths []string               `json:"incpaths"`
}

type IoAmount struct {
	Read  IoValues `json:"read"`
	Write IoValues `json:"write"`
}

type IoRate struct {
	Read  IoRateValues `json:"read"`
	Write IoRateValues `json:"write"`
}

type IoRateValues struct {
	// The rate at which IO syscalls were issued.  These may have been against
	// non-block devices, such as sockets, terminals or a psudo-filesystem
	// such as /proc.
	Syscalls float64 `json:"sysc"`

	// The rate at which bytes were transferred to or from a block device.
	// Even when a read or write is made against a block device, it may not be
	// counted in this number if it was served from cache, or if the file was
	// truncated or unlinked before it was synced to disk.
	BlockBytes float64 `json:"bytes"`
}

type IoStats struct {
	Total   IoAmount `json:"total,omitempty"`
	RateMax IoRate   `json:"max,omitempty"`
	RateDev IoRate   `json:"dev,omitempty"`
}

type IoStatsBuilder struct {
	IoStats
	// contains filtered or unexported fields
}

type IoValues struct {
	// The number of io syscalls issued.  These may have been against non-block
	// devices, such as sockets, terminals or a psudo-filesystem such as /proc.
	// For block IO operations, see rusage.
	Syscalls int64 `json:"sysc"`

	// The number of bytes transferred to or from a block device.  Even when a
	// read or write is made against a block device, it may not be counted in
	// this number if it was served from cache, or if the file was truncated
	// or unlinked before it was synced to disk.
	BlockBytes int64 `json:"bytes"`
}

type JobInfo struct {
	Name          string            `json:"name"`
	Pid           int               `json:"pid,omitempty"`
	Host          string            `json:"host,omitempty"`
	Type          string            `json:"type,omitempty"`
	Cwd           string            `json:"cwd,omitempty"`
	PythonInfo    *PythonInfo       `json:"python,omitempty"`
	RusageInfo    *RusageInfo       `json:"rusage,omitempty"`
	MemoryUsage   *ObservedMemory   `json:"used_bytes,omitempty"`
	IoStats       *IoStats          `json:"io,omitempty"`
	WallClockInfo *WallClockInfo    `json:"wallclock,omitempty"`
	Threads       int               `json:"threads,omitempty"`
	MemGB         int               `json:"memGB,omitempty"`
	ProfileMode   ProfileMode       `json:"profile_mode,omitempty"`
	Stackvars     string            `json:"stackvars_flag,omitempty"`
	Monitor       string            `json:"monitor_flag,omitempty"`
	Invocation    *InvocationData   `json:"invocation,omitempty"`
	Version       *VersionInfo      `json:"version,omitempty"`
	ClusterEnv    map[string]string `json:"sge,omitempty"`
}

type JobManager interface {
	GetSystemReqs(int, int) (int, int)
	GetMaxCores() int
	GetMaxMemGB() int
	GetSettings() *JobManagerSettings
	// contains filtered or unexported methods
}

type JobManagerJson struct {
	JobSettings *JobManagerSettings     `json:"settings"`
	JobModes    map[string]*JobModeJson `json:"jobmodes"`
}

type JobManagerSettings struct {
	ThreadsPerJob int      `json:"threads_per_job"`
	MemGBPerJob   int      `json:"memGB_per_job"`
	ThreadEnvs    []string `json:"thread_envs"`
}

type JobModeEnv struct {
	Name        string `json:"name"`
	Description string `json:"description"`
}

type JobModeJson struct {
	Cmd             string        `json:"cmd"`
	Args            []string      `json:"args,omitempty"`
	QueueQuery      string        `json:"queue_query,omitempty"`
	QueueQueryGrace int           `json:"queue_query_grace_secs,omitempty"`
	ResourcesOpt    string        `json:"resopt"`
	JobEnvs         []*JobModeEnv `json:"envs"`
}

type JobResources struct {
	Threads int    `json:"__threads,omitempty"`
	MemGB   int    `json:"__mem_gb,omitempty"`
	Special string `json:"__special,omitempty"`
}

type LocalJobManager struct {
	// contains filtered or unexported fields
}

type MaxJobsSemaphore struct {
	Limit int
	// contains filtered or unexported fields
}

type Metadata struct {
	// contains filtered or unexported fields
}

type MetadataFileName string

type MetadataInfo struct {
	// The filesystem path containing the metadata files.
	Path string `json:"path"`

	// The metadata file names which exist for this object.
	Names []string `json:"names"`
}

type MetadataState string

type MroCache struct {
	// contains filtered or unexported fields
}

type Nodable interface {

	// Gets the node's fully-qualified name.
	GetFQName() string

	// Returns the set of nodes which serve as prerequisites to this node,
	// as a mapping from fully-qualified name to node.
	GetPrenodes() map[string]Nodable

	// Returns the set of nodes which are able to run once this node
	// has completed.
	GetPostNodes() map[string]Nodable

	// Gets the mro AST object, if any, which will be executed for this node.
	Callable() syntax.Callable
	// contains filtered or unexported methods
}

type Node struct {
	// contains filtered or unexported fields
}

type NodeByteStamp struct {
	Timestamp   time.Time `json:"ts"`
	Bytes       int64     `json:"bytes"`
	Description string    `json:"desc"`
}

type NodeErrorInfo struct {
	FQname  string `json:"fqname"`
	Path    string `json:"path"`
	Summary string `json:"summary,omitempty"`
	Log     string `json:"log,omitempty"`
}

type NodeInfo struct {
	Name          string               `json:"name"`
	Fqname        string               `json:"fqname"`
	Type          string               `json:"type"`
	Path          string               `json:"path"`
	State         MetadataState        `json:"state"`
	Metadata      *MetadataInfo        `json:"metadata"`
	SweepBindings []*BindingInfo       `json:"sweepbindings"`
	Forks         []*ForkInfo          `json:"forks"`
	Edges         []EdgeInfo           `json:"edges"`
	StagecodeLang syntax.StageCodeType `json:"stagecodeLang"`
	StagecodeCmd  string               `json:"stagecodeCmd"`
	Error         *NodeErrorInfo       `json:"error,omitempty"`
}

type NodePerfInfo struct {
	Name      string           `json:"name"`
	Fqname    string           `json:"fqname"`
	Type      string           `json:"type"`
	Forks     []*ForkPerfInfo  `json:"forks"`
	MaxBytes  int64            `json:"maxbytes"`
	BytesHist []*NodeByteStamp `json:"bytehist"`
	HighMem   *ObservedMemory  `json:"highmem,omitempty"`
}

type ObservedMemory struct {
	Rss    int64 `json:"rss"`
	Shared int64 `json:"shared"`
	Vmem   int64 `json:"vmem"`
	Text   int64 `json:"text"`
	Stack  int64 `json:"stack"`
	Procs  int   `json:"proc_count"`
}

type PerfInfo struct {
	NumJobs         int       `json:"num_jobs"`
	NumThreads      int       `json:"num_threads"`
	Duration        float64   `json:"duration"`
	CoreHours       float64   `json:"core_hours"`
	MaxRss          int       `json:"maxrss"`
	MaxVmem         int       `json:"maxvmem"`
	InBlocks        int       `json:"in_blocks"`
	OutBlocks       int       `json:"out_blocks"`
	TotalBlocks     int       `json:"total_blocks"`
	InBlocksRate    float64   `json:"in_blocks_rate"`
	OutBlocksRate   float64   `json:"out_blocks_rate"`
	TotalBlocksRate float64   `json:"total_blocks_rate"`
	InBytes         int64     `json:"in_bytes"`
	OutBytes        int64     `json:"out_bytes"`
	InBytesRate     float64   `json:"in_bytes_rate"`
	OutBytesRate    float64   `json:"out_bytes_rate"`
	InBytesPeak     float64   `json:"in_bytes_peak"`
	OutBytesPeak    float64   `json:"out_bytes_peak"`
	Start           time.Time `json:"start"`
	End             time.Time `json:"end"`
	WallTime        float64   `json:"walltime"`
	UserTime        float64   `json:"usertime"`
	SystemTime      float64   `json:"systemtime"`
	TotalFiles      uint      `json:"total_files"`
	TotalBytes      uint64    `json:"total_bytes"`
	OutputFiles     uint      `json:"output_files"`
	OutputBytes     uint64    `json:"output_bytes"`
	VdrFiles        uint      `json:"vdr_files"`
	VdrBytes        uint64    `json:"vdr_bytes"`

	// Deviation for a single job is deviation over time as measured by mrjob.
	// For node aggregates, it's the deviation between child nodes.
	InBytesDev  float64 `json:"in_bytes_dev"`
	OutBytesDev float64 `json:"out_bytes_dev"`
}

type PerfInfoByStart []*PerfInfo

type Pipestance struct {
	// contains filtered or unexported fields
}

type PipestanceCopyingError struct {
	Psid string
}

type PipestanceExistsError struct {
	Psid string
}

type PipestanceFactory interface {
	ReattachToPipestance() (*Pipestance, error)
	InvokePipeline() (*Pipestance, error)
}

type PipestanceInvocationError struct {
	Psid           string
	InvocationPath string
}

type PipestanceJobModeError struct {
	Psid    string
	JobMode string
}

type PipestanceLockedError struct {
	Psid           string
	PipestancePath string
}

type PipestanceNotExistsError struct {
	Psid string
}

type PipestanceNotFailedError struct {
	Psid string
}

type PipestanceNotRunningError struct {
	Psid string
}

type PipestanceOverrides struct {
	// contains filtered or unexported fields
}

type PipestancePathError struct {
	Path string
}

type PipestanceSetup struct {
	Srcpath        string            // Path to the mro invocation file
	Psid           string            // pipestance ID
	PipestancePath string            // Path to put this pipestance
	MroPaths       []string          // Where to look for MROs
	MroVersion     string            // mro version
	Envs           map[string]string // mro environment vars to pass through
	JobMode        string            // jobmode to use
}

type PipestanceSizeError struct {
	Psid string
}

type PipestanceWipeError struct {
	Psid string
}

type ProfileMode string

type PythonInfo struct {
	BinPath string `json:"binpath"`
	Version string `json:"version"`
}

type RemoteJobManager struct {
	// contains filtered or unexported fields
}

type ResourceSemaphore struct {
	Name string
	// contains filtered or unexported fields
}

type Runtime struct {
	Config *RuntimeOptions

	MroCache        *MroCache
	JobManager      JobManager
	LocalJobManager JobManager
	// contains filtered or unexported fields
}

type RuntimeError struct {
	Msg string
}

type RuntimeOptions struct {
	// The runtime mode (required): either "local" or a named mode from
	// jobmanagers/config.json
	JobMode string

	// The volatile disk recovery mode (required): either "post",
	// "rolling", or "disable".
	VdrMode string

	// The profiling mode (required): "disable" or one of the available
	// constants.
	ProfileMode     ProfileMode
	MartianVersion  string
	LocalMem        int
	LocalCores      int
	MemPerCore      int
	MaxJobs         int
	JobFreqMillis   int
	ResourceSpecial string
	FullStageReset  bool
	StackVars       bool
	Zip             bool
	SkipPreflight   bool
	Monitor         bool
	Debug           bool
	StressTest      bool
	OnFinishHandler string
	Overrides       *PipestanceOverrides
	LimitLoadavg    bool
	NeverLocal      bool
}

type Rusage struct {
	MaxRss       int     `json:"ru_maxrss"`
	SharedRss    int     `json:"ru_ixrss"`
	UnsharedRss  int     `json:"ru_idrss"`
	MinorFaults  int     `json:"ru_minflt"`
	MajorFaults  int     `json:"ru_majflt"`
	SwapOuts     int     `json:"ru_nswap"`
	UserTime     float64 `json:"ru_utime"`
	SystemTime   float64 `json:"ru_stime"`
	InBlocks     int     `json:"ru_inblock"`
	OutBlocks    int     `json:"ru_oublock"`
	MessagesSent int     `json:"ru_msgsnd"`
	MessagesRcvd int     `json:"ru_msgrcv"`
	SignalsRcvd  int     `json:"ru_nsignals"`
}

type RusageInfo struct {
	Self     *Rusage `json:"self,omitempty"`
	Children *Rusage `json:"children,omitempty"`
}

type StageDefs struct {
	ChunkDefs []*ChunkDef   `json:"chunks"`
	JoinDef   *JobResources `json:"join,omitempty"`
}

type StageOverride map[string]interface{}

type StagePerfInfo struct {
	Name   string `json:"name"`
	Fqname string `json:"fqname"`
	Forki  int    `json:"forki"`
}

type Stagestance struct {
	// contains filtered or unexported fields
}

type StorageEvent struct {
	Timestamp time.Time
	Delta     int64
	Name      string
}

type StorageEventByTimestamp []*StorageEvent

type TopNode struct {
	// contains filtered or unexported fields
}

type VDRByTimestamp []*VDRKillReport

type VDRKillReport struct {
	Count     uint     `json:"count"`
	Size      uint64   `json:"size"`
	Timestamp string   `json:"timestamp"`
	Paths     []string `json:"paths"`
	Errors    []string `json:"errors"`
}

type VersionInfo struct {
	Martian   string `json:"martian"`
	Pipelines string `json:"pipelines"`
}

type WallClockInfo struct {
	Start    string  `json:"start"`
	End      string  `json:"end,omitempty"`
	Duration float64 `json:"duration_seconds,omitempty"`
}