dsort

type CreationPhaseMetadata struct {
	Shards    []*extract.Shard          `msg:"shards"`
	SendOrder map[string]*extract.Shard `msg:"send_order"`
}

type DetailedStats struct {
	*TimeStats
	*ThroughputStats
}

type JobInfo struct {
	ID                string        `json:"id"`
	Description       string        `json:"description"`
	StartedTime       time.Time     `json:"started_time,omitempty"`
	FinishTime        time.Time     `json:"finish_time,omitempty"`
	ExtractedDuration time.Duration `json:"started_meta_sorting,omitempty"`
	SortingDuration   time.Duration `json:"started_shard_creation,omitempty"`
	CreationDuration  time.Duration `json:"finished_shard_creation,omitempty"`
	Aborted           bool          `json:"aborted"`
	Archived          bool          `json:"archived"`
}

type LocalExtraction struct {
	PhaseInfo
	// TotalCnt is the number of shards DSort has to process in total.
	TotalCnt int64 `json:"total_count,string"`
	// ExtractedCnt describes number of extracted shards to given moment. At the
	// end, number should be roughly equal to TotalCnt/#Targets.
	ExtractedCnt int64 `json:"extracted_count,string"`
	// ExtractedSize describes uncompressed size of extracted shards to given moment.
	ExtractedSize int64 `json:"extracted_size,string"`
	// ExtractedRecordCnt describes number of records extracted from all shards.
	ExtractedRecordCnt int64 `json:"extracted_record_count,string"`
	// ExtractedToDiskCnt describes number of shards extracted to the disk. To
	// compute the number shards extracted to memory just subtract it from
	// ExtractedCnt.
	ExtractedToDiskCnt int64 `json:"extracted_to_disk_count,string"`
	// ExtractedToDiskSize describes uncompressed size of extracted shards to disk
	// to given moment.
	ExtractedToDiskSize int64 `json:"extracted_to_disk_size,string"`
	// ShardExtractionStats describes time statistics about single shard extraction.
	ShardExtractionStats *DetailedStats `json:"single_shard_stats,omitempty"`
}

type Manager struct {
	// Fields with json tags are the only fields which are persisted
	// into the disk once the dSort is finished.
	ManagerUUID string   `json:"manager_uuid"`
	Metrics     *Metrics `json:"metrics"`
	// contains filtered or unexported fields
}

type ManagerGroup struct {
	// contains filtered or unexported fields
}

type MetaSorting struct {
	PhaseInfo
	// SentStats describes time statistics about records sending to another target
	SentStats *TimeStats `json:"sent_stats,omitempty"`
	// RecvStats describes time statistics about records receiving from another target
	RecvStats *TimeStats `json:"recv_stats,omitempty"`
}

type Metrics struct {
	Extraction *LocalExtraction `json:"local_extraction,omitempty"`
	Sorting    *MetaSorting     `json:"meta_sorting,omitempty"`
	Creation   *ShardCreation   `json:"shard_creation,omitempty"`

	// Aborted specifies if the DSort has been aborted or not.
	Aborted atomic.Bool `json:"aborted,omitempty"`
	// Archived specifies if the DSort has been archived to persistent storage.
	Archived atomic.Bool `json:"archived,omitempty"`

	// Description of the job.
	Description string `json:"description,omitempty"`

	// Warnings which were produced during the job.
	Warnings []string `json:"warnings,omitempty"`
	// Errors which happened during the job.
	Errors []string `json:"errors,omitempty"`
	// contains filtered or unexported fields
}

type ParsedRequestSpec struct {
	Bck                 cmn.Bck               `json:"bck"`
	Description         string                `json:"description"`
	OutputBck           cmn.Bck               `json:"output_bck"`
	Extension           string                `json:"extension"`
	OutputShardSize     int64                 `json:"output_shard_size,string"`
	InputFormat         *parsedInputTemplate  `json:"input_format"`
	OutputFormat        *parsedOutputTemplate `json:"output_format"`
	Algorithm           *SortAlgorithm        `json:"algorithm"`
	OrderFileURL        string                `json:"order_file"`
	OrderFileSep        string                `json:"order_file_sep"`
	MaxMemUsage         cos.ParsedQuantity    `json:"max_mem_usage"`
	TargetOrderSalt     []byte                `json:"target_order_salt"`
	ExtractConcMaxLimit int                   `json:"extract_concurrency_max_limit"`
	CreateConcMaxLimit  int                   `json:"create_concurrency_max_limit"`
	StreamMultiplier    int                   `json:"stream_multiplier"` // TODO: should be removed
	ExtendedMetrics     bool                  `json:"extended_metrics"`

	// debug
	DSorterType string `json:"dsorter_type"`
	DryRun      bool   `json:"dry_run"`

	cmn.DSortConf
}

type PhaseInfo struct {
	Start time.Time `json:"started_time"`
	End   time.Time `json:"end_time"`
	// Elapsed time (in seconds) from start to given point of time or end when
	// phase has finished.
	Elapsed time.Duration `json:"elapsed"`
	// Running specifies if phase is in progress.
	Running bool `json:"running"`
	// Finished specifies if phase has finished. If running and finished is
	// false this means that the phase did not have started yet.
	Finished bool `json:"finished"`
	// contains filtered or unexported fields
}

type RemoteResponse struct {
	Record    *extract.Record    `msg:"r"`
	RecordObj *extract.RecordObj `msg:"o"`
}

type RequestSpec struct {
	// Required
	Bck             cmn.Bck `json:"bck" yaml:"bck"`
	Extension       string  `json:"extension" yaml:"extension"`
	InputFormat     string  `json:"input_format" yaml:"input_format"`
	OutputFormat    string  `json:"output_format" yaml:"output_format"`
	OutputShardSize string  `json:"output_shard_size" yaml:"output_shard_size"`

	// Optional
	Description string `json:"description" yaml:"description"`
	// Default: same as `bck` field
	OutputBck cmn.Bck `json:"output_bck" yaml:"output_bck"`
	// Default: alphanumeric, increasing
	Algorithm SortAlgorithm `json:"algorithm" yaml:"algorithm"`
	// Default: ""
	OrderFileURL string `json:"order_file" yaml:"order_file"`
	// Default: "\t"
	OrderFileSep string `json:"order_file_sep" yaml:"order_file_sep"`
	// Default: "80%"
	MaxMemUsage string `json:"max_mem_usage" yaml:"max_mem_usage"`
	// Default: calcMaxLimit()
	ExtractConcMaxLimit int `json:"extract_concurrency_max_limit" yaml:"extract_concurrency_max_limit"`
	// Default: calcMaxLimit()
	CreateConcMaxLimit int `json:"create_concurrency_max_limit" yaml:"create_concurrency_max_limit"`
	// Default: bundle.Multiplier
	StreamMultiplier int `json:"stream_multiplier" yaml:"stream_multiplier"`
	// Default: false
	ExtendedMetrics bool `json:"extended_metrics" yaml:"extended_metrics"`

	// debug
	DSorterType string `json:"dsorter_type"`
	DryRun      bool   `json:"dry_run"` // Default: false

	cmn.DSortConf
}

type ShardCreation struct {
	PhaseInfo
	// ToCreate specifies number of shards that have to be created in this phase.
	ToCreate int64 `json:"to_create,string"`
	// CreatedCnt specifies the number of shards that have been so far created.
	// Should match ToCreate when phase is finished.
	CreatedCnt int64 `json:"created_count,string"`
	// MovedShardCnt specifies the number of shards that have migrated from this
	// to another target in the cluster. Applies only when dealing with compressed
	// data. Sometimes it is faster to create a shard on a specific target and send it
	// over (rather than creating on a destination target).
	MovedShardCnt int64 `json:"moved_shard_count,string"`
	// RequestStats describes time statistics about request to other target.
	RequestStats *TimeStats `json:"req_stats,omitempty"`
	// ResponseStats describes time statistics about response to other target.
	ResponseStats *TimeStats `json:"resp_stats,omitempty"`
	// LocalSendStats describes time statistics about sending record content to other target.
	LocalSendStats *DetailedStats `json:"local_send_stats,omitempty"`
	// LocalRecvStats describes time statistics about receiving record content from other target.
	LocalRecvStats *DetailedStats `json:"local_recv_stats,omitempty"`
	// ShardCreationStats describes time statistics about single shard creation.
	ShardCreationStats *DetailedStats `json:"single_shard_stats,omitempty"`
}

type SortAlgorithm struct {
	Kind string `json:"kind"`

	// Kind: alphanumeric, content
	Decreasing bool `json:"decreasing"`

	// Kind: shuffle
	Seed string `json:"seed"` // seed provided to random generator

	// Kind: content
	Extension  string `json:"extension"`
	FormatType string `json:"format_type"`
}

type ThroughputStats struct {
	MinTp int64 `json:"min_throughput,string"`
	MaxTp int64 `json:"max_throughput,string"`
	AvgTp int64 `json:"avg_throughput,string"`
	// contains filtered or unexported fields
}

type TimeStats struct {
	// Total contains total number of milliseconds spend on
	// specific task.
	Total int64 `json:"total_ms,string"`
	// Count contains number of time specific task was triggered.
	Count int64 `json:"count,string"`
	MinMs int64 `json:"min_ms,string"`
	MaxMs int64 `json:"max_ms,string"`
	AvgMs int64 `json:"avg_ms,string"`
}