evaluator

type BinClassValidation interface {
	// Splitter divides data set into several subsets with some strategies (such as KFolds, LOO),
	// and hold out one subset as validation set and others as training set
	Splitter

	// SetPredictOut sets predicted probabilities from a prediction set to which `idx` refers.
	SetPredictOut(idx int, predProbas []float64) error

	// GetAllPredictOuts returns all prediction results has been stored.
	GetAllPredictOuts() map[int][]string

	// GetReport returns a json bytes of precision, recall, f1, true positive,
	// false positive, true negatives and false negatives for each class, and accuracy, over all split folds.
	GetOverallReport() (map[int][]byte, error)

	// GetROCAndAUC returns a json bytes of roc's points and auc.
	GetROCAndAUC(idx int) ([]byte, error)

	// GetAllROCAndAUC returns a map contains all split folds' json bytes of roc and auc.
	GetAllROCAndAUC() (map[int][]byte, error)
}

type Evaluator interface {
	// Start starts model evaluation, that is to segment the training set according to a certain strategy (cross validation, proportional random division),
	//  then start the training-validation process.
	// fileRows is returned by psi.IntersectParts after sample alignment.
	Start(fileRows [][]string) error

	// Stop deletes all the leaners created by Evaluator as well as other objects
	Stop()

	// SaveModel collects the results of the training in the evaluation phase,
	// that is, the model, for Evaluation of Model.
	// If the model is successfully trained,
	// it will trigger the local creation of a Model instance for validation.
	SaveModel(*pbCom.TrainTaskResult) error

	// SavePredictOut collects the prediction results in the evaluation phase.
	// If the prediction result is obtained, it will check how many prediction results have been obtained so far,
	//  and determine whether to start calculating the average scores for each metric.
	SavePredictOut(*pbCom.PredictTaskResult) error
}

type Mpc interface {
	// StartTask starts a specific task of training or prediction
	StartTask(*pbCom.StartTaskRequest) error
	// StopTask stops a specific task of training or prediction
	StopTask(*pbCom.StopTaskRequest) error
}

type RegressionValidation interface {
	// Splitter divides data set into several subsets with some strategies (such as KFolds, LOO),
	// and hold out one subset as validation set and others as training set
	Splitter

	// SetPredictOut sets prediction outcomes for a prediction set to which `idx` refers.
	SetPredictOut(idx int, yPred []float64) error

	// GetAllPredictOuts returns all prediction results has been stored.
	GetAllPredictOuts() map[int][]float64

	// GetAllRMSE returns scores of RMSE over all split folds,
	// and its Mean and Standard Deviation.
	GetAllRMSE() (map[int]float64, float64, float64, error)
}

type Splitter interface {
	// Split divides the file into two parts directly
	// based on percentage which denotes the first part of divisions.
	Split(percents int) error

	// ShuffleSplit shuffles the rows with `seed`,
	// then divides the file into two parts
	// based on `percents` which denotes the first part of divisions.
	ShuffleSplit(percents int, seed string) error

	// KFoldsSplit divides the file into `k` parts directly.
	// k is the number of parts that only could be 5 or 10.
	KFoldsSplit(k int) error

	// ShuffleKFoldsSplit shuffles the sorted rows with `seed`,
	// then divides the file into `k` parts.
	// k is the number of parts that only could be 5 or 10.
	ShuffleKFoldsSplit(k int, seed string) error

	// LooSplit sorts file rows by IDs which extracted from file by `idName`,
	// then divides each row into a subset.
	LooSplit() error

	// GetAllFolds returns all folds after split.
	// And could be only called successfully after split.
	GetAllFolds() ([][][]string, error)

	// GetTrainSet holds out the subset to which referred by `idxHO`
	// and returns the remaining as training set.
	GetTrainSet(idxHO int) ([][]string, error)

	// GetPredictSet returns the subset to which referred by `idx`
	// as predicting set (without label feature).
	GetPredictSet(idx int) ([][]string, error)

	// GetPredictSet returns the subset to which referred by `idx`
	// as validation set.
	GetValidSet(idx int) ([][]string, error)
}

type Trainer interface {
	// SavePredictAndEvaluatResult saves the training result and evaluation result for a Learner
	// and stops related task.
	SavePredictAndEvaluatResult(result *pbCom.TrainTaskResult)
}