stats

package
v0.0.0-...-366f0b0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2019 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Overview

Package stats provides functions to generate and query pre-computed test and task statistics. The statistics are aggregated per day and a combination of (test, task, variant, distro, project, requester) for tests and a combination of (task, variant, distro, project, requester) for tasks. For tests intermediate hourly statistics are also stored to avoid some re-computation.

Index

Constants

View Source 
const (
	MaxQueryLimit             = 1001
	GroupByTest       GroupBy = "test"
	GroupByTask       GroupBy = "task"
	GroupByVariant    GroupBy = "variant"
	GroupByDistro     GroupBy = "distro"
	SortEarliestFirst Sort    = "earliest"
	SortLatestFirst   Sort    = "latest"
)

Variables

View Source 
var (
	// BSON fields for the test stats struct
	TestStatsTestFileKey        = bsonutil.MustHaveTag(TestStats{}, "TestFile")
	TestStatsTaskNameKey        = bsonutil.MustHaveTag(TestStats{}, "TaskName")
	TestStatsBuildVariantKey    = bsonutil.MustHaveTag(TestStats{}, "BuildVariant")
	TestStatsDistroKey          = bsonutil.MustHaveTag(TestStats{}, "Distro")
	TestStatsDateKey            = bsonutil.MustHaveTag(TestStats{}, "Date")
	TestStatsNumPassKey         = bsonutil.MustHaveTag(TestStats{}, "NumPass")
	TestStatsNumFailKey         = bsonutil.MustHaveTag(TestStats{}, "NumFail")
	TestStatsAvgDurationPassKey = bsonutil.MustHaveTag(TestStats{}, "AvgDurationPass")
	TestStatsLastUpdateKey      = bsonutil.MustHaveTag(TestStats{}, "LastUpdate")

	// BSON fields for the task stats struct
	TaskStatsTaskNameKey           = bsonutil.MustHaveTag(TaskStats{}, "TaskName")
	TaskStatsBuildVariantKey       = bsonutil.MustHaveTag(TaskStats{}, "BuildVariant")
	TaskStatsDistroKey             = bsonutil.MustHaveTag(TaskStats{}, "Distro")
	TaskStatsDateKey               = bsonutil.MustHaveTag(TaskStats{}, "Date")
	TaskStatsNumSuccessKey         = bsonutil.MustHaveTag(TaskStats{}, "NumSuccess")
	TaskStatsNumFailedKey          = bsonutil.MustHaveTag(TaskStats{}, "NumFailed")
	TaskStatsNumTotalKey           = bsonutil.MustHaveTag(TaskStats{}, "NumTotal")
	TaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(TaskStats{}, "NumTestFailed")
	TaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(TaskStats{}, "NumSetupFailed")
	TaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(TaskStats{}, "NumSystemFailed")
	TaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(TaskStats{}, "NumTimeout")
	TaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(TaskStats{}, "AvgDurationSuccess")
	TaskStatsLastUpdateKey         = bsonutil.MustHaveTag(TaskStats{}, "LastUpdate")
)

Functions

func GenerateDailyTaskStats 

func GenerateDailyTaskStats(projectId string, requester string, day time.Time, tasks []string, jobRunTime time.Time) error

GenerateDailyTaskStats aggregates the hourly task stats present in the database and saves the resulting daily task stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateDailyTestStatsFromHourly 

func GenerateDailyTestStatsFromHourly(projectId string, requester string, day time.Time, tasks []string, jobRunTime time.Time) error

GenerateDailyTestStatsFromHourly aggregates the hourly test stats present in the database and saves the resulting daily test stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateHourlyTestStats 

func GenerateHourlyTestStats(projectId string, requester string, hour time.Time, tasks []string, jobRunTime time.Time) error

GenerateHourlyTestStats aggregates task and testresults prsent in the database and saves the resulting hourly test stats documents for the project, requester, hour, and tasks specified. The hour covered is the UTC hour corresponding to the given `hour` parameter.

func GetDailyTaskDoc 

func GetDailyTaskDoc(id DbTaskStatsId) (*dbTaskStats, error)

func GetDailyTestDoc 

func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func GetHourlyTestDoc 

func GetHourlyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func UpdateStatsStatus 

func UpdateStatsStatus(projectId string, lastJobRun time.Time, processedTasksUntil time.Time) error

UpdateStatsStatus updates the status of the stats pre-computations for a project.

Types

type DbTaskStatsId 

type DbTaskStatsId struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTaskStatsId represents the _id field for daily_task_stats documents.

type DbTestStatsId 

type DbTestStatsId struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTestStatsId represents the _id field for hourly_test_stats and daily_test_stats documents.

type GroupBy 

type GroupBy string

type Sort 

type Sort string

type StartAt 

type StartAt struct {
	Date         time.Time
	BuildVariant string
	Task         string
	Test         string
	Distro       string
}

StartAt represents parameters that allow a search query to resume at a specific point. Used for pagination.

func StartAtFromTaskStats 

func StartAtFromTaskStats(taskStats *TaskStats) StartAt

StartAtFromTaskStats creates a StartAt that can be used to resume a task stats query. Using the returned StartAt the given TaskStats will be the first result.

func StartAtFromTestStats 

func StartAtFromTestStats(testStats *TestStats) StartAt

StartAtFromTestStats creates a StartAt that can be used to resume a test stats query. Using the returned StartAt the given TestStats will be the first result.

type StatsFilter 

type StatsFilter struct {
	Project    string
	Requesters []string
	AfterDate  time.Time
	BeforeDate time.Time

	Tests         []string
	Tasks         []string
	BuildVariants []string
	Distros       []string

	GroupNumDays int
	GroupBy      GroupBy
	StartAt      *StartAt
	Limit        int
	Sort         Sort
}

StatsFilter represents search and aggregation parameters when querying the test or task statistics.

func (*StatsFilter) ValidateForTasks 

func (f *StatsFilter) ValidateForTasks() error

use with test stats validates that the StatsFilter struct is valid for use with task stats.

func (*StatsFilter) ValidateForTests 

func (f *StatsFilter) ValidateForTests() error

validateForTests validates that the StatsFilter struct is valid for use with test stats.

type StatsStatus 

type StatsStatus struct {
	ProjectId           string    `bson:"_id"`
	LastJobRun          time.Time `bson:"last_job_run"`
	ProcessedTasksUntil time.Time `bson:"processed_tasks_until"`
}

StatsStatus represents the status for stats pre-computations for a project. LastJobRun is the start date of the last successful pre-computation job that ran for the project. ProcessedTasksUntil is the date before which all finished tasks have been processed. It is usually the same as LastJobRun unless a previous job has failed and the stats computation has not caught up yet.

func GetStatsStatus 

func GetStatsStatus(projectId string) (StatsStatus, error)

GetStatsStatus retrieves the status of the stats pre-computations for a project.

type StatsToUpdate 

type StatsToUpdate struct {
	ProjectId string    `bson:"project"`
	Requester string    `bson:"requester"`
	Hour      time.Time `bson:"date"`
	Day       time.Time `bson:"day"`
	Tasks     []string  `bson:"task_names"`
}

func FindStatsToUpdate 

func FindStatsToUpdate(projectID string, requester []string, start, end time.Time) ([]StatsToUpdate, error)

FidnStatsToUpdate finds the stats that need to be updated as a result of tasks finishing between 'start' and 'end'. The results are ordered by project id, then hour, then requester.

type TaskStats 

type TaskStats struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumTotal           int       `bson:"num_total"`
	NumSuccess         int       `bson:"num_success"`
	NumFailed          int       `bson:"num_failed"`
	NumTimeout         int       `bson:"num_timeout"`
	NumTestFailed      int       `bson:"num_test_failed"`
	NumSystemFailed    int       `bson:"num_system_failed"`
	NumSetupFailed     int       `bson:"num_setup_failed"`
	AvgDurationSuccess float64   `bson:"avg_duration_success"`
	LastUpdate         time.Time `bson:"last_update"`
}

TaskStats represents task execution statistics.

func GetTaskStats 

func GetTaskStats(filter StatsFilter) ([]TaskStats, error)

GetTaskStats queries the precomputed task statistics using a filter.

func (*TaskStats) MarshalBSON 

func (s *TaskStats) MarshalBSON() ([]byte, error)

func (*TaskStats) UnmarshalBSON 

func (s *TaskStats) UnmarshalBSON(in []byte) error

type TestStats 

type TestStats struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumPass         int       `bson:"num_pass"`
	NumFail         int       `bson:"num_fail"`
	AvgDurationPass float64   `bson:"avg_duration_pass"`
	LastUpdate      time.Time `bson:"last_update"`
}

TestStats represents test execution statistics.

func GetTestStats 

func GetTestStats(filter StatsFilter) ([]TestStats, error)

GetTestStats queries the precomputed test statistics using a filter.

func (*TestStats) MarshalBSON 

func (s *TestStats) MarshalBSON() ([]byte, error)

func (*TestStats) UnmarshalBSON 

func (s *TestStats) UnmarshalBSON(in []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.