host

package
v0.0.0-...-ee3d319 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 16, 2018 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// Collection is the name of the MongoDB collection that stores hosts.
	Collection = "hosts"
)
View Source
const (
	MaxLCTInterval = 5 * time.Minute
)

Variables

View Source
var (
	IdKey                      = bsonutil.MustHaveTag(Host{}, "Id")
	DNSKey                     = bsonutil.MustHaveTag(Host{}, "Host")
	SecretKey                  = bsonutil.MustHaveTag(Host{}, "Secret")
	UserKey                    = bsonutil.MustHaveTag(Host{}, "User")
	TagKey                     = bsonutil.MustHaveTag(Host{}, "Tag")
	DistroKey                  = bsonutil.MustHaveTag(Host{}, "Distro")
	ProviderKey                = bsonutil.MustHaveTag(Host{}, "Provider")
	ProvisionedKey             = bsonutil.MustHaveTag(Host{}, "Provisioned")
	ProvisionTimeKey           = bsonutil.MustHaveTag(Host{}, "ProvisionTime")
	ExtIdKey                   = bsonutil.MustHaveTag(Host{}, "ExternalIdentifier")
	RunningTaskKey             = bsonutil.MustHaveTag(Host{}, "RunningTask")
	RunningTaskGroupKey        = bsonutil.MustHaveTag(Host{}, "RunningTaskGroup")
	RunningTaskBuildVariantKey = bsonutil.MustHaveTag(Host{}, "RunningTaskBuildVariant")
	RunningTaskVersionKey      = bsonutil.MustHaveTag(Host{}, "RunningTaskVersion")
	RunningTaskProjectKey      = bsonutil.MustHaveTag(Host{}, "RunningTaskProject")
	TaskDispatchTimeKey        = bsonutil.MustHaveTag(Host{}, "TaskDispatchTime")
	CreateTimeKey              = bsonutil.MustHaveTag(Host{}, "CreationTime")
	ExpirationTimeKey          = bsonutil.MustHaveTag(Host{}, "ExpirationTime")
	TerminationTimeKey         = bsonutil.MustHaveTag(Host{}, "TerminationTime")
	LTCTimeKey                 = bsonutil.MustHaveTag(Host{}, "LastTaskCompletedTime")
	LTCTaskKey                 = bsonutil.MustHaveTag(Host{}, "LastTask")
	LTCGroupKey                = bsonutil.MustHaveTag(Host{}, "LastGroup")
	LTCBVKey                   = bsonutil.MustHaveTag(Host{}, "LastBuildVariant")
	LTCVersionKey              = bsonutil.MustHaveTag(Host{}, "LastVersion")
	LTCProjectKey              = bsonutil.MustHaveTag(Host{}, "LastProject")
	StatusKey                  = bsonutil.MustHaveTag(Host{}, "Status")
	AgentRevisionKey           = bsonutil.MustHaveTag(Host{}, "AgentRevision")
	NeedsNewAgentKey           = bsonutil.MustHaveTag(Host{}, "NeedsNewAgent")
	StartedByKey               = bsonutil.MustHaveTag(Host{}, "StartedBy")
	InstanceTypeKey            = bsonutil.MustHaveTag(Host{}, "InstanceType")
	NotificationsKey           = bsonutil.MustHaveTag(Host{}, "Notifications")
	LastCommunicationTimeKey   = bsonutil.MustHaveTag(Host{}, "LastCommunicationTime")
	UserHostKey                = bsonutil.MustHaveTag(Host{}, "UserHost")
	ZoneKey                    = bsonutil.MustHaveTag(Host{}, "Zone")
	ProjectKey                 = bsonutil.MustHaveTag(Host{}, "Project")
	ProvisionOptionsKey        = bsonutil.MustHaveTag(Host{}, "ProvisionOptions")
	ProvisionAttemptsKey       = bsonutil.MustHaveTag(Host{}, "ProvisionAttempts")
	TaskCountKey               = bsonutil.MustHaveTag(Host{}, "TaskCount")
	StartTimeKey               = bsonutil.MustHaveTag(Host{}, "StartTime")
	TotalCostKey               = bsonutil.MustHaveTag(Host{}, "TotalCost")
	TotalIdleTimeKey           = bsonutil.MustHaveTag(Host{}, "TotalIdleTime")
)
View Source
var All = db.Query(nil)

All is a query that returns all hosts

IsActive is a query that returns all Evergreen hosts that are working or capable of being assigned work to do.

IsFree is a query that returns all running Evergreen hosts without an assigned task.

IsIdle is a query that returns all running Evergreen hosts with no task.

View Source
var IsRunningAndSpawned = db.Query(
	bson.M{
		StartedByKey: bson.M{"$ne": evergreen.User},
		StatusKey:    bson.M{"$ne": evergreen.HostTerminated},
	},
)

IsRunningAndSpawned is a query that returns all running hosts spawned by an Evergreen user.

View Source
var IsRunningTask = db.Query(
	bson.M{
		RunningTaskKey: bson.M{"$exists": true},
	},
)

IsRunningTask is a query that returns all running hosts with a running task

View Source
var IsTerminated = db.Query(
	bson.M{
		RunningTaskKey: bson.M{"$exists": false},
		StatusKey:      evergreen.HostTerminated},
)

IsTerminated is a query that returns all hosts that are terminated (and not running a task).

IsUninitialized is a query that returns all unstarted + uninitialized Evergreen hosts.

Functions

func ByDistroId

func ByDistroId(distroId string) db.Q

ByDistroId produces a query that returns all working hosts (not terminated and not quarantined) of the given distro.

func ByDistroIdDoc

func ByDistroIdDoc(distroId string) bson.M

func ByDynamicWithinTime

func ByDynamicWithinTime(startTime, endTime time.Time) db.Q

ByDynamicWithinTime is a query that returns all dynamic hosts running between a certain time and another time.

func ByExpiringBetween

func ByExpiringBetween(lowerBound time.Time, upperBound time.Time) db.Q

ByExpiringBetween produces a query that returns any user-spawned hosts that will expire between the specified times.

func ById

func ById(id string) db.Q

ById produces a query that returns a host with the given id.

func ByIds

func ByIds(ids []string) db.Q

ByIds produces a query that returns all hosts in the given list of ids.

func ByNotMonitoredSince

func ByNotMonitoredSince(threshold time.Time) db.Q

ByNotMonitoredSince produces a query that returns all hosts whose last reachability check was before the specified threshold, filtering out user-spawned hosts and hosts currently running tasks.

func ByRunningTaskId

func ByRunningTaskId(taskId string) db.Q

ByRunningTaskId returns a host running the task with the given id.

func ByUnprovisionedSince

func ByUnprovisionedSince(threshold time.Time) db.Q

ByUnprovisionedSince produces a query that returns all hosts Evergreen never finished setting up that were created before the given time.

func ByUserWithRunningStatus

func ByUserWithRunningStatus(user string) db.Q

ByUserWithRunningStatus produces a query that returns all running hosts for the given user id.

func ByUserWithUnterminatedStatus

func ByUserWithUnterminatedStatus(user string) db.Q

ByUserWithUnterminatedStatus produces a query that returns all running hosts for the given user id.

func Count

func Count(query db.Q) (int, error)

Count returns the number of hosts that satisfy the given query.

func DecommissionHostsWithDistroId

func DecommissionHostsWithDistroId(distroId string) error

func IsLive

func IsLive() bson.M

IsLive is a query that returns all working hosts started by Evergreen

func MarkInactiveStaticHosts

func MarkInactiveStaticHosts(activeStaticHosts []string) error

DecommissionInactiveStaticHosts marks static hosts in the database as terminated provided their ids aren't contained in the passed in activeStaticHosts slice. This is called in the scheduler, and marks any static host in the system that was removed from the distro as "terminated".

Previously this oepration marked these hosts as "decommissioned," which is not a state that makes sense for static hosts.

func NeedsNewAgent

func NeedsNewAgent(currentTime time.Time) db.Q

NeedsNewAgent returns hosts that are running and need a new agent, have no Last Commmunication Time, or have one that exists that is greater than the MaxLTCInterval duration away from the current time.

func NeedsProvisioning

func NeedsProvisioning() db.Q

NeedsProvisioning returns a query used by the hostinit process to determine hosts that have been started, but need additional provisioning.

It's likely true that Starting and Initializing are redundant, and EVG-2754 will address this issue.

func NumHostsByTaskSpec

func NumHostsByTaskSpec(group, bv, project, version string) (int, error)

ByTaskSpec returns a query that finds all running hosts that are running a task with the given group, buildvariant, project, and version.

func QueryWithFullTaskPipeline

func QueryWithFullTaskPipeline(match bson.M) []bson.M

QueryWithFullTaskPipeline returns a pipeline to match hosts and embeds the task document within the host, if it's running a task

func RemoveStaleInitializing

func RemoveStaleInitializing(distroID string) error

Removes host intents that have been been pending for more than 3 minutes for the specified distro.

If you pass the empty string as a distroID, it will remove stale host intents for *all* distros.

func UpdateAll

func UpdateAll(query interface{}, update interface{}) error

UpdateAll updates all hosts.

func UpdateOne

func UpdateOne(query interface{}, update interface{}) error

UpdateOne updates one host.

func UpsertOne

func UpsertOne(query interface{}, update interface{}) (*mgo.ChangeInfo, error)

UpsertOne upserts a host.

Types

type DistroStats

type DistroStats []StatsByDistro

func GetStatsByDistro

func GetStatsByDistro() (DistroStats, error)

GetStatsByDistro returns counts of up hosts broken down by distro

func (DistroStats) CountMap

func (d DistroStats) CountMap() map[string]int

func (DistroStats) TasksMap

func (d DistroStats) TasksMap() map[string]int

type Host

type Host struct {
	Id       string        `bson:"_id" json:"id"`
	Host     string        `bson:"host_id" json:"host"`
	User     string        `bson:"user" json:"user"`
	Secret   string        `bson:"secret" json:"secret"`
	Tag      string        `bson:"tag" json:"tag"`
	Distro   distro.Distro `bson:"distro" json:"distro"`
	Provider string        `bson:"host_type" json:"host_type"`

	// secondary (external) identifier for the host
	ExternalIdentifier string `bson:"ext_identifier" json:"ext_identifier"`

	// physical location of host
	Project string `bson:"project" json:"project"`
	Zone    string `bson:"zone" json:"zone"`

	// true if the host has been set up properly
	Provisioned       bool      `bson:"provisioned" json:"provisioned"`
	ProvisionAttempts int       `bson:"priv_attempts" json:"provision_attempts"`
	ProvisionTime     time.Time `bson:"prov_time,omitempty" json:"prov_time,omitempty"`

	ProvisionOptions *ProvisionOptions `bson:"provision_options,omitempty" json:"provision_options,omitempty"`

	// the task that is currently running on the host
	RunningTask             string `bson:"running_task,omitempty" json:"running_task,omitempty"`
	RunningTaskGroup        string `bson:"running_task_group,omitempty" json:"running_task_group,omitempty"`
	RunningTaskBuildVariant string `bson:"running_task_bv,omitempty" json:"running_task_bv,omitempty"`
	RunningTaskVersion      string `bson:"running_task_version,omitempty" json:"running_task_version,omitempty"`
	RunningTaskProject      string `bson:"running_task_project,omitempty" json:"running_task_project,omitempty"`

	// the task the most recently finished running on the host
	LastTask         string `bson:"last_task" json:"last_task"`
	LastGroup        string `bson:"last_group,omitempty" json:"last_group,omitempty"`
	LastBuildVariant string `bson:"last_bv,omitempty" json:"last_bv,omitempty"`
	LastVersion      string `bson:"last_version,omitempty" json:"last_version,omitempty"`
	LastProject      string `bson:"last_project,omitempty" json:"last_project,omitempty"`

	// the full task struct that is running on the host (only populated by certain aggregations)
	RunningTaskFull *task.Task `bson:"task_full,omitempty" json:"task_full,omitempty"`

	// duplicate of the DispatchTime field in the above task
	TaskDispatchTime time.Time `bson:"task_dispatch_time" json:"task_dispatch_time"`
	ExpirationTime   time.Time `bson:"expiration_time,omitempty" json:"expiration_time"`

	// creation is when the host document was inserted to the DB, start is when it was started on the cloud provider
	CreationTime    time.Time `bson:"creation_time" json:"creation_time"`
	StartTime       time.Time `bson:"start_time" json:"start_time"`
	TerminationTime time.Time `bson:"termination_time" json:"termination_time"`
	TaskCount       int       `bson:"task_count" json:"task_count"`

	LastTaskCompletedTime time.Time `bson:"last_task_completed_time" json:"last_task_completed_time"`
	LastCommunicationTime time.Time `bson:"last_communication" json:"last_communication"`

	Status    string `bson:"status" json:"status"`
	StartedBy string `bson:"started_by" json:"started_by"`
	// True if this host was created manually by a user (i.e. with spawnhost)
	UserHost      bool   `bson:"user_host" json:"user_host"`
	AgentRevision string `bson:"agent_revision" json:"agent_revision"`
	NeedsNewAgent bool   `bson:"needs_agent" json:"needs_agent"`
	// for ec2 dynamic hosts, the instance type requested
	InstanceType string `bson:"instance_type" json:"instance_type,omitempty"`
	// stores information on expiration notifications for spawn hosts
	Notifications map[string]bool `bson:"notifications,omitempty" json:"notifications,omitempty"`

	// incremented by task start and end stats collectors and
	// should reflect hosts total costs. Only populated for build-hosts
	// where host providers report costs.
	TotalCost float64 `bson:"total_cost,omitempty" json:"total_cost,omitempty"`

	// accrues the value of idle time.
	TotalIdleTime time.Duration `bson:"total_idle_time,omitempty" json:"total_idle_time,omitempty" yaml:"total_idle_time,omitempty"`
}

func Find

func Find(query db.Q) ([]Host, error)

Find gets all Hosts for the given query.

func FindHostsToTerminate

func FindHostsToTerminate() ([]Host, error)

func FindOne

func FindOne(query db.Q) (*Host, error)

FindOne gets one Host for the given query.

func FindOneId

func FindOneId(id string) (*Host, error)

func GetHostsByFromIdWithStatus

func GetHostsByFromIdWithStatus(id, status, user string, limit, sortDir int) ([]Host, error)

func (*Host) ClearRunningAndSetLastTask

func (h *Host) ClearRunningAndSetLastTask(t *task.Task) error

ClearRunningAndSetLastTask unsets the running task on the host and updates the last task fields.

func (*Host) ClearRunningTask

func (h *Host) ClearRunningTask() error

ClearRunningTask unsets the running task on the host.

func (*Host) CreateSecret

func (h *Host) CreateSecret() error

CreateSecret generates a host secret and updates the host both locally and in the database.

func (*Host) CurlCommand

func (h *Host) CurlCommand(url string) string

func (*Host) DisablePoisonedHost

func (h *Host) DisablePoisonedHost(logs string) error

func (*Host) GetElapsedCommunicationTime

func (h *Host) GetElapsedCommunicationTime() time.Duration

GetElapsedCommunicationTime returns how long since this host has communicated with evergreen or vice versa

func (*Host) IdleTime

func (h *Host) IdleTime() time.Duration

IdleTime returns how long has this host been idle

func (*Host) IncCost

func (h *Host) IncCost(amt float64) error

func (*Host) IncIdleTime

func (h *Host) IncIdleTime(dur time.Duration) error

func (*Host) IncProvisionAttempts

func (h *Host) IncProvisionAttempts() error

func (*Host) IncTaskCount

func (h *Host) IncTaskCount() error

func (*Host) Insert

func (h *Host) Insert() error

func (*Host) IsEphemeral

func (h *Host) IsEphemeral() bool

func (*Host) IsWaitingForAgent

func (h *Host) IsWaitingForAgent() bool

IsWaitingForAgent provides a local predicate for the logic in the "NeedsNewAgent" query.

func (*Host) MarkAsProvisioned

func (h *Host) MarkAsProvisioned() error

func (*Host) MarkReachable

func (h *Host) MarkReachable() error

func (*Host) Remove

func (h *Host) Remove() error

func (*Host) ResetLastCommunicated

func (h *Host) ResetLastCommunicated() error

ResetLastCommunicated sets the LastCommunicationTime to be zero.

func (*Host) RunSSHCommand

func (h *Host) RunSSHCommand(ctx context.Context, cmd string, sshOptions []string) (string, error)

RunSSHCommand runs an SSH command on a remote host.

func (*Host) SetAgentRevision

func (h *Host) SetAgentRevision(agentRevision string) error

SetAgentRevision sets the updated agent revision for the host

func (*Host) SetDNSName

func (h *Host) SetDNSName(dnsName string) error

SetDNSName updates the DNS name for a given host once

func (*Host) SetDecommissioned

func (h *Host) SetDecommissioned(user string, logs string) error

func (*Host) SetExpirationNotification

func (h *Host) SetExpirationNotification(thresholdKey string) error

SetExpirationNotification updates the notification time for a spawn host

func (*Host) SetExpirationTime

func (h *Host) SetExpirationTime(expirationTime time.Time) error

SetExpirationTime updates the expiration time of a spawn host

func (*Host) SetExtId

func (h *Host) SetExtId() error

func (*Host) SetInitializing

func (h *Host) SetInitializing() error

SetInitializing marks the host as initializing. Only allow this if the host is uninitialized.

func (*Host) SetNeedsNewAgent

func (h *Host) SetNeedsNewAgent(needsAgent bool) error

SetNeedsNewAgent sets the "needs new agent" flag on the host

func (*Host) SetQuarantined

func (h *Host) SetQuarantined(user string, logs string) error

func (*Host) SetRunning

func (h *Host) SetRunning(user string) error

func (*Host) SetStarting

func (h *Host) SetStarting() error

func (*Host) SetStatus

func (h *Host) SetStatus(status, user string, logs string) error

func (*Host) SetTerminated

func (h *Host) SetTerminated(user string) error

func (*Host) SetUnprovisioned

func (h *Host) SetUnprovisioned() error

func (*Host) SetupCommand

func (h *Host) SetupCommand() string

func (*Host) TearDownCommand

func (h *Host) TearDownCommand() string

TearDownCommand returns a command for running a teardown script on a host.

func (*Host) Terminate

func (h *Host) Terminate(user string) error

func (*Host) UpdateDocumentID

func (h *Host) UpdateDocumentID(newID string) (*Host, error)

UpdateDocumentID updates the host document corresponding to the current host to have a new ID by finding, deleting, and replacing the document with a new one.

func (*Host) UpdateLastCommunicated

func (h *Host) UpdateLastCommunicated() error

UpdateLastCommunicated sets the host's last communication time to the current time.

func (*Host) UpdateRunningTask

func (h *Host) UpdateRunningTask(t *task.Task) (bool, error)

UpdateRunningTask updates the running task in the host document, returns - true, nil on success - false, nil on duplicate key error, task is already assigned to another host - false, error on all other errors

func (*Host) Upsert

func (h *Host) Upsert() (*mgo.ChangeInfo, error)

type ProviderStats

type ProviderStats []StatsByProvider

func GetProviderCounts

func GetProviderCounts() (ProviderStats, error)

GetProvierCounts returns data on the number of hosts by different provider stats.

func (ProviderStats) Map

func (p ProviderStats) Map() map[string]int

type ProvisionOptions

type ProvisionOptions struct {
	// LoadCLI indicates (if set) that while provisioning the host, the CLI binary should
	// be placed onto the host after startup.
	LoadCLI bool `bson:"load_cli" json:"load_cli"`

	// TaskId if non-empty will trigger the CLI tool to fetch source and artifacts for the given task.
	// Ignored if LoadCLI is false.
	TaskId string `bson:"task_id" json:"task_id"`

	// Owner is the user associated with the host used to populate any necessary metadata.
	OwnerId string `bson:"owner_id" json:"owner_id"`
}

ProvisionOptions is struct containing options about how a new host should be set up.

type StatsByDistro

type StatsByDistro struct {
	// ID of the distro the below stats are for
	Distro string `bson:"distro" json:"distro,omitempty"`
	// Host status that the below stats are for
	Status string `bson:"status" json:"status"`
	// Number of hosts in this status
	Count int `bson:"count" json:"count"`
	// Number of tasks running on hosts in the above group (should only be nonzero for running hosts)
	NumTasks int `bson:"num_tasks_running" json:"num_tasks_running"`
	// MaxHosts reports the pool size of the distro.
	MaxHosts int `bson:"max_hosts" json:"max_hosts"`
}

type StatsByProvider

type StatsByProvider struct {
	// the name of a host provider
	Provider string `bson:"provider" json:"provider"`
	// Number of hosts with this provider
	Count int `bson:"count" json:"count"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL