termite

package
v0.0.0-...-4b497b0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 30, 2014 License: BSD-3-Clause Imports: 40 Imported by: 0

Documentation

Index

Constants

View Source
const (
	RPC_CHANNEL = "rpc......"
	HEADER_LEN  = 9
)

ids:

Variables

View Source
var Hostname string
View Source
var ShuttingDownError error

Functions

func ConnectionId

func ConnectionId() string

func DetectFiles

func DetectFiles(root string, cmd string) []string

func DumpAnnotations

func DumpAnnotations(req *WorkRequest, rep *WorkResponse, start time.Time,
	outDir string, topDir string)

func EscapeRegexp

func EscapeRegexp(s string) string

func FindSocket

func FindSocket() string

func HasDirPrefix

func HasDirPrefix(path, prefix string) bool

func HookedCopy

func HookedCopy(w io.Writer, r io.Reader, proc func([]byte)) error

Useful for debugging.

func HumanTrim

func HumanTrim(s string, l int) string

func IntToExponent

func IntToExponent(z int) uint

IntToExponent the smallest E such that 2^E >= Z.

func IsSpace

func IsSpace(b byte) bool

func MakeUnescape

func MakeUnescape(cmd string) string

func NewLazyLoopbackFile

func NewLazyLoopbackFile(n string) nodefs.File

func NewLocalDecider

func NewLocalDecider(dir string) *localDecider

func OpenSocketConnection

func OpenSocketConnection(socket string, channel string, timeout time.Duration) net.Conn

func ParseCommand

func ParseCommand(cmd string) []string

ParseCommand tries to parse quoting for a shell command line. It will give up and return nil when it returns shell-metacharacters ($, ` , etc.)

func PrintStdinSliceLen

func PrintStdinSliceLen(s []byte)

func RandomBytes

func RandomBytes(n int) []byte

func SplitPath

func SplitPath(name string) (dir, base string)

TODO - move into fuse

func Version

func Version() string

Types

type AnnotatingFS

type AnnotatingFS struct {
	pathfs.FileSystem
	// contains filtered or unexported fields
}

func NewAnnotatingFS

func NewAnnotatingFS(fs pathfs.FileSystem) *AnnotatingFS

func (*AnnotatingFS) Open

func (fs *AnnotatingFS) Open(name string, flags uint32, context *fuse.Context) (file nodefs.File, code fuse.Status)

func (*AnnotatingFS) Reap

func (fs *AnnotatingFS) Reap() []string

type Coordinator

type Coordinator struct {
	Mux *http.ServeMux
	// contains filtered or unexported fields
}

Coordinator implements CoordinatorService RPC interface for termite. Workers register here. A master looking for workers contacts the Coordinator to fetch a list of available workers. In addition, it has a HTTP interface to inspect each worker.

func NewCoordinator

func NewCoordinator(opts *CoordinatorOptions) *Coordinator

func (*Coordinator) List

func (c *Coordinator) List(req *ListRequest, rep *ListResponse) error

func (*Coordinator) PeriodicCheck

func (c *Coordinator) PeriodicCheck()

func (*Coordinator) Register

func (c *Coordinator) Register(req *RegistrationRequest, rep *Empty) error

func (*Coordinator) ServeHTTP

func (c *Coordinator) ServeHTTP(port int)

func (*Coordinator) Shutdown

func (c *Coordinator) Shutdown()

func (*Coordinator) WorkerCount

func (c *Coordinator) WorkerCount() int

type CoordinatorOptions

type CoordinatorOptions struct {
	// Secret is the password for coordinator, workers and master
	// to authenticate.
	Secret []byte

	// Password should be passed in the kill/restart URLs to make
	// sure web scrapers don't randomly shutdown workers.
	WebPassword string
}

type CoordinatorService

type CoordinatorService Coordinator

RPC interface for Coordinator

func (*CoordinatorService) List

func (cs *CoordinatorService) List(req *ListRequest, rep *ListResponse) error

func (*CoordinatorService) Register

func (cs *CoordinatorService) Register(req *RegistrationRequest, rep *Empty) error

type CreateMirrorRequest

type CreateMirrorRequest struct {
	// Ids of connections to use for RPC
	RpcId        string
	RevRpcId     string
	ContentId    string
	RevContentId string

	// The writable root for the mirror.
	WritableRoot string

	// Max number of processes to reserve.
	MaxJobCount int
}

type CreateMirrorResponse

type CreateMirrorResponse struct {
	GrantedJobCount int
}

type Empty

type Empty struct{}

type FuseFsStatus

type FuseFsStatus struct {
	Id    string
	Tasks []string
	Mem   string
}

type GetoptResult

type GetoptResult struct {
	Long  map[string]string
	Short map[byte]string
	Args  []string
}

func Getopt

func Getopt(args []string, longTakeArg []string, shortTakeArg []byte, reorder bool) (r GetoptResult)

func (*GetoptResult) HasLong

func (r *GetoptResult) HasLong(name string) bool

func (*GetoptResult) HasOptions

func (r *GetoptResult) HasOptions() bool

func (*GetoptResult) HasShort

func (r *GetoptResult) HasShort(name byte) bool

type ListRequest

type ListRequest struct {
	// Return changes after this time stamp.  Will halt if no
	// changes to report.
	Latest time.Time
}

type ListResponse

type ListResponse struct {
	Registrations []Registration
	LastChange    time.Time
}

type LocalMaster

type LocalMaster struct {
	// contains filtered or unexported fields
}

Expose functionality for the local tool to use.

func (*LocalMaster) InspectFile

func (m *LocalMaster) InspectFile(req *attr.AttrRequest, rep *attr.AttrResponse) error

func (*LocalMaster) RefreshAttributeCache

func (m *LocalMaster) RefreshAttributeCache(input *int, output *int) error

func (*LocalMaster) Run

func (m *LocalMaster) Run(req *WorkRequest, rep *WorkResponse) error

func (*LocalMaster) Shutdown

func (m *LocalMaster) Shutdown(req *int, rep *int) error

type LocalRule

type LocalRule struct {
	Regexp      string
	Local       bool
	Recurse     bool
	SkipRefresh bool
	Debug       bool
}

type LogRequest

type LogRequest struct {
	Whence int
	Off    int64
	Size   int64
}

type LogResponse

type LogResponse struct {
	Data []byte
}

type Master

type Master struct {
	// contains filtered or unexported fields
}

func NewMaster

func NewMaster(options *MasterOptions) *Master

func (*Master) CheckPrivate

func (m *Master) CheckPrivate()

func (*Master) FetchAll

func (m *Master) FetchAll()

Fetch in the background.

func (*Master) MaybeRunInMaster

func (m *Master) MaybeRunInMaster(req *WorkRequest, rep *WorkResponse) bool

func (*Master) ServeHTTP

func (m *Master) ServeHTTP(port int)

func (*Master) Start

func (m *Master) Start()

type MasterOptions

type MasterOptions struct {
	cba.StoreOptions

	WritableRoot string
	SourceRoot   string

	// How often a failed should be retried on another worker.
	RetryCount int

	// List of files that should not be served
	Excludes []string

	// If set, also serve files that have no group/other permissions
	ExposePrivate bool

	// Address of the coordinator.
	Coordinator string

	Secret []byte

	MaxJobs int

	// Turns on internal consistency checks. Expensive.
	Paranoia bool

	// On startup, fault-in all files.
	FetchAll bool

	// How often to do periodic householding work.
	Period time.Duration

	// How long to keep mirrors alive.
	KeepAlive time.Duration

	// Cache hashes in filesystem extended attributes.
	XAttrCache bool

	Uid int

	// The log file.  This is to ensure we don't export or hash
	// the log file.
	LogFile string

	// Path to the socket file.
	Socket string

	// Dump action graph data into this directory
	AnalysisDir string
}

Immutable state and options for master.

type Mirror

type Mirror struct {
	// contains filtered or unexported fields
}

State associated with one master.

func NewMirror

func NewMirror(worker *Worker, rpcConn, revConn, contentConn, revContentConn io.ReadWriteCloser) (*Mirror, error)

func (*Mirror) Run

func (m *Mirror) Run(req *WorkRequest, rep *WorkResponse) error

func (*Mirror) Status

func (m *Mirror) Status(req *MirrorStatusRequest, rep *MirrorStatusResponse) error

func (*Mirror) Update

func (m *Mirror) Update(req *UpdateRequest, rep *UpdateResponse) error

type MirrorStatusRequest

type MirrorStatusRequest struct {
}

type MirrorStatusResponse

type MirrorStatusResponse struct {
	Root         string
	Granted      int
	Fses         []FuseFsStatus
	Accepting    bool
	WaitingTasks int
	IdleFses     int
	RpcTimings   []string
}

type Registration

type Registration struct {
	Address        string
	Name           string
	Version        string
	HttpStatusPort int
}

type RegistrationRequest

type RegistrationRequest Registration

type RpcFs

type RpcFs struct {
	pathfs.FileSystem
	// contains filtered or unexported fields
}

func NewRpcFs

func NewRpcFs(attrClient *attr.Client, cache *cba.Store, contentConn io.ReadWriteCloser) *RpcFs

func (*RpcFs) Access

func (fs *RpcFs) Access(name string, mode uint32, context *fuse.Context) (code fuse.Status)

func (*RpcFs) Close

func (fs *RpcFs) Close()

func (*RpcFs) FetchHash

func (fs *RpcFs) FetchHash(a *attr.FileAttr) error

func (*RpcFs) GetAttr

func (fs *RpcFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status)

func (*RpcFs) Open

func (fs *RpcFs) Open(name string, flags uint32, context *fuse.Context) (nodefs.File, fuse.Status)

func (*RpcFs) OpenDir

func (fs *RpcFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry, fuse.Status)
func (fs *RpcFs) Readlink(name string, context *fuse.Context) (string, fuse.Status)

func (*RpcFs) String

func (fs *RpcFs) String() string

func (*RpcFs) Update

func (fs *RpcFs) Update(req *UpdateRequest, resp *UpdateResponse) error

type ShutdownRequest

type ShutdownRequest struct {
	Restart bool
	Kill    bool
}

type ShutdownResponse

type ShutdownResponse struct {
}

type Timing

type Timing struct {
	Name string
	Dt   float64
}

type UpdateRequest

type UpdateRequest struct {
	Files []*attr.FileAttr
}

type UpdateResponse

type UpdateResponse struct {
}

type User

type User struct {
	Uid int
	Gid int
}

type WorkRequest

type WorkRequest struct {
	// Unique id of this request.
	TaskId int

	// Id of connection streaming stdin.
	StdinId string

	// TODO - don't abuse RPC message for transporting this.
	StdinConn io.ReadWriteCloser

	Debug  bool
	Binary string
	Argv   []string
	Env    []string
	Dir    string

	// Signal that a command ran locally.  Used for logging in the master.
	RanLocally bool

	// if TrackReads is set, the worker will return files read by the task.
	TrackReads bool

	// If set, must run on this worker. Used for debugging.
	Worker string

	// The following is used with TrackReads and can be injected from the Makefile.
	DeclaredDeps   []string
	DeclaredTarget string
}

func (*WorkRequest) Summary

func (r *WorkRequest) Summary() string

type WorkResponse

type WorkResponse struct {
	Exit   syscall.WaitStatus
	Stderr string
	Stdout string

	Timings []Timing

	// Reaped files, if any
	*attr.FileSet

	// Task ids for which the fileset contains data.
	TaskIds []int

	// Files from the backing store that were read.
	Reads []string

	// Worker where this was processed.
	WorkerId string
}

func (*WorkResponse) String

func (r *WorkResponse) String() string

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

func NewWorker

func NewWorker(options *WorkerOptions) *Worker

func (*Worker) CreateMirror

func (w *Worker) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error

func (*Worker) DropMirror

func (w *Worker) DropMirror(mirror *Mirror)

func (*Worker) Log

func (w *Worker) Log(req *LogRequest, rep *LogResponse) error

func (*Worker) PeriodicHouseholding

func (w *Worker) PeriodicHouseholding()

func (*Worker) Report

func (w *Worker) Report()

func (*Worker) RunWorkerServer

func (w *Worker) RunWorkerServer()

func (*Worker) Shutdown

func (w *Worker) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error

func (*Worker) Status

func (w *Worker) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse) error

type WorkerMirrors

type WorkerMirrors struct {
	// contains filtered or unexported fields
}

func NewWorkerMirrors

func NewWorkerMirrors(w *Worker) *WorkerMirrors

func (*WorkerMirrors) DropMirror

func (wm *WorkerMirrors) DropMirror(mirror *Mirror)

func (*WorkerMirrors) Status

type WorkerOptions

type WorkerOptions struct {
	cba.StoreOptions

	// Address of the coordinator.
	Coordinator string

	// (starting) port to listen to.
	Port int

	// How many other ports try.
	PortRetry int

	Paranoia bool

	Secret  []byte
	TempDir string
	Jobs    int

	// If set, change user to this for running.
	User *User

	// How often to reap filesystems. If 1, use 1 FS per task.
	ReapCount int

	// Delay between contacting the coordinator for making reports.
	ReportInterval time.Duration
	LogFileName    string

	// If set, we restart once the heap usage passes this
	// threshold.
	HeapLimit uint64

	// How long to wait between the last task exit, and shutting
	// down the server.
	LameDuckPeriod time.Duration

	// full path to mkbox binary
	Mkbox string
}

type WorkerRegistration

type WorkerRegistration struct {
	Registration
	LastReported time.Time
}

type WorkerService

type WorkerService Worker

func (*WorkerService) CreateMirror

func (ws *WorkerService) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error

func (*WorkerService) Log

func (ws *WorkerService) Log(req *LogRequest, rep *LogResponse) error

func (*WorkerService) Shutdown

func (ws *WorkerService) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error

func (*WorkerService) Status

type WorkerStatusRequest

type WorkerStatusRequest struct {
}

type WorkerStatusResponse

type WorkerStatusResponse struct {
	MirrorStatus []MirrorStatusResponse
	Version      string
	MaxJobCount  int
	Accepting    bool

	// In chronological order.
	CpuStats  []stats.CpuStat
	TotalCpu  stats.CpuStat
	DiskStats []stats.DiskStat

	PhaseNames  []string
	PhaseCounts []int
	MemStat     stats.MemStat
}

type WorkerTask

type WorkerTask struct {
	// contains filtered or unexported fields
}

func (*WorkerTask) Kill

func (t *WorkerTask) Kill()

func (*WorkerTask) Run

func (t *WorkerTask) Run() error

func (*WorkerTask) String

func (t *WorkerTask) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL