Documentation ¶
Index ¶
- Constants
- Variables
- func ConnectionId() string
- func DetectFiles(root string, cmd string) []string
- func DumpAnnotations(req *WorkRequest, rep *WorkResponse, start time.Time, outDir string, ...)
- func EscapeRegexp(s string) string
- func FindSocket() string
- func HasDirPrefix(path, prefix string) bool
- func HookedCopy(w io.Writer, r io.Reader, proc func([]byte)) error
- func HumanTrim(s string, l int) string
- func IntToExponent(z int) uint
- func IsSpace(b byte) bool
- func MakeUnescape(cmd string) string
- func NewLazyLoopbackFile(n string) nodefs.File
- func NewLocalDecider(dir string) *localDecider
- func OpenSocketConnection(socket string, channel string, timeout time.Duration) net.Conn
- func ParseCommand(cmd string) []string
- func PrintStdinSliceLen(s []byte)
- func RandomBytes(n int) []byte
- func SplitPath(name string) (dir, base string)
- func Version() string
- type AnnotatingFS
- type Coordinator
- type CoordinatorOptions
- type CoordinatorService
- type CreateMirrorRequest
- type CreateMirrorResponse
- type Empty
- type FuseFsStatus
- type GetoptResult
- type ListRequest
- type ListResponse
- type LocalMaster
- type LocalRule
- type LogRequest
- type LogResponse
- type Master
- type MasterOptions
- type Mirror
- type MirrorStatusRequest
- type MirrorStatusResponse
- type Registration
- type RegistrationRequest
- type RpcFs
- func (fs *RpcFs) Access(name string, mode uint32, context *fuse.Context) (code fuse.Status)
- func (fs *RpcFs) Close()
- func (fs *RpcFs) FetchHash(a *attr.FileAttr) error
- func (fs *RpcFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status)
- func (fs *RpcFs) Open(name string, flags uint32, context *fuse.Context) (nodefs.File, fuse.Status)
- func (fs *RpcFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry, fuse.Status)
- func (fs *RpcFs) Readlink(name string, context *fuse.Context) (string, fuse.Status)
- func (fs *RpcFs) String() string
- func (fs *RpcFs) Update(req *UpdateRequest, resp *UpdateResponse) error
- type ShutdownRequest
- type ShutdownResponse
- type Timing
- type UpdateRequest
- type UpdateResponse
- type User
- type WorkRequest
- type WorkResponse
- type Worker
- func (w *Worker) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error
- func (w *Worker) DropMirror(mirror *Mirror)
- func (w *Worker) Log(req *LogRequest, rep *LogResponse) error
- func (w *Worker) PeriodicHouseholding()
- func (w *Worker) Report()
- func (w *Worker) RunWorkerServer()
- func (w *Worker) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error
- func (w *Worker) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse) error
- type WorkerMirrors
- type WorkerOptions
- type WorkerRegistration
- type WorkerService
- func (ws *WorkerService) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error
- func (ws *WorkerService) Log(req *LogRequest, rep *LogResponse) error
- func (ws *WorkerService) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error
- func (ws *WorkerService) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse) error
- type WorkerStatusRequest
- type WorkerStatusResponse
- type WorkerTask
Constants ¶
View Source
const ( RPC_CHANNEL = "rpc......" HEADER_LEN = 9 )
ids:
Variables ¶
View Source
var Hostname string
View Source
var ShuttingDownError error
Functions ¶
func ConnectionId ¶
func ConnectionId() string
func DetectFiles ¶
func DumpAnnotations ¶
func DumpAnnotations(req *WorkRequest, rep *WorkResponse, start time.Time, outDir string, topDir string)
func EscapeRegexp ¶
func FindSocket ¶
func FindSocket() string
func HasDirPrefix ¶
func HookedCopy ¶
Useful for debugging.
func IntToExponent ¶
IntToExponent the smallest E such that 2^E >= Z.
func MakeUnescape ¶
func NewLazyLoopbackFile ¶
func NewLocalDecider ¶
func NewLocalDecider(dir string) *localDecider
func OpenSocketConnection ¶
func ParseCommand ¶
ParseCommand tries to parse quoting for a shell command line. It will give up and return nil when it returns shell-metacharacters ($, ` , etc.)
func PrintStdinSliceLen ¶
func PrintStdinSliceLen(s []byte)
func RandomBytes ¶
Types ¶
type AnnotatingFS ¶
type AnnotatingFS struct { pathfs.FileSystem // contains filtered or unexported fields }
func NewAnnotatingFS ¶
func NewAnnotatingFS(fs pathfs.FileSystem) *AnnotatingFS
func (*AnnotatingFS) Reap ¶
func (fs *AnnotatingFS) Reap() []string
type Coordinator ¶
Coordinator implements CoordinatorService RPC interface for termite. Workers register here. A master looking for workers contacts the Coordinator to fetch a list of available workers. In addition, it has a HTTP interface to inspect each worker.
func NewCoordinator ¶
func NewCoordinator(opts *CoordinatorOptions) *Coordinator
func (*Coordinator) List ¶
func (c *Coordinator) List(req *ListRequest, rep *ListResponse) error
func (*Coordinator) PeriodicCheck ¶
func (c *Coordinator) PeriodicCheck()
func (*Coordinator) Register ¶
func (c *Coordinator) Register(req *RegistrationRequest, rep *Empty) error
func (*Coordinator) ServeHTTP ¶
func (c *Coordinator) ServeHTTP(port int)
func (*Coordinator) Shutdown ¶
func (c *Coordinator) Shutdown()
func (*Coordinator) WorkerCount ¶
func (c *Coordinator) WorkerCount() int
type CoordinatorOptions ¶
type CoordinatorService ¶
type CoordinatorService Coordinator
RPC interface for Coordinator
func (*CoordinatorService) List ¶
func (cs *CoordinatorService) List(req *ListRequest, rep *ListResponse) error
func (*CoordinatorService) Register ¶
func (cs *CoordinatorService) Register(req *RegistrationRequest, rep *Empty) error
type CreateMirrorRequest ¶
type CreateMirrorResponse ¶
type CreateMirrorResponse struct {
GrantedJobCount int
}
type FuseFsStatus ¶
type GetoptResult ¶
func Getopt ¶
func Getopt(args []string, longTakeArg []string, shortTakeArg []byte, reorder bool) (r GetoptResult)
func (*GetoptResult) HasLong ¶
func (r *GetoptResult) HasLong(name string) bool
func (*GetoptResult) HasOptions ¶
func (r *GetoptResult) HasOptions() bool
func (*GetoptResult) HasShort ¶
func (r *GetoptResult) HasShort(name byte) bool
type ListRequest ¶
type ListResponse ¶
type ListResponse struct { Registrations []Registration LastChange time.Time }
type LocalMaster ¶
type LocalMaster struct {
// contains filtered or unexported fields
}
Expose functionality for the local tool to use.
func (*LocalMaster) InspectFile ¶
func (m *LocalMaster) InspectFile(req *attr.AttrRequest, rep *attr.AttrResponse) error
func (*LocalMaster) RefreshAttributeCache ¶
func (m *LocalMaster) RefreshAttributeCache(input *int, output *int) error
func (*LocalMaster) Run ¶
func (m *LocalMaster) Run(req *WorkRequest, rep *WorkResponse) error
type LogRequest ¶
type LogResponse ¶
type LogResponse struct {
Data []byte
}
type Master ¶
type Master struct {
// contains filtered or unexported fields
}
func NewMaster ¶
func NewMaster(options *MasterOptions) *Master
func (*Master) CheckPrivate ¶
func (m *Master) CheckPrivate()
func (*Master) MaybeRunInMaster ¶
func (m *Master) MaybeRunInMaster(req *WorkRequest, rep *WorkResponse) bool
type MasterOptions ¶
type MasterOptions struct { cba.StoreOptions WritableRoot string SourceRoot string // How often a failed should be retried on another worker. RetryCount int // List of files that should not be served Excludes []string // If set, also serve files that have no group/other permissions ExposePrivate bool // Address of the coordinator. Coordinator string Secret []byte MaxJobs int // Turns on internal consistency checks. Expensive. Paranoia bool // On startup, fault-in all files. FetchAll bool // How often to do periodic householding work. Period time.Duration // How long to keep mirrors alive. KeepAlive time.Duration // Cache hashes in filesystem extended attributes. XAttrCache bool Uid int // The log file. This is to ensure we don't export or hash // the log file. LogFile string // Path to the socket file. Socket string // Dump action graph data into this directory AnalysisDir string }
Immutable state and options for master.
type Mirror ¶
type Mirror struct {
// contains filtered or unexported fields
}
State associated with one master.
func NewMirror ¶
func NewMirror(worker *Worker, rpcConn, revConn, contentConn, revContentConn io.ReadWriteCloser) (*Mirror, error)
func (*Mirror) Run ¶
func (m *Mirror) Run(req *WorkRequest, rep *WorkResponse) error
func (*Mirror) Status ¶
func (m *Mirror) Status(req *MirrorStatusRequest, rep *MirrorStatusResponse) error
func (*Mirror) Update ¶
func (m *Mirror) Update(req *UpdateRequest, rep *UpdateResponse) error
type MirrorStatusRequest ¶
type MirrorStatusRequest struct { }
type MirrorStatusResponse ¶
type Registration ¶
type RegistrationRequest ¶
type RegistrationRequest Registration
type RpcFs ¶
type RpcFs struct { pathfs.FileSystem // contains filtered or unexported fields }
func (*RpcFs) Update ¶
func (fs *RpcFs) Update(req *UpdateRequest, resp *UpdateResponse) error
type ShutdownRequest ¶
type ShutdownResponse ¶
type ShutdownResponse struct { }
type UpdateRequest ¶
type UpdateResponse ¶
type UpdateResponse struct { }
type WorkRequest ¶
type WorkRequest struct { // Unique id of this request. TaskId int // Id of connection streaming stdin. StdinId string // TODO - don't abuse RPC message for transporting this. StdinConn io.ReadWriteCloser Debug bool Binary string Argv []string Env []string Dir string // Signal that a command ran locally. Used for logging in the master. RanLocally bool // if TrackReads is set, the worker will return files read by the task. TrackReads bool // If set, must run on this worker. Used for debugging. Worker string // The following is used with TrackReads and can be injected from the Makefile. DeclaredDeps []string DeclaredTarget string }
func (*WorkRequest) Summary ¶
func (r *WorkRequest) Summary() string
type WorkResponse ¶
type WorkResponse struct { Exit syscall.WaitStatus Stderr string Stdout string Timings []Timing // Reaped files, if any *attr.FileSet // Task ids for which the fileset contains data. TaskIds []int // Files from the backing store that were read. Reads []string // Worker where this was processed. WorkerId string }
func (*WorkResponse) String ¶
func (r *WorkResponse) String() string
type Worker ¶
type Worker struct {
// contains filtered or unexported fields
}
func NewWorker ¶
func NewWorker(options *WorkerOptions) *Worker
func (*Worker) CreateMirror ¶
func (w *Worker) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error
func (*Worker) DropMirror ¶
func (*Worker) Log ¶
func (w *Worker) Log(req *LogRequest, rep *LogResponse) error
func (*Worker) PeriodicHouseholding ¶
func (w *Worker) PeriodicHouseholding()
func (*Worker) RunWorkerServer ¶
func (w *Worker) RunWorkerServer()
func (*Worker) Shutdown ¶
func (w *Worker) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error
func (*Worker) Status ¶
func (w *Worker) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse) error
type WorkerMirrors ¶
type WorkerMirrors struct {
// contains filtered or unexported fields
}
func NewWorkerMirrors ¶
func NewWorkerMirrors(w *Worker) *WorkerMirrors
func (*WorkerMirrors) DropMirror ¶
func (wm *WorkerMirrors) DropMirror(mirror *Mirror)
func (*WorkerMirrors) Status ¶
func (wm *WorkerMirrors) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse)
type WorkerOptions ¶
type WorkerOptions struct { cba.StoreOptions // Address of the coordinator. Coordinator string // (starting) port to listen to. Port int // How many other ports try. PortRetry int Paranoia bool Secret []byte TempDir string Jobs int // If set, change user to this for running. User *User // How often to reap filesystems. If 1, use 1 FS per task. ReapCount int // Delay between contacting the coordinator for making reports. ReportInterval time.Duration LogFileName string // If set, we restart once the heap usage passes this // threshold. HeapLimit uint64 // How long to wait between the last task exit, and shutting // down the server. LameDuckPeriod time.Duration // full path to mkbox binary Mkbox string }
type WorkerRegistration ¶
type WorkerRegistration struct { Registration LastReported time.Time }
type WorkerService ¶
type WorkerService Worker
func (*WorkerService) CreateMirror ¶
func (ws *WorkerService) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) error
func (*WorkerService) Log ¶
func (ws *WorkerService) Log(req *LogRequest, rep *LogResponse) error
func (*WorkerService) Shutdown ¶
func (ws *WorkerService) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) error
func (*WorkerService) Status ¶
func (ws *WorkerService) Status(req *WorkerStatusRequest, rep *WorkerStatusResponse) error
type WorkerStatusRequest ¶
type WorkerStatusRequest struct { }
type WorkerStatusResponse ¶
type WorkerTask ¶
type WorkerTask struct {
// contains filtered or unexported fields
}
func (*WorkerTask) Kill ¶
func (t *WorkerTask) Kill()
func (*WorkerTask) Run ¶
func (t *WorkerTask) Run() error
func (*WorkerTask) String ¶
func (t *WorkerTask) String() string
Source Files ¶
- analyze.go
- annotatingfs.go
- connection.go
- coordinator.go
- coordinatorhttp.go
- fuse.go
- getopt.go
- lazyfile.go
- localmaster.go
- localrun.go
- master.go
- masterrun.go
- masterstatus.go
- mirror.go
- mirrorconnection.go
- mirrors.go
- misc.go
- rpc.go
- rpcfs.go
- ssh-conn.go
- status.go
- task.go
- tcp-connection.go
- version.go
- worker.go
- workerconn.go
- workerstatus.go
Click to show internal directories.
Click to hide internal directories.