virtual

type Attributes ¶

type Attributes struct {
	// contains filtered or unexported fields
}

Attributes of a file, normally requested through stat() or readdir(). A bitmask is used to track which attributes are set.

func (a *Attributes) GetChangeID() uint64

func (a *Attributes) GetDeviceNumber() (filesystem.DeviceNumber, bool)

func (a *Attributes) GetFileHandle() []byte

func (a *Attributes) GetFileType() filesystem.FileType

func (a *Attributes) GetInodeNumber() uint64

func (a *Attributes) GetLastDataModificationTime() (time.Time, bool)

func (a *Attributes) GetLinkCount() uint32

func (a *Attributes) GetPermissions() (Permissions, bool)

func (a *Attributes) GetSizeBytes() (uint64, bool)

func (a *Attributes) SetChangeID(changeID uint64) *Attributes

func (a *Attributes) SetDeviceNumber(deviceNumber filesystem.DeviceNumber) *Attributes

func (a *Attributes) SetFileHandle(fileHandle []byte) *Attributes

func (a *Attributes) SetFileType(fileType filesystem.FileType) *Attributes

func (a *Attributes) SetInodeNumber(inodeNumber uint64) *Attributes

func (a *Attributes) SetLastDataModificationTime(lastDataModificationTime time.Time) *Attributes

func (a *Attributes) SetLinkCount(linkCount uint32) *Attributes

func (a *Attributes) SetPermissions(permissions Permissions) *Attributes

func (a *Attributes) SetSizeBytes(sizeBytes uint64) *Attributes

type AttributesMask ¶

type AttributesMask uint32

AttributesMask is a bitmask of status attributes that need to be requested through Node.VirtualGetAttributes().

const (
	// AttributesMaskChangeID requests the change ID, which clients
	// can use to determine if file data, directory contents, or
	// attributes of the node have changed.
	AttributesMaskChangeID AttributesMask = 1 << iota
	// AttributesMaskDeviceNumber requests the raw device number
	// (st_rdev).
	AttributesMaskDeviceNumber
	// AttributesMaskFileHandle requests an identifier of the file
	// that contains sufficient information to be able to resolve it
	// at a later point in time.
	AttributesMaskFileHandle
	// AttributesMaskFileType requests the file type (upper 4 bits
	// of st_mode).
	AttributesMaskFileType
	// AttributesMaskInodeNumber requests the inode number (st_ino).
	AttributesMaskInodeNumber
	// AttributesMaskLastDataModificationTime requests the last data
	// modification time (st_mtim).
	AttributesMaskLastDataModificationTime
	// AttributesMaskLinkCount requests the link count (st_nlink).
	AttributesMaskLinkCount
	// AttributesMaskPermissions requests the permissions (lowest 12
	// bits of set_mode).
	AttributesMaskPermissions
	// AttributesMaskSizeBytes requests the file size (st_size).
	AttributesMaskSizeBytes
)

type ByteRangeLock ¶

type ByteRangeLock[Owner comparable] struct {
	Start uint64
	End   uint64
	Owner Owner
	Type  ByteRangeLockType
}

ByteRangeLock holds information on a lock held on a non-empty range of bytes. Each lock has an owner associated with it. This field is only checked for equality, allowing it to merge adjacent or overlapping locks held by the same owner.

type ByteRangeLockSet ¶

type ByteRangeLockSet[Owner comparable] struct {
	// contains filtered or unexported fields
}

ByteRangeLockSet is a set for ByteRangeLocks applied against the same file. The set is modeled as a linked list, where entries are sorted by starting address. All entries describe disjoint non-empty ranges of bytes, except for shared locks with distinct owners.

func (ls *ByteRangeLockSet[Owner]) Initialize()

func (ls *ByteRangeLockSet[Owner]) Set(lProvided *ByteRangeLock[Owner]) int

func (ls *ByteRangeLockSet[Owner]) Test(lTest *ByteRangeLock[Owner]) *ByteRangeLock[Owner]

type ByteRangeLockType ¶

type ByteRangeLockType int

ByteRangeLockType is an enumeration that controls what kind of lock needs to be acquired.

const (
	// ByteRangeLockTypeUnlocked indicates that a byte range should
	// be unlocked. This value can only be provided to Set(); not
	// Test(). It is equivalent to POSIX's F_UNLCK.
	ByteRangeLockTypeUnlocked ByteRangeLockType = iota
	// ByteRangeLockTypeLockedExclusive indicates that a byte range
	// should be locked exclusively for writing. It is equivalent to
	// POSIX's F_WRLCK.
	ByteRangeLockTypeLockedExclusive
	// ByteRangeLockTypeLockedShared indicates that a byte range
	// should be locked shared for reading. It is equivalent to
	// F_RDLCK.
	ByteRangeLockTypeLockedShared
)

type ByteSliceID ¶

type ByteSliceID []byte

ByteSliceID is a helper type for consumers of StatelessHandleAllocator and ResolvableHandleAllocator. It uses the contents of a byte slice as an object ID. The byte slice will be prefixed with its own length, so that no ambiguity exists if allocators are nested.

func (data ByteSliceID) WriteTo(w io.Writer) (nTotal int64, err error)

type CASFileFactory ¶

type CASFileFactory interface {
	LookupFile(digest digest.Digest, isExecutable bool, readMonitor FileReadMonitor) NativeLeaf
}

CASFileFactory is a factory type for files whose contents correspond with an object stored in the Content Addressable Storage (CAS).

func NewBlobAccessCASFileFactory(ctx context.Context, contentAddressableStorage blobstore.BlobAccess, errorLogger util.ErrorLogger) CASFileFactory

func NewResolvableHandleAllocatingCASFileFactory(base CASFileFactory, allocation StatelessHandleAllocation) CASFileFactory

func NewStatelessHandleAllocatingCASFileFactory(base CASFileFactory, allocation StatelessHandleAllocation) CASFileFactory

type ChangeInfo ¶

type ChangeInfo struct {
	Before uint64
	After  uint64
}

ChangeInfo contains a pair of change IDs of a directory, before and after performing a directory mutating operation. This information needs to be returned by various NFSv4 operations.

type CharacterDeviceFactory ¶

type CharacterDeviceFactory interface {
	LookupCharacterDevice(deviceNumber filesystem.DeviceNumber) NativeLeaf
}

CharacterDeviceFactory is a factory type for character devices. Character devices are immutable files; it is not possible to change the device after it has been created.

var BaseCharacterDeviceFactory CharacterDeviceFactory = baseCharacterDeviceFactory{}

func NewHandleAllocatingCharacterDeviceFactory(base CharacterDeviceFactory, allocation ResolvableHandleAllocation) CharacterDeviceFactory

type Child ¶

type Child[TDirectory any, TLeaf any, TNode any] struct {
	// contains filtered or unexported fields
}

Child is a variant type that either contains a directory or leaf object.

TODO: In principle it should be possible to eliminate the 'kind' field, if it weren't for the fact that we can't compare TDirectory and TLeaf against nil. There is no nullable constraint. https://github.com/golang/go/issues/53656

func (Child[TDirectory, TLeaf, TNode]) FromDirectory(directory TDirectory) Child[TDirectory, TLeaf, TNode]

func (Child[TDirectory, TLeaf, TNode]) FromLeaf(leaf TLeaf) Child[TDirectory, TLeaf, TNode]

func (c Child[TDirectory, TLeaf, TNode]) GetNode() TNode

func (c Child[TDirectory, TLeaf, TNode]) GetPair() (TDirectory, TLeaf)

func (c Child[TDirectory, TLeaf, TNode]) IsSet() bool

type ChildFilter ¶

type ChildFilter func(node InitialNode, remove ChildRemover) bool

ChildFilter is a callback that is invoked by PrepopulatedDirectory.FilterChildren() for each of the children underneath the current directory hierarchy.

For each of the children, an InitialNode object is provided that describes the contents of that file or directory. In addition to that, a callback is provided that can remove the file or the contents of the directory. This callback may be invoked synchronously or asynchronously, potentially after FilterChildren() has completed.

The boolean return value of this function signals whether traversal should continue. When false, traversal will stop immediately.

type ChildRemover ¶

type ChildRemover func() error

ChildRemover is a callback that is provided to ChildFilter to remove the provided file or contents of the provided directory.

type DigestHandleResolver ¶

type DigestHandleResolver func(blobDigest digest.Digest, remainder io.ByteReader) (DirectoryChild, Status)

DigestHandleResolver is a handle resolver that needs to be implemented by users of ResolvableDigestHandleAllocator. This callback is responsible for looking up a file or directory that corresponds to a given REv2 digest that was reobtained from a file handle.

type Directory ¶

type Directory interface {
	Node

	// VirtualOpenChild opens a regular file within the directory.
	//
	// When createAttributes is nil, this method will fail with
	// StatusErrNoEnt if the file does not exist. When not nil, a
	// file will be created.
	//
	// When existingOptions is nil, this method will fail with
	// StatusErrExist if the file already exists. When not nil, an
	// existing file will be opened.
	//
	// Either one or both of createAttributes and existingOptions
	// need to be provided.
	VirtualOpenChild(ctx context.Context, name path.Component, shareAccess ShareMask, createAttributes *Attributes, existingOptions *OpenExistingOptions, requested AttributesMask, openedFileAttributes *Attributes) (Leaf, AttributesMask, ChangeInfo, Status)
	// VirtualLink links an existing file into the directory.
	VirtualLink(ctx context.Context, name path.Component, leaf Leaf, requested AttributesMask, attributes *Attributes) (ChangeInfo, Status)
	// VirtualLookup obtains the inode corresponding with a child
	// stored within the directory.
	//
	// TODO: Can't use DirectoryChild in the return type here, due to
	// https://github.com/golang/go/issues/50259.
	VirtualLookup(ctx context.Context, name path.Component, requested AttributesMask, out *Attributes) (Child[Directory, Leaf, Node], Status)
	// VirtualMkdir creates an empty directory within the current
	// directory.
	VirtualMkdir(name path.Component, requested AttributesMask, attributes *Attributes) (Directory, ChangeInfo, Status)
	// VirtualMknod creates a character FIFO or UNIX domain socket
	// within the current directory.
	VirtualMknod(ctx context.Context, name path.Component, fileType filesystem.FileType, requested AttributesMask, attributes *Attributes) (Leaf, ChangeInfo, Status)
	// VirtualReadDir reports files and directories stored within
	// the directory.
	VirtualReadDir(ctx context.Context, firstCookie uint64, requested AttributesMask, reporter DirectoryEntryReporter) Status
	// VirtualRename renames a file stored in the current directory,
	// potentially moving it to another directory.
	VirtualRename(oldName path.Component, newDirectory Directory, newName path.Component) (ChangeInfo, ChangeInfo, Status)
	// VirtualRemove removes an empty directory or leaf node stored
	// within the current directory. Depending on the parameters,
	// this method behaves like rmdir(), unlink() or a mixture of
	// the two. The latter is needed by NFSv4.
	VirtualRemove(name path.Component, removeDirectory, removeLeaf bool) (ChangeInfo, Status)
	// VirtualSymlink creates a symbolic link within the current
	// directory.
	VirtualSymlink(ctx context.Context, pointedTo []byte, linkName path.Component, requested AttributesMask, attributes *Attributes) (Leaf, ChangeInfo, Status)
}

Directory node that is exposed through FUSE using SimpleRawFileSystem, or through NFSv4. The names of all of these operations are prefixed with 'Virtual' to ensure they don't collide with filesystem.Directory.

func NewStaticDirectory(directories map[path.Component]DirectoryChild) Directory

type DirectoryChild ¶

type DirectoryChild = Child[Directory, Leaf, Node]

DirectoryChild is either a Directory or a Leaf, as returned by Directory.VirtualLookup().

type DirectoryEntryReporter ¶

type DirectoryEntryReporter interface {
	// TODO: Can't use DirectoryChild in the arguments here, due to
	// https://github.com/golang/go/issues/50259.
	ReportEntry(nextCookie uint64, name path.Component, child Child[Directory, Leaf, Node], attributes *Attributes) bool
}

DirectoryEntryReporter is used by VirtualReadDir() to report individual directory entries. These methods may be called while locks on the underlying directory are held. This means that it's not safe to call methods of the child directory, as that could cause deadlocks.

type DirectoryPrepopulatedDirEntry ¶

type DirectoryPrepopulatedDirEntry struct {
	Child PrepopulatedDirectory
	Name  path.Component
}

DirectoryPrepopulatedDirEntry contains information about a directory node that is stored in a PrepopulatedDirectory.

type FUSERemovalNotifier ¶

type FUSERemovalNotifier func(parent uint64, name path.Component)

FUSERemovalNotifier is a callback method that can be registered to report the removal of files from stateful directories.

type FUSERemovalNotifierRegistrar ¶

type FUSERemovalNotifierRegistrar func(removalNotifier FUSERemovalNotifier)

FUSERemovalNotifierRegistrar has the same signature as FUSEStatefulHandleAllocator.RegisterRemovalNotifier(). It has been added to aid testing.

type FUSEStatefulHandleAllocator ¶

type FUSEStatefulHandleAllocator struct {
	// contains filtered or unexported fields
}

FUSEStatefulHandleAllocator creates a handle allocator for the purpose of exposing the virtual file system through FUSE. It is responsible for decorating all files in the file system, so that they have inode numbers and link counts. Inode numbers are unique for stateful (mutable) files, while they are identical for stateless files that share the same identifiers, meaning they can be deduplicated by the kernel.

The FUSE protocol is stateful, in the sense that the kernel and the userspace service share state on which nodes in the file system have been resolved. The kernel will never issue requests against objects that have not been resolved yet, or for which the kernel has issued a FORGET operation. This means that this handle allocator does not need to provide any mechanism for resolving arbitrary files present in the file system, making its implementation simple. There is thus no distinction between stateless and resolvable files.

func NewFUSEHandleAllocator(randomNumberGenerator random.ThreadSafeGenerator) *FUSEStatefulHandleAllocator

func (hr *FUSEStatefulHandleAllocator) New() StatefulHandleAllocation

func (hr *FUSEStatefulHandleAllocator) RegisterRemovalNotifier(removalNotifier FUSERemovalNotifier)

type FileAllocator ¶

type FileAllocator interface {
	NewFile(isExecutable bool, size uint64, shareAccess ShareMask) (NativeLeaf, Status)
}

FileAllocator is called into by InMemoryPrepopulatedDirectory to create new files within the file system. Such files could either be stored in memory, on disk, remotely, etc.

Files returned by this interface should have a link count of 1, and are opened using the provided share access mask.

func NewHandleAllocatingFileAllocator(base FileAllocator, allocator StatefulHandleAllocator) FileAllocator

func NewPoolBackedFileAllocator(pool re_filesystem.FilePool, errorLogger util.ErrorLogger) FileAllocator

type FileReadMonitor ¶

type FileReadMonitor func()

FileReadMonitor is used by the regular files created through the InitialContentsFetcher to indicate that one or more calls against VirtualRead() have occurred. This is used by AccessMonitoringInitialContentsFetcher to monitor file access.

type FileReadMonitorFactory ¶

type FileReadMonitorFactory func(name path.Component) FileReadMonitor

FileReadMonitorFactory is a factory type for FileReadMonitor that is provided to the InitialContentsFetcher, so that the InitialContentsFetcher can attach the resulting monitors to any files that are returned.

If this function returns nil, no monitor is attached to the file.

type HandleResolver ¶

type HandleResolver func(r io.ByteReader) (DirectoryChild, Status)

HandleResolver is a method that is used by ResolvableHandleAllocator to reconstruct files based on the identifiers provided to New().

TODO: Implementations of this method must currently make sure that directories and leaves that are returned are decorated with a handle allocation. Can't we let implementations of ResolvableHandleAllocator do this? That way resolvers may remain simple.

type InitialContentsFetcher ¶

type InitialContentsFetcher interface {
	FetchContents(fileReadMonitorFactory FileReadMonitorFactory) (map[path.Component]InitialNode, error)

	// GetContainingDigests() returns a set of digests of objects in
	// the Content Addressable Storage that back the directories and
	// leaf nodes yielded by this InitialContentsFetcher.
	//
	// The set returned by this function may be passed to
	// ContentAddressableStorage.FindMissingBlobs() to check whether
	// the all files underneath this directory still exist, and to
	// prevent them from being removed in the nearby future.
	//
	// This API assumes that the resulting set is small enough to
	// fit in memory. For hierarchies backed by Tree objects, this
	// will generally hold. It may not be safe to call this method
	// on InitialContentsFetchers that expand to infinitely big
	// hierarchies.
	GetContainingDigests(ctx context.Context) (digest.Set, error)
}

InitialContentsFetcher is called into by PrepopulatedDirectory when a directory whose contents need to be instantiated lazily is accessed. The results returned by FetchContents() are used to populate the directory.

FetchContents() should be called until it succeeds at most once. It may be possible FetchContents() is never called. This may happen if the directory in question is never accessed.

var EmptyInitialContentsFetcher InitialContentsFetcher = emptyInitialContentsFetcher{}

func NewAccessMonitoringInitialContentsFetcher(base InitialContentsFetcher, rootDirectoryMonitor access.UnreadDirectoryMonitor) InitialContentsFetcher

func NewCASInitialContentsFetcher(ctx context.Context, directoryWalker cas.DirectoryWalker, casFileFactory CASFileFactory, symlinkFactory SymlinkFactory, digestFunction digest.Function) InitialContentsFetcher

type InitialNode ¶

type InitialNode = Child[InitialContentsFetcher, NativeLeaf, any]

InitialNode is the value type of the map of directory entries returned by InitialContentsFetcher.FetchContents(). Either Directory or Leaf is set, but not both.

type Leaf ¶

type Leaf interface {
	Node

	VirtualAllocate(off, size uint64) Status
	VirtualSeek(offset uint64, regionType filesystem.RegionType) (*uint64, Status)
	VirtualOpenSelf(ctx context.Context, shareAccess ShareMask, options *OpenExistingOptions, requested AttributesMask, attributes *Attributes) Status
	VirtualRead(buf []byte, offset uint64) (n int, eof bool, s Status)
	VirtualReadlink(ctx context.Context) ([]byte, Status)
	VirtualClose(shareAccess ShareMask)
	VirtualWrite(buf []byte, offset uint64) (int, Status)
}

Leaf node that is exposed through FUSE using SimpleRawFileSystem, or through NFSv4. Examples of leaf nodes are regular files, sockets, FIFOs, symbolic links and devices.

TODO: Should all methods take an instance of Context?

type LeafPrepopulatedDirEntry ¶

type LeafPrepopulatedDirEntry struct {
	Child NativeLeaf
	Name  path.Component
}

LeafPrepopulatedDirEntry contains information about a leaf node that is stored in a PrepopulatedDirectory.

type NFSStatefulHandleAllocator ¶

type NFSStatefulHandleAllocator struct {
	// contains filtered or unexported fields
}

NFSStatefulHandleAllocator creates a handle allocator for the purpose of exposing the virtual file system through NFS. It is responsible for decorating all files in the file system, so that they have file handles, inode numbers and link counts. File handles and inode numbers are unique for stateful (mutable) files, while they are identical for stateless files that share the same identifiers, meaning they can be deduplicated by the kernel.

The NFS protocol is stateless, in the sense that the client and server share no state on which nodes in the file system have been resolved. The client does not inform the server that it has released files from its cache. This means that the server needs to be able to resolve all file handles that are either still present in the file system or are still opened by the client. This handle allocator is capable of only doing the former. The latter can be supported at a higher level.

To work well with infinitely big directory structures (e.g., bb_clientd's "cas" directory), this implementation makes use of the handle resolver function provided to AsResolvableAllocator(). Instead of tracking these nodes explicitly, it generates longer file handles that have the value provided to ResolvableHandleAllocator.New() as a suffix, making it possible to regenerate these nodes on the fly.

func NewNFSHandleAllocator(randomNumberGenerator random.SingleThreadedGenerator) *NFSStatefulHandleAllocator

func (hr *NFSStatefulHandleAllocator) New() StatefulHandleAllocation

func (hr *NFSStatefulHandleAllocator) ResolveHandle(r io.ByteReader) (DirectoryChild, Status)

type NativeLeaf ¶

type NativeLeaf interface {
	Leaf

	// Operations called into by implementations of
	// PrepopulatedDirectory. The Link() operation may fail, for the
	// reason that Directory.VirtualLink() may be called on leaf
	// nodes that have been removed concurrently.
	Link() Status
	Unlink()

	// Additional operations that are used by consumers of
	// PrepopulatedDirectory.
	//
	// TODO: Remove these once Go supports generics. We could turn
	// PrepopulatedDirectory and InitialContentsFetcher into
	// parameterized types, where the leaves could be any type
	// that's based on NativeLeaf.
	Readlink() (path.Parser, error)
	UploadFile(ctx context.Context, contentAddressableStorage blobstore.BlobAccess, digestFunction digest.Function, writableFileUploadDelay <-chan struct{}) (digest.Digest, error)
	// GetContainingDigests() returns a set of digests of objects in
	// the Content Addressable Storage that back the contents of
	// this file.
	//
	// The set returned by this function may be passed to
	// ContentAddressableStorage.FindMissingBlobs() to check whether
	// the file still exists in its entirety, and to prevent that
	// the file is removed in the nearby future.
	GetContainingDigests() digest.Set
	// GetBazelOutputServiceStat() returns the status of the leaf
	// node in the form of a Status message that is used by the
	// Bazel Output Service protocol.
	GetBazelOutputServiceStat(digestFunction *digest.Function) (*bazeloutputservice.BatchStatResponse_Stat, error)
	// AppendOutputPathPersistencyDirectoryNode() appends a FileNode
	// or SymlinkNode entry to a Directory message that is used to
	// persist the state of a Bazel Output Service output path to
	// disk.
	AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component)
}

NativeLeaf objects are non-directory nodes that can be placed in a PrepopulatedDirectory.

func NewSpecialFile(fileType filesystem.FileType, deviceNumber *filesystem.DeviceNumber) NativeLeaf

type Node ¶

type Node interface {
	VirtualGetAttributes(ctx context.Context, requested AttributesMask, attributes *Attributes)
	VirtualSetAttributes(ctx context.Context, in *Attributes, requested AttributesMask, attributes *Attributes) Status
}

Node is the intersection between Directory and Leaf. These are the operations that can be applied to both kinds of objects.

type OpenExistingOptions ¶

type OpenExistingOptions struct {
	Truncate bool
}

OpenExistingOptions contains options that describe what should happen with a file when opened. The Truncate option corresponds to open()'s O_TRUNC option. This option has no effect on freshly created files, as those are always empty.

func (o *OpenExistingOptions) ToAttributesMask() (m AttributesMask)

type Permissions ¶

type Permissions uint8

Permissions of a file. Unlike regular UNIX file system, no distinction is made between owner, group and all permissions. This is because the virtual file system is effectively single user.

const (
	// PermissionsRead indicates that file contents may be read, or
	// that files in a directory may be listed.
	PermissionsRead Permissions = 1 << iota
	// PermissionsWrite indicates that file contents may be written
	// to, or that files in a directory may be added, removed or
	// renamed.
	PermissionsWrite
	// PermissionsExecute indicates that a file is executable, or
	// that files in a directory may be looked up.
	PermissionsExecute
)

func NewPermissionsFromMode(m uint32) (p Permissions)

func (p Permissions) ToMode() (m uint32)

type PrepopulatedDirectory ¶

type PrepopulatedDirectory interface {
	Directory

	// LookupChild() looks up a file or directory contained in a
	// PrepopulatedDirectory. This method is similar to
	// VirtualLookup(), except that it returns the native types
	// managed by PrepopulatedDirectory.
	//
	// TODO: Can't use PrepopulatedDirectoryChild in the return type
	// here, due to https://github.com/golang/go/issues/50259.
	LookupChild(name path.Component) (Child[PrepopulatedDirectory, NativeLeaf, Node], error)
	// LookupAllChildren() looks up all files and directories
	// contained in a PrepopulatedDirectory. This method is similar
	// to VirtualReadDir(), except that it returns the native types
	// managed by PrepopulatedDirectory. Entries are returned in
	// alphabetical order.
	LookupAllChildren() ([]DirectoryPrepopulatedDirEntry, []LeafPrepopulatedDirEntry, error)
	// CreateChildren() creates one or more files or directories in
	// the current directory.
	//
	// If the overwrite flag is set, existing files and directories
	// will be replaced. If the overwrite flag is not set, the call
	// will fail if one or more entries already exist. No changes
	// will be made to the directory in that case.
	CreateChildren(children map[path.Component]InitialNode, overwrite bool) error
	// CreateAndEnterPrepopulatedDirectory() is similar to
	// LookupChild(), except that it creates the specified directory
	// if it does not yet exist. If a file already exists, it will
	// be removed.
	CreateAndEnterPrepopulatedDirectory(name path.Component) (PrepopulatedDirectory, error)
	// RemoveAllChildren() removes all files and directories
	// contained in the current directory.
	//
	// This method is identical to the one filesystem.Directory,
	// except that the forbidNewChildren flag may be set to
	// permanently mark the directory in such a way that no further
	// files may be added. When called on the root directory, all
	// resources associated with the directory hierarchy will be
	// released.
	RemoveAllChildren(forbidNewChildren bool) error
	// InstallHooks sets up hooks for creating files and logging
	// errors that occur under the directory subtree.
	//
	// This function is identical to BuildDirectory.InstallHooks(),
	// except that it uses the FUSE specific FileAllocator instead
	// of FilePool.
	InstallHooks(fileAllocator FileAllocator, errorLogger util.ErrorLogger)
	// FilterChildren() can be used to traverse over all of the
	// InitialContentsFetcher and NativeLeaf objects stored in this
	// directory hierarchy. For each of the objects, a callback is
	// provided that can be used to remove the file or the contents
	// of the directory associated with this object.
	//
	// This function can be used by bb_clientd to purge files or
	// directories that are no longer present in the Content
	// Addressable Storage at the start of the build.
	FilterChildren(childFilter ChildFilter) error

	// Functions inherited from filesystem.Directory.
	ReadDir() ([]filesystem.FileInfo, error)
	RemoveAll(name path.Component) error
	Remove(name path.Component) error
}

PrepopulatedDirectory is a Directory that is writable and can contain files of type NativeLeaf.

By making use of InitialContentsFetcher, it is possible to create subdirectories that are prepopulated with files and directories. These will be instantiated only when accessed. This feature is used by bb_worker to lazily load the input root while a build action is being executed. Similarly, it is used by bb_clientd to lazily instantiate the contents of a Tree object.

func NewInMemoryPrepopulatedDirectory(fileAllocator FileAllocator, symlinkFactory SymlinkFactory, errorLogger util.ErrorLogger, handleAllocator StatefulHandleAllocator, initialContentsSorter Sorter, hiddenFilesMatcher StringMatcher, clock clock.Clock) PrepopulatedDirectory

type PrepopulatedDirectoryChild ¶

type PrepopulatedDirectoryChild = Child[PrepopulatedDirectory, NativeLeaf, Node]

PrepopulatedDirectoryChild is either a PrepopulatedDirectory or a NativeLeaf, as returned by PrepopulatedDirectory.LookupChild().

type ReadOnlyDirectory ¶

type ReadOnlyDirectory struct{}

ReadOnlyDirectory can be embedded into a Directory to disable all operations that mutate the directory contents.

func (ReadOnlyDirectory) VirtualLink(ctx context.Context, name path.Component, leaf Leaf, requested AttributesMask, out *Attributes) (ChangeInfo, Status)

func (ReadOnlyDirectory) VirtualMkdir(name path.Component, requested AttributesMask, out *Attributes) (Directory, ChangeInfo, Status)

func (ReadOnlyDirectory) VirtualMknod(ctx context.Context, name path.Component, fileType filesystem.FileType, requested AttributesMask, out *Attributes) (Leaf, ChangeInfo, Status)

func (ReadOnlyDirectory) VirtualRemove(name path.Component, removeDirectory, removeLeaf bool) (ChangeInfo, Status)

func (ReadOnlyDirectory) VirtualRename(oldName path.Component, newDirectory Directory, newName path.Component) (ChangeInfo, ChangeInfo, Status)

func (ReadOnlyDirectory) VirtualSetAttributes(ctx context.Context, in *Attributes, requested AttributesMask, out *Attributes) Status

func (ReadOnlyDirectory) VirtualSymlink(ctx context.Context, pointedTo []byte, linkName path.Component, requested AttributesMask, out *Attributes) (Leaf, ChangeInfo, Status)

type ResolvableDigestHandleAllocator ¶

type ResolvableDigestHandleAllocator struct {
	// contains filtered or unexported fields
}

ResolvableDigestHandleAllocator is a convenience type for the handle allocator API, used for identifying objects by REv2 digest. It is used by bb_clientd's "cas" directory to give unique file handles to objects stored in the Content Addressable Storage.

Because REv2 objects that include their instance name are too long to fit in NFS file handles, this type uses a stateless allocator for the instance name part. It uses a resolvable allocator only for the object hash and size, as those are generally small enough to fit in the file handle.

This means that every time a new instance name is used, a resolver gets leaked. This is acceptable for most use cases, as the instance names in use tend to be limited.

func NewResolvableDigestHandleAllocator(allocation StatelessHandleAllocation, resolver DigestHandleResolver) *ResolvableDigestHandleAllocator

func (a *ResolvableDigestHandleAllocator) New(blobDigest digest.Digest) ResolvableHandleAllocation

type ResolvableHandleAllocation ¶

type ResolvableHandleAllocation interface {
	AsResolvableAllocator(resolver HandleResolver) ResolvableHandleAllocator
	AsStatelessDirectory(directory Directory) Directory
	AsNativeLeaf(leaf NativeLeaf) NativeLeaf
	AsLeaf(leaf Leaf) Leaf
}

ResolvableHandleAllocation corresponds to an allocation of a file handle that is not only stateless, but can also be trivially reconstructed based on a small, bounded of information.

TODO: ResolvableHandleAllocators can be nested. The resolver provided to AsResolvableAllocator() is only used for the first level. We could eliminate the argument, but that makes composition harder.

type ResolvableHandleAllocator ¶

type ResolvableHandleAllocator interface {
	New(id io.WriterTo) ResolvableHandleAllocation
}

ResolvableHandleAllocator is responsible for allocating file handles that are not only stateless, but can also be trivially reconstructed based on a small, bounded amount of information.

This kind of allocator is generally used by bb_clientd's cas/ subdirectory. This directory allows for the exploration of arbitrary objects stored in the Content Addressable Storage (CAS). This allocator can be used to store the digest of such objects in the file handle, meaning bb_clientd doesn't need to track state for each of the files individually.

The exact amount of space that can be stored in a file handle is protocol specific. NFSv3 and NFSv4 use file handles that are 64 and 128 bytes in size, respectively.

type ShareMask ¶

type ShareMask uint32

ShareMask is a bitmask of operations that are permitted against a Leaf that has been opened.

const (
	// ShareMaskRead permits calls to VirtualRead().
	ShareMaskRead ShareMask = 1 << iota
	// ShareMaskWrite permits calls to VirtualWrite().
	ShareMaskWrite
)

func (sm ShareMask) Count() uint

type Sorter ¶

type Sorter func(data sort.Interface)

Sorter is a function type for a sorting algorithm. Its signature is identical to the sort.Sort() function.

This type is used by InMemoryPrepopulatedDirectory to make the policy for sorting the results of VirtualReadDir() configurable. Depending on the use case, it is desirable to use a deterministic algorithm (e.g., alphabetic sorting) or an undeterministic one (e.g., random shuffling).

type StatefulDirectoryHandle ¶

type StatefulDirectoryHandle interface {
	GetAttributes(requested AttributesMask, attributes *Attributes)
	NotifyRemoval(name path.Component)
	Release()
}

StatefulDirectoryHandle is a handle that needs to be embedded into stateful directories. It can be used to report mutations to the directory through NotifyRemoval(), or report deletion through Release().

The directory type that embeds the handle must call GetAttributes() as part of Directory.VirtualGetAttributes() to augment the attributes with an inode number and/or file handle.

type StatefulHandleAllocation ¶

type StatefulHandleAllocation interface {
	StatelessHandleAllocation

	AsStatefulDirectory(directory Directory) StatefulDirectoryHandle
}

StatefulHandleAllocation corresponds to an allocation of a file handle that is unique, meaning it can be used to identify stateful, mutable files or directories.

Exactly one call to one of the methods on this interface needs to be performed to convert the allocation to an actual use of the file handle.

type StatefulHandleAllocator ¶

type StatefulHandleAllocator interface {
	New() StatefulHandleAllocation
}

StatefulHandleAllocator is responsible for allocating new file handles, giving files and directories stored in the file system their own identity and lifetime. The exact meaning of identity is implementation specific. In the case of FUSE it can be an inode number/node ID, while in the case of NFSv4 it corresponds to the value of a 128-byte nfs_fh4.

type StatelessHandleAllocation ¶

type StatelessHandleAllocation interface {
	ResolvableHandleAllocation

	AsStatelessAllocator() StatelessHandleAllocator
}

StatelessHandleAllocation corresponds to an allocation of a file handle that is stateless, meaning it can be used to identify stateless files or directories.

type StatelessHandleAllocator ¶

type StatelessHandleAllocator interface {
	New(id io.WriterTo) StatelessHandleAllocation
}

StatelessHandleAllocator is responsible for allocating file handles of files that are stateless, immutable files.

For every handle that is allocated, an identifier needs to be provided in the form of an io.WriterTo. This may be used to give files with the same identifier the same underlying file handle or inode number.

NOTE: Care must be taken that the provided identifier is properly terminated or is prefixed with its own length. Stateless handle allocators may be nested, causing their identifiers to be concatenated. This could cause ambiguity if this rule is not followed.

type Status ¶

type Status int

Status response of operations applied against Node objects.

const (
	// StatusOK indicates that the operation succeeded.
	StatusOK Status = iota
	// StatusErrAccess indicates that the operation failed due to
	// permission being denied.
	StatusErrAccess
	// StatusErrBadHandle indicates that the provided file handle
	// failed internal consistency checks.
	StatusErrBadHandle
	// StatusErrExist indicates that a file system object of the
	// specified target name (when creating, renaming or linking)
	// already exists.
	StatusErrExist
	// StatusErrInval indicates that the arguments for this
	// operation are not valid.
	StatusErrInval
	// StatusErrIO indicates that the operation failed due to an I/O
	// error.
	StatusErrIO
	// StatusErrIsDir indicates that a request is made against a
	// directory when the current operation does not allow a
	// directory as a target.
	StatusErrIsDir
	// StatusErrNoEnt indicate sthat the operation failed due to a
	// file not existing.
	StatusErrNoEnt
	// StatusErrNotDir indicates that a request is made against a
	// leaf when the current operation does not allow a leaf as a
	// target.
	StatusErrNotDir
	// StatusErrNotEmpty indicates that attempt was made to remove a
	// directory that was not empty.
	StatusErrNotEmpty
	// StatusErrNXIO indicates that a request is made beyond the
	// limits of the file or device.
	StatusErrNXIO
	// StatusErrPerm indicates that the operation was not allowed
	// because the caller is neither a privileged user (root) nor
	// the owner of the target of the operation.
	StatusErrPerm
	// StatusErrROFS indicates that a modifying operation was
	// attempted on a read-only file system.
	StatusErrROFS
	// StatusErrStale indicates that the file system object referred
	// to by the file handle no longer exists, or access to it has
	// been revoked.
	StatusErrStale
	// StatusErrSymlink indicates that a request is made against a
	// symbolic link when the current operation does not allow a
	// symbolic link as a target.
	StatusErrSymlink
	// StatusErrWrongType that a request is made against an object
	// that is of an invalid type for the current operation, and
	// there is no more specific error (such as StatusErrIsDir or
	// StatusErrSymlink) that applies.
	StatusErrWrongType
	// StatusErrXDev indicates an attempt to do an operation, such
	// as linking, that inappropriately crosses a boundary.
	StatusErrXDev
)

type StringMatcher ¶

type StringMatcher func(s string) bool

StringMatcher is a function type that has the same signature as regexp.Regexp's MatchString() method. It is used by InMemoryPrepopulatedDirectory to determine which files should be hidden from directory listings.

type SymlinkFactory ¶

type SymlinkFactory interface {
	LookupSymlink(target []byte) NativeLeaf
}

SymlinkFactory is a factory type for symbolic links. Symbolic links are immutable files; the target to which they point can only be altered by replacing the node entirely (e.g., by first unlinking it from the directory).

var BaseSymlinkFactory SymlinkFactory = symlinkFactory{}

func NewHandleAllocatingSymlinkFactory(base SymlinkFactory, allocation StatelessHandleAllocation) SymlinkFactory

type UserSettableSymlink ¶

type UserSettableSymlink struct {
	// contains filtered or unexported fields
}

UserSettableSymlink is an implementation of a symbolic link, whose target can be modified using the TemporaryDirectoryInstaller gRPC API.

Instead of just pointing to a single target, this type is capable of storing one target per user. Both when reading and adjusting the symbolic link's target, the public authentication metadata is used to identify the user.

func NewUserSettableSymlink(buildDirectory *path.Builder) *UserSettableSymlink

func (f *UserSettableSymlink) AppendOutputPathPersistencyDirectoryNode(directory *outputpathpersistency.Directory, name path.Component)

func (f *UserSettableSymlink) CheckReadiness(ctx context.Context, request *emptypb.Empty) (*emptypb.Empty, error)

func (f *UserSettableSymlink) GetBazelOutputServiceStat(digestFunction *digest.Function) (*bazeloutputservice.BatchStatResponse_Stat, error)

func (UserSettableSymlink) GetContainingDigests() digest.Set

func (f *UserSettableSymlink) InstallTemporaryDirectory(ctx context.Context, request *tmp_installer.InstallTemporaryDirectoryRequest) (*emptypb.Empty, error)

func (UserSettableSymlink) Link() Status

func (f *UserSettableSymlink) Readlink() (path.Parser, error)

func (UserSettableSymlink) Unlink()

func (UserSettableSymlink) UploadFile(ctx context.Context, contentAddressableStorage blobstore.BlobAccess, digestFunction digest.Function, writableFileUploadDelay <-chan struct{}) (digest.Digest, error)

func (UserSettableSymlink) VirtualAllocate(off, size uint64) Status

func (UserSettableSymlink) VirtualClose(shareAccess ShareMask)

func (f *UserSettableSymlink) VirtualGetAttributes(ctx context.Context, requested AttributesMask, attributes *Attributes)

func (UserSettableSymlink) VirtualOpenSelf(ctx context.Context, shareAccess ShareMask, options *OpenExistingOptions, requested AttributesMask, attributes *Attributes) Status

func (UserSettableSymlink) VirtualRead(buf []byte, offset uint64) (int, bool, Status)

func (f *UserSettableSymlink) VirtualReadlink(ctx context.Context) ([]byte, Status)

func (UserSettableSymlink) VirtualSeek(offset uint64, regionType filesystem.RegionType) (*uint64, Status)

func (f *UserSettableSymlink) VirtualSetAttributes(ctx context.Context, in *Attributes, requested AttributesMask, out *Attributes) Status

func (UserSettableSymlink) VirtualWrite(buf []byte, off uint64) (int, Status)