filer

package
v0.0.0-...-0b345dd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 6, 2022 License: Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LogFlushInterval = time.Minute
	PaginationSize   = 1024
	FilerStoreId     = "filer.store.id"
)
View Source 
const (
	DirectoryEtcRoot      = "/etc"
	DirectoryEtcSeaweedFS = "/etc/seaweedfs"
	DirectoryEtcRemote    = "/etc/remote"
	FilerConfName         = "filer.conf"
	IamConfigDirecotry    = "/etc/iam"
	IamIdentityFile       = "identity.json"
	IamPoliciesFile       = "policies.json"
)
View Source 
const (
	TopicsDir    = "/topics"
	SystemLogDir = TopicsDir + "/.system/log"
)
View Source 
const CountEntryChunksForGzip = 50
View Source 
const (
	HARD_LINK_MARKER = '\x01'
)
View Source 
const (
	ManifestBatch = 10000
)
View Source 
const (
	MetaOffsetPrefix = "Meta"
)
View Source 
const (
	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
)
View Source 
const REMOTE_STORAGE_CONF_SUFFIX = ".conf"
View Source 
const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"

Variables

View Source 
var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())
)
View Source 
var (
	ErrUnsupportedListDirectoryPrefixed      = errors.New("unsupported directory prefix listing")
	ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing")
	ErrKvNotImplemented                      = errors.New("kv not implemented yet")
	ErrKvNotFound                            = errors.New("kv: not found")
)
View Source 
var (
	Root = &Entry{
		FullPath: "/",
		Attr: Attr{
			Mtime:  time.Now(),
			Crtime: time.Now(),
			Mode:   os.ModeDir | 0755,
			Uid:    OS_UID,
			Gid:    OS_GID,
		},
	}
)
View Source 
var (
	Stores []FilerStore
)

Functions

func CacheRemoteObjectToLocalCluster 

func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error

func CompactFileChunks 

func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk)

func DeleteMountMapping 

func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error)

func DetectMountInfo 

func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error)

func DoMinusChunks 

func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk)

func DoMinusChunksBySourceFileId 

func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk)

func ETag 

func ETag(entry *filer_pb.Entry) (etag string)

func ETagChunks 

func ETagChunks(chunks []*filer_pb.FileChunk) (etag string)

func ETagEntry 

func ETagEntry(entry *Entry) (etag string)

func EntryAttributeToPb 

func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes

func EqualEntry 

func EqualEntry(a, b *Entry) bool

func FileSize 

func FileSize(entry *filer_pb.Entry) (size uint64)

func FromPbEntryToExistingEntry 

func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry)

func HasChunkManifest 

func HasChunkManifest(chunks []*filer_pb.FileChunk) bool

func HasData 

func HasData(entry *filer_pb.Entry) bool

func InsertMountMapping 

func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStorageLocation *remote_pb.RemoteStorageLocation) (err error)

func IsSameData 

func IsSameData(a, b *filer_pb.Entry) bool

func LookupByMasterClientFn 

func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]*operation.LookupResult, error)

func LookupFn 

func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionType

func MapFullPathToRemoteStorageLocation 

func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, fp util.FullPath) *remote_pb.RemoteStorageLocation

func MapRemoteStorageLocationPathToFullPath 

func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, remoteLocationPath string) (fp util.FullPath)

func MaybeManifestize 

func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error)

func MinusChunks 

func MinusChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error)

func NewFileReader 

func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.Reader

func ParseS3ConfigurationFromBytes 

func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error

func ProtoToText 

func ProtoToText(writer io.Writer, config proto.Message) error

func ReadAll 

func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) error

func ReadEachLogEntry 

func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error)

func ReadEntry 

func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error

func ReadInsideFiler 

func ReadInsideFiler(filerClient filer_pb.SeaweedFilerClient, dir, name string) (content []byte, err error)

func ReadMountMappings 

func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error)

func ReadRemoteStorageConf 

func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error)

func Replay 

func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) error

func ResolveChunkManifest 

func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error)

func ResolveOneChunkManifest 

func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error)

func SaveInsideFiler 

func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, content []byte) error

func SeparateManifestChunks 

func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonManifestChunks []*filer_pb.FileChunk)

func StreamContent 

func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error

func TotalSize 

func TotalSize(chunks []*filer_pb.FileChunk) (size uint64)

func UnmarshalRemoteStorageMappings 

func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.RemoteStorageMapping, err error)

func VolumeId 

func VolumeId(fileId string) string

Types

type Attr 

type Attr struct {
	Mtime         time.Time   // time of last modification
	Crtime        time.Time   // time of creation (OS X only)
	Mode          os.FileMode // file mode
	Uid           uint32      // owner uid
	Gid           uint32      // group gid
	Mime          string      // mime type
	Replication   string      // replication
	Collection    string      // collection name
	TtlSec        int32       // ttl in seconds
	DiskType      string
	UserName      string
	GroupNames    []string
	SymlinkTarget string
	Md5           []byte
	FileSize      uint64
	Rdev          uint32
	Inode         uint64
}

func PbToEntryAttribute 

func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr

func (Attr) IsDirectory 

func (attr Attr) IsDirectory() bool

type BucketAware 

type BucketAware interface {
	OnBucketCreation(bucket string)
	OnBucketDeletion(bucket string)
	CanDropWholeBucket() bool
}

type BucketName 

type BucketName string

type BucketOption 

type BucketOption struct {
	Name        BucketName
	Replication string
	// contains filtered or unexported fields
}

type ChunkReadAt 

type ChunkReadAt struct {
	// contains filtered or unexported fields
}

func NewChunkReaderAtFromClient 

func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt

func (*ChunkReadAt) Close 

func (c *ChunkReadAt) Close() error

func (*ChunkReadAt) ReadAt 

func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error)

type ChunkStreamReader 

type ChunkStreamReader struct {
	// contains filtered or unexported fields
}

---------------- ChunkStreamReader ----------------------------------

func NewChunkStreamReader 

func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader

func NewChunkStreamReaderFromFiler 

func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader

func (*ChunkStreamReader) Close 

func (c *ChunkStreamReader) Close()

func (*ChunkStreamReader) Read 

func (c *ChunkStreamReader) Read(p []byte) (n int, err error)

func (*ChunkStreamReader) ReadAt 

func (c *ChunkStreamReader) ReadAt(p []byte, off int64) (n int, err error)

func (*ChunkStreamReader) Seek 

func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error)

type ChunkView 

type ChunkView struct {
	FileId      string
	Offset      int64
	Size        uint64
	LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
	ChunkSize   uint64
	CipherKey   []byte
	IsGzipped   bool
}

func ViewFromChunks 

func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView)

func ViewFromVisibleIntervals 

func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView)

func (*ChunkView) IsFullChunk 

func (cv *ChunkView) IsFullChunk() bool

type Debuggable 

type Debuggable interface {
	Debug(writer io.Writer)
}

type Entry 

type Entry struct {
	util.FullPath

	Attr
	Extended map[string][]byte

	// the following is for files
	Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`

	HardLinkId      HardLinkId
	HardLinkCounter int32
	Content         []byte
	Remote          *filer_pb.RemoteEntry
	Quota           int64
}

func FromPbEntry 

func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry

func (*Entry) DecodeAttributesAndChunks 

func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error

func (*Entry) EncodeAttributesAndChunks 

func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error)

func (*Entry) IsInRemoteOnly 

func (entry *Entry) IsInRemoteOnly() bool

func (*Entry) ShallowClone 

func (entry *Entry) ShallowClone() *Entry

func (*Entry) Size 

func (entry *Entry) Size() uint64

func (*Entry) Timestamp 

func (entry *Entry) Timestamp() time.Time

func (*Entry) ToExistingProtoEntry 

func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry)

func (*Entry) ToProtoEntry 

func (entry *Entry) ToProtoEntry() *filer_pb.Entry

func (*Entry) ToProtoFullEntry 

func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry

type Filer 

type Filer struct {
	Store        VirtualFilerStore
	MasterClient *wdclient.MasterClient

	GrpcDialOption grpc.DialOption
	DirBucketsPath string
	FsyncBuckets   []string

	Cipher             bool
	LocalMetaLogBuffer *log_buffer.LogBuffer

	MetaAggregator *MetaAggregator
	Signature      int32
	FilerConf      *FilerConf
	RemoteStorage  *FilerRemoteStorage
	UniqueFileId   uint32
	// contains filtered or unexported fields
}

func NewFiler 

func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
	filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer

func (*Filer) AggregateFromPeers 

func (f *Filer) AggregateFromPeers(self pb.ServerAddress)

func (*Filer) BeginTransaction 

func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error)

func (*Filer) CanRename 

func (f *Filer) CanRename(source, target util.FullPath) error

func (*Filer) CommitTransaction 

func (f *Filer) CommitTransaction(ctx context.Context) error

func (*Filer) CreateEntry 

func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error

func (*Filer) DeleteChunks 

func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk)

func (*Filer) DeleteChunksNotRecursive 

func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk)

func (*Filer) DeleteEntryMetaAndData 

func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error)

func (*Filer) DetectBucket 

func (f *Filer) DetectBucket(source util.FullPath) (bucket string)

func (*Filer) DirectDeleteChunks 

func (f *Filer) DirectDeleteChunks(chunks []*filer_pb.FileChunk)

func (*Filer) FindEntry 

func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, err error)

func (*Filer) GetMaster 

func (fs *Filer) GetMaster() pb.ServerAddress

func (*Filer) GetStore 

func (f *Filer) GetStore() (store FilerStore)

func (*Filer) KeepMasterClientConnected 

func (fs *Filer) KeepMasterClientConnected()

func (*Filer) ListDirectoryEntries 

func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error)

For now, prefix and namePattern are mutually exclusive

func (*Filer) ListExistingPeerUpdates 

func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate)

func (*Filer) LoadBuckets 

func (f *Filer) LoadBuckets()

func (*Filer) LoadConfiguration 

func (f *Filer) LoadConfiguration(config *util.ViperProxy)

func (*Filer) LoadFilerConf 

func (f *Filer) LoadFilerConf()

func (*Filer) LoadRemoteStorageConfAndMapping 

func (f *Filer) LoadRemoteStorageConfAndMapping()

////////////////////////////////// load and maintain remote storages //////////////////////////////////

func (*Filer) NotifyUpdateEvent 

func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32)

func (*Filer) ReadBucketOption 

func (f *Filer) ReadBucketOption(buketName string) (replication string, fsync bool)

func (*Filer) ReadPersistedLogBuffer 

func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error)

func (*Filer) RollbackTransaction 

func (f *Filer) RollbackTransaction(ctx context.Context) error

func (*Filer) SetStore 

func (f *Filer) SetStore(store FilerStore)

func (*Filer) Shutdown 

func (f *Filer) Shutdown()

func (*Filer) StreamListDirectoryEntries 

func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)

For now, prefix and namePattern are mutually exclusive

func (*Filer) UpdateEntry 

func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error)

type FilerBuckets 

type FilerBuckets struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type FilerConf 

type FilerConf struct {
	// contains filtered or unexported fields
}

func NewFilerConf 

func NewFilerConf() (fc *FilerConf)

func ReadFilerConf 

func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error)

func (*FilerConf) AddLocationConf 

func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error)

func (*FilerConf) DeleteLocationConf 

func (fc *FilerConf) DeleteLocationConf(locationPrefix string)

func (*FilerConf) GetCollectionTtls 

func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string)

func (*FilerConf) LoadFromBytes 

func (fc *FilerConf) LoadFromBytes(data []byte) (err error)

func (*FilerConf) MatchStorageRule 

func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf)

func (*FilerConf) ToProto 

func (fc *FilerConf) ToProto() *filer_pb.FilerConf

func (*FilerConf) ToText 

func (fc *FilerConf) ToText(writer io.Writer) error

type FilerRemoteStorage 

type FilerRemoteStorage struct {
	// contains filtered or unexported fields
}

func NewFilerRemoteStorage 

func NewFilerRemoteStorage() (rs *FilerRemoteStorage)

func (*FilerRemoteStorage) FindMountDirectory 

func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation)

func (*FilerRemoteStorage) FindRemoteStorageClient 

func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool)

func (*FilerRemoteStorage) GetRemoteStorageClient 

func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool)

func (*FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping 

func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *Filer) (err error)

type FilerStore 

type FilerStore interface {
	// GetName gets the name to locate the configuration in filer.toml file
	GetName() string
	// Initialize initializes the file store
	Initialize(configuration util.Configuration, prefix string) error
	InsertEntry(context.Context, *Entry) error
	UpdateEntry(context.Context, *Entry) (err error)
	// err == filer_pb.ErrNotFound if not found
	FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
	DeleteEntry(context.Context, util.FullPath) (err error)
	DeleteFolderChildren(context.Context, util.FullPath) (err error)
	ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
	ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)

	BeginTransaction(ctx context.Context) (context.Context, error)
	CommitTransaction(ctx context.Context) error
	RollbackTransaction(ctx context.Context) error

	KvPut(ctx context.Context, key []byte, value []byte) (err error)
	KvGet(ctx context.Context, key []byte) (value []byte, err error)
	KvDelete(ctx context.Context, key []byte) (err error)

	Shutdown()
}

type FilerStorePathTranlator 

type FilerStorePathTranlator struct {
	// contains filtered or unexported fields
}

func NewFilerStorePathTranlator 

func NewFilerStorePathTranlator(storeRoot string, store FilerStore) *FilerStorePathTranlator

func (*FilerStorePathTranlator) BeginTransaction 

func (t *FilerStorePathTranlator) BeginTransaction(ctx context.Context) (context.Context, error)

func (*FilerStorePathTranlator) CommitTransaction 

func (t *FilerStorePathTranlator) CommitTransaction(ctx context.Context) error

func (*FilerStorePathTranlator) DeleteEntry 

func (t *FilerStorePathTranlator) DeleteEntry(ctx context.Context, fp util.FullPath) (err error)

func (*FilerStorePathTranlator) DeleteFolderChildren 

func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error)

func (*FilerStorePathTranlator) DeleteOneEntry 

func (t *FilerStorePathTranlator) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error)

func (*FilerStorePathTranlator) FindEntry 

func (t *FilerStorePathTranlator) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error)

func (*FilerStorePathTranlator) GetName 

func (t *FilerStorePathTranlator) GetName() string

func (*FilerStorePathTranlator) Initialize 

func (t *FilerStorePathTranlator) Initialize(configuration util.Configuration, prefix string) error

func (*FilerStorePathTranlator) InsertEntry 

func (t *FilerStorePathTranlator) InsertEntry(ctx context.Context, entry *Entry) error

func (*FilerStorePathTranlator) KvDelete 

func (t *FilerStorePathTranlator) KvDelete(ctx context.Context, key []byte) (err error)

func (*FilerStorePathTranlator) KvGet 

func (t *FilerStorePathTranlator) KvGet(ctx context.Context, key []byte) (value []byte, err error)

func (*FilerStorePathTranlator) KvPut 

func (t *FilerStorePathTranlator) KvPut(ctx context.Context, key []byte, value []byte) (err error)

func (*FilerStorePathTranlator) ListDirectoryEntries 

func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error)

func (*FilerStorePathTranlator) ListDirectoryPrefixedEntries 

func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error)

func (*FilerStorePathTranlator) RollbackTransaction 

func (t *FilerStorePathTranlator) RollbackTransaction(ctx context.Context) error

func (*FilerStorePathTranlator) Shutdown 

func (t *FilerStorePathTranlator) Shutdown()

func (*FilerStorePathTranlator) UpdateEntry 

func (t *FilerStorePathTranlator) UpdateEntry(ctx context.Context, entry *Entry) error

type FilerStoreWrapper 

type FilerStoreWrapper struct {
	// contains filtered or unexported fields
}

func NewFilerStoreWrapper 

func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper

func (*FilerStoreWrapper) AddPathSpecificStore 

func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore)

func (*FilerStoreWrapper) BeginTransaction 

func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error)

func (*FilerStoreWrapper) CanDropWholeBucket 

func (fsw *FilerStoreWrapper) CanDropWholeBucket() bool

func (*FilerStoreWrapper) CommitTransaction 

func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error

func (*FilerStoreWrapper) Debug 

func (fsw *FilerStoreWrapper) Debug(writer io.Writer)

func (*FilerStoreWrapper) DeleteEntry 

func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) (err error)

func (*FilerStoreWrapper) DeleteFolderChildren 

func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error)
func (fsw *FilerStoreWrapper) DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error

func (*FilerStoreWrapper) DeleteOneEntry 

func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error)
func (fsw *FilerStoreWrapper) DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) (err error)

func (*FilerStoreWrapper) FindEntry 

func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error)

func (*FilerStoreWrapper) GetName 

func (fsw *FilerStoreWrapper) GetName() string

func (*FilerStoreWrapper) Initialize 

func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration, prefix string) error

func (*FilerStoreWrapper) InsertEntry 

func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error

func (*FilerStoreWrapper) KvDelete 

func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error)

func (*FilerStoreWrapper) KvGet 

func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []byte, err error)

func (*FilerStoreWrapper) KvPut 

func (fsw *FilerStoreWrapper) KvPut(ctx context.Context, key []byte, value []byte) (err error)

func (*FilerStoreWrapper) ListDirectoryEntries 

func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error)

func (*FilerStoreWrapper) ListDirectoryPrefixedEntries 

func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)

func (*FilerStoreWrapper) OnBucketCreation 

func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string)

func (*FilerStoreWrapper) OnBucketDeletion 

func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string)

func (*FilerStoreWrapper) RollbackTransaction 

func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error

func (*FilerStoreWrapper) Shutdown 

func (fsw *FilerStoreWrapper) Shutdown()

func (*FilerStoreWrapper) UpdateEntry 

func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error

type HardLinkId 

type HardLinkId []byte // 16 bytes + 1 marker byte

func NewHardLinkId 

func NewHardLinkId() HardLinkId

type ListEachEntryFunc 

type ListEachEntryFunc func(entry *Entry) bool

type MetaAggregator 

type MetaAggregator struct {
	MetaLogBuffer *log_buffer.LogBuffer

	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
	// contains filtered or unexported fields
}

func NewMetaAggregator 

func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator

MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk. The old data comes from what each LocalMetadata persisted on disk.

func (*MetaAggregator) OnPeerUpdate 

func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate)

type OnChunksFunc 

type OnChunksFunc func([]*filer_pb.FileChunk) error

type OnHardLinkIdsFunc 

type OnHardLinkIdsFunc func([]HardLinkId) error

type Point 

type Point struct {
	// contains filtered or unexported fields
}

type ReaderCache 

type ReaderCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*ReaderCache) MaybeCache 

func (rc *ReaderCache) MaybeCache(chunkViews []*ChunkView)

func (*ReaderCache) ReadChunkAt 

func (rc *ReaderCache) ReadChunkAt(buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error)

func (*ReaderCache) UnCache 

func (rc *ReaderCache) UnCache(fileId string)

type ReaderPattern 

type ReaderPattern struct {
	// contains filtered or unexported fields
}

func NewReaderPattern 

func NewReaderPattern() *ReaderPattern

func (*ReaderPattern) IsRandomMode 

func (rp *ReaderPattern) IsRandomMode() bool

func (*ReaderPattern) IsStreamingMode 

func (rp *ReaderPattern) IsStreamingMode() bool

func (*ReaderPattern) MonitorReadAt 

func (rp *ReaderPattern) MonitorReadAt(offset int64, size int)

type SaveDataAsChunkFunctionType 

type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error)

type SingleChunkCacher 

type SingleChunkCacher struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type VirtualFilerStore 

type VirtualFilerStore interface {
	FilerStore
	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
	DeleteOneEntry(ctx context.Context, entry *Entry) error
	DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) error
	AddPathSpecificStore(path string, storeId string, store FilerStore)
	OnBucketCreation(bucket string)
	OnBucketDeletion(bucket string)
	CanDropWholeBucket() bool
}

type VisibleInterval 

type VisibleInterval struct {
	// contains filtered or unexported fields
}

func MergeIntoVisibles 

func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval)

func NonOverlappingVisibleIntervals 

func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error)

NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory If the file chunk content is a chunk manifest

Source Files

View all Source files

Directories

Path Synopsis
elastic
v7
Package elastic is for elastic filer store.
 Package elastic is for elastic filer store.
stored_procedure
Package sqlite is for sqlite filer store.
 Package sqlite is for sqlite filer store.
Package ydb is for YDB filer store.
 Package ydb is for YDB filer store.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.