model

package
v3.0.9+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 8, 2022 License: AGPL-3.0 Imports: 16 Imported by: 108

Documentation

Overview

Package model gathers the main structs and interfaces used in sync library.

Index

Constants

View Source
const (
	// Use unique path separator everywhere
	InternalPathSeparator = "/"
	GlobSeparator         = '/'
	// Minio default Etag when a new file is detected
	DefaultEtag = "00000000000000000000000000000000-1"

	DirectionLeft  DirectionType = 1
	DirectionRight DirectionType = 2
	DirectionBi    DirectionType = 3

	Interrupt SyncCmd = iota
	Pause
	Resume
)
View Source
const (
	StatusScopeTask = iota
	StatusScopeProcessing
)

Variables

This section is empty.

Functions

func Ignores

func Ignores(endpoint Endpoint, name string) bool

Ignores checks if a specific name should be ignored by the given Endpoint

func IsConfigError

func IsConfigError(e error) bool

func IsFolderHiddenFile

func IsFolderHiddenFile(name string) bool

IsFolderHiddenFile checks if file is .pydio

func IsIgnoredFile

func IsIgnoredFile(path string, ignores ...glob.Glob) (ignored bool)

func NewConfigError

func NewConfigError(err error) error

func NodeRequiresChecksum

func NodeRequiresChecksum(node *tree.Node) bool

func Retry

func Retry(f func() error, seconds ...time.Duration) error

Retry tries to apply an operation as many time as required

func RetryWithCtx

func RetryWithCtx(ctx context.Context, f func(retry int) error, seconds ...time.Duration) error

RetryWithCtx does like Retry with an additionnal cancellable context

func StringContentToETag

func StringContentToETag(uuid string) string

func ZapEndpoint

func ZapEndpoint(key string, e Endpoint) zapcore.Field

Types

type AuthRequester

type AuthRequester interface {
	RequiresAuth() bool
}

type BulkLoader

type BulkLoader interface {
	BulkLoadNodes(ctx context.Context, nodes map[string]string) (map[string]interface{}, error)
}

A BulkLoader can stream calls to ReadNode - Better use CachedBranchProvider

type CachedBranchProvider

type CachedBranchProvider interface {
	Endpoint
	GetCachedBranches(ctx context.Context, roots ...string) PathSyncSource
}

A CachedBranchProvider can quickly load a full branch recursively in memory and expose it as a PathSyncSource

type ChecksumProvider

type ChecksumProvider interface {
	Endpoint
	ComputeChecksum(node *tree.Node) error
}

ChecksumProvider is able to compute a checksum for a given node (typically an Etag)

type Command

type Command struct {
	sync.Mutex
	// contains filtered or unexported fields
}

Command is a pub/sub for dispatching SyncCmd

func NewCommand

func NewCommand() *Command

NewCommand creates a new command

func (*Command) Publish

func (c *Command) Publish(cmd SyncCmd)

func (*Command) Stop

func (c *Command) Stop()

Stop closes internal channels

func (*Command) Subscribe

func (c *Command) Subscribe() (chan SyncCmd, chan bool)

Subscribe creates a new listener to the Input and returns a chan for closing subscription

type ConfigError

type ConfigError struct {
	// contains filtered or unexported fields
}

func (*ConfigError) HasConfigError

func (c *ConfigError) HasConfigError() bool

type Conflicter

type Conflicter interface {
	Conflict() (bool, interface{})
}

type DataSyncSource

type DataSyncSource interface {
	PathSyncSource
	// GetReaderOn provides a ReadCloser for reading content of a node located at a given path
	GetReaderOn(path string) (out io.ReadCloser, err error)
}

DataSyncSource provides a way to read the actual content of the nodes

func AsDataSyncSource

func AsDataSyncSource(endpoint Endpoint) (DataSyncSource, bool)

AsDataSyncSource tries to cast an Endpoint to a DataSyncSource

type DataSyncTarget

type DataSyncTarget interface {
	PathSyncTarget
	// GetWriterOn provides a writeCloser for writing content to a given path.
	GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)
}

DataSyncTarget provides a way to write some actual content to the nodes

func AsDataSyncTarget

func AsDataSyncTarget(endpoint Endpoint) (DataSyncTarget, bool)

AsDataSyncTarget tries to cast an Endpoint to a DataSyncTarget

type DirectionType

type DirectionType int

type Endpoint

type Endpoint interface {
	// LoadNode loads a given node by its path from this endpoint
	LoadNode(ctx context.Context, path string, extendedStats ...bool) (node *tree.Node, err error)
	// GetEndpointInfo returns static informations about this endpoint
	GetEndpointInfo() EndpointInfo
}

Endpoint is the most basic interface for representing an endpoint for synchronization. It is just able to return some info and to load a node

type EndpointInfo

type EndpointInfo struct {
	URI                   string
	RequiresNormalization bool
	RequiresFoldersRescan bool
	IsAsynchronous        bool
	EchoTime              time.Duration
	Ignores               []string
}

EndpointInfo provides static info about a given Endpoint (returned by GetEndpointInfo method)

type EndpointOptions

type EndpointOptions struct {
	BrowseOnly bool
	Properties map[string]string
}

EndpointOptions is used to configure an Endpoint at creation time

type EndpointRootStat

type EndpointRootStat struct {
	HasChildrenInfo bool
	HasSizeInfo     bool

	Size    int64
	Folders int64
	Files   int64

	PgSize     int64 `json:"-"`
	PgChildren int64 `json:"-"`
	PgFolders  int64 `json:"-"`
	PgFiles    int64 `json:"-"`

	LastPg float64 `json:"-"`
}

EndpointRootStat gives information about the size/files/folders of an endpoint

func (*EndpointRootStat) Children

func (e *EndpointRootStat) Children() int64

Children returns the sum of Folders and Files

func (*EndpointRootStat) IsEmpty

func (e *EndpointRootStat) IsEmpty() bool

IsEmpty checks if the given stat seems empty. Call to IsKnown should be done before otherwise it can be a false negative

func (*EndpointRootStat) IsKnown

func (e *EndpointRootStat) IsKnown() bool

IsKnown returns true if either HasSizeInfo or HasChildrenInfo is set

type EndpointStatus

type EndpointStatus struct {
	EndpointInfo
	WatchConnection WatchConnectionInfo
	Stats           *EndpointRootStat
}

type ErrorType

type ErrorType int
const (
	SyncErrorUnknown ErrorType = iota
	SyncErrorNetworking
	SyncErrorAuthentication
	SyncErrorConflicts
)

type EventInfo

type EventInfo struct {
	Time           string
	Size           int64
	Etag           string
	Folder         bool
	Path           string
	Source         PathSyncSource `json:"-"`
	Type           EventType
	Host           string
	Port           string
	UserAgent      string
	OperationId    string
	ScanEvent      bool
	ScanSourceNode *tree.Node
	Metadata       map[string]string
	MoveSource     *tree.Node
	MoveTarget     *tree.Node
}

EventInfo contains the information of the event that occurred and the source IP:PORT of the client which triggerred the event.

func NodeToEventInfo

func NodeToEventInfo(ctx context.Context, path string, node *tree.Node, eventType EventType) (eventInfo EventInfo)

func (EventInfo) CreateContext

func (e EventInfo) CreateContext(ctx context.Context) context.Context

type EventType

type EventType string

EventType represents the type of the event occurred.

const (
	// EventCreate notifies when a new object is created
	EventCreate EventType = "ObjectCreated"
	// EventRename notifies when a new object is renamed
	EventRename EventType = "ObjectRenamed"
	// EventRemove notifies when a new object is deleted
	EventRemove EventType = "ObjectRemoved"
	// EventAccessed notifies when an object is accessed.
	EventAccessed EventType = "ObjectAccessed"
	// EventAccessedRead notifies when an object is accessed (specifically read).
	EventAccessedRead EventType = "ObjectAccessed:Read"
	// EventAccessedStat notifies when an object is accessed (specifically stat).
	EventAccessedStat EventType = "ObjectAccessed:Stat"
	// EventOther notifies any other events
	EventOther EventType = "ObjectOther"
	// EventOther notifies any other events
	EventSureMove EventType = "NodeMoved"
)

type HashStoreReader

type HashStoreReader interface {
	// SetRefHashStore passes a reference to a loaded snapshot
	SetRefHashStore(source PathSyncSource)
}

HashStoreReader can maintain a reference to a snapshot to quickly find hashes for nodes directly from the snapshot if they have not been modified

type LockBranchProvider

type LockBranchProvider interface {
	// Lock set a lock on a branch, with a preset sessionUUID and an expiration time
	LockBranch(ctx context.Context, node *tree.Node, sessionUUID string, expireAfter time.Duration) error
	// Unlock removes lock manually from this branch, if it was not expired already
	UnlockBranch(ctx context.Context, sessionUUID string) error
}

LockBranchProvider can set/remove a lock on a branch, with automatic expiration

type LockEvent

type LockEvent struct {
	Type        LockEventType
	Source      PathSyncSource
	Path        string
	OperationId string
}

LockEvent is a simple struct for signaling lock/unlock operation

type LockEventType

type LockEventType int
const (
	LockEventLock LockEventType = iota
	LockEventUnlock
)

type MetadataProvider

type MetadataProvider interface {
	// ProvidesMetadataNamespaces returns a list of patterns to check on provider nodes MetaStore.
	ProvidesMetadataNamespaces() ([]glob.Glob, bool)
}

MetadataProvider declares metadata namespaces that may be mapped to target metadata

type MetadataReceiver

type MetadataReceiver interface {
	// CreateMetadata add a metadata to the node
	CreateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
	// UpdateMetadata updates an existing metadata value
	UpdateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
	// DeleteMetadata deletes a metadata by namespace
	DeleteMetadata(ctx context.Context, node *tree.Node, namespace string) error
}

MetadataReceiver implements methods for updating nodes metadata

type MultiStater

type MultiStater map[string]Stater

func NewMultiStater

func NewMultiStater() MultiStater

func (MultiStater) Stats

func (m MultiStater) Stats() map[string]interface{}

func (MultiStater) String

func (m MultiStater) String() string

type Networking

type Networking interface {
	NetworkError() error
}

type PathSyncSource

type PathSyncSource interface {
	Endpoint
	// Walk walks the nodes with a callback
	Walk(walknFc WalkNodesFunc, root string, recursive bool) (err error)
	// Watch setup an event watcher on the nodes
	Watch(recursivePath string) (*WatchObject, error)
}

PathSyncSource is a type of endpoint that can be used as a source of tree.Nodes for synchronization. It can browse and watch the nodes, but not get the nodes actual content (see DataSyncSource).

func AsPathSyncSource

func AsPathSyncSource(endpoint Endpoint) (PathSyncSource, bool)

AsPathSyncSource tries to cast an Endpoint to a PathSyncSource

type PathSyncTarget

type PathSyncTarget interface {
	Endpoint
	// CreateNode is used to create a node in the tree
	CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)
	// DeleteNode is used to remove a node (and all its children) from the tree
	DeleteNode(ctx context.Context, path string) (err error)
	// MoveNode is used to move a node (and all its children) from one place to another in the tree.
	MoveNode(ctx context.Context, oldPath string, newPath string) (err error)
}

PathSyncTarget is a type of endpoint that can be used as a target for synchronization, typically an Index. It can be updated with a tree of nodes, but cannot store actual data (see DataSyncTarget)

func AsPathSyncTarget

func AsPathSyncTarget(endpoint Endpoint) (PathSyncTarget, bool)

AsPathSyncTarget tries to cast an Endpoint to a PathSyncTarget

type ProcessingStatus

type ProcessingStatus struct {
	// contains filtered or unexported fields
}

func NewProcessingStatus

func NewProcessingStatus(info string) *ProcessingStatus

func (*ProcessingStatus) AtomicProgress

func (p *ProcessingStatus) AtomicProgress() bool

func (*ProcessingStatus) EndpointURI

func (p *ProcessingStatus) EndpointURI() string

func (*ProcessingStatus) Error

func (p *ProcessingStatus) Error() error

func (*ProcessingStatus) IsError

func (p *ProcessingStatus) IsError() bool

func (*ProcessingStatus) MarshalJSON

func (p *ProcessingStatus) MarshalJSON() ([]byte, error)

MarshalJSON implements custom JSON marshalling

func (*ProcessingStatus) Node

func (p *ProcessingStatus) Node() *tree.Node

func (*ProcessingStatus) Progress

func (p *ProcessingStatus) Progress() float32

func (*ProcessingStatus) SetEndpoint

func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus

func (*ProcessingStatus) SetError

func (p *ProcessingStatus) SetError(e error) *ProcessingStatus

func (*ProcessingStatus) SetNode

func (p *ProcessingStatus) SetNode(node *tree.Node) *ProcessingStatus

func (*ProcessingStatus) SetProgress

func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus

func (*ProcessingStatus) Status

func (p *ProcessingStatus) Status() int

func (*ProcessingStatus) String

func (p *ProcessingStatus) String() string

func (*ProcessingStatus) Type

func (p *ProcessingStatus) Type() StatusScope

func (*ProcessingStatus) UnmarshalJSON

func (p *ProcessingStatus) UnmarshalJSON(data []byte) error

UnmarshalJSON implements custom JSON unmarshalling

type ProcessorEvent

type ProcessorEvent struct {
	Type string
	Data interface{}
}

type SessionProvider

type SessionProvider interface {
	// StartSession opens a new indexation session and returns it
	StartSession(ctx context.Context, rootNode *tree.Node, silent bool) (*tree.IndexationSession, error)
	// FlushSession calls the Flush method on the underlying service without closing the session yet
	FlushSession(ctx context.Context, sessionUuid string) error
	// FinishSession closes the indexation session
	FinishSession(ctx context.Context, sessionUuid string) error
}

SessionProvider has internal mechanism to start/flush/finish an IndexationSession

func AsSessionProvider

func AsSessionProvider(endpoint Endpoint) (SessionProvider, bool)

AsSessionProvider tries to cast an Endpoint to a SessionProvider

type SnapshotFactory

type SnapshotFactory interface {
	// Load creates the snapshot and return it
	Load(source PathSyncSource) (Snapshoter, error)
	Close(ctx context.Context) error
	Reset(ctx context.Context) error
}

SnapshotFactory provides dependency injection for creating snapshots using a specific persistence layer.

type SnapshotUpdater

type SnapshotUpdater interface {
	// SetUpdateSnapshot stores internal reference to a Snapshot
	SetUpdateSnapshot(PathSyncTarget)
	// PatchUpdateSnaptshot applies a patch of operations to the internal snapshot
	PatchUpdateSnapshot(ctx context.Context, patch interface{})
}

SnapshotUpdater is an endpoint that can embed a reference to a snapshot and update it afterward

type Snapshoter

type Snapshoter interface {
	PathSyncSource
	// IsEmpty indicates whether this snapshot is properly initialized or not (e.g. underlying DB not found)
	IsEmpty() bool
	// Capture walks the source and stores all paths inside snapshot. The paths parameter allows to filter the branches to capture.
	Capture(ctx context.Context, source PathSyncSource, paths ...string) error
}

Snapshoter is an extended version of PathSyncSource that can capture another source at once.

type Stater

type Stater interface {
	fmt.Stringer
	Stats() map[string]interface{}
}

Stater can provide a json-serializable description of its content

type Status

type Status interface {
	fmt.Stringer

	Type() StatusScope
	Status() int
	IsError() bool
	Error() error

	Progress() float32
	AtomicProgress() bool

	EndpointURI() string
	Node() *tree.Node
}

type StatusProvider

type StatusProvider interface {
	// SetupChannels register channels for listening to status and done infos
	SetupChannels(status chan Status, done chan interface{}, cmd *Command)
	// Status notify of a new ProcessStatus
	Status(s Status)
	// Done notify the patch is processed, can send any useful info to the associated channel
	Done(info interface{})
}

StatusProvider can register channels to send status/done events during processing

type StatusScope

type StatusScope int

type SyncCmd

type SyncCmd int

type TaskStatus

type TaskStatus int
const (
	TaskStatusIdle TaskStatus = iota
	TaskStatusPaused
	TaskStatusDisabled
	TaskStatusProcessing
	TaskStatusError
	TaskStatusRestarting
	TaskStatusStopping
	TaskStatusRemoved
)

type UuidFoldersRefresher

type UuidFoldersRefresher interface {
	// ExistingFolders lists all folders with their UUID
	ExistingFolders(ctx context.Context) (map[string][]*tree.Node, error)
	// UpdateFolderUuid refreshes a given folder UUID and return it.
	UpdateFolderUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}

UuidFoldersRefresher provides tools to detect UUID duplicates and update them if necessary

type UuidProvider

type UuidProvider interface {
	// LoadNodeByUuid loads a node by UUID.
	LoadNodeByUuid(ctx context.Context, uuid string) (node *tree.Node, err error)
}

UuidProvider declares an endpoint to be able to load a node by its unique UUID

type UuidReceiver

type UuidReceiver interface {
	// UpdateNodeUuid refresh node UUID and returns the new node
	UpdateNodeUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}

UuidReceiver is able to update an existing node UUID

type Versioner

type Versioner interface {
	Commit(node *tree.Node)
	ListVersions(node *tree.Node) (versions map[int]string, lastVersion int)
}

type WalkNodesFunc

type WalkNodesFunc func(path string, node *tree.Node, err error)

type WatchConnectionInfo

type WatchConnectionInfo int
const (
	WatchConnected WatchConnectionInfo = iota
	WatchDisconnected
	WatchActive
	WatchIdle
	WatchStats
)

type WatchObject

type WatchObject struct {
	// eventInfo will be put on this chan
	EventInfoChan chan EventInfo
	// errors will be put on this chan
	ErrorChan chan error
	// will stop the watcher goroutines
	DoneChan chan bool
	// Provides info about the internal watcher connection
	// Can be nill if the internal watcher does not support such notifications
	ConnectionInfo chan WatchConnectionInfo
}

func (*WatchObject) Close

func (w *WatchObject) Close()

Close the watcher, will stop all goroutines

func (*WatchObject) ConnectionInfos

func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo

func (WatchObject) Done

func (w WatchObject) Done() <-chan bool

Done returns a channel that unblocks when Close has been called

func (*WatchObject) Errors

func (w *WatchObject) Errors() chan error

Errors returns the chan receiving errors

func (*WatchObject) Events

func (w *WatchObject) Events() chan EventInfo

Events returns the chan receiving events

func (WatchObject) NextError

func (w WatchObject) NextError() error

NextError pops the next error from the ErrorChan

func (WatchObject) NextEvent

func (w WatchObject) NextEvent() EventInfo

NextEvent pops the next event from the EventInfoChan

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL