operations

package
v1.52.3-0...-16af9ab Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 26, 2020 License: MIT Imports: 34 Imported by: 0

Documentation

Overview

Package operations does generic operations on filesystems and objects

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func BackupDir

func BackupDir(fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error)

BackupDir returns the correctly configured --backup-dir

func CanServerSideMove

func CanServerSideMove(fdst fs.Fs) bool

CanServerSideMove returns true if fdst support server side moves or server side copies

Some remotes simulate rename by server-side copy and delete, so include remotes that implements either Mover or Copier.

func Cat

func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error

Cat any files to the io.Writer

if offset == 0 it will be ignored if offset > 0 then the file will be seeked to that offset if offset < 0 then the file will be seeked that far from the end

if count < 0 then it will be ignored if count >= 0 then only that many characters will be output

func Check

func Check(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error

Check the files in fsrc and fdst according to Size and hash

func CheckDownload

func CheckDownload(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error

CheckDownload checks the files in fsrc and fdst according to Size and the actual contents of the files.

func CheckEqualReaders

func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error)

CheckEqualReaders checks to see if in1 and in2 have the same content when read.

it returns true if differences were found

func CheckFn

func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool) error

CheckFn checks the files in fsrc and fdst according to Size and hash using checkFunction on each file to check the hashes.

checkFunction sees if dst and src are identical

it returns true if differences were found it also returns whether it couldn't be hashed

func CheckHashes

func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error)

CheckHashes checks the two files to see if they have common known hash types and compares them

Returns

equal - which is equality of the hashes

hash - the HashType. This is HashNone if either of the hashes were unset or a compatible hash couldn't be found.

err - may return an error which will already have been logged

If an error is returned it will return equal as false

func CheckIdentical

func CheckIdentical(ctx context.Context, dst, src fs.Object) (differ bool, err error)

CheckIdentical checks to see if dst and src are identical by reading all their bytes if necessary.

it returns true if differences were found

func CleanUp

func CleanUp(ctx context.Context, f fs.Fs) error

CleanUp removes the trash for the Fs

func CommonHash

func CommonHash(fa, fb fs.Info) (hash.Type, *fs.HashesOption)

CommonHash returns a single hash.Type and a HashOption with that type which is in common between the two fs.Fs.

func CompareOrCopyDest

func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error)

CompareOrCopyDest checks --compare-dest and --copy-dest to see if src does not need to be copied

Returns True if src does not need to be copied

func ConfigMaxDepth

func ConfigMaxDepth(recursive bool) int

ConfigMaxDepth returns the depth to use for a recursive or non recursive listing.

func Copy

func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error)

Copy src object to dst or f if nil. If dst is nil then it uses remote as the name of the new object.

It returns the destination object if possible. Note that this may be nil.

func CopyFile

func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error)

CopyFile moves a single file possibly to a new name

func CopyURL

func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool, noClobber bool) (dst fs.Object, err error)

CopyURL copies the data from the url to (fdst, dstFileName)

func CopyURLToWriter

func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error)

CopyURLToWriter copies the data from the url to the io.Writer supplied

func Count

func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error)

Count counts the objects and their sizes in the Fs

Obeys includes and excludes

func Deduplicate

func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode) error

Deduplicate interactively finds duplicate files and offers to delete all but one or rename them to be different. Only useful with Google Drive which can have duplicate file names.

func Delete

func Delete(ctx context.Context, f fs.Fs) error

Delete removes all the contents of a container. Unlike Purge, it obeys includes and excludes.

func DeleteFile

func DeleteFile(ctx context.Context, dst fs.Object) (err error)

DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.

If useBackupDir is set and --backup-dir is in effect then it moves the file to there instead of deleting

func DeleteFileWithBackupDir

func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error)

DeleteFileWithBackupDir deletes a single file respecting --dry-run and accumulating stats and errors.

If backupDir is set then it moves the file to there instead of deleting

func DeleteFiles

func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error

DeleteFiles removes all the files passed in the channel

func DeleteFilesWithBackupDir

func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error

DeleteFilesWithBackupDir removes all the files passed in the channel

If backupDir is set the files will be placed into that directory instead of being deleted.

func DirMove

func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error)

DirMove renames srcRemote to dstRemote

It does this by loading the directory tree into memory (using ListR if available) and doing renames in parallel.

func Equal

func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool

Equal checks to see if the src and dst objects are equal by looking at size, mtime and hash

If the src and dst size are different then it is considered to be not equal. If --size-only is in effect then this is the only check that is done. If --ignore-size is in effect then this check is skipped and the files are considered the same size.

If the size is the same and the mtime is the same then it is considered to be equal. This check is skipped if using --checksum.

If the size is the same and mtime is different, unreadable or --checksum is set and the hash is the same then the file is considered to be equal. In this case the mtime on the dst is updated if --checksum is not set.

Otherwise the file is considered to be not equal including if there were errors reading info.

func GetCompareDest

func GetCompareDest() (CompareDest fs.Fs, err error)

GetCompareDest sets up --compare-dest

func GetCopyDest

func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error)

GetCopyDest sets up --copy-dest

func HashLister

func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error

HashLister does an md5sum equivalent for the hash type passed in

func HashListerBase64

func HashListerBase64(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error

HashListerBase64 does an md5sum equivalent for the hash type passed in with base64 encoded

func List

func List(ctx context.Context, f fs.Fs, w io.Writer) error

List the Fs to the supplied writer

Shows size and path - obeys includes and excludes

Lists in parallel which may get them out of order

func ListDir

func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error

ListDir lists the directories/buckets/containers in the Fs to the supplied writer

func ListFn

func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error

ListFn lists the Fs to the supplied function

Lists in parallel which may get them out of order

func ListJSON

func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error

ListJSON lists fsrc using the options in opt calling callback for each item

func ListLong

func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error

ListLong lists the Fs to the supplied writer

Shows size, mod time and path - obeys includes and excludes

Lists in parallel which may get them out of order

func Md5sum

func Md5sum(ctx context.Context, f fs.Fs, w io.Writer) error

Md5sum list the Fs to the supplied writer

Produces the same output as the md5sum command - obeys includes and excludes

Lists in parallel which may get them out of order

func Mkdir

func Mkdir(ctx context.Context, f fs.Fs, dir string) error

Mkdir makes a destination directory or container

func Move

func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error)

Move src object to dst or fdst if nil. If dst is nil then it uses remote as the name of the new object.

Note that you must check the destination does not exist before calling this and pass it as dst. If you pass dst=nil and the destination does exist then this may create duplicates or return errors.

It returns the destination object if possible. Note that this may be nil.

func MoveBackupDir

func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error)

MoveBackupDir moves a file to the backup dir

func MoveFile

func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error)

MoveFile moves a single file possibly to a new name

func NeedTransfer

func NeedTransfer(ctx context.Context, dst, src fs.Object) bool

NeedTransfer checks to see if src needs to be copied to dst using the current config.

Returns a flag which indicates whether the file needs to be transferred or not.

func NewReOpen

func NewReOpen(ctx context.Context, src fs.Object, maxTries int, options ...fs.OpenOption) (rc io.ReadCloser, err error)

NewReOpen makes a handle which will reopen itself and seek to where it was on errors

If hashOption is set this will be applied when reading from the start

If rangeOption is set then this will applied when reading from the start, and updated on retries.

func Overlapping

func Overlapping(fdst, fsrc fs.Info) bool

Overlapping returns true if fdst and fsrc point to the same underlying Fs and they overlap.

func PublicLink(ctx context.Context, f fs.Fs, remote string) (string, error)

PublicLink adds a "readable by anyone with link" permission on the given file or folder.

func Purge

func Purge(ctx context.Context, f fs.Fs, dir string) error

Purge removes a directory and all of its contents

func Rcat

func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error)

Rcat reads data from the Reader until EOF and uploads it to a file on remote

func RcatSize

func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error)

RcatSize reads data from the Reader until EOF and uploads it to a file on remote. Pass in size >=0 if known, <0 if not known

func Rmdir

func Rmdir(ctx context.Context, f fs.Fs, dir string) error

Rmdir removes a container but not if not empty

func Rmdirs

func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error

Rmdirs removes any empty directories (or directories only containing empty directories) under f, including f.

func Same

func Same(fdst, fsrc fs.Info) bool

Same returns true if fdst and fsrc point to the same underlying Fs

func SameConfig

func SameConfig(fdst, fsrc fs.Info) bool

SameConfig returns true if fdst and fsrc are using the same config file entry

func SameDir

func SameDir(fdst, fsrc fs.Info) bool

SameDir returns true if fdst and fsrc point to the same underlying Fs and they are the same directory.

func SameObject

func SameObject(src, dst fs.Object) bool

SameObject returns true if src and dst could be pointing to the same object.

func SameRemoteType

func SameRemoteType(fdst, fsrc fs.Info) bool

SameRemoteType returns true if fdst and fsrc are the same type

func SetTier

func SetTier(ctx context.Context, fsrc fs.Fs, tier string) error

SetTier changes tier of object in remote

func Sha1sum

func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error

Sha1sum list the Fs to the supplied writer

Obeys includes and excludes

Lists in parallel which may get them out of order

func SuffixName

func SuffixName(remote string) string

SuffixName adds the current --suffix to the remote, obeying --suffix-keep-extension if set

func TryRmdir

func TryRmdir(ctx context.Context, f fs.Fs, dir string) error

TryRmdir removes a container but not if not empty. It doesn't count errors but may return one.

Types

type DeduplicateMode

type DeduplicateMode int

DeduplicateMode is how the dedupe command chooses what to do

const (
	DeduplicateInteractive DeduplicateMode = iota // interactively ask the user
	DeduplicateSkip                               // skip all conflicts
	DeduplicateFirst                              // choose the first object
	DeduplicateNewest                             // choose the newest object
	DeduplicateOldest                             // choose the oldest object
	DeduplicateRename                             // rename the objects
	DeduplicateLargest                            // choose the largest object
	DeduplicateSmallest                           // choose the smallest object
)

Deduplicate modes

func (*DeduplicateMode) Set

func (x *DeduplicateMode) Set(s string) error

Set a DeduplicateMode from a string

func (DeduplicateMode) String

func (x DeduplicateMode) String() string

func (*DeduplicateMode) Type

func (x *DeduplicateMode) Type() string

Type of the value

type FsInfo

type FsInfo struct {
	// Name of the remote (as passed into NewFs)
	Name string

	// Root of the remote (as passed into NewFs)
	Root string

	// String returns a description of the FS
	String string

	// Precision of the ModTimes in this Fs in Nanoseconds
	Precision time.Duration

	// Returns the supported hash types of the filesystem
	Hashes []string

	// Features returns the optional features of this Fs
	Features map[string]bool
}

FsInfo provides information about a remote

func GetFsInfo

func GetFsInfo(f fs.Fs) *FsInfo

GetFsInfo gets the information (FsInfo) about a given Fs

type ListFormat

type ListFormat struct {
	// contains filtered or unexported fields
}

ListFormat defines files information print format

func (*ListFormat) AddEncrypted

func (l *ListFormat) AddEncrypted()

AddEncrypted adds the encrypted path to file to output

func (*ListFormat) AddHash

func (l *ListFormat) AddHash(ht hash.Type)

AddHash adds the hash of the type given to the output

func (*ListFormat) AddID

func (l *ListFormat) AddID()

AddID adds file's ID to the output if known

func (*ListFormat) AddMimeType

func (l *ListFormat) AddMimeType()

AddMimeType adds file's MimeType to the output if known

func (*ListFormat) AddModTime

func (l *ListFormat) AddModTime()

AddModTime adds file's Mod Time to output

func (*ListFormat) AddOrigID

func (l *ListFormat) AddOrigID()

AddOrigID adds file's Original ID to the output if known

func (*ListFormat) AddPath

func (l *ListFormat) AddPath()

AddPath adds path to file to output

func (*ListFormat) AddSize

func (l *ListFormat) AddSize()

AddSize adds file's size to output

func (*ListFormat) AddTier

func (l *ListFormat) AddTier()

AddTier adds file's Tier to the output if known

func (*ListFormat) AppendOutput

func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string)

AppendOutput adds string generated by specific function to printed output

func (*ListFormat) Format

func (l *ListFormat) Format(entry *ListJSONItem) (result string)

Format prints information about the DirEntry in the format defined

func (*ListFormat) SetAbsolute

func (l *ListFormat) SetAbsolute(absolute bool)

SetAbsolute prints a leading slash in front of path names

func (*ListFormat) SetCSV

func (l *ListFormat) SetCSV(useCSV bool)

SetCSV defines if the output should be csv

Note that you should call SetSeparator before this if you want a custom separator

func (*ListFormat) SetDirSlash

func (l *ListFormat) SetDirSlash(dirSlash bool)

SetDirSlash defines if slash should be printed

func (*ListFormat) SetOutput

func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string)

SetOutput sets functions used to create files information

func (*ListFormat) SetSeparator

func (l *ListFormat) SetSeparator(separator string)

SetSeparator changes separator in struct

type ListJSONItem

type ListJSONItem struct {
	Path          string
	Name          string
	EncryptedPath string `json:",omitempty"`
	Encrypted     string `json:",omitempty"`
	Size          int64
	MimeType      string    `json:",omitempty"`
	ModTime       Timestamp //`json:",omitempty"`
	IsDir         bool
	Hashes        map[string]string `json:",omitempty"`
	ID            string            `json:",omitempty"`
	OrigID        string            `json:",omitempty"`
	Tier          string            `json:",omitempty"`
	IsBucket      bool              `json:",omitempty"`
}

ListJSONItem in the struct which gets marshalled for each line

type ListJSONOpt

type ListJSONOpt struct {
	Recurse       bool     `json:"recurse"`
	NoModTime     bool     `json:"noModTime"`
	NoMimeType    bool     `json:"noMimeType"`
	ShowEncrypted bool     `json:"showEncrypted"`
	ShowOrigIDs   bool     `json:"showOrigIDs"`
	ShowHash      bool     `json:"showHash"`
	DirsOnly      bool     `json:"dirsOnly"`
	FilesOnly     bool     `json:"filesOnly"`
	HashTypes     []string `json:"hashTypes"` // hash types to show if ShowHash is set, eg "MD5", "SHA-1"
}

ListJSONOpt describes the options for ListJSON

type OverrideRemote

type OverrideRemote struct {
	fs.ObjectInfo
	// contains filtered or unexported fields
}

OverrideRemote is a wrapper to override the Remote for an ObjectInfo

func NewOverrideRemote

func NewOverrideRemote(oi fs.ObjectInfo, remote string) *OverrideRemote

NewOverrideRemote returns an OverrideRemoteObject which will return the remote specified

func (*OverrideRemote) GetTier

func (o *OverrideRemote) GetTier() string

GetTier returns storage tier or class of the Object

func (*OverrideRemote) ID

func (o *OverrideRemote) ID() string

ID returns the ID of the Object if known, or "" if not

func (*OverrideRemote) MimeType

func (o *OverrideRemote) MimeType(ctx context.Context) string

MimeType returns the mime type of the underlying object or "" if it can't be worked out

func (*OverrideRemote) Remote

func (o *OverrideRemote) Remote() string

Remote returns the overridden remote name

func (*OverrideRemote) UnWrap

func (o *OverrideRemote) UnWrap() fs.Object

UnWrap returns the Object that this Object is wrapping or nil if it isn't wrapping anything

type ReOpen

type ReOpen struct {
	// contains filtered or unexported fields
}

ReOpen is a wrapper for an object reader which reopens the stream on error

func (*ReOpen) Close

func (h *ReOpen) Close() error

Close the stream

func (*ReOpen) Read

func (h *ReOpen) Read(p []byte) (n int, err error)

Read bytes retrying as necessary

type Timestamp

type Timestamp struct {
	When   time.Time
	Format string
}

Timestamp a time in the provided format

func (Timestamp) MarshalJSON

func (t Timestamp) MarshalJSON() (out []byte, err error)

MarshalJSON turns a Timestamp into JSON

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL