Documentation ¶
Overview ¶
Generic file system interface for rclone object storage systems
Index ¶
- Constants
- Variables
- func CalculateModifyWindow(fs ...Fs)
- func Check(fdst, fsrc Fs) error
- func CheckMd5sums(src, dst Object) (equal bool, unset bool, err error)
- func Choose(what string, defaults, help []string, newOk bool) string
- func ChooseOption(o *Option) string
- func ChooseRemote() string
- func Command(commands []string) byte
- func Confirm() bool
- func Copy(f Fs, dst, src Object)
- func CopyDir(fdst, fsrc Fs) error
- func Debug(o interface{}, text string, args ...interface{})
- func DeleteFiles(to_be_deleted ObjectsChan)
- func DeleteRemote(name string)
- func EditConfig()
- func EditRemote(name string)
- func Equal(src, dst Object) bool
- func ErrorLog(o interface{}, text string, args ...interface{})
- func FsSame(fdst, fsrc Fs) bool
- func List(f Fs, w io.Writer) error
- func ListDir(f Fs, w io.Writer) error
- func ListFn(f Fs, fn func(Object)) error
- func ListLong(f Fs, w io.Writer) error
- func LoadConfig()
- func Log(o interface{}, text string, args ...interface{})
- func Md5sum(f Fs, w io.Writer) error
- func Md5sumsEqual(src, dst string) bool
- func MimeType(o Object) string
- func Mkdir(f Fs) error
- func MoveDir(fdst, fsrc Fs) error
- func NewLoggedTransport(transport http.RoundTripper, logBody bool) *loggedTransport
- func NewRemote(name string)
- func Obscure(x string) string
- func OkRemote(name string) bool
- func OutputLog(o interface{}, text string, args ...interface{})
- func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup)
- func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup)
- func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup)
- func Purge(f Fs) error
- func ReadLine() string
- func Register(info *FsInfo)
- func RemoteConfig(name string)
- func RetryError(err error) error
- func RetryErrorf(format string, a ...interface{}) error
- func Reveal(y string) string
- func Rmdir(f Fs) error
- func SaveConfig()
- func ShowRemote(name string)
- func ShowRemotes()
- func Sync(fdst, fsrc Fs) error
- type Account
- type ConfigInfo
- type Copier
- type Dir
- type DirChan
- type DirMover
- type Fs
- type FsInfo
- type Limited
- func (f *Limited) Copy(src Object, remote string) (Object, error)
- func (f *Limited) List() ObjectsChan
- func (f *Limited) ListDir() DirChan
- func (f *Limited) Mkdir() error
- func (f *Limited) Name() string
- func (f *Limited) NewFsObject(remote string) Object
- func (f *Limited) Precision() time.Duration
- func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
- func (f *Limited) Rmdir() error
- func (f *Limited) Root() string
- func (f *Limited) String() string
- type Mover
- type Object
- type ObjectPair
- type ObjectPairChan
- type Objects
- type ObjectsChan
- type Option
- type OptionExample
- type Purger
- type Retry
- type SizeSuffix
- type StatsInfo
- func (s *StatsInfo) Bytes(bytes int64)
- func (s *StatsInfo) Checking(o Object)
- func (s *StatsInfo) DoneChecking(o Object)
- func (s *StatsInfo) DoneTransferring(o Object)
- func (s *StatsInfo) Error()
- func (s *StatsInfo) Errored() bool
- func (s *StatsInfo) Errors(errors int64)
- func (s *StatsInfo) GetErrors() int64
- func (s *StatsInfo) GetTransfers() int64
- func (s *StatsInfo) Log()
- func (s *StatsInfo) ResetCounters()
- func (s *StatsInfo) ResetErrors()
- func (s *StatsInfo) String() string
- func (s *StatsInfo) Transferring(o Object)
Constants ¶
const ( // User agent for Fs which can set it UserAgent = "rclone/" + Version // Very large precision value to show mod time isn't supported ModTimeNotSupported = 100 * 365 * 24 * time.Hour )
Constants
const Version = "v1.20"
Variables ¶
var ( // Config file ConfigFile *goconfig.ConfigFile // Home directory HomeDir = configHome() // Config file path ConfigPath = path.Join(HomeDir, configFileName) // Global config Config = &ConfigInfo{} )
Global
var ( // Error returned by NewFs if not found in config file NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file") ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes") ErrorCantMove = fmt.Errorf("Can't copy object - incompatible remotes") ErrorCantDirMove = fmt.Errorf("Can't copy directory - incompatible remotes") ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists") )
Globals
var (
Stats = NewStats()
)
Globals
Functions ¶
func CalculateModifyWindow ¶
func CalculateModifyWindow(fs ...Fs)
Work out modify window for fses passed in - sets Config.ModifyWindow
This is the largest modify window of all the fses in use, and the user configured value
func CheckMd5sums ¶
Check the two files to see if the MD5sums are the same
Returns two bools, the first of which is equality and the second of which is true if either of the MD5SUMs were unset.
May return an error which will already have been logged ¶
If an error is returned it will return equal as false
func Copy ¶
Copy src object to dst or f if nil
If dst is nil then the object must not exist already. If you do call Copy() with dst nil on a pre-existing file then some filing systems (eg Drive) may duplicate the file.
func Debug ¶
func Debug(o interface{}, text string, args ...interface{})
Write debuging output for this Object or Fs
func DeleteFiles ¶
func DeleteFiles(to_be_deleted ObjectsChan)
Delete all the files passed in the channel
func Equal ¶
Checks to see if the src and dst objects are equal by looking at size, mtime and MD5SUM
If the src and dst size are different then it is considered to be not equal. If --size-only is in effect then this is the only check that is done.
If the size is the same and the mtime is the same then it is considered to be equal. This check is skipped if using --checksum.
If the size is the same and mtime is different, unreadable or --checksum is set and the MD5SUM is the same then the file is considered to be equal. In this case the mtime on the dst is updated if --checksum is not set.
Otherwise the file is considered to be not equal including if there were errors reading info.
func ErrorLog ¶
func ErrorLog(o interface{}, text string, args ...interface{})
Write error log output for this Object or Fs Unconditionally logs a message regardless of Config.Quiet or Config.Verbose
func List ¶
List the Fs to the supplied writer
Shows size and path ¶
Lists in parallel which may get them out of order
func ListFn ¶
List the Fs to the supplied function
Lists in parallel which may get them out of order
func ListLong ¶
List the Fs to the supplied writer
Shows size, mod time and path ¶
Lists in parallel which may get them out of order
func Log ¶
func Log(o interface{}, text string, args ...interface{})
Write log output for this Object or Fs
func Md5sum ¶
List the Fs to the supplied writer
Produces the same output as the md5sum command ¶
Lists in parallel which may get them out of order
func Md5sumsEqual ¶
Md5sumsEqual checks to see if src == dst, but ignores empty strings
func NewLoggedTransport ¶
func NewLoggedTransport(transport http.RoundTripper, logBody bool) *loggedTransport
NewLoggedTransport wraps the transport passed in and logs all roundtrips including the body if logBody is set.
func OutputLog ¶
func OutputLog(o interface{}, text string, args ...interface{})
Outputs log for object
func PairChecker ¶
func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup)
Read Objects~s on in send to out if they need uploading
FIXME potentially doing lots of MD5SUMS at once
func PairCopier ¶
func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup)
Read Objects on in and copy them
func PairMover ¶
func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup)
Read Objects on in and move them if possible, or copy them if not
func Register ¶
func Register(info *FsInfo)
Register a filesystem
Fs modules should use this in an init() function
func RetryError ¶
RetryError makes an error which indicates it would like to be retried
func RetryErrorf ¶
RetryErrorf makes an error which indicates it would like to be retried
Types ¶
type Account ¶
type Account struct {
// contains filtered or unexported fields
}
Account limits and accounts for one transfer
func NewAccount ¶
func NewAccount(in io.ReadCloser, obj Object) *Account
NewAccount makes a Account reader for an object
func (*Account) ETA ¶
ETA returns the ETA of the current operation, rounded to full seconds. If the ETA cannot be determined 'ok' returns false.
func (*Account) Progress ¶
Returns bytes read as well as the size. Size can be <= 0 if the size is unknown.
type ConfigInfo ¶
type ConfigInfo struct { Verbose bool Quiet bool DryRun bool CheckSum bool SizeOnly bool ModifyWindow time.Duration Checkers int Transfers int ConnectTimeout time.Duration // Connect timeout Timeout time.Duration // Data channel timeout DumpHeaders bool DumpBodies bool }
Filesystem config options
func (*ConfigInfo) Client ¶
func (ci *ConfigInfo) Client() *http.Client
Transport returns an http.Client with the correct timeouts
func (*ConfigInfo) Transport ¶
func (ci *ConfigInfo) Transport() http.RoundTripper
Transport returns an http.RoundTripper with the correct timeouts
type Copier ¶
type Copier interface { // Copy src to this remote using server side copy operations. // // This is stored with the remote path given // // It returns the destination Object and a possible error // // Will only be called if src.Fs().Name() == f.Name() // // If it isn't possible then return fs.ErrorCantCopy Copy(src Object, remote string) (Object, error) }
type Dir ¶
type Dir struct { Name string // name of the directory When time.Time // modification or creation time - IsZero for unknown Bytes int64 // size of directory and contents -1 for unknown Count int64 // number of objects -1 for unknown }
A structure of directory/container/bucket lists
type Fs ¶
type Fs interface { // The name of the remote (as passed into NewFs) Name() string // The root of the remote (as passed into NewFs) Root() string // String returns a description of the FS String() string // List the Fs into a channel List() ObjectsChan // List the Fs directories/buckets/containers into a channel ListDir() DirChan // Find the Object at remote. Returns nil if can't be found NewFsObject(remote string) Object // Put in to the remote path with the modTime given of the given size // // May create the object even if it returns an error - if so // will return the object and the error, otherwise will return // nil and the error Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) // Make the directory (container, bucket) // // Shouldn't return an error if it already exists Mkdir() error // Remove the directory (container, bucket) if empty // // Return an error if it doesn't exist or isn't empty Rmdir() error // Precision of the ModTimes in this Fs Precision() time.Duration }
A Filesystem, describes the local filesystem and the remote object store
func NewFs ¶
NewFs makes a new Fs object from the path
The path is of the form remote:path
Remotes are looked up in the config file. If the remote isn't found then NotFoundInConfigFile will be returned.
On Windows avoid single character remote names as they can be mixed up with drive letters.
func NewLimited ¶
NewLimited maks a limited Fs limited to the objects passed in
type FsInfo ¶
type FsInfo struct { // Name of this fs Name string // Create a new file system. If root refers to an existing // object, then it should return a Fs which only returns that // object. NewFs func(name string, root string) (Fs, error) // Function to call to help with config Config func(string) // Options for the Fs configuration Options []Option }
Filesystem info
type Limited ¶
type Limited struct {
// contains filtered or unexported fields
}
This defines a Limited Fs which can only return the Objects passed in from the Fs passed in
func (*Limited) Copy ¶
Copy src to this remote using server side copy operations.
This is stored with the remote path given ¶
It returns the destination Object and a possible error ¶
Will only be called if src.Fs().Name() == f.Name()
If it isn't possible then return fs.ErrorCantCopy
func (*Limited) NewFsObject ¶
Find the Object at remote. Returns nil if can't be found
func (*Limited) Put ¶
Put in to the remote path with the modTime given of the given size
May create the object even if it returns an error - if so will return the object and the error, otherwise will return nil and the error
type Mover ¶
type Mover interface { // Move src to this remote using server side move operations. // // This is stored with the remote path given // // It returns the destination Object and a possible error // // Will only be called if src.Fs().Name() == f.Name() // // If it isn't possible then return fs.ErrorCantMove Move(src Object, remote string) (Object, error) }
type Object ¶
type Object interface { // String returns a description of the Object String() string // Fs returns the Fs that this object is part of Fs() Fs // Remote returns the remote path Remote() string // Md5sum returns the md5 checksum of the file // If no Md5sum is available it returns "" Md5sum() (string, error) // ModTime returns the modification date of the file // It should return a best guess if one isn't available ModTime() time.Time // SetModTime sets the metadata on the object to set the modification date SetModTime(time.Time) // Size returns the size of the file Size() int64 // Open opens the file for read. Call Close() on the returned io.ReadCloser Open() (io.ReadCloser, error) // Update in to the object with the modTime given of the given size Update(in io.Reader, modTime time.Time, size int64) error // Storable says whether this object can be stored Storable() bool // Removes this object Remove() error }
A filesystem like object which can either be a remote object or a local file/directory
type ObjectPair ¶
type ObjectPair struct {
// contains filtered or unexported fields
}
A pair of Objects
type Option ¶
type Option struct { Name string Help string Optional bool Examples []OptionExample }
An options for a Fs
type OptionExample ¶
An example for an option
type Purger ¶
type Purger interface { // Purge all files in the root and the root directory // // Implement this if you have a way of deleting all the files // quicker than just running Remove() on the result of List() // // Return an error if it doesn't exist Purge() error }
Optional interfaces
type Retry ¶
An optional interface for error as to whether the operation should be retried
This should be returned from Update or Put methods as required
type SizeSuffix ¶
type SizeSuffix int64
type StatsInfo ¶
type StatsInfo struct {
// contains filtered or unexported fields
}
Stats limits and accounts all transfers
func (*StatsInfo) DoneChecking ¶
DoneChecking removes a check from the stats
func (*StatsInfo) DoneTransferring ¶
DoneTransferring removes a transfer from the stats
func (*StatsInfo) GetTransfers ¶
GetTransfers reads the number of transfers
func (*StatsInfo) ResetCounters ¶
func (s *StatsInfo) ResetCounters()
ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
func (*StatsInfo) ResetErrors ¶
func (s *StatsInfo) ResetErrors()
ResetErrors sets the errors count to 0
func (*StatsInfo) Transferring ¶
Transferring adds a transfer into the stats