filetransfer

package
v0.0.0-...-d9a1f69 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 21, 2024 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source
const (
	RunFileKindOther = RunFileKind(iota)

	// An internal W&B file.
	RunFileKindWandb

	// An artifact file.
	RunFileKindArtifact

	// A media file.
	RunFileKindMedia
)
View Source
const (
	DefaultConcurrencyLimit = 128
)

Variables

This section is empty.

Functions

func FileTransferRetryPolicy

func FileTransferRetryPolicy(
	ctx context.Context,
	resp *http.Response,
	err error,
) (bool, error)

FileTransferRetryPolicy is the retry policy to be used for file operations.

Types

type DefaultFileTransfer

type DefaultFileTransfer struct {
	// contains filtered or unexported fields
}

DefaultFileTransfer uploads or downloads files to/from the server

func NewDefaultFileTransfer

func NewDefaultFileTransfer(
	client *retryablehttp.Client,
	logger *observability.CoreLogger,
	fileTransferStats FileTransferStats,
) *DefaultFileTransfer

NewDefaultFileTransfer creates a new fileTransfer

func (*DefaultFileTransfer) Download

func (ft *DefaultFileTransfer) Download(task *Task) error

Download downloads a file from the server

func (*DefaultFileTransfer) Upload

func (ft *DefaultFileTransfer) Upload(task *Task) error

Upload uploads a file to the server

type FileTransfer

type FileTransfer interface {
	Upload(task *Task) error
	Download(task *Task) error
}

type FileTransferManager

type FileTransferManager interface {
	// Asynchronously begins the main loop of the file transfer manager.
	Start()

	// Waits for all asynchronous work to finish.
	Close()

	// Schedules a file upload operation.
	AddTask(task *Task)
}

A manager of asynchronous file upload tasks.

func NewFileTransferManager

func NewFileTransferManager(opts ...FileTransferManagerOption) FileTransferManager

type FileTransferManagerOption

type FileTransferManagerOption func(fm *fileTransferManager)

func WithFileTransfer

func WithFileTransfer(fileTransfer FileTransfer) FileTransferManagerOption

func WithFileTransferStats

func WithFileTransferStats(fileTransferStats FileTransferStats) FileTransferManagerOption

func WithSettings

func WithSettings(settings *service.Settings) FileTransferManagerOption

type FileTransferStats

type FileTransferStats interface {
	// GetFilesStats returns byte counts for uploads.
	GetFilesStats() *service.FilePusherStats

	// GetFileCounts returns a breakdown of the kinds of files uploaded.
	GetFileCounts() *service.FileCounts

	// IsDone returns whether all uploads finished.
	IsDone() bool

	// SetDone marks all uploads as finished.
	SetDone()

	// UpdateUploadStats updates the upload stats for a file.
	UpdateUploadStats(newInfo FileUploadInfo)
}

FileTransferStats reports file upload/download progress and totals.

func NewFileTransferStats

func NewFileTransferStats() FileTransferStats

type FileUploadInfo

type FileUploadInfo struct {
	// The local path to the file being uploaded.
	Path string

	// The kind of file this is.
	FileKind RunFileKind

	// The number of bytes uploaded so far.
	UploadedBytes int64

	// The total number of bytes being uploaded.
	TotalBytes int64
}

FileUploadInfo is information about an in-progress file upload.

type ProgressReader

type ProgressReader struct {
	// Note: this turns ProgressReader into a ReadSeeker, not just a Reader!
	// The retryablehttp client will seek to 0 on every retry.
	*os.File
	// contains filtered or unexported fields
}

func NewProgressReader

func NewProgressReader(file *os.File, size int64, callback func(processed, total int)) (*ProgressReader, error)

func (*ProgressReader) Len

func (pr *ProgressReader) Len() int

func (*ProgressReader) Read

func (pr *ProgressReader) Read(p []byte) (int, error)

type RunFileKind

type RunFileKind int64

RunFileKind is the category of a file saved with a run.

func RunFileKindFromProto

func RunFileKindFromProto(enum service.FilesItem_FileType) RunFileKind

RunFileKindFromProto converts the FilesItem.FileType enum to RunFileKind.

type Storage

type Storage int

type Task

type Task struct {
	// FileKind is the category of file being uploaded or downloaded
	FileKind RunFileKind

	// Type is the type of task (upload or download)
	Type TaskType

	// Path is the local path to the file
	Path string

	// Name is the name of the file
	Name string

	// Url is the endpoint to upload to/download from
	Url string

	// Headers to send on the upload
	Headers []string

	// Size is the size of the file
	Size int64

	// Error, if any.
	Err error

	// Callback to execute after completion (success or failure).
	CompletionCallback func(*Task)

	// ProgressCallback is a callback to execute on progress updates
	ProgressCallback func(int, int)

	// This can be used to cancel the file upload or download if it is no longer needed.
	Context context.Context
}

Task is a task to upload/download a file

func (*Task) SetCompletionCallback

func (ut *Task) SetCompletionCallback(callback func(*Task))

func (*Task) SetProgressCallback

func (ut *Task) SetProgressCallback(callback func(int, int))

func (*Task) String

func (ut *Task) String() string

type TaskType

type TaskType int
const (
	UploadTask TaskType = iota
	DownloadTask
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL