s3util

package
v0.0.0-...-c537d22 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 19, 2024 License: MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const DefaultChunkSizeLimit = 1024 * 1024 * 50
View Source 
const DefaultIncompleteExpiry = time.Hour * 24
View Source 
const (
	TempDirectoryEnv = "TEMP_DIR" // Would just use default temp directory if not specified
)

Variables

View Source 
var (
	ErrResumedUploadOffsetInvalid = errors.New("offset does not match uploaded status")
	ErrTimeout                    = errors.New("timeout")
)

Functions

func DownloadRangeS3 

func DownloadRangeS3(dlr Downloader, w io.WriterAt, bucket, key string, start, finish int) error

func DownloadS3 

func DownloadS3(downloader Downloader, bucket, key string) (*os.File, error)

func GetObjectInputFromConfig 

func GetObjectInputFromConfig(c *data.SignedUrlConfig, bucket, key string) *s3.GetObjectInput

func SignedUrl 

func SignedUrl(info S3Info, key string, duration time.Duration, cfg *data.SignedUrlConfig) (string, error)

func StreamS3 

func StreamS3(_s3 S3Info, hash string) (io.ReadCloser, error)

func UploadS3 

func UploadS3(
	info S3Info,
	r io.Reader,
	ct string,
	perms string,
	key string,
) error

Types

type ChunkFileService 

type ChunkFileService struct {
	Bucket string
	Client *s3.S3

	// Filepath stored from initialization
	Filepath string
	// contains filtered or unexported fields
}

ChunkFileService is a data.ChunkFileService specific for FS.

func (*ChunkFileService) Cleanup 

func (s *ChunkFileService) Cleanup()

Cleanup should delete the ChunkFileService and do any other cleanup necessary. It is only called when an incomplete upload has timed out.

func (*ChunkFileService) CompleteUpload 

func (s *ChunkFileService) CompleteUpload() error

CompleteUpload should perform any necessary steps to finalize the multipart upload.

func (*ChunkFileService) InitializeChunk 

func (s *ChunkFileService) InitializeChunk(u *data.ChunkUpload) error

InitializeChunk initializes the chunk upload operation.

func (*ChunkFileService) IsInitialized 

func (s *ChunkFileService) IsInitialized() bool

IsInitialized should return true if the ChunkFileService has already been initialized.

func (*ChunkFileService) UploadPart 

func (s *ChunkFileService) UploadPart(part []byte) error

UploadPart should upload the provided bytes to the cloud. Uploaded parts should be completely sequential.

type Downloader 

type Downloader interface {
	Download(w io.WriterAt, i *s3.GetObjectInput, options ...func(*s3manager.Downloader)) (int64, error)
}

Wrapper around Service Downloader for stubbing

func NewConcurrentDownloader 

func NewConcurrentDownloader(info S3Info, count int) Downloader

func NewDownloader 

func NewDownloader(info S3Info) Downloader

func NewOrderedDownloader 

func NewOrderedDownloader(info S3Info) Downloader

type InitError 

type InitError struct {
	// contains filtered or unexported fields
}

func (InitError) Error 

func (e InitError) Error() string

type S3Baked 

type S3Baked struct {
	Bucket     string
	HashPrefix string
	Region     string
	// contains filtered or unexported fields
}

func (*S3Baked) DefaultBucket 

func (s *S3Baked) DefaultBucket() string

func (*S3Baked) GetSession 

func (s *S3Baked) GetSession() *session.Session

func (*S3Baked) Init 

func (s *S3Baked) Init(id, secret string) error

func (*S3Baked) MkHashKey 

func (s *S3Baked) MkHashKey(hash string) string

func (*S3Baked) MkURL 

func (s *S3Baked) MkURL(key string) string

MkURL creates a full s3 url from a file hashKey which is usually a prefix/hash combination

type S3Info 

type S3Info interface {
	GetSession() *session.Session
	// The default operational bucket
	DefaultBucket() string
	// Allows prefixing, suffixing around keys based on hash sums
	MkHashKey(string) string
	// Creates an Service URL that utilizes the DefaultBucket
	MkURL(string) string
}

type Service 

type Service struct {
	data.ChunkManager

	Bucket      string
	Region      string
	Timeout     time.Duration
	Concurrency int // Concurrent downloading

	Session *session.Session
	Client  *s3.S3
	// contains filtered or unexported fields
}

Adheres to the data.Service interface

func (*Service) Delete 

func (s *Service) Delete(filepath string) error

Delete the file from FS

func (*Service) Download 

func (s *Service) Download(filepath string, w io.WriterAt, p *data.DownloadParams) error

func (*Service) DownloadRange 

func (s *Service) DownloadRange(filepath string, w io.WriterAt, start, finish int) error

func (*Service) Exists 

func (s *Service) Exists(filepath string) (bool, error)

Checks if the file exists

func (*Service) Get 

func (s *Service) Get(filepath string) (io.ReadCloser, error)

Get gets a file reader from Service

func (*Service) Head 

func (s *Service) Head(filepath string) (*data.HeadInfo, error)

func (*Service) Init 

func (s *Service) Init(id, secret string) error

func (*Service) Put 

func (s *Service) Put(filepath string, r io.Reader) error

func (*Service) SetConcurrency 

func (s *Service) SetConcurrency(concurrency int)

func (*Service) SetTimeout 

func (s *Service) SetTimeout(tm time.Duration)

func (*Service) SignedUrl 

func (s *Service) SignedUrl(filename string, tm time.Duration, config *data.SignedUrlConfig) (string, error)

func (*Service) Stream 

func (s *Service) Stream(filepath string, w io.Writer) error

func (*Service) Upload 

func (s *Service) Upload(filepath string, filesize int, r io.Reader) error

Upload here is generally only used if you do not care about progress or status updates as it hides this information from the user. You only get the final error or non-error code.

type UploadError 

type UploadError struct {
	// contains filtered or unexported fields
}

func (UploadError) Error 

func (e UploadError) Error() string

type Uploader 

type Uploader interface {
	Upload(input *s3manager.UploadInput, options ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
}

Wrapper around Service Uploader for stubbing

func NewUploader 

func NewUploader(info S3Info) Uploader

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.