objcli

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 4, 2024 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Overview

Package objcli exposes a unified 'Client' interface for accessing/managing objects stored in the cloud.

Index

Constants

View Source
const (
	// NoUploadID is a readability definition which should be used by client implementations for cloud providers which
	// do not utilize uploads ids.
	NoUploadID = ""

	// NoPartNumber is a readability definition which should be used by client implementations for cloud providers that
	// do not need numbers to order parts for multipart uploads.
	NoPartNumber = 0
)

Variables

View Source
var (
	// ErrIncludeAndExcludeAreMutuallyExclusive is returned if the user attempts to supply both the include and exclude
	// arguments to 'IterateObjects' at once.
	ErrIncludeAndExcludeAreMutuallyExclusive = errors.New("include/exclude are mutually exclusive")

	// ErrExpectedNoUploadID is returned if the user has provided an upload id for a client which doesn't generate or
	// require upload ids.
	ErrExpectedNoUploadID = errors.New("received an unexpected upload id, cloud provider doesn't required upload ids")
)

Functions

func ShouldIgnore

func ShouldIgnore(query string, include, exclude []*regexp.Regexp) bool

ShouldIgnore uses the given regular expressions to determine if we should skip listing the provided file.

func TestDownloadJSON

func TestDownloadJSON(t *testing.T, client Client, key string, data any)

TestDownloadJSON downloads the given object, unmarshaling it into the provided interface.

func TestDownloadRAW

func TestDownloadRAW(t *testing.T, client Client, key string) []byte

TestDownloadRAW downloads the object as raw data.

func TestListObjects

func TestListObjects(t *testing.T, client Client, prefix string) []*objval.ObjectAttrs

TestListObjects returns the attributes of all the existing objects.

func TestRequireKeyExists

func TestRequireKeyExists(t *testing.T, client Client, key string)

TestRequireKeyExists asserts that the given key exists.

func TestRequireKeyNotFound

func TestRequireKeyNotFound(t *testing.T, client Client, key string)

TestRequireKeyNotFound asserts that the given key does not exist.

func TestUploadJSON

func TestUploadJSON(t *testing.T, client Client, key string, body any)

TestUploadJSON uploads the given data as JSON.

func TestUploadRAW

func TestUploadRAW(t *testing.T, client Client, key string, body []byte)

TestUploadRAW uploads the given raw data.

Types

type AbortMultipartUploadOptions

type AbortMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

AbortMultipartUploadOptions encapsulates the options available when using the 'AbortMultipartUpload' function.

type AppendToObjectOptions

type AppendToObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Body is the data that will be appended.
	Body io.ReadSeeker
}

AppendToObjectOptions encapsulates the options available when using the 'AppendToObject' function.

type Client

type Client interface {
	// Provider returns the cloud provider this client is interfacing with.
	//
	// NOTE: This may be used to change high level behavior which may be cloud provider specific.
	Provider() objval.Provider

	// GetObject retrieves an object form the cloud, an optional byte range argument may be supplied which causes only
	// the requested byte range to be returned.
	//
	// NOTE: The returned objects body must be closed to avoid resource leaks.
	GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

	// GetObjectAttrs returns general metadata about the object with the given key.
	GetObjectAttrs(ctx context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

	// PutObject creates an object in the cloud with the given key/options.
	PutObject(ctx context.Context, opts PutObjectOptions) error

	// CopyObject copies an object from one location to another, this may be within the same bucket.
	//
	// NOTE: Each cloud provider has limitations on the max size for copied objects therefore using this function
	// directly is not recommend; see 'objutil.CopyObject' which handles these nuances.
	CopyObject(ctx context.Context, opts CopyObjectOptions) error

	// AppendToObject appends the provided data to the object with the given key, this is a binary concatenation.
	//
	// NOTE: If the given object does not already exist, it will be created.
	AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

	// DeleteObjects deletes all the objects with the given keys ignoring any errors for keys which are not found.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

	// DeleteDirectory deletes all the objects which have the given prefix.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

	// IterateObjects iterates through the objects a bucket running the provided iteration function for each object
	// which matches the given filtering parameters.
	IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

	// CreateMultipartUpload creates a new multipart upload for the given key.
	//
	// NOTE: Not all clients directly support multipart uploads, the interface exposed should be used as if they do. The
	// underlying client will handle any nuances.
	CreateMultipartUpload(ctx context.Context, opts CreateMultipartUploadOptions) (string, error)

	// ListParts returns the list of parts staged or uploaded for the given upload id/key pair.
	//
	// NOTE: The returned parts will not have their part number populated as this is not stored by all cloud providers.
	ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

	// UploadPart creates/uploads a new part for the multipart upload with the given id.
	UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

	// UploadPartCopy creates a new part for the multipart upload using an existing object (or part of an existing
	// object).
	UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

	// CompleteMultipartUpload completes the multipart upload with the given id, the given parts should be provided in
	// the order that they should be constructed.
	CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

	// AbortMultipartUpload aborts the multipart upload with the given id whilst cleaning up any abandoned parts.
	AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

	// Close the underlying client/SDK where applicable; use of the client, or the underlying SDK after a call to Close
	// has undefined behavior. This is required to stop memory leaks in GCP.
	Close() error
}

Client is a unified interface for accessing/managing objects stored in the cloud.

type CompleteMultipartUploadOptions

type CompleteMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Parts is an ordered list of parts that should be constructed into the completed object.
	Parts []objval.Part
}

CompleteMultipartUploadOptions encapsulates the options available when using the 'CompleteMultipartUpload' function.

type CopyObjectOptions

type CopyObjectOptions struct {
	// DestinationBucket is the bucket the will be copied into.
	DestinationBucket string

	// DestinationKey is the key for the copied object.
	DestinationKey string

	// SourceBucket is the bucket containing the object being copied.
	SourceBucket string

	// SourceKey is the key of the object being copied.
	SourceKey string
}

CopyObjectOptions encapsulates the options available when using the 'CopyObject' function.

type CreateMultipartUploadOptions

type CreateMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

CreateMultipartUploadOptions encapsulates the options available when using the 'CreateMultipartUpload' function.

type DeleteDirectoryOptions

type DeleteDirectoryOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Prefix is the prefix that will be operated on.
	Prefix string
}

DeleteDirectoryOptions encapsulates the options available when using the 'DeleteDirectory' function.

type DeleteObjectsOptions

type DeleteObjectsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Keys are the keys that will be deleted.
	Keys []string
}

DeleteObjectsOptions encapsulates the options available when using the 'DeleteObjects' function.

type GetObjectAttrsOptions

type GetObjectAttrsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

GetObjectAttrsOptions encapsulates the options available when using the 'GetObjectAttrs' function.

type GetObjectOptions

type GetObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// ByteRange allows specifying a start/end offset to be operated on.
	ByteRange *objval.ByteRange
}

GetObjectOptions encapsulates the options available when using the 'GetObject' function.

type IterateFunc

type IterateFunc func(attrs *objval.ObjectAttrs) error

IterateFunc is the function used when iterating over objects, this function will be called once for each object whose key matches the provided filtering.

type IterateObjectsOptions

type IterateObjectsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Prefix is the prefix that will listed.
	Prefix string

	// Delimiter use to group keys e.g. '/' causes listing to only occur within a "directory".
	Delimiter string

	// Include objects where the keys match any of the given regular expressions.
	Include []*regexp.Regexp

	// Exclude objects where the keys match any of the given regular expressions.
	Exclude []*regexp.Regexp

	// Func is executed for each object listed.
	Func IterateFunc
}

IterateObjectsOptions encapsulates the options available when using the 'IterateObjects' function.

type ListPartsOptions

type ListPartsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID - string
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

ListPartsOptions encapsulates the options available when using the 'ListParts' function.

type MockClient

type MockClient struct {
	mock.Mock
}

MockClient is an autogenerated mock type for the Client type

func NewMockClient

func NewMockClient(t interface {
	mock.TestingT
	Cleanup(func())
},
) *MockClient

NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockClient) AbortMultipartUpload

func (_m *MockClient) AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

AbortMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) AppendToObject

func (_m *MockClient) AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

AppendToObject provides a mock function with given fields: ctx, opts

func (*MockClient) Close

func (_m *MockClient) Close() error

Close provides a mock function with given fields:

func (*MockClient) CompleteMultipartUpload

func (_m *MockClient) CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

CompleteMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) CopyObject

func (_m *MockClient) CopyObject(ctx context.Context, opts CopyObjectOptions) error

CopyObject provides a mock function with given fields: ctx, opts

func (*MockClient) CreateMultipartUpload

func (_m *MockClient) CreateMultipartUpload(ctx context.Context, opts CreateMultipartUploadOptions) (string, error)

CreateMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) DeleteDirectory

func (_m *MockClient) DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

DeleteDirectory provides a mock function with given fields: ctx, opts

func (*MockClient) DeleteObjects

func (_m *MockClient) DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

DeleteObjects provides a mock function with given fields: ctx, opts

func (*MockClient) GetObject

func (_m *MockClient) GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

GetObject provides a mock function with given fields: ctx, opts

func (*MockClient) GetObjectAttrs

func (_m *MockClient) GetObjectAttrs(ctx context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

GetObjectAttrs provides a mock function with given fields: ctx, opts

func (*MockClient) IterateObjects

func (_m *MockClient) IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

IterateObjects provides a mock function with given fields: ctx, opts

func (*MockClient) ListParts

func (_m *MockClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

ListParts provides a mock function with given fields: ctx, opts

func (*MockClient) Provider

func (_m *MockClient) Provider() objval.Provider

Provider provides a mock function with given fields:

func (*MockClient) PutObject

func (_m *MockClient) PutObject(ctx context.Context, opts PutObjectOptions) error

PutObject provides a mock function with given fields: ctx, opts

func (*MockClient) UploadPart

func (_m *MockClient) UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

UploadPart provides a mock function with given fields: ctx, opts

func (*MockClient) UploadPartCopy

func (_m *MockClient) UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

UploadPartCopy provides a mock function with given fields: ctx, opts

type PutObjectOptions

type PutObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Body is the data that will be uploaded.
	//
	// NOTE: Required to be a 'ReadSeeker' to support checksum calculation/validation.
	Body io.ReadSeeker
}

PutObjectOptions encapsulates the options available when using the 'PutObject' function.

type RateLimitedClient

type RateLimitedClient struct {
	// contains filtered or unexported fields
}

RateLimitedClient implements objcli.Client interface mostly by deferring to the underlying Client, but where the methods which involve uploading/downloading objects, the rate limiter is used to control the rate of data transfer.

The rate-limited methods are:

- GetObject - PutObject - AppendToObject - UploadPart

func NewRateLimitedClient

func NewRateLimitedClient(c Client, rl *rate.Limiter) *RateLimitedClient

NewRateLimitedClient returns a RateLimitedClient.

func (*RateLimitedClient) AbortMultipartUpload

func (r *RateLimitedClient) AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

func (*RateLimitedClient) AppendToObject

func (r *RateLimitedClient) AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

func (*RateLimitedClient) CompleteMultipartUpload

func (r *RateLimitedClient) CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

func (*RateLimitedClient) CreateMultipartUpload

func (r *RateLimitedClient) CreateMultipartUpload(
	ctx context.Context,
	opts CreateMultipartUploadOptions,
) (string, error)

func (*RateLimitedClient) DeleteDirectory

func (r *RateLimitedClient) DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

func (*RateLimitedClient) DeleteObjects

func (r *RateLimitedClient) DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

func (*RateLimitedClient) GetObject

func (r *RateLimitedClient) GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

func (*RateLimitedClient) GetObjectAttrs

func (r *RateLimitedClient) GetObjectAttrs(
	ctx context.Context,
	opts GetObjectAttrsOptions,
) (*objval.ObjectAttrs, error)

func (*RateLimitedClient) IterateObjects

func (r *RateLimitedClient) IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

func (*RateLimitedClient) ListParts

func (r *RateLimitedClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

func (*RateLimitedClient) Provider

func (r *RateLimitedClient) Provider() objval.Provider

func (*RateLimitedClient) PutObject

func (r *RateLimitedClient) PutObject(ctx context.Context, opts PutObjectOptions) error

func (*RateLimitedClient) UploadPart

func (r *RateLimitedClient) UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

func (*RateLimitedClient) UploadPartCopy

func (r *RateLimitedClient) UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

type TestClient

type TestClient struct {

	// Buckets is the in memory state maintained by the client. Internally, access is guarded by a mutex, however, it's
	// not safe/recommended to access this attribute whilst a test is running; it should only be used to inspect state
	// (to perform assertions) once testing is complete.
	Buckets objval.TestBuckets
	// contains filtered or unexported fields
}

TestClient implementation of the 'Client' interface which stores state in memory, and can be used to avoid having to manually mock a client during unit testing.

func NewTestClient

func NewTestClient(t *testing.T, provider objval.Provider) *TestClient

NewTestClient returns a new test client, which has no buckets/objects.

func (*TestClient) AbortMultipartUpload

func (t *TestClient) AbortMultipartUpload(_ context.Context, opts AbortMultipartUploadOptions) error

func (*TestClient) AppendToObject

func (t *TestClient) AppendToObject(_ context.Context, opts AppendToObjectOptions) error

func (*TestClient) Close

func (t *TestClient) Close() error

func (*TestClient) CompleteMultipartUpload

func (t *TestClient) CompleteMultipartUpload(_ context.Context, opts CompleteMultipartUploadOptions) error

func (*TestClient) CopyObject

func (t *TestClient) CopyObject(_ context.Context, opts CopyObjectOptions) error

func (*TestClient) CreateMultipartUpload

func (t *TestClient) CreateMultipartUpload(_ context.Context, _ CreateMultipartUploadOptions) (string, error)

func (*TestClient) DeleteDirectory

func (t *TestClient) DeleteDirectory(_ context.Context, opts DeleteDirectoryOptions) error

func (*TestClient) DeleteObjects

func (t *TestClient) DeleteObjects(_ context.Context, opts DeleteObjectsOptions) error

func (*TestClient) GetObject

func (t *TestClient) GetObject(_ context.Context, opts GetObjectOptions) (*objval.Object, error)

func (*TestClient) GetObjectAttrs

func (t *TestClient) GetObjectAttrs(_ context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

func (*TestClient) IterateObjects

func (t *TestClient) IterateObjects(_ context.Context, opts IterateObjectsOptions) error

func (*TestClient) ListParts

func (t *TestClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

func (*TestClient) Provider

func (t *TestClient) Provider() objval.Provider

func (*TestClient) PutObject

func (t *TestClient) PutObject(_ context.Context, opts PutObjectOptions) error

func (*TestClient) UploadPart

func (t *TestClient) UploadPart(_ context.Context, opts UploadPartOptions) (objval.Part, error)

func (*TestClient) UploadPartCopy

func (t *TestClient) UploadPartCopy(_ context.Context, opts UploadPartCopyOptions) (objval.Part, error)

type UploadPartCopyOptions

type UploadPartCopyOptions struct {
	// DestinationBucket is the bucket the will be copied into.
	DestinationBucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// DestinationKey is the key for the copied object.
	DestinationKey string

	// SourceBucket is the bucket containing the object being copied.
	SourceBucket string

	// SourceKey is the key of the object being copied.
	SourceKey string

	// Number is the number that will be assigned to the part.
	//
	// NOTE: Should be between 1-10,000 and is used for the ordering of parts upon completion.
	Number int

	// ByteRange allows specifying a start/end offset to be operated on.
	//
	// NOTE: Not supported by all cloud providers.
	ByteRange *objval.ByteRange
}

UploadPartCopyOptions encapsulates the options available when using the 'UploadPartCopy' function.

type UploadPartOptions

type UploadPartOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID - string
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Number is the number that will be assigned to the part.
	//
	// NOTE: Should be between 1-10,000 and is used for the ordering of parts upon completion.
	Number int

	// Body is the data that will be uploaded.
	Body io.ReadSeeker
}

UploadPartOptions encapsulates the options available when using the 'UploadPart' function.

Directories

Path Synopsis
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL