objcli

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 17, 2023 License: Apache-2.0 Imports: 21 Imported by: 2

Documentation

Overview

Package objcli exposes a unified 'Client' interface for accessing/managing objects stored in the cloud.

Index

Constants

View Source
const (
	// NoUploadID is a readability definition which should be used by client implementations for cloud providers which
	// do not utilize uploads ids.
	NoUploadID = ""

	// NoPartNumber is a readability definition which should be used by client implementations for cloud providers that
	// do not need numbers to order parts for multipart uploads.
	NoPartNumber = 0
)

Variables

View Source
var (
	// ErrIncludeAndExcludeAreMutuallyExclusive is returned if the user attempts to supply both the include and exclude
	// arguments to 'IterateObjects' at once.
	ErrIncludeAndExcludeAreMutuallyExclusive = errors.New("include/exclude are mutually exclusive")

	// ErrExpectedNoUploadID is returned if the user has provided an upload id for a client which doesn't generate or
	// require upload ids.
	ErrExpectedNoUploadID = errors.New("received an unexpected upload id, cloud provider doesn't required upload ids")
)

Functions

func ShouldIgnore

func ShouldIgnore(query string, include, exclude []*regexp.Regexp) bool

ShouldIgnore uses the given regular expressions to determine if we should skip listing the provided file.

func TestDownloadJSON

func TestDownloadJSON(t *testing.T, client Client, key string, data any)

TestDownloadJSON downloads the given object, unmarshaling it into the provided interface.

func TestDownloadRAW

func TestDownloadRAW(t *testing.T, client Client, key string) []byte

TestDownloadRAW downloads the object as raw data.

func TestListObjects

func TestListObjects(t *testing.T, client Client, prefix string) []*objval.ObjectAttrs

TestListObjects returns the attributes of all the existing objects.

func TestRequireKeyExists

func TestRequireKeyExists(t *testing.T, client Client, key string)

TestRequireKeyExists asserts that the given key exists.

func TestRequireKeyNotFound

func TestRequireKeyNotFound(t *testing.T, client Client, key string)

TestRequireKeyNotFound asserts that the given key does not exist.

func TestUploadJSON

func TestUploadJSON(t *testing.T, client Client, key string, body any)

TestUploadJSON uploads the given data as JSON.

func TestUploadRAW

func TestUploadRAW(t *testing.T, client Client, key string, body []byte)

TestUploadRAW uploads the given raw data.

Types

type Client

type Client interface {
	// Provider returns the cloud provider this client is interfacing with.
	//
	// NOTE: This may be used to change high level behavior which may be cloud provider specific.
	Provider() objval.Provider

	// GetObject retrieves an object form the cloud, an optional byte range argument may be supplied which causes only
	// the requested byte range to be returned.
	//
	// NOTE: The returned objects body must be closed to avoid resource leaks.
	GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)

	// GetObjectAttrs returns general metadata about the object with the given key.
	GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)

	// PutObject creates an object in the cloud with the given key/options.
	//
	// NOTE: The body is required to be a 'ReadSeeker' to support checksum calculation/validation.
	PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error

	// AppendToObject appends the provided data to the object with the given key, this is a binary concatenation.
	//
	// NOTE: If the given object does not already exist, it will be created.
	AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error

	// DeleteObjects deletes all the objects with the given keys ignoring any errors for keys which are not found.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteObjects(ctx context.Context, bucket string, keys ...string) error

	// DeleteDirectory deletes all the objects which have the given prefix.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteDirectory(ctx context.Context, bucket, prefix string) error

	// IterateObjects iterates through the objects a bucket running the provided iteration function for each object
	// which matches the given filtering parameters.
	IterateObjects(
		ctx context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc,
	) error

	// CreateMultipartUpload creates a new multipart upload for the given key.
	//
	// NOTE: Not all clients directly support multipart uploads, the interface exposed should be used as if they do. The
	// underlying client will handle any nuances.
	CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)

	// ListParts returns the list of parts staged or uploaded for the given upload id/key pair.
	//
	// NOTE: The returned parts will not have their part number populated as this is not stored by all cloud providers.
	ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)

	// UploadPart creates/uploads a new part for the multipart upload with the given id.
	//
	// NOTE: The part 'number' should be between 1-10,000 and is used for the ordering of parts upon completion.
	UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)

	// UploadPartCopy creates a new part for the multipart upload using an existing object (or part of an existing
	// object).
	//
	// NOTE: Not all cloud providers support providing a byte range.
	UploadPartCopy(
		ctx context.Context, bucket, id, dst, src string, number int, br *objval.ByteRange,
	) (objval.Part, error)

	// CompleteMultipartUpload completes the multipart upload with the given id, the given parts should be provided in
	// the order that they should be constructed.
	CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error

	// AbortMultipartUpload aborts the multipart upload with the given id whilst cleaning up any abandoned parts.
	AbortMultipartUpload(ctx context.Context, bucket, id, key string) error
}

Client is a unified interface for accessing/managing objects stored in the cloud.

type IterateFunc

type IterateFunc func(attrs *objval.ObjectAttrs) error

IterateFunc is the function used when iterating over objects, this function will be called once for each object whose key matches the provided filtering.

type MockClient

type MockClient struct {
	mock.Mock
}

MockClient is an autogenerated mock type for the Client type

func NewMockClient

func NewMockClient(t mockConstructorTestingTNewMockClient) *MockClient

NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.

func (*MockClient) AbortMultipartUpload

func (_m *MockClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error

AbortMultipartUpload provides a mock function with given fields: ctx, bucket, id, key

func (*MockClient) AppendToObject

func (_m *MockClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error

AppendToObject provides a mock function with given fields: ctx, bucket, key, data

func (*MockClient) CompleteMultipartUpload

func (_m *MockClient) CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error

CompleteMultipartUpload provides a mock function with given fields: ctx, bucket, id, key, parts

func (*MockClient) CreateMultipartUpload

func (_m *MockClient) CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)

CreateMultipartUpload provides a mock function with given fields: ctx, bucket, key

func (*MockClient) DeleteDirectory

func (_m *MockClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error

DeleteDirectory provides a mock function with given fields: ctx, bucket, prefix

func (*MockClient) DeleteObjects

func (_m *MockClient) DeleteObjects(ctx context.Context, bucket string, keys ...string) error

DeleteObjects provides a mock function with given fields: ctx, bucket, keys

func (*MockClient) GetObject

func (_m *MockClient) GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)

GetObject provides a mock function with given fields: ctx, bucket, key, br

func (*MockClient) GetObjectAttrs

func (_m *MockClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)

GetObjectAttrs provides a mock function with given fields: ctx, bucket, key

func (*MockClient) IterateObjects

func (_m *MockClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc) error

IterateObjects provides a mock function with given fields: ctx, bucket, prefix, delimiter, include, exclude, fn

func (*MockClient) ListParts

func (_m *MockClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)

ListParts provides a mock function with given fields: ctx, bucket, id, key

func (*MockClient) Provider

func (_m *MockClient) Provider() objval.Provider

Provider provides a mock function with given fields:

func (*MockClient) PutObject

func (_m *MockClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error

PutObject provides a mock function with given fields: ctx, bucket, key, body

func (*MockClient) UploadPart

func (_m *MockClient) UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)

UploadPart provides a mock function with given fields: ctx, bucket, id, key, number, body

func (*MockClient) UploadPartCopy

func (_m *MockClient) UploadPartCopy(ctx context.Context, bucket, id, dst, src string, number int, br *objval.ByteRange) (objval.Part, error)

UploadPartCopy provides a mock function with given fields: ctx, bucket, id, dst, src, number, br

type RateLimitedClient

type RateLimitedClient struct {
	// contains filtered or unexported fields
}

RateLimitedClient implements objcli.Client interface mostly by deferring to the underlying Client, but where the methods which involve uploading/downloading objects, the rate limiter is used to control the rate of data transfer.

The rate-limited methods are:

* GetObject

* PutObject

* AppendToObject

* UploadPart

func NewRateLimitedClient

func NewRateLimitedClient(c Client, rl *rate.Limiter) *RateLimitedClient

NewRateLimitedClient returns a RateLimitedClient.

func (*RateLimitedClient) AbortMultipartUpload

func (r *RateLimitedClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error

func (*RateLimitedClient) AppendToObject

func (r *RateLimitedClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error

func (*RateLimitedClient) CompleteMultipartUpload

func (r *RateLimitedClient) CompleteMultipartUpload(ctx context.Context, bucket, id, key string,
	parts ...objval.Part,
) error

func (*RateLimitedClient) CreateMultipartUpload

func (r *RateLimitedClient) CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)

func (*RateLimitedClient) DeleteDirectory

func (r *RateLimitedClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error

func (*RateLimitedClient) DeleteObjects

func (r *RateLimitedClient) DeleteObjects(ctx context.Context, bucket string, keys ...string) error

func (*RateLimitedClient) GetObject

func (r *RateLimitedClient) GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object,
	error,
)

func (*RateLimitedClient) GetObjectAttrs

func (r *RateLimitedClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)

func (*RateLimitedClient) IterateObjects

func (r *RateLimitedClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, include,
	exclude []*regexp.Regexp, fn IterateFunc,
) error

func (*RateLimitedClient) ListParts

func (r *RateLimitedClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)

func (*RateLimitedClient) Provider

func (r *RateLimitedClient) Provider() objval.Provider

func (*RateLimitedClient) PutObject

func (r *RateLimitedClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error

func (*RateLimitedClient) UploadPart

func (r *RateLimitedClient) UploadPart(ctx context.Context, bucket, id, key string, number int,
	body io.ReadSeeker,
) (objval.Part, error)

func (*RateLimitedClient) UploadPartCopy

func (r *RateLimitedClient) UploadPartCopy(ctx context.Context, bucket, id, dst, src string, number int,
	br *objval.ByteRange,
) (objval.Part, error)

type TestClient

type TestClient struct {

	// Buckets is the in memory state maintained by the client. Internally, access is guarded by a mutex, however, it's
	// not safe/recommended to access this attribute whilst a test is running; it should only be used to inspect state
	// (to perform assertions) once testing is complete.
	Buckets objval.TestBuckets
	// contains filtered or unexported fields
}

TestClient implementation of the 'Client' interface which stores state in memory, and can be used to avoid having to manually mock a client during unit testing.

func NewTestClient

func NewTestClient(t *testing.T, provider objval.Provider) *TestClient

NewTestClient returns a new test client, which has no buckets/objects.

func (*TestClient) AbortMultipartUpload

func (t *TestClient) AbortMultipartUpload(_ context.Context, bucket, id, key string) error

func (*TestClient) AppendToObject

func (t *TestClient) AppendToObject(_ context.Context, bucket, key string, data io.ReadSeeker) error

func (*TestClient) CompleteMultipartUpload

func (t *TestClient) CompleteMultipartUpload(_ context.Context, bucket, id, key string, parts ...objval.Part) error

func (*TestClient) CreateMultipartUpload

func (t *TestClient) CreateMultipartUpload(_ context.Context, _, _ string) (string, error)

func (*TestClient) DeleteDirectory

func (t *TestClient) DeleteDirectory(_ context.Context, bucket, prefix string) error

func (*TestClient) DeleteObjects

func (t *TestClient) DeleteObjects(_ context.Context, bucket string, keys ...string) error

func (*TestClient) GetObject

func (t *TestClient) GetObject(_ context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)

func (*TestClient) GetObjectAttrs

func (t *TestClient) GetObjectAttrs(_ context.Context, bucket, key string) (*objval.ObjectAttrs, error)

func (*TestClient) IterateObjects

func (t *TestClient) IterateObjects(_ context.Context, bucket, prefix, delimiter string, include,
	exclude []*regexp.Regexp, fn IterateFunc,
) error

func (*TestClient) ListParts

func (t *TestClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)

func (*TestClient) Provider

func (t *TestClient) Provider() objval.Provider

func (*TestClient) PutObject

func (t *TestClient) PutObject(_ context.Context, bucket, key string, body io.ReadSeeker) error

func (*TestClient) UploadPart

func (t *TestClient) UploadPart(
	_ context.Context,
	bucket, id, key string,
	number int,
	body io.ReadSeeker,
) (objval.Part, error)

func (*TestClient) UploadPartCopy

func (t *TestClient) UploadPartCopy(_ context.Context, bucket, id, dst, src string, number int,
	br *objval.ByteRange,
) (objval.Part, error)

Directories

Path Synopsis
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL