bucket

package

v0.0.0 Latest Latest Go to latest Published: Apr 9, 2020 License: MIT Imports: 20 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/atsu/goat

Documentation ¶

Index ¶

Constants
type Bucket
- func NewBucket(name string) (Bucket, error)
type BucketIterator
type GCS
type PipelineObject
- func NewPipelineObject(key string, etag string) *PipelineObject
- func (oc PipelineObject) MatchesBucket(bc Bucket) bool
- func (oc PipelineObject) MatchesStream(sc *stream.StreamConfig) booldeprecated
type S3
type UploadCallback

Constants ¶

View Source

const DefaultStorageClass = "STANDARD"

View Source

const GoogleStorageClass = "STANDARD"

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type Bucket ¶

type Bucket interface {
	GetName() string
	SetName(string)
	GetOwner() string
	GetPipeline() string

	NewSession() error
	GetObject(string) (io.ReadCloser, error)
	GetObjectIfMatch(string, string) (io.ReadCloser, error)

	Path() string
	KeyPrefix(time.Time) string
	Upload(*bytes.Buffer) error
	ListObjects(string, string, BucketIterator) error

	SetFlags()
}

func NewBucket ¶

func NewBucket(name string) (Bucket, error)

NewBucket takes s3://foo or gs://bar and returns either S3{} or GCS{}, respectively.

Note: prefix (s3:// or gs://) is stripped.

The default is S3{}

NewBucket returns error if illegal characters are present.

type BucketIterator ¶

type BucketIterator func(*PipelineObject) bool

type GCS ¶

type GCS struct {
	Pipeline string `default:"pipeline.unset" json:"pipeline"`
	Owner    string `default:"data" json:"owner"`
	Bucket   string `default:"atsu.unset" json:"bucket"`
	Suffix   string `default:"json.gz" json:"suffix"`
	Name     string `json:"name"`

	Objects int `json:"objects"`
	Bytes   int `json:"bytes"`
	// contains filtered or unexported fields
}

uprovides GCS-related configuration

func (*GCS) GetName ¶

func (g *GCS) GetName() string

func (*GCS) GetObject ¶

func (g *GCS) GetObject(key string) (io.ReadCloser, error)

func (*GCS) GetObjectIfMatch ¶

func (g *GCS) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)

func (*GCS) GetOwner ¶

func (g *GCS) GetOwner() string

func (*GCS) GetPipeline ¶

func (g *GCS) GetPipeline() string

func (*GCS) GetSession ¶

func (g *GCS) GetSession() *storage.Client

func (*GCS) Key ¶

func (g *GCS) Key(t time.Time) string

Key returns the full GCS key

func (*GCS) KeyPrefix ¶

func (g *GCS) KeyPrefix(t time.Time) string

KeyPrefix generates the GCS prefix, including time-based prefix

func (*GCS) ListObjects ¶

func (g *GCS) ListObjects(path string, latest string, helper BucketIterator) error

func (*GCS) NewSession ¶

func (g *GCS) NewSession() error

NewSession creates a new Google session; requires GOOGLE_APPLICATION_CREDENTIALS

func (*GCS) Path ¶

func (g *GCS) Path() string

Path generates the GCS prefix, excluding time-based prefix

func (*GCS) SetFlags ¶

func (g *GCS) SetFlags()

SetFlags to install various command-line flag(s)

func (*GCS) SetName ¶

func (g *GCS) SetName(o string)

func (*GCS) String ¶

func (g *GCS) String() string

String returns JSON representation

func (*GCS) Upload ¶

func (g *GCS) Upload(b *bytes.Buffer) error

Upload uploads the provided buffer via GCS

func (*GCS) UploadCallback ¶

func (g *GCS) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error

func (*GCS) UploadKey ¶

func (g *GCS) UploadKey(key string, b *bytes.Buffer) error

func (*GCS) UploadKeyCallback ¶

func (g *GCS) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error

type PipelineObject ¶

type PipelineObject struct {
	Owner    string
	Pipeline string
	Name     string
	Date     time.Time
	File     string

	Key  string
	Etag string
}

PipelineObject represents an individual object stored in a Bucket

func NewPipelineObject ¶

func NewPipelineObject(key string, etag string) *PipelineObject

NewPipelineObject parses an S3 key and returns an PipelineObject

func (PipelineObject) MatchesBucket ¶

func (oc PipelineObject) MatchesBucket(bc Bucket) bool

Ensure this object belongs to us

func (PipelineObject) MatchesStream deprecated

func (oc PipelineObject) MatchesStream(sc *stream.StreamConfig) bool

Deprecated: Use MatchesBucket instead

type S3 ¶

type S3 struct {
	Pipeline string `default:"pipeline.unset" json:"pipeline"`
	Owner    string `default:"unset" json:"owner"`
	Bucket   string `default:"atsu.unset" json:"bucket"`
	Suffix   string `default:"json.gz" json:"suffix"`
	Name     string `json:"name"`

	Objects int `json:"objects"`
	Bytes   int `json:"bytes"`
	// contains filtered or unexported fields
}

S3 provides S3-related configuration

func (*S3) GetCredentials ¶

func (s *S3) GetCredentials() *credentials.Credentials

func (*S3) GetName ¶

func (s *S3) GetName() string

func (*S3) GetObject ¶

func (s *S3) GetObject(key string) (io.ReadCloser, error)

func (*S3) GetObjectIfMatch ¶

func (s *S3) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)

func (*S3) GetOwner ¶

func (s *S3) GetOwner() string

func (*S3) GetPipeline ¶

func (s *S3) GetPipeline() string

func (*S3) GetSession ¶

func (s *S3) GetSession() *session.Session

func (*S3) Key ¶

func (s *S3) Key(t time.Time) string

Key returns the full S3 key

func (*S3) KeyPrefix ¶

func (s *S3) KeyPrefix(t time.Time) string

KeyPrefix generates the S3 prefix, including time-based prefix

func (*S3) ListObjects ¶

func (s *S3) ListObjects(path string, latest string, iterator BucketIterator) error

func (*S3) NewSession ¶

func (s *S3) NewSession() error

NewSession creates a new AWS session using "atsu.io" credentials

func (*S3) Path ¶

func (s *S3) Path() string

Path generates the S3 prefix, excluding time-based prefix

func (*S3) SetFlags ¶

func (s *S3) SetFlags()

SetFlags to install various command-line flag(s)

func (*S3) SetName ¶

func (s *S3) SetName(o string)

func (*S3) String ¶

func (s *S3) String() string

String returns JSON representation

func (*S3) Upload ¶

func (s *S3) Upload(b *bytes.Buffer) error

Upload uploads the provided buffer via S3

func (*S3) UploadCallback ¶

func (s *S3) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error

func (*S3) UploadKey ¶

func (s *S3) UploadKey(key string, b *bytes.Buffer) error

func (*S3) UploadKeyCallback ¶

func (s *S3) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error

type UploadCallback ¶

type UploadCallback func(string, *bytes.Buffer, interface{}, error) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL