dstore

package module
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 27, 2020 License: Apache-2.0 Imports: 28 Imported by: 78

README

dfuse Storage Abstraction

reference License

dstore is a simple abstraction on top of Local storage and Cloud storage. It handles commonly used functions to store things (locally, or on cloud storage providers), list files, delete, etc..

It is used by dfuse.

Contributing

Issues and PR in this repo related strictly to the dstore library.

Report any protocol-specific issues in their respective repositories

Please first refer to the general dfuse contribution guide, if you wish to contribute to this code base.

License

Apache 2.0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var StopIteration = errors.New("stop iteration")

Functions

This section is empty.

Types

type AzureStore 

type AzureStore struct {
	// contains filtered or unexported fields
}

func NewAzureStore 

func NewAzureStore(baseURL *url.URL, extension, compressionType string, overwrite bool) (*AzureStore, error)

func (AzureStore) DeleteObject 

func (a AzureStore) DeleteObject(ctx context.Context, base string) error

func (AzureStore) FileExists 

func (a AzureStore) FileExists(ctx context.Context, base string) (bool, error)

func (AzureStore) ListFiles 

func (a AzureStore) ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

func (AzureStore) ObjectPath 

func (a AzureStore) ObjectPath(name string) string

func (AzureStore) OpenObject 

func (a AzureStore) OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)

func (AzureStore) Overwrite 

func (c AzureStore) Overwrite() bool

func (AzureStore) PushLocalFile 

func (a AzureStore) PushLocalFile(ctx context.Context, localFile, toBaseName string) (err error)

func (AzureStore) SetOverwrite 

func (c AzureStore) SetOverwrite(in bool)

func (AzureStore) Walk 

func (a AzureStore) Walk(ctx context.Context, prefix, ignoreSuffix string, f func(filename string) (err error)) error

func (AzureStore) WriteObject 

func (a AzureStore) WriteObject(ctx context.Context, base string, f io.Reader) (err error)

type BufferedFileReadCloser 

type BufferedFileReadCloser struct {
	// contains filtered or unexported fields
}

func NewBufferedFileReadCloser 

func NewBufferedFileReadCloser(file *os.File) *BufferedFileReadCloser

func (*BufferedFileReadCloser) Close 

func (readCloser *BufferedFileReadCloser) Close() error

func (*BufferedFileReadCloser) Read 

func (readCloser *BufferedFileReadCloser) Read(p []byte) (n int, err error)

type GSStore 

type GSStore struct {
	// contains filtered or unexported fields
}

func NewGSStore 

func NewGSStore(baseURL *url.URL, extension, compressionType string, overwrite bool) (*GSStore, error)

func (*GSStore) DeleteObject 

func (s *GSStore) DeleteObject(ctx context.Context, base string) error

func (*GSStore) FileExists 

func (s *GSStore) FileExists(ctx context.Context, base string) (bool, error)

func (*GSStore) ListFiles 

func (s *GSStore) ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

func (*GSStore) ObjectPath 

func (s *GSStore) ObjectPath(name string) string

func (*GSStore) OpenObject 

func (s *GSStore) OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)

func (GSStore) Overwrite 

func (c GSStore) Overwrite() bool

func (*GSStore) PushLocalFile 

func (s *GSStore) PushLocalFile(ctx context.Context, localFile, toBaseName string) (err error)

func (GSStore) SetOverwrite 

func (c GSStore) SetOverwrite(in bool)

func (*GSStore) Walk 

func (s *GSStore) Walk(ctx context.Context, prefix, _ string, f func(filename string) (err error)) error

func (*GSStore) WriteObject 

func (s *GSStore) WriteObject(ctx context.Context, base string, f io.Reader) (err error)

type GZipReadCloser 

type GZipReadCloser struct {
	*gzip.Reader
	// contains filtered or unexported fields
}

func NewGZipReadCloser 

func NewGZipReadCloser(src io.ReadCloser) (*GZipReadCloser, error)

func (*GZipReadCloser) Close 

func (g *GZipReadCloser) Close() error

type LocalStore 

type LocalStore struct {
	// contains filtered or unexported fields
}

func NewLocalStore 

func NewLocalStore(basePath, extension, compressionType string, overwrite bool) (*LocalStore, error)

func (*LocalStore) DeleteObject 

func (s *LocalStore) DeleteObject(ctx context.Context, base string) error

func (*LocalStore) FileExists 

func (s *LocalStore) FileExists(ctx context.Context, base string) (bool, error)

func (*LocalStore) ListFiles 

func (s *LocalStore) ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

func (*LocalStore) ObjectPath 

func (s *LocalStore) ObjectPath(name string) string

func (*LocalStore) OpenObject 

func (s *LocalStore) OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)

func (LocalStore) Overwrite 

func (c LocalStore) Overwrite() bool

func (*LocalStore) PushLocalFile 

func (s *LocalStore) PushLocalFile(ctx context.Context, localFile, toBaseName string) (err error)

func (LocalStore) SetOverwrite 

func (c LocalStore) SetOverwrite(in bool)

func (*LocalStore) Walk 

func (s *LocalStore) Walk(ctx context.Context, prefix, ignoreSuffix string, f func(filename string) (err error)) error

func (*LocalStore) WriteObject 

func (s *LocalStore) WriteObject(ctx context.Context, base string, reader io.Reader) (err error)

type MockStore 

type MockStore struct {
	// contains filtered or unexported fields
}

func NewMockStore 

func NewMockStore(writeFunc func(base string, f io.Reader) (err error)) *MockStore

func (*MockStore) DeleteObject 

func (m *MockStore) DeleteObject(ctx context.Context, base string) error

func (*MockStore) FileExists 

func (m *MockStore) FileExists(ctx context.Context, base string) (bool, error)

func (*MockStore) ListFiles 

func (s *MockStore) ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

func (*MockStore) ObjectPath 

func (m *MockStore) ObjectPath(base string) string

func (*MockStore) OpenObject 

func (m *MockStore) OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)

func (*MockStore) Overwrite 

func (m *MockStore) Overwrite() bool

func (*MockStore) PushLocalFile 

func (m *MockStore) PushLocalFile(ctx context.Context, localFile string, toBaseName string) (err error)

func (*MockStore) SetFile 

func (m *MockStore) SetFile(name string, content []byte)

SetFile sets the content of a file. Set the value "err" to trigger an error when reading this file.

func (*MockStore) SetOverwrite 

func (s *MockStore) SetOverwrite(in bool)

func (*MockStore) Walk 

func (m *MockStore) Walk(ctx context.Context, prefix, _ string, f func(filename string) error) error

func (*MockStore) WriteObject 

func (m *MockStore) WriteObject(ctx context.Context, base string, f io.Reader) (err error)

type S3Store 

type S3Store struct {
	// contains filtered or unexported fields
}

func NewS3Store 

func NewS3Store(baseURL *url.URL, extension, compressionType string, overwrite bool) (*S3Store, error)

func (*S3Store) DeleteObject 

func (s *S3Store) DeleteObject(ctx context.Context, base string) error

func (*S3Store) FileExists 

func (s *S3Store) FileExists(ctx context.Context, base string) (bool, error)

func (*S3Store) ListFiles 

func (s *S3Store) ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

func (*S3Store) ObjectPath 

func (s *S3Store) ObjectPath(name string) string

func (*S3Store) OpenObject 

func (s *S3Store) OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)

func (S3Store) Overwrite 

func (c S3Store) Overwrite() bool

func (*S3Store) PushLocalFile 

func (s *S3Store) PushLocalFile(ctx context.Context, localFile, toBaseName string) (err error)

func (S3Store) SetOverwrite 

func (c S3Store) SetOverwrite(in bool)

func (*S3Store) Walk 

func (s *S3Store) Walk(ctx context.Context, prefix, _ string, f func(filename string) (err error)) error

func (*S3Store) WriteObject 

func (s *S3Store) WriteObject(ctx context.Context, base string, f io.Reader) (err error)

type SimpleGStore 

type SimpleGStore struct {
	// contains filtered or unexported fields
}

func NewSimpleGStore 

func NewSimpleGStore(baseURL string) (*SimpleGStore, error)

func (*SimpleGStore) Context 

func (s *SimpleGStore) Context() (ctx context.Context, cancel func())

func (*SimpleGStore) FileExists 

func (s *SimpleGStore) FileExists(base string) (bool, error)

func (*SimpleGStore) ListFiles 

func (s *SimpleGStore) ListFiles(prefix string, max int) (out []string, err error)

func (*SimpleGStore) ObjectPath 

func (s *SimpleGStore) ObjectPath(name string) string

func (*SimpleGStore) OpenObject 

func (s *SimpleGStore) OpenObject(name string) (out io.ReadCloser, err error)

func (*SimpleGStore) SetOperationTimeout 

func (s *SimpleGStore) SetOperationTimeout(d time.Duration)

func (*SimpleGStore) WriteObject 

func (s *SimpleGStore) WriteObject(base string, f io.Reader) (err error)

type Store 

type Store interface {
	OpenObject(ctx context.Context, name string) (out io.ReadCloser, err error)
	FileExists(ctx context.Context, base string) (bool, error)
	ObjectPath(base string) string

	WriteObject(ctx context.Context, base string, f io.Reader) (err error)
	PushLocalFile(ctx context.Context, localFile, toBaseName string) (err error)

	Overwrite() bool
	SetOverwrite(enabled bool)

	Walk(ctx context.Context, prefix, ignoreSuffix string, f func(filename string) (err error)) error
	ListFiles(ctx context.Context, prefix, ignoreSuffix string, max int) ([]string, error)

	DeleteObject(ctx context.Context, base string) error
}

func NewDBinStore 

func NewDBinStore(baseURL string) (Store, error)

func NewJSONLStore 

func NewJSONLStore(baseURL string) (Store, error)

func NewSimpleStore 

func NewSimpleStore(baseURL string) (Store, error)

func NewStore 

func NewStore(baseURL, extension, compressionType string, overwrite bool) (Store, error)

NewStore creates a new Store instance. The baseURL is always a directory, and does not end with a `/`.

Source Files

View all Source files

Directories

Path Synopsis
cmd
dstore

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.