import "github.com/pachyderm/pachyderm/src/server/pkg/obj"
amazon_client.go google_client.go local_client.go microsoft_client.go minio_client.go monkey_client.go obj.go tracing.go
const (
StorageBackendEnvVar = "STORAGE_BACKEND"
PachRootEnvVar = "PACH_ROOT"
)Environment variables for determining storage backend and pathing
const (
Minio = "MINIO"
Amazon = "AMAZON"
Google = "GOOGLE"
Microsoft = "MICROSOFT"
Local = "LOCAL"
)Valid object storage backends
const (
GoogleBucketEnvVar = "GOOGLE_BUCKET"
GoogleCredEnvVar = "GOOGLE_CRED"
)Google environment variables
const (
MicrosoftContainerEnvVar = "MICROSOFT_CONTAINER"
MicrosoftIDEnvVar = "MICROSOFT_ID"
MicrosoftSecretEnvVar = "MICROSOFT_SECRET"
)Microsoft environment variables
const (
MinioBucketEnvVar = "MINIO_BUCKET"
MinioEndpointEnvVar = "MINIO_ENDPOINT"
MinioIDEnvVar = "MINIO_ID"
MinioSecretEnvVar = "MINIO_SECRET"
MinioSecureEnvVar = "MINIO_SECURE"
MinioSignatureEnvVar = "MINIO_SIGNATURE"
)Minio environment variables
const (
AmazonRegionEnvVar = "AMAZON_REGION"
AmazonBucketEnvVar = "AMAZON_BUCKET"
AmazonIDEnvVar = "AMAZON_ID"
AmazonSecretEnvVar = "AMAZON_SECRET"
AmazonTokenEnvVar = "AMAZON_TOKEN"
AmazonVaultAddrEnvVar = "AMAZON_VAULT_ADDR"
AmazonVaultRoleEnvVar = "AMAZON_VAULT_ROLE"
AmazonVaultTokenEnvVar = "AMAZON_VAULT_TOKEN"
AmazonDistributionEnvVar = "AMAZON_DISTRIBUTION"
CustomEndpointEnvVar = "CUSTOM_ENDPOINT"
)Amazon environment variables
var EnvVarToSecretKey = map[string]string{ GoogleBucketEnvVar: "google-bucket", GoogleCredEnvVar: "google-cred", MicrosoftContainerEnvVar: "microsoft-container", MicrosoftIDEnvVar: "microsoft-id", MicrosoftSecretEnvVar: "microsoft-secret", MinioBucketEnvVar: "minio-bucket", MinioEndpointEnvVar: "minio-endpoint", MinioIDEnvVar: "minio-id", MinioSecretEnvVar: "minio-secret", MinioSecureEnvVar: "minio-secure", MinioSignatureEnvVar: "minio-signature", AmazonRegionEnvVar: "amazon-region", AmazonBucketEnvVar: "amazon-bucket", AmazonIDEnvVar: "amazon-id", AmazonSecretEnvVar: "amazon-secret", AmazonTokenEnvVar: "amazon-token", AmazonVaultAddrEnvVar: "amazon-vault-addr", AmazonVaultRoleEnvVar: "amazon-vault-role", AmazonVaultTokenEnvVar: "amazon-vault-token", AmazonDistributionEnvVar: "amazon-distribution", CustomEndpointEnvVar: "custom-endpoint", }
EnvVarToSecretKey is an environment variable name to secret key mapping This is being used to temporarily bridge the gap as we transition to a model where object storage access in the workers is based on environment variables and a library rather than mounting a secret to a sidecar container which accesses object storage
BlockPathFromEnv gets the path to an object storage block based on environment variables.
func DisableMonkeyTest()
DisableMonkeyTest disables sporadic request failures.
func EnableMonkeyTest()
EnableMonkeyTest enables sporadic request failures.
InitMonkeyTest sets up this package for monkey testing. Object storage clients will be wrapped with a client that sporadically fails requests.
IsMonkeyError checks if an error was caused by a monkey client.
IsRetryable determines if an operation should be retried given an error
func NewExponentialBackOffConfig() *backoff.ExponentialBackOff
NewExponentialBackOffConfig creates an exponential back-off config with longer wait times than the default.
StorageRootFromEnv gets the storage root based on environment variables.
TestStorage is a defensive method for checking to make sure that storage is properly configured.
type AmazonCreds struct {
// Direct credentials. Only applicable if Pachyderm is given its own permanent
// AWS credentials
ID string // Access Key ID
Secret string // Secret Access Key
Token string // Access token (if using temporary security credentials
// Vault options (if getting AWS credentials from Vault)
VaultAddress string // normally addresses come from env, but don't have vault service name
VaultRole string
VaultToken string
}AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment
type BackoffReadCloser struct {
// contains filtered or unexported fields
}BackoffReadCloser retries with exponential backoff in the case of failures
func (b *BackoffReadCloser) Close() error
Close closes the ReaderCloser contained in b.
func (b *BackoffReadCloser) Read(data []byte) (int, error)
type BackoffWriteCloser struct {
// contains filtered or unexported fields
}BackoffWriteCloser retries with exponential backoff in the case of failures
func (b *BackoffWriteCloser) Close() error
Close closes the WriteCloser contained in b.
func (b *BackoffWriteCloser) Write(data []byte) (int, error)
type Client interface {
// Writer returns a writer which writes to an object.
// It should error if the object already exists or we don't have sufficient
// permissions to write it.
Writer(ctx context.Context, name string) (io.WriteCloser, error)
// Reader returns a reader which reads from an object.
// If `size == 0`, the reader should read from the offset till the end of the object.
// It should error if the object doesn't exist or we don't have sufficient
// permission to read it.
Reader(ctx context.Context, name string, offset uint64, size uint64) (io.ReadCloser, error)
// Delete deletes an object.
// It should error if the object doesn't exist or we don't have sufficient
// permission to delete it.
Delete(ctx context.Context, name string) error
// Walk calls `fn` with the names of objects which can be found under `prefix`.
Walk(ctx context.Context, prefix string, fn func(name string) error) error
// Exsits checks if a given object already exists
Exists(ctx context.Context, name string) bool
// IsRetryable determines if an operation should be retried given an error
IsRetryable(err error) bool
// IsNotExist returns true if err is a non existence error
IsNotExist(err error) bool
// IsIgnorable returns true if the error can be ignored
IsIgnorable(err error) bool
}Client is an interface to object storage.
func NewAmazonClient(region, bucket string, creds *AmazonCreds, distribution string, endpoint string, reversed ...bool) (Client, error)
NewAmazonClient creates an amazon client with the following credentials:
bucket - S3 bucket name distribution - cloudfront distribution ID id - AWS access key id secret - AWS secret access key token - AWS access token region - AWS region endpoint - Custom endpoint (generally used for S3 compatible object stores)
NewAmazonClientFromEnv creates a Amazon client based on environment variables.
NewAmazonClientFromSecret constructs an amazon client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.
NewClientFromEnv creates a client based on environment variables.
NewClientFromSecret creates a client based on mounted secret files.
func NewClientFromURLAndSecret(url *ObjectStoreURL, reversed ...bool) (c Client, err error)
NewClientFromURLAndSecret constructs a client by parsing `URL` and then constructing the correct client for that URL using secrets.
NewGoogleClient creates a google client with the given bucket name.
NewGoogleClientFromEnv creates a Google client based on environment variables.
NewGoogleClientFromSecret creates a google client by reading credentials from a mounted GoogleSecret. You may pass "" for bucket in which case it will read the bucket from the secret.
NewLocalClient returns a Client that stores data on the local file system
NewMicrosoftClient creates a microsoft client:
container - Azure Blob Container name accountName - Azure Storage Account name accountKey - Azure Storage Account key
NewMicrosoftClientFromEnv creates a Microsoft client based on environment variables.
NewMicrosoftClientFromSecret creates a microsoft client by reading credentials from a mounted MicrosoftSecret. You may pass "" for container in which case it will read the container from the secret.
NewMinioClient creates an s3 compatible client with the following credentials:
endpoint - S3 compatible endpoint bucket - S3 bucket name id - AWS access key id secret - AWS secret access key secure - Set to true if connection is secure. isS3V2 - Set to true if client follows S3V2
NewMinioClientFromEnv creates a Minio client based on environment variables.
NewMinioClientFromSecret constructs an s3 compatible client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.
TracingObjClient wraps the given object client 'c', adding tracing to all calls made by the returned interface
type ObjectStoreURL struct {
// The object store, e.g. s3, gcs, as...
Store string
// The "bucket" (in AWS parlance) or the "container" (in Azure parlance).
Bucket string
// The object itself.
Object string
}ObjectStoreURL represents a parsed URL to an object in an object store.
func ParseURL(urlStr string) (*ObjectStoreURL, error)
ParseURL parses an URL into ObjectStoreURL.
RetryError is used to log retry attempts.
Package obj imports 44 packages (graph) and is imported by 11 packages. Updated 2019-08-13. Refresh now. Tools for package owners.