cockroach: Index | Files | Directories

package cloudimpl

import ""


Package Files

aws_kms.go azure_storage.go external_storage.go file_table_storage.go gcs_storage.go http_storage.go nodelocal_storage.go s3_storage.go workload_storage.go


const (
    // AWSAccessKeyParam is the query parameter for access_key in an AWS URI.
    AWSAccessKeyParam = "AWS_ACCESS_KEY_ID"
    // AWSSecretParam is the query parameter for the 'secret' in an AWS URI.
    // AWSTempTokenParam is the query parameter for session_token in an AWS URI.
    AWSTempTokenParam = "AWS_SESSION_TOKEN"
    // AWSEndpointParam is the query parameter for the 'endpoint' in an AWS URI.
    AWSEndpointParam = "AWS_ENDPOINT"

    // S3RegionParam is the query parameter for the 'endpoint' in an S3 URI.
    S3RegionParam = "AWS_REGION"

    // KMSRegionParam is the query parameter for the 'region' in every KMS URI.
    KMSRegionParam = "REGION"

    // AzureAccountNameParam is the query parameter for account_name in an azure URI.
    AzureAccountNameParam = "AZURE_ACCOUNT_NAME"
    // AzureAccountKeyParam is the query parameter for account_key in an azure URI.
    AzureAccountKeyParam = "AZURE_ACCOUNT_KEY"

    // GoogleBillingProjectParam is the query parameter for the billing project
    // in a gs URI.
    GoogleBillingProjectParam = "GOOGLE_BILLING_PROJECT"

    // AuthParam is the query parameter for the cluster settings named
    // key in a URI.
    AuthParam = "AUTH"
    // AuthParamImplicit is the query parameter for the implicit authentication
    // mode in a URI.
    AuthParamImplicit = "implicit"
    // AuthParamDefault is the query parameter for the default authentication
    // mode in a URI.
    AuthParamDefault = "default"
    // AuthParamSpecified is the query parameter for the specified authentication
    // mode in a URI.
    AuthParamSpecified = "specified"

    // CredentialsParam is the query parameter for the base64-encoded contents of
    // the Google Application Credentials JSON file.
    CredentialsParam = "CREDENTIALS"

    // CloudstorageGSDefaultKey is the setting whose value is the JSON key to use
    // during Google Cloud Storage operations.
    CloudstorageGSDefaultKey = cloudstorageGSDefault + cloudstorageKey

    // CloudstorageHTTPCASetting is the setting whose value is the custom root CA
    // (appended to system's default CAs) for verifying certificates when
    // interacting with HTTPS storage.
    CloudstorageHTTPCASetting = cloudstorageHTTP + ".custom_ca"
const MaxDelayedRetryAttempts = 3

MaxDelayedRetryAttempts is the number of times the delayedRetry method will re-run the provided function.


var ErrFileDoesNotExist = errors.New("external_storage: file doesn't exist")

ErrFileDoesNotExist is a sentinel error for indicating that a specified bucket/object/key/file (depending on storage terminology) does not exist. This error is raised by the ReadFile method.

var ErrListingUnsupported = errors.New("listing is not supported")

ErrListingUnsupported is a marker for indicating listing is unsupported.

var (
    // GcsDefault is the setting which defines the JSON key to use during GCS
    // operations.
    GcsDefault = settings.RegisterPublicStringSetting(
        "if set, JSON key to use during Google Cloud Storage operations",
var HTTPRetryOptions = retry.Options{
    InitialBackoff: 100 * time.Millisecond,
    MaxBackoff:     2 * time.Second,
    MaxRetries:     32,
    Multiplier:     4,

HTTPRetryOptions defines the tunable settings which control the retry of HTTP operations.

func ExternalStorageConfFromURI Uses

func ExternalStorageConfFromURI(path, user string) (roachpb.ExternalStorage, error)

ExternalStorageConfFromURI generates an ExternalStorage config from a URI string.

func ExternalStorageFromURI Uses

func ExternalStorageFromURI(
    ctx context.Context,
    uri string,
    externalConfig base.ExternalIODirConfig,
    settings *cluster.Settings,
    blobClientFactory blobs.BlobClientFactory,
    user string,
    ie *sql.InternalExecutor,
    kvDB *kv.DB,
) (cloud.ExternalStorage, error)

ExternalStorageFromURI returns an ExternalStorage for the given URI.

func MakeAWSKMS Uses

func MakeAWSKMS(uri string, env cloud.KMSEnv) (cloud.KMS, error)

MakeAWSKMS is the factory method which returns a configured, ready-to-use AWS KMS object.

func MakeExternalStorage Uses

func MakeExternalStorage(
    ctx context.Context,
    dest roachpb.ExternalStorage,
    conf base.ExternalIODirConfig,
    settings *cluster.Settings,
    blobClientFactory blobs.BlobClientFactory,
    ie *sql.InternalExecutor,
    kvDB *kv.DB,
) (cloud.ExternalStorage, error)

MakeExternalStorage creates an ExternalStorage from the given config.

func MakeHTTPStorage Uses

func MakeHTTPStorage(
    base string, settings *cluster.Settings, ioConf base.ExternalIODirConfig,
) (cloud.ExternalStorage, error)

MakeHTTPStorage returns an instance of HTTPStorage ExternalStorage.

func MakeLocalStorageURI Uses

func MakeLocalStorageURI(path string) string

MakeLocalStorageURI converts a local path (should always be relative) to a valid nodelocal URI.

func MakeS3Storage Uses

func MakeS3Storage(
    ctx context.Context,
    ioConf base.ExternalIODirConfig,
    conf *roachpb.ExternalStorage_S3,
    settings *cluster.Settings,
) (cloud.ExternalStorage, error)

MakeS3Storage returns an instance of S3 ExternalStorage.

func MakeSQLConnFileTableStorage Uses

func MakeSQLConnFileTableStorage(
    ctx context.Context, cfg roachpb.ExternalStorage_FileTable, conn cloud.SQLConnI,
) (cloud.ExternalStorage, error)

MakeSQLConnFileTableStorage returns an instance of a FileTableStorage which uses a network connection backed SQL executor. This is used by the CLI to interact with the underlying FileToTableSystem. It only supports a subset of methods compared to the internal SQL connection backed FileTableStorage.

func MakeUserFileStorageURI Uses

func MakeUserFileStorageURI(qualifiedTableName, filename string) string

MakeUserFileStorageURI converts a qualified table name and filename to a valid userfile URI.

func ParseWorkloadConfig Uses

func ParseWorkloadConfig(uri *url.URL) (*roachpb.ExternalStorage_Workload, error)

ParseWorkloadConfig parses a workload config URI to a proto config.

func SanitizeExternalStorageURI Uses

func SanitizeExternalStorageURI(path string, extraParams []string) (string, error)

SanitizeExternalStorageURI returns the external storage URI with with some secrets redacted, for use when showing these URIs in the UI, to provide some protection from shoulder-surfing. The param is still present -- just redacted -- to make it clearer that that value is indeed persisted interally. extraParams which should be scrubbed -- for params beyond those that the various clound-storage URIs supported by this package know about -- can be passed allowing this function to be used to scrub other URIs too (such as non-cloudstorage changefeed sinks).

func URINeedsGlobExpansion Uses

func URINeedsGlobExpansion(uri string) bool

URINeedsGlobExpansion checks if URI can be expanded by checking if it contains wildcard characters. This should be used before passing a URI into ListFiles().



Package cloudimpl imports 46 packages (graph) and is imported by 6 packages. Updated 2020-08-12. Refresh now. Tools for package owners.