pachyderm: Index | Files

package assets

import ""


Package Files



const (
    // DefaultRequireCriticalServersOnly is the default for requiring critical servers only.
    // (bryce) this default is set here and in the service env config, need to figure out how to refactor
    // this to be in one place.
    DefaultRequireCriticalServersOnly = false
const (
    // DefaultUploadConcurrencyLimit is the default maximum number of concurrent object storage uploads.
    // (bryce) this default is set here and in the service env config, need to figure out how to refactor
    // this to be in one place.
    DefaultUploadConcurrencyLimit = 100
const (
    // RequireCriticalServersOnlyEnvVar is the environment variable for requiring critical servers only.
    RequireCriticalServersOnlyEnvVar = "REQUIRE_CRITICAL_SERVERS_ONLY"
const (
    // UploadConcurrencyLimitEnvVar is the environment variable for the upload concurrency limit.
    UploadConcurrencyLimitEnvVar = "STORAGE_UPLOAD_CONCURRENCY_LIMIT"
const (
    // WorkerSAName is the name of the service account that pachyderm
    // pipelines use to create an s3 gateway service for each job
    WorkerSAName = "pachyderm-worker"


var (

    // ServiceAccountName is the name of Pachyderm's service account.
    // It's public because it's needed by pps.APIServer to create the RCs for
    // workers.
    ServiceAccountName = "pachyderm"

    // PrometheusPort hosts the prometheus stats for scraping
    PrometheusPort = 656

    // IAMAnnotation is the annotation used for the IAM role, this can work
    // with something like kube2iam as an alternative way to provide
    // credentials.
    IAMAnnotation = ""

func AddRegistry Uses

func AddRegistry(registry string, imageName string) string

AddRegistry switches the registry that an image is targeting, unless registry is blank

func AmazonIAMRoleSecret Uses

func AmazonIAMRoleSecret(region, bucket, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonIAMRoleSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func AmazonSecret Uses

func AmazonSecret(region, bucket, id, secret, token, distribution, endpoint string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
id             - AWS access key id
secret         - AWS secret access key
token          - AWS access token
distribution   - cloudfront distribution
endpoint       - Custom endpoint (generally used for S3 compatible object stores)
advancedConfig - advanced configuration

func AmazonVaultSecret Uses

func AmazonVaultSecret(region, bucket, vaultAddress, vaultRole, vaultToken, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonVaultSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
vaultAddress   - address/hostport of vault
vaultRole      - pachd's role in vault
vaultToken     - pachd's vault token
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func ClusterRole Uses

func ClusterRole(opts *AssetOpts) *rbacv1.ClusterRole

ClusterRole returns a ClusterRole that should be bound to the Pachyderm service account.

func ClusterRoleBinding Uses

func ClusterRoleBinding(opts *AssetOpts) *rbacv1.ClusterRoleBinding

ClusterRoleBinding returns a ClusterRoleBinding that binds Pachyderm's ClusterRole to its ServiceAccount.

func DashDeployment Uses

func DashDeployment(opts *AssetOpts) *apps.Deployment

DashDeployment creates a Deployment for the pachyderm dashboard.

func DashService Uses

func DashService(opts *AssetOpts) *v1.Service

DashService creates a Service for the pachyderm dashboard.

func EtcdDeployment Uses

func EtcdDeployment(opts *AssetOpts, hostPath string) *apps.Deployment

EtcdDeployment returns an etcd k8s Deployment.

func EtcdHeadlessService Uses

func EtcdHeadlessService(opts *AssetOpts) *v1.Service

EtcdHeadlessService returns a headless etcd service, which is only for DNS resolution.

func EtcdNodePortService Uses

func EtcdNodePortService(local bool, opts *AssetOpts) *v1.Service

EtcdNodePortService returns a NodePort etcd service. This will let non-etcd pods talk to etcd

func EtcdStatefulSet Uses

func EtcdStatefulSet(opts *AssetOpts, backend backend, diskSpace int) interface{}

EtcdStatefulSet returns a stateful set that manages an etcd cluster

func EtcdStorageClass Uses

func EtcdStorageClass(opts *AssetOpts, backend backend) (interface{}, error)

EtcdStorageClass creates a storage class used for dynamic volume provisioning. Currently dynamic volume provisioning only works on AWS and GCE.

func EtcdVolume Uses

func EtcdVolume(persistentDiskBackend backend, opts *AssetOpts,
    hostPath string, name string, size int) (*v1.PersistentVolume, error)

EtcdVolume creates a persistent volume backed by a volume with name "name"

func EtcdVolumeClaim Uses

func EtcdVolumeClaim(size int, opts *AssetOpts) *v1.PersistentVolumeClaim

EtcdVolumeClaim creates a persistent volume claim of 'size' GB.

Note that if you're controlling Etcd with a Stateful Set, this is unnecessary (the stateful set controller will create PVCs automatically).

func GetBackendSecretVolumeAndMount Uses

func GetBackendSecretVolumeAndMount(backend string) (v1.Volume, v1.VolumeMount)

GetBackendSecretVolumeAndMount returns a properly configured Volume and VolumeMount object given a backend. The backend needs to be one of the constants defined in pfs/server.

func GetSecretEnvVars Uses

func GetSecretEnvVars(storageBackend string) []v1.EnvVar

GetSecretEnvVars returns the environment variable specs for the storage secret.

func GithookService Uses

func GithookService(namespace string) *v1.Service

GithookService returns a k8s service that exposes a public IP

func GoogleSecret Uses

func GoogleSecret(bucket string, cred string) map[string][]byte

GoogleSecret creates a google secret with a bucket name.

func Images Uses

func Images(opts *AssetOpts) []string

Images returns a list of all the images that are used by a pachyderm deployment.

func LocalSecret Uses

func LocalSecret() map[string][]byte

LocalSecret creates an empty secret.

func MicrosoftSecret Uses

func MicrosoftSecret(container string, id string, secret string) map[string][]byte

MicrosoftSecret creates a microsoft secret with following parameters:

container - Azure blob container
id    	   - Azure storage account name
secret    - Azure storage account key

func MinioSecret Uses

func MinioSecret(bucket string, id string, secret string, endpoint string, secure, isS3V2 bool) map[string][]byte

MinioSecret creates an amazon secret with the following parameters:

bucket - S3 bucket name
id     - S3 access key id
secret - S3 secret access key
endpoint  - S3 compatible endpoint
secure - set to true for a secure connection.
isS3V2 - Set to true if client follows S3V2

func PachdDeployment Uses

func PachdDeployment(opts *AssetOpts, objectStoreBackend backend, hostPath string) *apps.Deployment

PachdDeployment returns a pachd k8s Deployment.

func PachdPeerService Uses

func PachdPeerService(opts *AssetOpts) *v1.Service

PachdPeerService returns an internal pachd service. This service will reference the PeerPorr, which does not employ TLS even if cluster TLS is enabled. Because of this, the service is a `ClusterIP` type (i.e. not exposed outside of the cluster.)

func PachdService Uses

func PachdService(opts *AssetOpts) *v1.Service

PachdService returns a pachd service.

func Role Uses

func Role(opts *AssetOpts) *rbacv1.Role

Role returns a Role that should be bound to the Pachyderm service account.

func RoleBinding Uses

func RoleBinding(opts *AssetOpts) *rbacv1.RoleBinding

RoleBinding returns a RoleBinding that binds Pachyderm's Role to its ServiceAccount.

func ServiceAccounts Uses

func ServiceAccounts(opts *AssetOpts) []*v1.ServiceAccount

ServiceAccounts returns a kubernetes service account for use with Pachyderm.

func WriteAmazonAssets Uses

func WriteAmazonAssets(encoder serde.Encoder, opts *AssetOpts, region string, bucket string, volumeSize int, creds *AmazonCreds, cloudfrontDistro string, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteAmazonAssets writes assets to an amazon backend.

func WriteAssets Uses

func WriteAssets(encoder serde.Encoder, opts *AssetOpts, objectStoreBackend backend,
    persistentDiskBackend backend, volumeSize int,
    hostPath string) error

WriteAssets writes the assets to encoder.

func WriteCustomAssets Uses

func WriteCustomAssets(encoder serde.Encoder, opts *AssetOpts, args []string, objectStoreBackend string,
    persistentDiskBackend string, secure, isS3V2 bool, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteCustomAssets writes assets to a custom combination of object-store and persistent disk.

func WriteDashboardAssets Uses

func WriteDashboardAssets(encoder serde.Encoder, opts *AssetOpts) error

WriteDashboardAssets writes the k8s config for deploying the Pachyderm dashboard to 'encoder'

func WriteGoogleAssets Uses

func WriteGoogleAssets(encoder serde.Encoder, opts *AssetOpts, bucket string, cred string, volumeSize int) error

WriteGoogleAssets writes assets to a google backend.

func WriteLocalAssets Uses

func WriteLocalAssets(encoder serde.Encoder, opts *AssetOpts, hostPath string) error

WriteLocalAssets writes assets to a local backend.

func WriteMicrosoftAssets Uses

func WriteMicrosoftAssets(encoder serde.Encoder, opts *AssetOpts, container string, id string, secret string, volumeSize int) error

WriteMicrosoftAssets writes assets to a microsoft backend

func WriteSecret Uses

func WriteSecret(encoder serde.Encoder, data map[string][]byte, opts *AssetOpts) error

WriteSecret writes a JSON-encoded k8s secret to the given writer. The secret uses the given map as data.

func WriteTLSSecret Uses

func WriteTLSSecret(encoder serde.Encoder, opts *AssetOpts) error

WriteTLSSecret creates a new TLS secret in the kubernetes manifest (equivalent to one generate by 'kubectl create secret tls'). This will be mounted by the pachd pod and used as its TLS public certificate and private key

type AmazonCreds Uses

type AmazonCreds struct {
    // Direct credentials. Only applicable if Pachyderm is given its own permanent
    // AWS credentials
    ID     string // Access Key ID
    Secret string // Secret Access Key
    Token  string // Access token (if using temporary security credentials

    // Vault options (if getting AWS credentials from Vault)
    VaultAddress string // normally addresses come from env, but don't have vault service name
    VaultRole    string
    VaultToken   string

AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment

type AssetOpts Uses

type AssetOpts struct {
    PachdShards uint64
    Version     string
    LogLevel    string
    Metrics     bool
    Dynamic     bool
    EtcdNodes   int
    EtcdVolume  string
    DashOnly    bool
    NoDash      bool
    DashImage   string
    Registry    string
    EtcdPrefix  string
    PachdPort   int32
    TracePort   int32
    HTTPPort    int32
    PeerPort    int32

    // NoGuaranteed will not generate assets that have both resource limits and
    // resource requests set which causes kubernetes to give the pods
    // guaranteed QoS. Guaranteed QoS generally leads to more stable clusters
    // but on smaller test clusters such as those run on minikube it doesn't
    // help much and may cause more instability than it prevents.
    NoGuaranteed bool

    // DisableAuthentication stops Pachyderm's authentication service
    // from talking to GitHub, for testing. Instead users can authenticate
    // simply by providing a username.
    DisableAuthentication bool

    // BlockCacheSize is the amount of memory each PachD node allocates towards
    // its cache of PFS blocks. If empty, assets.go will choose a default size.
    BlockCacheSize string

    // PachdCPURequest is the amount of CPU we request for each pachd node. If
    // empty, assets.go will choose a default size.
    PachdCPURequest string

    // PachdNonCacheMemRequest is the amount of memory we request for each
    // pachd node in addition to BlockCacheSize. If empty, assets.go will choose
    // a default size.
    PachdNonCacheMemRequest string

    // EtcdCPURequest is the amount of CPU (in cores) we request for each etcd
    // node. If empty, assets.go will choose a default size.
    EtcdCPURequest string

    // EtcdMemRequest is the amount of memory we request for each etcd node. If
    // empty, assets.go will choose a default size.
    EtcdMemRequest string

    // EtcdStorageClassName is the name of an existing StorageClass to use when
    // creating a StatefulSet for dynamic etcd storage. If unset, a new
    // StorageClass will be created for the StatefulSet.
    EtcdStorageClassName string

    // IAM role that the Pachyderm deployment should assume when talking to AWS
    // services (if using kube2iam + metadata service + IAM role to delegate
    // permissions to pachd via its instance).
    // This is in AssetOpts rather than AmazonCreds because it must be passed
    // as an annotation on the pachd pod rather than as a k8s secret
    IAMRole string

    // ImagePullSecret specifies an image pull secret that gets attached to the
    // various deployments so that their images can be pulled from a private
    // registry.
    ImagePullSecret string

    // NoRBAC, if true, will disable creation of RBAC assets.
    NoRBAC bool

    // LocalRoles, if true, uses Role and RoleBinding instead of ClusterRole and
    // ClusterRoleBinding.
    LocalRoles bool

    // Namespace is the kubernetes namespace to deploy to.
    Namespace string

    // NoExposeDockerSocket if true prevents pipelines from accessing the docker socket.
    NoExposeDockerSocket bool

    // ExposeObjectAPI, if set, causes pachd to serve Object/Block API requests on
    // its public port. This should generally be false in production (it breaks
    // auth) but is needed by tests
    ExposeObjectAPI bool

    // If set, the files indictated by 'TLS.ServerCert' and 'TLS.ServerKey' are
    // placed into a Kubernetes secret and used by pachd nodes to authenticate
    // during TLS
    TLS *TLSOpts

    // Sets the cluster deployment ID. If unset, this will be a randomly
    // generated UUID without dashes.
    ClusterDeploymentID string

    // RequireCriticalServersOnly is true when only the critical Pachd servers
    // are required to startup and run without error.
    RequireCriticalServersOnly bool

AssetOpts are options that are applicable to all the asset types.

type FeatureFlags Uses

type FeatureFlags struct {
    // NewStorageLayer, if true, will make Pachyderm use the new storage layer.
    NewStorageLayer bool

FeatureFlags are flags for experimental features.

type StorageOpts Uses

type StorageOpts struct {
    UploadConcurrencyLimit int

StorageOpts are options that are applicable to the storage layer.

type TLSOpts Uses

type TLSOpts struct {
    ServerCert string
    ServerKey  string

TLSOpts indicates the cert and key file that Pachd should use to authenticate with clients

Package assets imports 20 packages (graph) and is imported by 6 packages. Updated 2020-04-03. Refresh now. Tools for package owners.