pachyderm: github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets Index | Files

package assets

import "github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"

Index

Package Files

assets.go

Variables

var (

    // ServiceAccountName is the name of Pachyderm's service account.
    // It's public because it's needed by pps.APIServer to create the RCs for
    // workers.
    ServiceAccountName = "pachyderm"

    // PrometheusPort hosts the prometheus stats for scraping
    PrometheusPort = 9091

    // IAMAnnotation is the annotation used for the IAM role, this can work
    // with something like kube2iam as an alternative way to provide
    // credentials.
    IAMAnnotation = "iam.amazonaws.com/role"
)

func AddRegistry Uses

func AddRegistry(registry string, imageName string) string

AddRegistry switches the registry that an image is targeting, unless registry is blank

func AmazonIAMRoleSecret Uses

func AmazonIAMRoleSecret(region, bucket, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonIAMRoleSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func AmazonSecret Uses

func AmazonSecret(region, bucket, id, secret, token, distribution, endpoint string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
id             - AWS access key id
secret         - AWS secret access key
token          - AWS access token
distribution   - cloudfront distribution
endpoint       - Custom endpoint (generally used for S3 compatible object stores)
advancedConfig - advanced configuration

func AmazonVaultSecret Uses

func AmazonVaultSecret(region, bucket, vaultAddress, vaultRole, vaultToken, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonVaultSecret creates an amazon secret with the following parameters:

region         - AWS region
bucket         - S3 bucket name
vaultAddress   - address/hostport of vault
vaultRole      - pachd's role in vault
vaultToken     - pachd's vault token
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func ClusterRole Uses

func ClusterRole(opts *AssetOpts) *rbacv1.ClusterRole

ClusterRole returns a ClusterRole that should be bound to the Pachyderm service account.

func ClusterRoleBinding Uses

func ClusterRoleBinding(opts *AssetOpts) *rbacv1.ClusterRoleBinding

ClusterRoleBinding returns a ClusterRoleBinding that binds Pachyderm's ClusterRole to its ServiceAccount.

func DashDeployment Uses

func DashDeployment(opts *AssetOpts) *apps.Deployment

DashDeployment creates a Deployment for the pachyderm dashboard.

func DashService Uses

func DashService(opts *AssetOpts) *v1.Service

DashService creates a Service for the pachyderm dashboard.

func EtcdDeployment Uses

func EtcdDeployment(opts *AssetOpts, hostPath string) *apps.Deployment

EtcdDeployment returns an etcd k8s Deployment.

func EtcdHeadlessService Uses

func EtcdHeadlessService(opts *AssetOpts) *v1.Service

EtcdHeadlessService returns a headless etcd service, which is only for DNS resolution.

func EtcdNodePortService Uses

func EtcdNodePortService(local bool, opts *AssetOpts) *v1.Service

EtcdNodePortService returns a NodePort etcd service. This will let non-etcd pods talk to etcd

func EtcdStatefulSet Uses

func EtcdStatefulSet(opts *AssetOpts, backend backend, diskSpace int) interface{}

EtcdStatefulSet returns a stateful set that manages an etcd cluster

func EtcdStorageClass Uses

func EtcdStorageClass(opts *AssetOpts, backend backend) (interface{}, error)

EtcdStorageClass creates a storage class used for dynamic volume provisioning. Currently dynamic volume provisioning only works on AWS and GCE.

func EtcdVolume Uses

func EtcdVolume(persistentDiskBackend backend, opts *AssetOpts,
    hostPath string, name string, size int) (*v1.PersistentVolume, error)

EtcdVolume creates a persistent volume backed by a volume with name "name"

func EtcdVolumeClaim Uses

func EtcdVolumeClaim(size int, opts *AssetOpts) *v1.PersistentVolumeClaim

EtcdVolumeClaim creates a persistent volume claim of 'size' GB.

Note that if you're controlling Etcd with a Stateful Set, this is unnecessary (the stateful set controller will create PVCs automatically).

func GetBackendSecretVolumeAndMount Uses

func GetBackendSecretVolumeAndMount(backend string) (v1.Volume, v1.VolumeMount)

GetBackendSecretVolumeAndMount returns a properly configured Volume and VolumeMount object given a backend. The backend needs to be one of the constants defined in pfs/server.

func GetSecretEnvVars Uses

func GetSecretEnvVars(storageBackend string) []v1.EnvVar

GetSecretEnvVars returns the environment variable specs for the storage secret.

func GithookService Uses

func GithookService(namespace string) *v1.Service

GithookService returns a k8s service that exposes a public IP

func GoogleSecret Uses

func GoogleSecret(bucket string, cred string) map[string][]byte

GoogleSecret creates a google secret with a bucket name.

func Images Uses

func Images(opts *AssetOpts) []string

Images returns a list of all the images that are used by a pachyderm deployment.

func LocalSecret Uses

func LocalSecret() map[string][]byte

LocalSecret creates an empty secret.

func MicrosoftSecret Uses

func MicrosoftSecret(container string, id string, secret string) map[string][]byte

MicrosoftSecret creates a microsoft secret with following parameters:

container - Azure blob container
id    	   - Azure storage account name
secret    - Azure storage account key

func MinioSecret Uses

func MinioSecret(bucket string, id string, secret string, endpoint string, secure, isS3V2 bool) map[string][]byte

MinioSecret creates an amazon secret with the following parameters:

bucket - S3 bucket name
id     - S3 access key id
secret - S3 secret access key
endpoint  - S3 compatible endpoint
secure - set to true for a secure connection.
isS3V2 - Set to true if client follows S3V2

func PachdDeployment Uses

func PachdDeployment(opts *AssetOpts, objectStoreBackend backend, hostPath string) *apps.Deployment

PachdDeployment returns a pachd k8s Deployment.

func PachdService Uses

func PachdService(opts *AssetOpts) *v1.Service

PachdService returns a pachd service.

func Role Uses

func Role(opts *AssetOpts) *rbacv1.Role

Role returns a Role that should be bound to the Pachyderm service account.

func RoleBinding Uses

func RoleBinding(opts *AssetOpts) *rbacv1.RoleBinding

RoleBinding returns a RoleBinding that binds Pachyderm's Role to its ServiceAccount.

func ServiceAccount Uses

func ServiceAccount(opts *AssetOpts) *v1.ServiceAccount

ServiceAccount returns a kubernetes service account for use with Pachyderm.

func WriteAmazonAssets Uses

func WriteAmazonAssets(encoder Encoder, opts *AssetOpts, region string, bucket string, volumeSize int, creds *AmazonCreds, cloudfrontDistro string, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteAmazonAssets writes assets to an amazon backend.

func WriteAssets Uses

func WriteAssets(encoder Encoder, opts *AssetOpts, objectStoreBackend backend,
    persistentDiskBackend backend, volumeSize int,
    hostPath string) error

WriteAssets writes the assets to encoder.

func WriteCustomAssets Uses

func WriteCustomAssets(encoder Encoder, opts *AssetOpts, args []string, objectStoreBackend string,
    persistentDiskBackend string, secure, isS3V2 bool, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteCustomAssets writes assets to a custom combination of object-store and persistent disk.

func WriteDashboardAssets Uses

func WriteDashboardAssets(encoder Encoder, opts *AssetOpts) error

WriteDashboardAssets writes the k8s config for deploying the Pachyderm dashboard to 'encoder'

func WriteGoogleAssets Uses

func WriteGoogleAssets(encoder Encoder, opts *AssetOpts, bucket string, cred string, volumeSize int) error

WriteGoogleAssets writes assets to a google backend.

func WriteLocalAssets Uses

func WriteLocalAssets(encoder Encoder, opts *AssetOpts, hostPath string) error

WriteLocalAssets writes assets to a local backend.

func WriteMicrosoftAssets Uses

func WriteMicrosoftAssets(encoder Encoder, opts *AssetOpts, container string, id string, secret string, volumeSize int) error

WriteMicrosoftAssets writes assets to a microsoft backend

func WriteSecret Uses

func WriteSecret(encoder Encoder, data map[string][]byte, opts *AssetOpts) error

WriteSecret writes a JSON-encoded k8s secret to the given writer. The secret uses the given map as data.

func WriteTLSSecret Uses

func WriteTLSSecret(encoder Encoder, opts *AssetOpts) error

WriteTLSSecret creates a new TLS secret in the kubernetes manifest (equivalent to one generate by 'kubectl create secret tls'). This will be mounted by the pachd pod and used as its TLS public certificate and private key

type AmazonCreds Uses

type AmazonCreds struct {
    // Direct credentials. Only applicable if Pachyderm is given its own permanent
    // AWS credentials
    ID     string // Access Key ID
    Secret string // Secret Access Key
    Token  string // Access token (if using temporary security credentials

    // Vault options (if getting AWS credentials from Vault)
    VaultAddress string // normally addresses come from env, but don't have vault service name
    VaultRole    string
    VaultToken   string
}

AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment

type AssetOpts Uses

type AssetOpts struct {
    FeatureFlags
    PachdShards uint64
    Version     string
    LogLevel    string
    Metrics     bool
    Dynamic     bool
    EtcdNodes   int
    EtcdVolume  string
    DashOnly    bool
    NoDash      bool
    DashImage   string
    Registry    string
    EtcdPrefix  string
    PachdPort   int32
    TracePort   int32
    HTTPPort    int32
    PeerPort    int32

    // NoGuaranteed will not generate assets that have both resource limits and
    // resource requests set which causes kubernetes to give the pods
    // guaranteed QoS. Guaranteed QoS generally leads to more stable clusters
    // but on smaller test clusters such as those run on minikube it doesn't
    // help much and may cause more instability than it prevents.
    NoGuaranteed bool

    // DisableAuthentication stops Pachyderm's authentication service
    // from talking to GitHub, for testing. Instead users can authenticate
    // simply by providing a username.
    DisableAuthentication bool

    // BlockCacheSize is the amount of memory each PachD node allocates towards
    // its cache of PFS blocks. If empty, assets.go will choose a default size.
    BlockCacheSize string

    // PachdCPURequest is the amount of CPU we request for each pachd node. If
    // empty, assets.go will choose a default size.
    PachdCPURequest string

    // PachdNonCacheMemRequest is the amount of memory we request for each
    // pachd node in addition to BlockCacheSize. If empty, assets.go will choose
    // a default size.
    PachdNonCacheMemRequest string

    // EtcdCPURequest is the amount of CPU (in cores) we request for each etcd
    // node. If empty, assets.go will choose a default size.
    EtcdCPURequest string

    // EtcdMemRequest is the amount of memory we request for each etcd node. If
    // empty, assets.go will choose a default size.
    EtcdMemRequest string

    // EtcdStorageClassName is the name of an existing StorageClass to use when
    // creating a StatefulSet for dynamic etcd storage. If unset, a new
    // StorageClass will be created for the StatefulSet.
    EtcdStorageClassName string

    // IAM role that the Pachyderm deployment should assume when talking to AWS
    // services (if using kube2iam + metadata service + IAM role to delegate
    // permissions to pachd via its instance).
    // This is in AssetOpts rather than AmazonCreds because it must be passed
    // as an annotation on the pachd pod rather than as a k8s secret
    IAMRole string

    // ImagePullSecret specifies an image pull secret that gets attached to the
    // various deployments so that their images can be pulled from a private
    // registry.
    ImagePullSecret string

    // NoRBAC, if true, will disable creation of RBAC assets.
    NoRBAC bool

    // LocalRoles, if true, uses Role and RoleBinding instead of ClusterRole and
    // ClusterRoleBinding.
    LocalRoles bool

    // Namespace is the kubernetes namespace to deploy to.
    Namespace string

    // NoExposeDockerSocket if true prevents pipelines from accessing the docker socket.
    NoExposeDockerSocket bool

    // ExposeObjectAPI, if set, causes pachd to serve Object/Block API requests on
    // its public port. This should generally be false in production (it breaks
    // auth) but is needed by tests
    ExposeObjectAPI bool

    // If set, the files indictated by 'TLS.ServerCert' and 'TLS.ServerKey' are
    // placed into a Kubernetes secret and used by pachd nodes to authenticate
    // during TLS
    TLS *TLSOpts
}

AssetOpts are options that are applicable to all the asset types.

type Encoder Uses

type Encoder interface {
    // Encodes the given struct to the wrapped output stream. This also will write out a separator
    // value, suitable for differentiating multiple objects in the stream.
    Encode(interface{}) (err error)
}

Encoder is the interface for writing out assets. This is assumed to wrap an output writer.

type FeatureFlags Uses

type FeatureFlags struct {
    // NewHashTree, if true, will make Pachyderm use 1.9 hash trees.
    NewHashTree bool
}

FeatureFlags are flags for experimental features.

type TLSOpts Uses

type TLSOpts struct {
    ServerCert string
    ServerKey  string
}

TLSOpts indicates the cert and key file that Pachd should use to authenticate with clients

Package assets imports 18 packages (graph) and is imported by 6 packages. Updated 2019-10-06. Refresh now. Tools for package owners.