config

type AWSBatch struct {
	// JobDefinition can be either a name or the Amazon Resource Name (ARN).
	JobDefinition string
	// JobQueue can be either a name or the Amazon Resource Name (ARN).
	JobQueue string
	// Turn off task state reconciler. When enabled, Funnel communicates with AWS Batch
	// to find tasks that never started and updates task state accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by AWS Batch
	ReconcileRate Duration
	AWSConfig
}

type AWSConfig struct {
	// An optional endpoint URL (hostname only or fully qualified URI)
	// that overrides the default generated endpoint for a client.
	Endpoint string
	// The region to send requests to. This parameter is required and must
	// be configured on a per-client basis for all AWS services in funnel with the
	// exception of S3Storage.
	Region string
	// The maximum number of times that a request will be retried for failures.
	// By default defers the max retry setting to the service
	// specific configuration. Set to a value > 0 to override.
	MaxRetries int
	// If both the key and secret are empty AWS credentials will be read from
	// the environment.
	Key    string
	Secret string
	// Don't automatically read AWS credentials from the environment.
	DisableAutoCredentialLoad bool
}

type AmazonS3Storage struct {
	Disabled bool
	SSE      struct {
		CustomerKeyFile string
		KMSKey          string
	}
	AWSConfig
}

type Badger struct {
	// Path to database directory.
	Path string
}

type BasicCredential struct {
	User     string
	Password string
}

type BoltDB struct {
	Path string
}

type Config struct {
	// component selectors
	EventWriters []string
	Database     string
	Compute      string
	// funnel components
	Server    Server
	RPCClient RPCClient
	Scheduler Scheduler
	Node      Node
	Worker    Worker
	Logger    logger.Config
	// databases / event handlers
	BoltDB    BoltDB
	Badger    Badger
	DynamoDB  DynamoDB
	Elastic   Elastic
	MongoDB   MongoDB
	Kafka     Kafka
	PubSub    PubSub
	Datastore Datastore
	// compute
	HTCondor   HPCBackend
	Slurm      HPCBackend
	PBS        HPCBackend
	GridEngine struct {
		Template     string
		TemplateFile string
	}
	AWSBatch   AWSBatch
	Kubernetes Kubernetes
	// storage
	LocalStorage  LocalStorage
	AmazonS3      AmazonS3Storage
	GenericS3     []GenericS3Storage
	GoogleStorage GoogleCloudStorage
	Swift         SwiftStorage
	HTTPStorage   HTTPStorage
	FTPStorage    FTPStorage
}

type Datastore struct {
	Project string
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

type Duration time.Duration

type DynamoDB struct {
	TableBasename string
	AWSConfig
}

type Elastic struct {
	IndexPrefix string
	URL         string
}

type FTPStorage struct {
	Disabled bool
	// Timeout duration for http GET calls
	Timeout  Duration
	User     string
	Password string
}

type GenericS3Storage struct {
	Disabled bool
	Endpoint string
	Key      string
	Secret   string
}

type GoogleCloudStorage struct {
	Disabled bool
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

type HPCBackend struct {
	// Turn off task state reconciler. When enabled, Funnel communicates with the HPC
	// scheduler to find tasks that are stuck in a queued state or errored and
	// updates the task state accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by the backend
	ReconcileRate Duration
	// Template is the template to use for submissions to the HPC backend
	Template string
	// TemplateFile is the path to the template to use for submissions to the HPC backend
	TemplateFile string
}

type HTTPStorage struct {
	Disabled bool
	// Timeout duration for http GET calls
	Timeout Duration
}

type Kafka struct {
	Servers []string
	Topic   string
}

type Kubernetes struct {
	// Turn off task state reconciler. When enabled, Funnel communicates with Kuberenetes
	// to find tasks that are stuck in a queued state or errored and updates the task state
	// accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by the backend
	ReconcileRate Duration
	// Disable cleanup of complete/failed jobs. Cleanup is run during reconcile loop.
	DisableJobCleanup bool
	// Batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch
	Template string
	// TemplateFile is the path to the job template.
	TemplateFile string
	// Path to the Kubernetes configuration file, otherwise assumes the Funnel server is running in a pod and
	// attempts to use https://godoc.org/k8s.io/client-go/rest#InClusterConfig to infer configuration.
	ConfigFile string
	// Namespace to spawn jobs within
	Namespace string
}

type LocalStorage struct {
	Disabled    bool
	AllowedDirs []string
}

type MongoDB struct {
	// Addrs holds the addresses for the seed servers.
	Addrs []string
	// Database is the database name used within MongoDB to store funnel data.
	Database string
	// Timeout is the amount of time to wait for a server to respond when
	// first connecting and on follow up operations in the session. If
	// timeout is zero, the call may block forever waiting for a connection
	// to be established.
	Timeout Duration
	// Username and Password inform the credentials for the initial authentication
	// done on the database defined by the Database field.
	Username string
	Password string
}

type Node struct {
	ID string
	// A Node will automatically try to detect what resources are available to it.
	// Defining Resources in the Node configuration overrides this behavior.
	Resources struct {
		Cpus   uint32
		RamGb  float64 // nolint
		DiskGb float64
	}
	// If the node has been idle for longer than the timeout, it will shut down.
	// -1 means there is no timeout. 0 means timeout immediately after the first task.
	Timeout Duration
	// How often the node sends update requests to the server.
	UpdateRate Duration
	Metadata   map[string]string
}

type PubSub struct {
	Topic   string
	Project string
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

type RPCClient struct {
	BasicCredential
	ServerAddress string
	// The timeout to use for making RPC client connections in nanoseconds
	// This timeout is Only enforced when used in conjunction with the
	// grpc.WithBlock dial option.
	Timeout Duration
	// The maximum number of times that a request will be retried for failures.
	// Time between retries follows an exponential backoff starting at 5 seconds
	// up to 1 minute
	MaxRetries uint
}

type Scheduler struct {
	// How often to run a scheduler iteration.
	ScheduleRate Duration
	// How many tasks to schedule in one iteration.
	ScheduleChunk int
	// How long to wait for a node ping before marking it as dead
	NodePingTimeout Duration
	// How long to wait for node initialization before marking it dead
	NodeInitTimeout Duration
	// How long to wait before deleting a dead node from the DB.
	NodeDeadTimeout Duration
}

type Server struct {
	ServiceName      string
	HostName         string
	HTTPPort         string
	RPCPort          string
	BasicAuth        []BasicCredential
	DisableHTTPCache bool
}

type SwiftStorage struct {
	Disabled   bool
	UserName   string
	Password   string
	AuthURL    string
	TenantName string
	TenantID   string
	RegionName string
	// Size of chunks to use for large object creation.
	// Defaults to 500 MB if not set or set below 10 MB.
	// The max number of chunks for a single object is 1000.
	ChunkSizeBytes int64
	// The maximum number of times to retry on error.
	// Defaults to 3.
	MaxRetries int
}

type Worker struct {
	// Directory to write task files to
	WorkDir string
	// How often the worker should poll for cancel signals
	PollingRate Duration
	// How often to update stdout/stderr log fields.
	// Setting this to 0 will result in these fields being updated a single time
	// after the executor exits.
	LogUpdateRate Duration
	// Max bytes of stdout/stderr to store in the database.
	// Setting this to 0 turns off stdout/stderr logging.
	LogTailSize int64
	// Normally the worker cleans up its working directory after executing.
	// This option disables that behavior.
	LeaveWorkDir bool
	// Limit the number of concurrent downloads/uploads
	MaxParallelTransfers int
}