Documentation ¶
Index ¶
- Constants
- Variables
- func AnalyzeError(appContext ApplicationContext, assetID string, err error)
- func CheckDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) ([]*fapp.FybrikModule, []string)
- func ConstructOpenAPIReq(datasetID string, resourceMetadata *datacatalog.ResourceMetadata, ...) *policymanager.GetPolicyDecisionsRequest
- func CopyMap(m map[string]interface{}) map[string]interface{}
- func CreateDataRequest(application *fappv1.FybrikApplication, dataCtx *fappv1.DataContext, ...) adminconfig.DataRequest
- func GetDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) ([]*fapp.FybrikModule, error)
- func LookupPolicyDecisions(datasetID string, resourceMetadata *datacatalog.ResourceMetadata, ...) ([]taxonomy.Action, string, error)
- func SetMapField(obj map[string]interface{}, k string, v interface{}) bool
- func SupportsDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) bool
- func ValidateFybrikModule(module *fapp.FybrikModule, taxonomyFile string) error
- func ValidatePolicyDecisionsResponse(response *policymanager.GetPolicyDecisionsResponse, taxonomyFile string) error
- type ApplicationContext
- type BlueprintReconciler
- type ContextInterface
- type FybrikApplicationReconciler
- func (r *FybrikApplicationReconciler) Environment() (*datapath.Environment, error)
- func (r *FybrikApplicationReconciler) GetAllModules() (map[string]*fappv1.FybrikModule, error)
- func (r *FybrikApplicationReconciler) GetWorkloadCluster(appContext ApplicationContext, env *datapath.Environment) (multicluster.Cluster, error)
- func (r *FybrikApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
- func (r *FybrikApplicationReconciler) RegisterAsset(assetID string, catalogID string, info *fapp.DatasetDetails, ...) (string, error)
- func (r *FybrikApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error
- func (r *FybrikApplicationReconciler) ValidateAssetResponse(response interface{}, taxonomyFile, datasetID string) error
- type FybrikModuleReconciler
- type HelmValues
- type ModuleInstanceSpec
- type NewAssetInfo
- type PathBuilder
- type PlotterGenerator
- type PlotterInterface
- func (c *PlotterInterface) CreateOrUpdateResource(owner, ref *fapp.ResourceReference, plotterSpec *fapp.PlotterSpec, ...) error
- func (c *PlotterInterface) CreateResourceReference(owner *fapp.ResourceReference) *fapp.ResourceReference
- func (c *PlotterInterface) DeleteResource(ref *fapp.ResourceReference) error
- func (c *PlotterInterface) GetManagedObject() runtime.Object
- func (c *PlotterInterface) GetResourceSignature(ref *fapp.ResourceReference) *fapp.Plotter
- func (c *PlotterInterface) GetResourceStatus(ref *fapp.ResourceReference) (fapp.ObservedState, error)
- func (c *PlotterInterface) ResourceExists(ref *fapp.ResourceReference) bool
- type PlotterModulesSpec
- type PlotterReconciler
- func (r *PlotterReconciler) GenerateBlueprint(instances []ModuleInstanceSpec, clusterName string, plotter *fapp.Plotter) fapp.BlueprintSpec
- func (r *PlotterReconciler) GenerateBlueprints(instances []ModuleInstanceSpec, plotter *fapp.Plotter) map[string]fapp.BlueprintSpec
- func (r *PlotterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
- func (r *PlotterReconciler) RefineInstances(instances []ModuleInstanceSpec) []ModuleInstanceSpec
- func (r *PlotterReconciler) SetupWithManager(mgr ctrl.Manager) error
Constants ¶
const ( // ReadyCondition means that access to a dataset is granted ReadyConditionIndex int64 = 0 // DenyCondition means that access to a dataset is denied DenyConditionIndex int64 = 1 // ErrorCondition means that an error was encountered during blueprint construction ErrorConditionIndex int64 = 2 )
Condition indices are static. Conditions always present in the status.
const ( FybrikApplicationKind = "FybrikApplication" PlotterUpdatePrefix = "plotter_" Separator = " ; " )
const ( ReadAccessDenied string = "governance policies forbid access to the data" CopyNotAllowed string = "copy of the data is required but can not be done according to the governance policies" WriteNotAllowed string = "governance policies forbid writing of the data" StorageAccountUndefined string = "no storage account has been defined" ModuleNotFound string = "no module has been registered" InsufficientStorage string = "no bucket was provisioned for implicit copy" InvalidClusterConfiguration string = "cluster configuration does not support the requirements" )
ErrorMessages that are reported to the user
const ( ModuleValidationConditionIndex = 0 FybrikModuleKind = "FybrikModule" )
const ( PlotterKind string = "Plotter" PlotterFinalizerName string = "Plotter.finalizer" )
const (
BlueprintFinalizerName string = "Blueprint.finalizer"
)
const Transform = "transform"
Temporary hard-coded capability representing Actions of read/copy capability. A change to modules is requested to add "transform" as an additional capability to the existing read and copy modules.
Variables ¶
var ApplicationTaxonomy = environment.GetDataDir() + "/taxonomy/fybrik_application.json"
var DataCatalogCreateAssetResponseTaxonomy = environment.GetDataDir() + "/taxonomy/datacatalog.json#/definitions/CreateAssetResponse"
var DataCatalogGetAssetResponseTaxonomy = environment.GetDataDir() + "/taxonomy/datacatalog.json#/definitions/GetAssetResponse"
var ModuleTaxonomy = environment.GetDataDir() + "/taxonomy/fybrik_module.json"
var PolicyManagerTaxonomy = environment.GetDataDir() + "/taxonomy/policymanager.json#/definitions/GetPolicyDecisionsResponse"
Functions ¶
func AnalyzeError ¶
func AnalyzeError(appContext ApplicationContext, assetID string, err error)
AnalyzeError analyzes whether the given error is fatal, or a retrial attempt can be made. Reasons for retrial can be either communication problems with external services, or kubernetes problems to perform some action on a resource. A retrial is achieved by returning an error to the reconcile method
func CheckDependencies ¶ added in v0.6.0
func CheckDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) ([]*fapp.FybrikModule, []string)
CheckDependencies returns dependent modules
func ConstructOpenAPIReq ¶ added in v0.5.0
func ConstructOpenAPIReq(datasetID string, resourceMetadata *datacatalog.ResourceMetadata, input *fapp.FybrikApplication, operation *policymanager.RequestAction) *policymanager.GetPolicyDecisionsRequest
func CreateDataRequest ¶ added in v0.6.0
func CreateDataRequest(application *fappv1.FybrikApplication, dataCtx *fappv1.DataContext, assetMetadata *datacatalog.ResourceMetadata) adminconfig.DataRequest
CreateDataRequest generates a new DataRequest object for a specific asset based on FybrikApplication and asset metadata
func GetDependencies ¶ added in v0.6.0
func GetDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) ([]*fapp.FybrikModule, error)
GetDependencies returns dependencies of a selected module
func LookupPolicyDecisions ¶
func LookupPolicyDecisions(datasetID string, resourceMetadata *datacatalog.ResourceMetadata, policyManager connectors.PolicyManager, appContext ApplicationContext, op *policymanager.RequestAction) ([]taxonomy.Action, string, error)
LookupPolicyDecisions provides a list of governance actions for the given dataset and the given operation Input: - asset ID - asset metadata - policy manager facade - application info - data flow and locations Output: - a list of governance actions (upon a successful response) - a message from the connector (upon a successful response) - an error from the connector or an error formulated by Fybrik in case of Deny
func SetMapField ¶
SetMapField updates a map
func SupportsDependencies ¶ added in v0.6.0
func SupportsDependencies(module *fapp.FybrikModule, moduleMap map[string]*fapp.FybrikModule) bool
SupportsDependencies checks whether the module supports the dependency requirements
func ValidateFybrikModule ¶ added in v0.6.0
func ValidateFybrikModule(module *fapp.FybrikModule, taxonomyFile string) error
func ValidatePolicyDecisionsResponse ¶ added in v0.6.0
func ValidatePolicyDecisionsResponse(response *policymanager.GetPolicyDecisionsResponse, taxonomyFile string) error
Types ¶
type ApplicationContext ¶ added in v0.6.1
type ApplicationContext struct { Log *zerolog.Logger Application *fappv1.FybrikApplication UUID string }
type BlueprintReconciler ¶
type BlueprintReconciler struct { client.Client Name string Log zerolog.Logger Scheme *runtime.Scheme Helmer helm.Interface }
BlueprintReconciler reconciles a Blueprint object
func NewBlueprintReconciler ¶
func NewBlueprintReconciler(mgr ctrl.Manager, name string, helmer helm.Interface) *BlueprintReconciler
NewBlueprintReconciler creates a new reconciler for Blueprint resources
func (*BlueprintReconciler) SetupWithManager ¶
func (r *BlueprintReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager registers Blueprint controller
type ContextInterface ¶
type ContextInterface interface { ResourceExists(ref *fapp.ResourceReference) bool CreateOrUpdateResource(owner *fapp.ResourceReference, ref *fapp.ResourceReference, plotterSpec *fapp.PlotterSpec, labels map[string]string, uuid string) error DeleteResource(ref *fapp.ResourceReference) error GetResourceStatus(ref *fapp.ResourceReference) (fapp.ObservedState, error) CreateResourceReference(owner *fapp.ResourceReference) *fapp.ResourceReference GetManagedObject() runtime.Object }
ContextInterface is an interface for communication with a generated resource
type FybrikApplicationReconciler ¶
type FybrikApplicationReconciler struct { client.Client Name string Log zerolog.Logger Scheme *runtime.Scheme PolicyManager pmclient.PolicyManager DataCatalog dcclient.DataCatalog ResourceInterface ContextInterface ClusterManager multicluster.ClusterLister StorageManager storage.StorageManagerInterface ConfigEvaluator adminconfig.EvaluatorInterface Infrastructure *infrastructure.AttributeManager }
FybrikApplicationReconciler reconciles a FybrikApplication object
func NewFybrikApplicationReconciler ¶
func NewFybrikApplicationReconciler(mgr ctrl.Manager, name string, policyManager pmclient.PolicyManager, catalog dcclient.DataCatalog, cm multicluster.ClusterLister, storageManager storage.StorageManagerInterface, evaluator adminconfig.EvaluatorInterface, attributeManager *infrastructure.AttributeManager) *FybrikApplicationReconciler
NewFybrikApplicationReconciler creates a new reconciler for FybrikApplications
func (*FybrikApplicationReconciler) Environment ¶ added in v0.7.0
func (r *FybrikApplicationReconciler) Environment() (*datapath.Environment, error)
func (*FybrikApplicationReconciler) GetAllModules ¶
func (r *FybrikApplicationReconciler) GetAllModules() (map[string]*fappv1.FybrikModule, error)
GetAllModules returns all CRDs of the kind FybrikModule mapped by their name
func (*FybrikApplicationReconciler) GetWorkloadCluster ¶ added in v0.6.0
func (r *FybrikApplicationReconciler) GetWorkloadCluster(appContext ApplicationContext, env *datapath.Environment) (multicluster.Cluster, error)
GetWorkloadCluster returns a workload cluster If no cluster has been specified for a workload, a local cluster is assumed.
func (*FybrikApplicationReconciler) Reconcile ¶
func (r *FybrikApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
Reconcile reconciles FybrikApplication CRD It receives FybrikApplication CRD and selects the appropriate modules that will run The outcome is a Plotter containing multiple Blueprints that run on different clusters
func (*FybrikApplicationReconciler) RegisterAsset ¶
func (r *FybrikApplicationReconciler) RegisterAsset(assetID string, catalogID string, info *fapp.DatasetDetails, input *fapp.FybrikApplication) (string, error)
RegisterAsset registers a new asset in the specified catalog Input arguments: - assetID: DataSetID as it appears in fybrik-application - catalogID: the destination catalog identifier - info: connection and credential details Returns: - an error if happened - the new asset identifier
func (*FybrikApplicationReconciler) SetupWithManager ¶
func (r *FybrikApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager registers FybrikApplication controller
func (*FybrikApplicationReconciler) ValidateAssetResponse ¶ added in v0.6.0
func (r *FybrikApplicationReconciler) ValidateAssetResponse(response interface{}, taxonomyFile, datasetID string) error
type FybrikModuleReconciler ¶ added in v0.6.0
type FybrikModuleReconciler struct { client.Client Name string Log zerolog.Logger Scheme *runtime.Scheme }
FybrikModuleReconciler reconciles a FybrikModule object
func NewFybrikModuleReconciler ¶ added in v0.6.0
func NewFybrikModuleReconciler(mgr ctrl.Manager, name string) *FybrikModuleReconciler
NewFybrikModuleReconciler creates a new reconciler for FybrikModules
func (*FybrikModuleReconciler) Reconcile ¶ added in v0.6.0
func (r *FybrikModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
Reconcile validates FybrikModule CRD
func (*FybrikModuleReconciler) SetupWithManager ¶ added in v0.6.0
func (r *FybrikModuleReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager registers Module controller
type HelmValues ¶ added in v1.0.1
type HelmValues struct { // Asset specific arguments such as data stores and transformations fapp.ModuleArguments `json:",inline"` // Application details such as workload selector and user information *fapp.ApplicationDetails `json:",inline"` // Application and debug labels Labels map[string]string `json:"labels"` // Application unique identifier UUID string `json:"uuid"` }
HelmValues are the values passed to modules during orchestration of the data plane
type ModuleInstanceSpec ¶ added in v0.6.0
type ModuleInstanceSpec struct { Module fapp.BlueprintModule ClusterName string Scope fapp.CapabilityScope }
ModuleInstanceSpec consists of the module spec and arguments
type NewAssetInfo ¶
type NewAssetInfo struct { StorageAccount *fappv2.FybrikStorageAccountSpec Details *fappv1.DataStore Persistent bool }
NewAssetInfo points to the provisioned storage and holds information about the new asset
type PathBuilder ¶ added in v0.7.0
component responsible for data path construction
func (*PathBuilder) FindPaths ¶ added in v0.7.0
func (p *PathBuilder) FindPaths() []datapath.Solution
FindPaths finds all valid data paths between the data source and the workload First, data paths are constructed using interface connections, starting from data source. Then, transformations are added to the found paths, and clusters are matched to satisfy restrictions from admin config policies. Optimization is done by the shortest path (the paths are sorted by the length). To be changed in future versions.
type PlotterGenerator ¶ added in v0.6.0
type PlotterGenerator struct { Client client.Client Log *zerolog.Logger UUID string Owner types.NamespacedName StorageManager storage.StorageManagerInterface ProvisionedStorage map[string]NewAssetInfo }
PlotterGenerator constructs a plotter based on the requirements (governance actions, data location) and the existing set of FybrikModules
func (*PlotterGenerator) AddFlowInfoForAsset ¶ added in v0.6.0
func (p *PlotterGenerator) AddFlowInfoForAsset(item *datapath.DataInfo, application *fappv1.FybrikApplication, selection *datapath.Solution, plotterSpec *fappv1.PlotterSpec) error
Adds the asset details, flows and templates to the given plotter spec.
func (*PlotterGenerator) Provision ¶ added in v0.6.0
func (p *PlotterGenerator) Provision(item *datapath.DataInfo, destinationInterface *taxonomy.Interface, account *fappv2.FybrikStorageAccountSpec) (*fappv1.DataStore, error)
Provision allocates storage based on the selected account and generates the destination data store for the plotter
type PlotterInterface ¶
PlotterInterface context implementation for communication with a single Plotter resource
func NewPlotterInterface ¶
func NewPlotterInterface(cl client.Client) *PlotterInterface
NewPlotterInterface creates a new plotter interface for FybrikApplication controller
func (*PlotterInterface) CreateOrUpdateResource ¶
func (c *PlotterInterface) CreateOrUpdateResource(owner, ref *fapp.ResourceReference, plotterSpec *fapp.PlotterSpec, labels map[string]string, uuid string) error
CreateOrUpdateResource creates a new Plotter resource or updates an existing one
func (*PlotterInterface) CreateResourceReference ¶
func (c *PlotterInterface) CreateResourceReference(owner *fapp.ResourceReference) *fapp.ResourceReference
CreateResourceReference returns an identifier (name and namespace) of the generated resource.
func (*PlotterInterface) DeleteResource ¶
func (c *PlotterInterface) DeleteResource(ref *fapp.ResourceReference) error
DeleteResource deletes the generated Plotter resource
func (*PlotterInterface) GetManagedObject ¶
func (c *PlotterInterface) GetManagedObject() runtime.Object
GetManagedObject returns the type of the managed runtime object
func (*PlotterInterface) GetResourceSignature ¶
func (c *PlotterInterface) GetResourceSignature(ref *fapp.ResourceReference) *fapp.Plotter
GetResourceSignature returns the namespaced information of the generated Plotter resource
func (*PlotterInterface) GetResourceStatus ¶
func (c *PlotterInterface) GetResourceStatus(ref *fapp.ResourceReference) (fapp.ObservedState, error)
GetResourceStatus returns the generated Plotter status
func (*PlotterInterface) ResourceExists ¶
func (c *PlotterInterface) ResourceExists(ref *fapp.ResourceReference) bool
ResourceExists checks whether the Plotter resource generated by FybrikApplication controller is active
type PlotterModulesSpec ¶ added in v0.5.0
type PlotterModulesSpec struct { ClusterName string VaultAuthPath string AssetID string ModuleName string ModuleArguments *fapp.StepParameters FlowType taxonomy.DataFlow Chart fapp.ChartSpec Scope fapp.CapabilityScope Capability taxonomy.Capability }
PlotterModulesSpec consists of module details extracted from the Plotter structure
type PlotterReconciler ¶
type PlotterReconciler struct { client.Client Name string Log zerolog.Logger Scheme *runtime.Scheme ClusterManager multicluster.ClusterManager }
PlotterReconciler reconciles a Plotter object
func NewPlotterReconciler ¶
func NewPlotterReconciler(mgr ctrl.Manager, name string, manager multicluster.ClusterManager) *PlotterReconciler
NewPlotterReconciler creates a new reconciler for Plotter resources
func (*PlotterReconciler) GenerateBlueprint ¶ added in v0.5.0
func (r *PlotterReconciler) GenerateBlueprint(instances []ModuleInstanceSpec, clusterName string, plotter *fapp.Plotter) fapp.BlueprintSpec
GenerateBlueprint creates the Blueprint spec based on the datasets and the governance actions required, which dictate the modules that must run in the fybrik Credentials for accessing data set are stored in a credential management system (such as vault) and the paths for accessing them are included in the blueprint. The credentials themselves are not included in the blueprint.
func (*PlotterReconciler) GenerateBlueprints ¶ added in v0.5.0
func (r *PlotterReconciler) GenerateBlueprints(instances []ModuleInstanceSpec, plotter *fapp.Plotter) map[string]fapp.BlueprintSpec
GenerateBlueprints creates Blueprint specs (one per cluster)
func (*PlotterReconciler) RefineInstances ¶ added in v0.5.0
func (r *PlotterReconciler) RefineInstances(instances []ModuleInstanceSpec) []ModuleInstanceSpec
RefineInstances collects all instances of the same read/write module with non "Asset" scope and creates a new instance instead, with accumulated arguments.
func (*PlotterReconciler) SetupWithManager ¶
func (r *PlotterReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager registers Plotter controller