jindofsx

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 18, 2024 License: Apache-2.0 Imports: 43 Imported by: 0

Documentation

Index

Constants

View Source
const (
	CSI_DRIVER = "fuse.csi.fluid.io"

	Mount_TYPE = "mount_type"

	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Disk Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Disk Capacity: "

	SUMMARY_PREFIX_TOTAL_MEM_CAPACITY = "Total MEM Capacity: "

	SUMMARY_PREFIX_USED_MEM_CAPACITY = "Used MEM Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	JINDO_MASTERNUM_DEFAULT = 1
	JINDO_HA_MASTERNUM      = 3

	DEFAULT_MASTER_RPC_PORT = 8101
	DEFAULT_WORKER_RPC_PORT = 6101
	DEFAULT_RAFT_RPC_PORT   = 8103

	FuseOnly = "fuseOnly"

	QueryUfsTotal = "QUERY_UFS_TOTAL"
)

Variables

This section is empty.

Functions

func GetReservedPorts

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for JindoRuntime

func Precheck

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type Fuse

type Fuse struct {
	Args           []string            `json:"args"`
	HostPath       string              `json:"hostPath"`
	NodeSelector   map[string]string   `json:"nodeSelector,omitempty"`
	FuseProperties map[string]string   `json:"properties"`
	Global         bool                `json:"global,omitempty"`
	RunAs          string              `json:"runAs,omitempty"`
	Tolerations    []corev1.Toleration `json:"tolerations,omitempty"`
	Labels         map[string]string   `json:"labels,omitempty"`
	Annotations    map[string]string   `json:"annotations,omitempty"`
	CriticalPod    bool                `json:"criticalPod,omitempty"`
	Resources      Resources           `json:"resources,omitempty"`
	MountPath      string              `json:"mountPath,omitempty"`
	Mode           string              `json:"mode,omitempty"`
	Env            map[string]string   `json:"env,omitempty"`
	HostPID        bool                `json:"hostPID,omitempty"`
}

type HadoopConfig

type HadoopConfig struct {
	ConfigMap       string `json:"configMap"`
	IncludeHdfsSite bool   `json:"includeHdfsSite"`
	IncludeCoreSite bool   `json:"includeCoreSite"`
}

type Jindo

type Jindo struct {
	Image               string                 `json:"image"`
	ImageTag            string                 `json:"imageTag"`
	ImagePullPolicy     string                 `json:"imagePullPolicy"`
	FuseImage           string                 `json:"fuseImage"`
	FuseImageTag        string                 `json:"fuseImageTag"`
	FuseImagePullPolicy string                 `json:"fuseImagePullPolicy"`
	User                int                    `json:"user"`
	Group               int                    `json:"group"`
	UseHostNetwork      bool                   `json:"useHostNetwork"`
	Properties          map[string]string      `json:"properties"`
	Master              Master                 `json:"master"`
	Worker              Worker                 `json:"worker"`
	Fuse                Fuse                   `json:"fuse"`
	Mounts              Mounts                 `json:"mounts"`
	HadoopConfig        HadoopConfig           `json:"hadoopConfig,omitempty"`
	Secret              string                 `json:"secret,omitempty"`
	Tolerations         []corev1.Toleration    `json:"tolerations,omitempty"`
	InitPortCheck       common.InitPortCheck   `json:"initPortCheck,omitempty"`
	LogConfig           map[string]string      `json:"logConfig,omitempty"`
	FuseLogConfig       map[string]string      `json:"fuseLogConfig,omitempty"`
	PlacementMode       string                 `json:"placement,omitempty"`
	Owner               *common.OwnerReference `json:"owner,omitempty"`
	RuntimeIdentity     common.RuntimeIdentity `json:"runtimeIdentity"`
	ClusterDomain       string                 `json:"clusterDomain,omitempty"`
	UFSVolumes          []UFSVolume            `json:"ufsVolumes,omitempty"`
	SecretKey           string                 `json:"secretKey,omitempty"`
	SecretValue         string                 `json:"secretValue,omitempty"`
	UseStsToken         bool                   `json:"UseStsToken"`
	MountType           string                 `json:"mountType,omitempty"`
}

type JindoFSxEngine

type JindoFSxEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	Recorder record.EventRecorder
	*ctrl.Helper
	// contains filtered or unexported fields
}

func (*JindoFSxEngine) AssignNodesToCache

func (e *JindoFSxEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)

func (*JindoFSxEngine) BindToDataset

func (e *JindoFSxEngine) BindToDataset() (err error)

func (*JindoFSxEngine) CheckAndUpdateRuntimeStatus

func (e *JindoFSxEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*JindoFSxEngine) CheckMasterReady

func (e *JindoFSxEngine) CheckMasterReady() (ready bool, err error)

func (*JindoFSxEngine) CheckRuntimeHealthy

func (e *JindoFSxEngine) CheckRuntimeHealthy() (err error)

func (*JindoFSxEngine) CheckRuntimeReady

func (e *JindoFSxEngine) CheckRuntimeReady() (ready bool)

func (*JindoFSxEngine) CheckWorkersReady

func (e *JindoFSxEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady checks if the workers are ready

func (*JindoFSxEngine) CreateDataLoadJob

func (e *JindoFSxEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error)

CreateDataLoadJob creates the job to load data

func (*JindoFSxEngine) CreateDataMigrateJob added in v0.9.0

func (e *JindoFSxEngine) CreateDataMigrateJob(ctx cruntime.ReconcileRequestContext, targetDataMigrate datav1alpha1.DataMigrate) error

func (*JindoFSxEngine) CreateVolume

func (e *JindoFSxEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*JindoFSxEngine) DeleteVolume

func (e *JindoFSxEngine) DeleteVolume() (err error)

DeleteVolume

func (*JindoFSxEngine) FreeStorageBytes

func (e *JindoFSxEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of Jindo in bytes

func (*JindoFSxEngine) GetDataOperationValueFile added in v0.9.0

func (e *JindoFSxEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*JindoFSxEngine) GetHCFSStatus

func (e *JindoFSxEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*JindoFSxEngine) GetHCFSStatusWithoutMaster

func (e *JindoFSxEngine) GetHCFSStatusWithoutMaster() (status *datav1alpha1.HCFSStatus, err error)

func (*JindoFSxEngine) GetReportSummary

func (e *JindoFSxEngine) GetReportSummary() (summary string, err error)

report jindo summary

func (*JindoFSxEngine) HasDeprecatedCommonLabelname

func (e *JindoFSxEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (*JindoFSxEngine) PrepareUFS

func (e *JindoFSxEngine) PrepareUFS() (err error)

PrepareUFS do all the UFS preparations

func (*JindoFSxEngine) SetupMaster

func (e *JindoFSxEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*JindoFSxEngine) SetupWorkers

func (e *JindoFSxEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*JindoFSxEngine) ShouldCheckUFS

func (e *JindoFSxEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*JindoFSxEngine) ShouldSetupMaster

func (e *JindoFSxEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*JindoFSxEngine) ShouldSetupWorkers

func (e *JindoFSxEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*JindoFSxEngine) ShouldUpdateUFS

func (e *JindoFSxEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

JindoFSxEngine hasn't support UpdateOnUFSChange

func (*JindoFSxEngine) Shutdown

func (e *JindoFSxEngine) Shutdown() (err error)

Shutdown shuts down the Jindo engine

func (*JindoFSxEngine) SyncMetadata

func (e *JindoFSxEngine) SyncMetadata() (err error)

func (JindoFSxEngine) SyncReplicas

func (e JindoFSxEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

func (*JindoFSxEngine) SyncRuntime

func (e *JindoFSxEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*JindoFSxEngine) SyncScheduleInfoToCacheNodes

func (e *JindoFSxEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs the cache info of the nodes by labeling the nodes And the Application pod can leverage such info for scheduling

func (*JindoFSxEngine) TotalFileNums

func (e *JindoFSxEngine) TotalFileNums() (value int64, err error)

return the total num of files in Jindo

func (*JindoFSxEngine) TotalJindoStorageBytes

func (e *JindoFSxEngine) TotalJindoStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoFSxEngine) TotalStorageBytes

func (e *JindoFSxEngine) TotalStorageBytes() (value int64, err error)

return total storage size of Jindo in bytes

func (*JindoFSxEngine) UpdateCacheOfDataset

func (e *JindoFSxEngine) UpdateCacheOfDataset() (err error)

func (*JindoFSxEngine) UpdateDatasetStatus

func (e *JindoFSxEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

func (*JindoFSxEngine) UpdateOnUFSChange

func (e *JindoFSxEngine) UpdateOnUFSChange(*utils.UFSToUpdate) (updateReady bool, err error)

func (*JindoFSxEngine) UsedStorageBytes

func (e *JindoFSxEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of Jindo in bytes

type Level

type Level struct {
	Path       string `json:"path,omitempty"`
	Type       string `json:"type,omitempty"`
	MediumType string `json:"mediumType,omitempty"`
	Quota      string `json:"quota,omitempty"`
}

type Master

type Master struct {
	ReplicaCount        int                  `json:"replicaCount"`
	Resources           Resources            `json:"resources"`
	NodeSelector        map[string]string    `json:"nodeSelector,omitempty"`
	MasterProperties    map[string]string    `json:"properties"`
	FileStoreProperties map[string]string    `json:"fileStoreProperties"`
	TokenProperties     map[string]string    `json:"secretProperties"`
	Port                Ports                `json:"ports,omitempty"`
	OssKey              string               `json:"osskey,omitempty"`
	OssSecret           string               `json:"osssecret,omitempty"`
	Tolerations         []corev1.Toleration  `json:"tolerations,omitempty"`
	DnsServer           string               `json:"dnsServer,omitempty"`
	NameSpace           string               `json:"namespace,omitempty"`
	Labels              map[string]string    `json:"labels,omitempty"`
	Annotations         map[string]string    `json:"annotations,omitempty"`
	ServiceCount        int                  `json:"svccount"`
	Env                 map[string]string    `json:"env,omitempty"`
	VolumeMounts        []corev1.VolumeMount `json:"volumeMounts,omitempty"`
	Volumes             []corev1.Volume      `json:"volumes,omitempty"`
}

type Mounts

type Mounts struct {
	Master            map[string]*Level `json:"master"`
	WorkersAndClients map[string]*Level `json:"workersAndClients"`
}

type Ports

type Ports struct {
	Rpc  int `json:"rpc,omitempty"`
	Raft int `json:"raft,omitempty"`
}

type Resource

type Resource struct {
	CPU    string `json:"cpu"`
	Memory string `json:"memory"`
}

type Resources

type Resources struct {
	Limits   Resource `json:"limits"`
	Requests Resource `json:"requests"`
}

type UFSVolume added in v0.9.0

type UFSVolume struct {
	Name          string `json:"name"`
	SubPath       string `json:"subPath,omitempty"`
	ContainerPath string `json:"containerPath"`
	ReadOnly      bool   `json:"readOnly"`
}

type Worker

type Worker struct {
	ReplicaCount     int                 `json:"replicaCount"`
	Resources        Resources           `json:"resources,omitempty"`
	NodeSelector     map[string]string   `json:"nodeSelector,omitempty"`
	WorkerProperties map[string]string   `json:"properties"`
	Port             Ports               `json:"ports,omitempty"`
	Tolerations      []corev1.Toleration `json:"tolerations,omitempty"`
	// Affinity         corev1.Affinity       `json:"affinity,omitempty"`
	Labels      map[string]string `json:"labels,omitempty"`
	Annotations map[string]string `json:"annotations,omitempty"`
	Path        string            `json:"dataPath"`
	Env         map[string]string `json:"env,omitempty"`
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL