goosefs

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 18, 2024 License: Apache-2.0 Imports: 46 Imported by: 0

Documentation

Index

Constants

View Source
const (

	// goosefsUser string = "fluid"
	METRICS_PREFIX_BYTES_READ_LOCAL = "Cluster.BytesReadLocal "

	METRICS_PREFIX_BYTES_READ_REMOTE = "Cluster.BytesReadRemote "

	METRICS_PREFIX_BYTES_READ_UFS_ALL = "Cluster.BytesReadUfsAll "

	METRICS_PREFIX_BYTES_READ_LOCAL_THROUGHPUT = "Cluster.BytesReadLocalThroughput "

	METRICS_PREFIX_BYTES_READ_REMOTE_THROUGHPUT = "Cluster.BytesReadRemoteThroughput "

	METRICS_PREFIX_BYTES_READ_UFS_THROUGHPUT = "Cluster.BytesReadUfsThroughput "

	SUMMARY_PREFIX_TOTAL_CAPACITY = "Total Capacity: "

	SUMMARY_PREFIX_USED_CAPACITY = "Used Capacity: "

	METADATA_SYNC_NOT_DONE_MSG = "[Calculating]"

	GOOSEFS_RUNTIME_METRICS_LABEL = "goosefs_runtime_metrics"

	CHECK_METADATA_SYNC_DONE_TIMEOUT_MILLISEC = 500

	AUTO_SELECT_PORT_MIN = 20000
	AUTO_SELECT_PORT_MAX = 30000

	PORT_NUM = 9

	CACHE_HIT_QUERY_INTERVAL_MIN = 1

	HADOOP_CONF_HDFS_SITE_FILENAME = "hdfs-site.xml"

	HADOOP_CONF_CORE_SITE_FILENAME = "core-site.xml"

	HADOOP_CONF_MOUNT_PATH = "/hdfs-config"

	WOKRER_POD_ROLE = "goosefs-worker"
)

Variables

This section is empty.

Functions

func Build

Build function builds the GooseFS Engine

func ContainsString added in v0.7.0

func ContainsString(s []string, v string) bool

ContainsString returns true if a string is present in a iteratee.

func GetReservedPorts

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for goosefsRuntime

func Precheck added in v0.8.0

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type APIGateway

type APIGateway struct {
	Enabled bool  `yaml:"enabled,omitempty"`
	Ports   Ports `yaml:"ports,omitempty"`
}

type Affinity

type Affinity struct {
	NodeAffinity *NodeAffinity `yaml:"nodeAffinity"`
}

type Fuse

type Fuse struct {
	Image              string            `yaml:"image,omitempty"`
	NodeSelector       map[string]string `yaml:"nodeSelector,omitempty"`
	ImageTag           string            `yaml:"imageTag,omitempty"`
	ImagePullPolicy    string            `yaml:"imagePullPolicy,omitempty"`
	Properties         map[string]string `yaml:"properties,omitempty"`
	Env                map[string]string `yaml:"env,omitempty"`
	JvmOptions         []string          `yaml:"jvmOptions,omitempty"`
	MountPath          string            `yaml:"mountPath,omitempty"`
	ShortCircuitPolicy string            `yaml:"shortCircuitPolicy,omitempty"`
	Args               []string          `yaml:"args,omitempty"`
	HostNetwork        bool              `yaml:"hostNetwork,omitempty"`
	HostPID            bool              `json:"hostPID,omitempty"`
	Enabled            bool              `yaml:"enabled,omitempty"`
	Resources          common.Resources  `yaml:"resources,omitempty"`
	Global             bool              `yaml:"global,omitempty"`
	CriticalPod        bool              `yaml:"criticalPod,omitempty"`
	Annotations        map[string]string `json:"annotations,omitempty"`
}

type GooseFS

type GooseFS struct {
	FullnameOverride string `yaml:"fullnameOverride"`

	common.ImageInfo `yaml:",inline"`
	common.UserInfo  `yaml:",inline"`

	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`

	Properties map[string]string `yaml:"properties,omitempty"`

	Master Master `yaml:"master,omitempty"`

	JobMaster JobMaster `yaml:"jobMaster,omitempty"`

	Worker Worker `yaml:"worker,omitempty"`

	JobWorker JobWorker `yaml:"jobWorker,omitempty"`

	Fuse Fuse `yaml:"fuse,omitempty"`

	APIGateway APIGateway `yaml:"apiGateway,omitempty"`

	Tieredstore Tieredstore `yaml:"tieredstore,omitempty"`

	Metastore Metastore `yaml:"metastore,omitempty"`

	Journal Journal `yaml:"journal,omitempty"`

	ShortCircuit ShortCircuit `yaml:"shortCircuit,omitempty"`

	UFSPaths []UFSPath `yaml:"ufsPaths,omitempty"`

	UFSVolumes []UFSVolume `yaml:"ufsVolumes,omitempty"`

	InitUsers common.InitUsers `yaml:"initUsers,omitempty"`

	Monitoring string `yaml:"monitoring,omitempty"`

	HadoopConfig HadoopConfig `yaml:"hadoopConfig,omitempty"`

	Tolerations []v1.Toleration `yaml:"tolerations,omitempty"`

	PlacementMode string `yaml:"placement,omitempty"`
}

The value yaml file

type GooseFSEngine

type GooseFSEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	UnitTest bool

	*ctrl.Helper
	Recorder record.EventRecorder
	// contains filtered or unexported fields
}

GooseFSEngine implements the Engine interface.

func (*GooseFSEngine) AssignNodesToCache

func (e *GooseFSEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)

AssignNodesToCache finds nodes to place the cache engine

func (*GooseFSEngine) BindToDataset

func (e *GooseFSEngine) BindToDataset() (err error)

func (*GooseFSEngine) CheckAndUpdateRuntimeStatus

func (e *GooseFSEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*GooseFSEngine) CheckMasterReady

func (e *GooseFSEngine) CheckMasterReady() (ready bool, err error)

CheckMasterReady checks if the master is ready

func (*GooseFSEngine) CheckRuntimeHealthy

func (e *GooseFSEngine) CheckRuntimeHealthy() (err error)

CheckRuntimeHealthy checks the healthy of the runtime

func (*GooseFSEngine) CheckRuntimeReady

func (e *GooseFSEngine) CheckRuntimeReady() (ready bool)

func (*GooseFSEngine) CheckWorkersReady

func (e *GooseFSEngine) CheckWorkersReady() (ready bool, err error)

CheckWorkersReady check the workers ready.

func (*GooseFSEngine) CreateDataLoadJob

func (e *GooseFSEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error)

CreateDataLoadJob creates the job to load data

func (*GooseFSEngine) CreateDataMigrateJob added in v0.9.0

func (e *GooseFSEngine) CreateDataMigrateJob(ctx cruntime.ReconcileRequestContext, targetDataMigrate datav1alpha1.DataMigrate) error

func (*GooseFSEngine) CreateVolume

func (e *GooseFSEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*GooseFSEngine) DeleteVolume

func (e *GooseFSEngine) DeleteVolume() (err error)

DeleteVolume creates volume

func (*GooseFSEngine) FreeStorageBytes

func (e *GooseFSEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of GooseFS in bytes

func (*GooseFSEngine) GetAPIGatewayStatus

func (e *GooseFSEngine) GetAPIGatewayStatus() (status *datav1alpha1.APIGatewayStatus, err error)

Query the API Gateway status

func (*GooseFSEngine) GetCacheHitStates

func (e *GooseFSEngine) GetCacheHitStates() (cacheHitStates cacheHitStates)

GetCacheHitStates gets cache hit related info by parsing GooseFS metrics

func (*GooseFSEngine) GetDataOperationValueFile added in v0.9.0

func (e *GooseFSEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*GooseFSEngine) GetHCFSStatus

func (e *GooseFSEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*GooseFSEngine) GetMetadataFileName

func (e *GooseFSEngine) GetMetadataFileName() string

func (*GooseFSEngine) GetMetadataInfoFile

func (e *GooseFSEngine) GetMetadataInfoFile() string

func (*GooseFSEngine) GetMetadataInfoFileName

func (e *GooseFSEngine) GetMetadataInfoFileName() string

func (*GooseFSEngine) GetReportMetrics

func (e *GooseFSEngine) GetReportMetrics() (summary string, err error)

reportMetrics reports goosefs metrics

func (*GooseFSEngine) GetReportSummary

func (e *GooseFSEngine) GetReportSummary() (summary string, err error)

reportSummary reports goosefs summary

func (*GooseFSEngine) GetWorkerUsedCapacity

func (e *GooseFSEngine) GetWorkerUsedCapacity() (map[string]int64, error)

GetWorkerUsedCapacity gets cache capacity usage for each worker as a map. It parses result from stdout when executing `goosefs fsadmin report capacity` command and extracts worker name(IP or hostname) along with used capacity for that worker

func (*GooseFSEngine) HasDeprecatedCommonLabelname

func (e *GooseFSEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (GooseFSEngine) ParseReportMetric

func (e GooseFSEngine) ParseReportMetric(metrics string, cacheHitStates, lastCacheHitStates *cacheHitStates)

parse goosefs report metric to cacheHitStates

func (GooseFSEngine) ParseReportSummary

func (e GooseFSEngine) ParseReportSummary(s string) cacheStates

parse goosefs report summary to cacheStates

func (*GooseFSEngine) PrepareUFS

func (e *GooseFSEngine) PrepareUFS() (err error)

PrepareUFS does all the UFS preparations

func (*GooseFSEngine) RestoreMetadataInternal

func (e *GooseFSEngine) RestoreMetadataInternal() (err error)

RestoreMetadataInternal restore metadata from backup there are three kinds of data to be restored 1. metadata of dataset 2. ufsTotal info of dataset 3. fileNum info of dataset if 1 fails, the goosefs master will fail directly, if 2 or 3 fails, fluid will get the info from goosefs again

func (*GooseFSEngine) SetupMaster

func (e *GooseFSEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*GooseFSEngine) SetupWorkers

func (e *GooseFSEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*GooseFSEngine) ShouldCheckUFS

func (e *GooseFSEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*GooseFSEngine) ShouldSetupMaster

func (e *GooseFSEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*GooseFSEngine) ShouldSetupWorkers

func (e *GooseFSEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*GooseFSEngine) ShouldUpdateUFS

func (e *GooseFSEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

func (*GooseFSEngine) Shutdown

func (e *GooseFSEngine) Shutdown() (err error)

shut down the GooseFS engine

func (*GooseFSEngine) SyncMetadata

func (e *GooseFSEngine) SyncMetadata() (err error)

SyncMetadata syncs metadata if necessary For GooseFS Engine, metadata sync is an asynchronous operation, which means you should call this function periodically to make sure the function actually takes effect.

func (*GooseFSEngine) SyncReplicas

func (e *GooseFSEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

SyncReplicas syncs the replicas

func (*GooseFSEngine) SyncRuntime added in v0.8.0

func (e *GooseFSEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*GooseFSEngine) SyncScheduleInfoToCacheNodes added in v0.7.0

func (e *GooseFSEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs schedule info to nodes

func (*GooseFSEngine) TotalFileNums

func (e *GooseFSEngine) TotalFileNums() (value int64, err error)

TotalFileNums returns the total num of files in GooseFS

func (*GooseFSEngine) TotalStorageBytes

func (e *GooseFSEngine) TotalStorageBytes() (value int64, err error)

TotalStorageBytes return total storage size of GooseFS in bytes

func (*GooseFSEngine) UpdateCacheOfDataset

func (e *GooseFSEngine) UpdateCacheOfDataset() (err error)

UpdateCacheOfDataset updates the CacheStates and Runtimes of the dataset.

func (*GooseFSEngine) UpdateDatasetStatus

func (e *GooseFSEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

UpdateDatasetStatus updates the status of the dataset

func (*GooseFSEngine) UpdateOnUFSChange

func (e *GooseFSEngine) UpdateOnUFSChange(ufsToUpdate *utils.UFSToUpdate) (updateReady bool, err error)

func (*GooseFSEngine) UsedStorageBytes

func (e *GooseFSEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of GooseFS in bytes

type HadoopConfig

type HadoopConfig struct {
	ConfigMap       string `yaml:"configMap"`
	IncludeHdfsSite bool   `yaml:"includeHdfsSite"`
	IncludeCoreSite bool   `yaml:"includeCoreSite"`
}

type JobMaster

type JobMaster struct {
	Ports     Ports            `yaml:"ports,omitempty"`
	Resources common.Resources `yaml:"resources,omitempty"`
}

type JobWorker

type JobWorker struct {
	Ports     Ports            `yaml:"ports,omitempty"`
	Resources common.Resources `yaml:"resources,omitempty"`
}

type Journal

type Journal struct {
	VolumeType string `yaml:"volumeType,omitempty"`
	Size       string `yaml:"size,omitempty"`
}

type Level

type Level struct {
	Alias      string `yaml:"alias,omitempty"`
	Level      int    `yaml:"level"`
	Mediumtype string `yaml:"mediumtype,omitempty"`
	Type       string `yaml:"type,omitempty"`
	Path       string `yaml:"path,omitempty"`
	Quota      string `yaml:"quota,omitempty"`
	High       string `yaml:"high,omitempty"`
	Low        string `yaml:"low,omitempty"`
}

type Master

type Master struct {
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`
	Env          map[string]string `yaml:"env,omitempty"`
	Affinity     Affinity          `yaml:"affinity"`
	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	Properties   map[string]string `yaml:"properties,omitempty"`
	Replicas     int32             `yaml:"replicaCount,omitempty"`
	HostNetwork  bool              `yaml:"hostNetwork,omitempty"`
	Resources    common.Resources  `yaml:"resources,omitempty"`
	Ports        Ports             `yaml:"ports,omitempty"`
	BackupPath   string            `yaml:"backupPath,omitempty"`
	Restore      Restore           `yaml:"restore,omitempty"`
	Annotations  map[string]string `json:"annotations,omitempty"`
}

type Metastore

type Metastore struct {
	VolumeType string `yaml:"volumeType,omitempty"`
	Size       string `yaml:"size,omitempty"`
}

type NodeAffinity

type NodeAffinity struct {

	// If the affinity requirements specified by this field are not met at
	// scheduling time, the pod will not be scheduled onto the node.
	// If the affinity requirements specified by this field cease to be met
	// at some point during pod execution (e.g. due to an update), the system
	// may or may not try to eventually evict the pod from its node.
	// +optional
	RequiredDuringSchedulingIgnoredDuringExecution *NodeSelector `yaml:"requiredDuringSchedulingIgnoredDuringExecution"`
}

type NodeSelector

type NodeSelector struct {
	//Required. A list of node selector terms.
	NodeSelectorTerms []NodeSelectorTerm `yaml:"nodeSelectorTerms"`
}

NodeSelector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.

type NodeSelectorRequirement

type NodeSelectorRequirement struct {
	// The label key that the selector applies to.
	Key string `yaml:"key,omitempty"`
	// Represents a key's relationship to a set of values.
	// Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
	Operator string `yaml:"operator,omitempty"`
	// +optional
	Values []string `yaml:"values,omitempty"`
}

NodeSelectorRequirement is a selector that contains values, a key, and an operator that relates the key and values.

type NodeSelectorTerm

type NodeSelectorTerm struct {
	// A list of node selector requirements by node's labels.
	MatchExpressions []NodeSelectorRequirement `yaml:"matchExpressions"`
}

NodeSelectorTerm represents expressions and fields required to select nodes. A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.

type Ports

type Ports struct {
	Rpc      int `yaml:"rpc,omitempty"`
	Web      int `yaml:"web,omitempty"`
	Embedded int `yaml:"embedded,omitempty"`
	Data     int `yaml:"data,omitempty"`
	Rest     int `yaml:"rest,omitempty"`
}

type Restore

type Restore struct {
	Enabled bool   `yaml:"enabled,omitempty"`
	Path    string `yaml:"path,omitempty"`
	PVCName string `yaml:"pvcName,omitempty"`
}

type ShortCircuit

type ShortCircuit struct {
	Enable     bool   `yaml:"enable,omitempty"`
	Policy     string `yaml:"policy,omitempty"`
	VolumeType string `yaml:"volumeType,omitempty"`
}

type Tieredstore

type Tieredstore struct {
	Levels []Level `yaml:"levels,omitempty"`
}

type UFSPath

type UFSPath struct {
	HostPath  string `yaml:"hostPath"`
	UFSVolume `yaml:",inline"`
}

type UFSVolume

type UFSVolume struct {
	Name          string `yaml:"name"`
	SubPath       string `yaml:"subPath,omitempty"`
	ContainerPath string `yaml:"containerPath"`
}

type Worker

type Worker struct {
	JvmOptions   []string          `yaml:"jvmOptions,omitempty"`
	Env          map[string]string `yaml:"env,omitempty"`
	NodeSelector map[string]string `yaml:"nodeSelector,omitempty"`
	Properties   map[string]string `yaml:"properties,omitempty"`
	HostNetwork  bool              `yaml:"hostNetwork,omitempty"`
	Resources    common.Resources  `yaml:"resources,omitempty"`
	Ports        Ports             `yaml:"ports,omitempty"`
	Annotations  map[string]string `json:"annotations,omitempty"`
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL