gf_images_jobs_core

package
v0.0.0-...-a1b0e2b Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 26, 2024 License: GPL-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source
const JOB_STATUS__COMPLETED job_status_val = "completed"
View Source
const JOB_STATUS__FAILED job_status_val = "failed"
View Source
const JOB_STATUS__FAILED_PARTIAL job_status_val = "failed_partial"
View Source
const JOB_UPDATE_TYPE__COMPLETED job_update_type_val = "completed"
View Source
const JOB_UPDATE_TYPE__ERROR job_update_type_val = "error"
View Source
const JOB_UPDATE_TYPE__OK job_update_type_val = "ok"

Variables

This section is empty.

Functions

This section is empty.

Types

type GF_jobs_lifecycle_callbacks

type GF_jobs_lifecycle_callbacks struct {
	Job_type__transform_imgs__fun func() *gf_core.GFerror
	Job_type__uploaded_imgs__fun  func() *gf_core.GFerror
}

type GFimageClassificationToProcess

type GFimageClassificationToProcess struct {
	GFimageIDstr gf_images_core.GFimageID
}

type GFimageExternToProcess

type GFimageExternToProcess struct {
	SourceURLstr     string `bson:"source_url_str"` // FIX!! - rename this to Origin_url_str to be consistent with other origin_url naming
	OriginPageURLstr string `bson:"origin_page_url_str"`
}

type GFimageLocalToProcess

type GFimageLocalToProcess struct {
	LocalFilePathStr string
}

type GFimageUploadedToProcess

type GFimageUploadedToProcess struct {
	GFimageIDstr  gf_images_core.GFimageID
	S3filePathStr string                 // path to image in S3 in a bucket that it was originally uploaded to by client
	MetaMap       map[string]interface{} // metadata user might include for this image
}

type GFjobRunning

type GFjobRunning struct {
	Id              primitive.ObjectID `bson:"_id,omitempty"`
	IDstr           string             `bson:"id_str"`
	T_str           string             `bson:"t"`
	Client_type_str string             `bson:"client_type_str"`
	Status_str      string             `bson:"status_str"` // "running"|"complete"
	Start_time_f    float64            `bson:"start_time_f"`
	End_time_f      float64            `bson:"end_time_f"`

	// LEGACY!! - update "images_to_process_lst" to "images_extern_to_process_lst" in the DB (bson)
	Images_extern_to_process_lst   []GFimageExternToProcess   `bson:"images_to_process_lst"`
	Images_uploaded_to_process_lst []GFimageUploadedToProcess `bson:"images_uploaded_to_process_lst"`

	Errors_lst []JobError `bson:"errors_lst"`

	// user that submited the job
	UserID gf_core.GF_ID `bson:"user_id"`
	// contains filtered or unexported fields
}

func JobsMngrCreateRunningJob

func JobsMngrCreateRunningJob(pClientTypeStr string,
	p_job_updates_ch chan JobUpdateMsg,
	pUserID gf_core.GF_ID,
	pRuntimeSys *gf_core.RuntimeSys) (*GFjobRunning, *gf_core.GFerror)

type GFjobRuntime

type GFjobRuntime struct {
	// contains filtered or unexported fields
}

type GFmetrics

type GFmetrics struct {
	Cmd__start_job_local_imgs__count     prometheus.Counter
	Cmd__start_job_transform_imgs__count prometheus.Counter
	Cmd__start_job_uploaded_imgs__count  prometheus.Counter
	Cmd__start_job_extern_imgs__count    prometheus.Counter
	CmdStartJobClassifyImagesCount       prometheus.Counter
	ImagesPluginsMetrics                 *gf_images_plugins.GFmetrics
}

func MetricsCreate

func MetricsCreate(pNamespaceStr string) *GFmetrics

type JobError

type JobError struct {
	Type_str             string `bson:"type_str"`  //"fetcher_error"|"transformer_error"
	Error_str            string `bson:"error_str"` //serialization of the golang error
	Image_source_url_str string `bson:"image_source_url_str"`
}

type JobMsg

type JobMsg struct {
	JobIDstr        string // if its an existing job. for new jobs the mngr creates a new job ID
	UserID          gf_core.GF_ID
	Client_type_str string // type of client that is sending this message to the Job manager
	CmdStr          string // "start_job" | "get_running_job_ids"

	Job_init_ch    chan *GFjobRunning // used by clients for receiving outputs of job initialization by jobs_mngr
	Job_updates_ch chan JobUpdateMsg  // used by jobs_mngr to send job_updates to

	//------------------------
	// RESPONSE_CH - channel passed by the client, which is to be used for the response.
	//		used as a general mechanism for clients to receive responses from the jobs_mngr
	//		after the job has been processed.
	ResponseCh chan interface{}

	Images_extern_to_process_lst   []GFimageExternToProcess
	Images_uploaded_to_process_lst []GFimageUploadedToProcess
	Images_local_to_process_lst    []GFimageLocalToProcess
	ImagesToClassifyLst            []GFimageClassificationToProcess

	// FLOW_NAMES
	Flows_names_lst []string
}

type JobUpdateMsg

type JobUpdateMsg struct {
	Name_str             string                        `json:"name_str"`
	Type_str             job_update_type_val           `json:"type_str"` // "ok" | "error" | "complete"
	ImageIDstr           gf_images_core.GFimageID      `json:"image_id_str"`
	Image_source_url_str string                        `json:"image_source_url_str"`
	Err_str              string                        `json:"err_str,omitempty"` // if the update indicates an error, this is its value
	Image_thumbs         *gf_images_core.GFimageThumbs `json:"-"`
}

type JobsMngr

type JobsMngr chan JobMsg

func JobsMngrInit

func JobsMngrInit(pImagesStoreLocalDirPathStr string,
	pImagesThumbnailsStoreLocalDirPathStr string,
	pVideoStoreLocalDirPathStr string,
	pMediaDomainStr string,
	pLifecycleCallbacks *GF_jobs_lifecycle_callbacks,
	pConfig *gf_images_core.GFconfig,
	pImageStorage *gf_images_storage.GFimageStorage,
	pS3info *gf_aws.GFs3Info,
	pMetricsCore *gf_images_core.GFmetrics,
	pRuntimeSys *gf_core.RuntimeSys) JobsMngr

func TgetJobsMngr

func TgetJobsMngr(pImagesStoreLocalDirPathStr string,
	pImagesThumbnailsStoreLocalDirPathStr string,
	pVideoStoreLocalDirPathStr string,
	pMediaDomainStr string,
	pPluginsPyDirPathStr string,
	pImagesClassifyPyDirPathStr string,
	pRuntimeSys *gf_core.RuntimeSys) JobsMngr

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL