gf_crawl_lib

type GFcrawler struct {
	Name_str    string
	StartURLstr string
}

type GFcrawlerConfig struct {
	Crawled_images_s3_bucket_name_str string
	Images_s3_bucket_name_str         string
	Images_local_dir_path_str         string
	Cluster_node_type_str             string
	Crawl_config_file_path_str        string
	ImagesUseNewStorageEngineBool     bool
}

type GFcrawlerCycleRun struct {
	Id                   primitive.ObjectID `bson:"_id,omitempty"`
	Id_str               string             `bson:"id_str"`
	T_str                string             `bson:"t"` //"crawler_cycle_run"
	Creation_unix_time_f float64            `bson:"creation_unix_time_f"`
	Crawler_name_str     string             `bson:"crawler_name_str"`
	Target_domain_str    string             `bson:"targit_domain_str"`
	Target_url_str       string             `bson:"target_url_str"`
	Start_time_f         float64            `bson:"start_time_f"`
	End_time_f           float64            `bson:"end_time_f"`
}