frontier

type Cookie struct {

	// Cookie name.
	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
	// Cookie value.
	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
	// Cookie domain.
	Domain string `protobuf:"bytes,3,opt,name=domain,proto3" json:"domain,omitempty"`
	// Cookie path.
	Path string `protobuf:"bytes,4,opt,name=path,proto3" json:"path,omitempty"`
	// Cookie expiration date as the number of seconds since the UNIX epoch.
	Expires float64 `protobuf:"fixed64,5,opt,name=expires,proto3" json:"expires,omitempty"`
	// Cookie size.
	Size int32 `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"`
	// True if cookie is http-only.
	HttpOnly bool `protobuf:"varint,7,opt,name=http_only,json=httpOnly,proto3" json:"http_only,omitempty"`
	// True if cookie is secure.
	Secure bool `protobuf:"varint,8,opt,name=secure,proto3" json:"secure,omitempty"`
	// True in case of session cookie.
	Session bool `protobuf:"varint,9,opt,name=session,proto3" json:"session,omitempty"`
	// Cookie SameSite type.
	SameSite string `protobuf:"bytes,10,opt,name=same_site,json=sameSite,proto3" json:"same_site,omitempty"`
	// contains filtered or unexported fields
}

type CountResponse struct {
	Count int64 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"`
	// contains filtered or unexported fields
}

type CrawlExecutionId struct {
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// contains filtered or unexported fields
}

type CrawlExecutionStatus struct {
	Id                  string                     `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	State               CrawlExecutionStatus_State `protobuf:"varint,2,opt,name=state,proto3,enum=veidemann.api.frontier.v1.CrawlExecutionStatus_State" json:"state,omitempty"`
	JobId               string                     `protobuf:"bytes,3,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	SeedId              string                     `protobuf:"bytes,4,opt,name=seed_id,json=seedId,proto3" json:"seed_id,omitempty"`
	StartTime           *timestamppb.Timestamp     `protobuf:"bytes,6,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` // When this crawl execution started crawling
	EndTime             *timestamppb.Timestamp     `protobuf:"bytes,7,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"`       // When this crawl execution ended
	DocumentsCrawled    int64                      `protobuf:"varint,8,opt,name=documents_crawled,json=documentsCrawled,proto3" json:"documents_crawled,omitempty"`
	BytesCrawled        int64                      `protobuf:"varint,9,opt,name=bytes_crawled,json=bytesCrawled,proto3" json:"bytes_crawled,omitempty"`
	UrisCrawled         int64                      `protobuf:"varint,10,opt,name=uris_crawled,json=urisCrawled,proto3" json:"uris_crawled,omitempty"`
	DocumentsFailed     int64                      `protobuf:"varint,11,opt,name=documents_failed,json=documentsFailed,proto3" json:"documents_failed,omitempty"`
	DocumentsOutOfScope int64                      `protobuf:"varint,12,opt,name=documents_out_of_scope,json=documentsOutOfScope,proto3" json:"documents_out_of_scope,omitempty"`
	DocumentsRetried    int64                      `protobuf:"varint,13,opt,name=documents_retried,json=documentsRetried,proto3" json:"documents_retried,omitempty"`
	DocumentsDenied     int64                      `protobuf:"varint,14,opt,name=documents_denied,json=documentsDenied,proto3" json:"documents_denied,omitempty"`
	LastChangeTime      *timestamppb.Timestamp     `protobuf:"bytes,15,opt,name=last_change_time,json=lastChangeTime,proto3" json:"last_change_time,omitempty"` // When this record was last updated
	CreatedTime         *timestamppb.Timestamp     `protobuf:"bytes,16,opt,name=created_time,json=createdTime,proto3" json:"created_time,omitempty"`            // When this crawl execution was created
	CurrentUriId        []string                   `protobuf:"bytes,20,rep,name=current_uri_id,json=currentUriId,proto3" json:"current_uri_id,omitempty"`
	JobExecutionId      string                     `protobuf:"bytes,21,opt,name=job_execution_id,json=jobExecutionId,proto3" json:"job_execution_id,omitempty"`
	Error               *v11.Error                 `protobuf:"bytes,22,opt,name=error,proto3" json:"error,omitempty"` // Extra description of error state
	DesiredState        CrawlExecutionStatus_State ``                                                                 // Used when external process want to abort execution
	/* 157-byte string literal not displayed */
	// contains filtered or unexported fields
}

type CrawlExecutionStatusChange struct {
	Id                     string                     `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	State                  CrawlExecutionStatus_State `protobuf:"varint,2,opt,name=state,proto3,enum=veidemann.api.frontier.v1.CrawlExecutionStatus_State" json:"state,omitempty"`
	EndTime                *timestamppb.Timestamp     `protobuf:"bytes,4,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` // When this crawl execution ended
	AddDocumentsCrawled    int64                      `protobuf:"varint,5,opt,name=add_documents_crawled,json=addDocumentsCrawled,proto3" json:"add_documents_crawled,omitempty"`
	AddBytesCrawled        int64                      `protobuf:"varint,6,opt,name=add_bytes_crawled,json=addBytesCrawled,proto3" json:"add_bytes_crawled,omitempty"`
	AddUrisCrawled         int64                      `protobuf:"varint,7,opt,name=add_uris_crawled,json=addUrisCrawled,proto3" json:"add_uris_crawled,omitempty"`
	AddDocumentsFailed     int64                      `protobuf:"varint,8,opt,name=add_documents_failed,json=addDocumentsFailed,proto3" json:"add_documents_failed,omitempty"`
	AddDocumentsOutOfScope int64                      `` /* 134-byte string literal not displayed */
	AddDocumentsRetried    int64                      `protobuf:"varint,10,opt,name=add_documents_retried,json=addDocumentsRetried,proto3" json:"add_documents_retried,omitempty"`
	AddDocumentsDenied     int64                      `protobuf:"varint,11,opt,name=add_documents_denied,json=addDocumentsDenied,proto3" json:"add_documents_denied,omitempty"`
	AddCurrentUri          *QueuedUri                 `protobuf:"bytes,12,opt,name=add_current_uri,json=addCurrentUri,proto3" json:"add_current_uri,omitempty"`
	DeleteCurrentUri       *QueuedUri                 `protobuf:"bytes,13,opt,name=delete_current_uri,json=deleteCurrentUri,proto3" json:"delete_current_uri,omitempty"`
	Error                  *v11.Error                 `protobuf:"bytes,14,opt,name=error,proto3" json:"error,omitempty"` // Extra description of error state
	// contains filtered or unexported fields
}

type CrawlExecutionStatus_State int32

type CrawlHostGroup struct {

	// If IP-address is not resolved this is SHA-1 of URL.
	// After IP-resolution this is SHA-1 hash of IP or CrawlHostGroupConfig id if one such config matched.
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// Lover limit for time between pageloads from this CrawlHostGroup.
	MinTimeBetweenPageLoadMs int64 `` /* 142-byte string literal not displayed */
	// Upper limit for time between pageloads from this CrawlHostGroup.
	// This is the upper limit for calculation of dealy time, but actual time might be higher depending on
	// the harvesters capacity.
	MaxTimeBetweenPageLoadMs int64 `` /* 142-byte string literal not displayed */
	// The fetch time of the URI is multiplied with this value to get the delay time before fetching the next URI.
	// If min_time_between_page_load_ms and/or max_time_between_page_load_ms are set, then those values are used as
	// the upper/lower limits for delay.
	// If delay_factor is unset or zero, then a delay_facor of one is assumed. If delay_factor is negative,
	// a delay_factor of zero is assumed.
	DelayFactor float32 `protobuf:"fixed32,4,opt,name=delay_factor,json=delayFactor,proto3" json:"delay_factor,omitempty"`
	// The maximum number of retries before giving up fetching a uri.
	MaxRetries int32 `protobuf:"varint,5,opt,name=max_retries,json=maxRetries,proto3" json:"max_retries,omitempty"`
	// The minimum time before a failed page load is retried.
	RetryDelaySeconds int32 `protobuf:"varint,6,opt,name=retry_delay_seconds,json=retryDelaySeconds,proto3" json:"retry_delay_seconds,omitempty"`
	// The number of queued Uri's belonging to this CrawlHostGroup
	QueuedUriCount int64 `protobuf:"varint,7,opt,name=queued_uri_count,json=queuedUriCount,proto3" json:"queued_uri_count,omitempty"`
	// If this CrawlHostGroup is busy, this field contains the id of the uri currently beeing fetched.
	CurrentUriId string `protobuf:"bytes,8,opt,name=current_uri_id,json=currentUriId,proto3" json:"current_uri_id,omitempty"`
	// Token to guard against two harvesters responding to the same request.
	SessionToken string `protobuf:"bytes,9,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"`
	// The time when frontier sent a PageHarvestSpec to a harvester.
	FetchStartTimeStamp *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=fetch_start_time_stamp,json=fetchStartTimeStamp,proto3" json:"fetch_start_time_stamp,omitempty"`
	// contains filtered or unexported fields
}

type CrawlSeedRequest struct {
	JobExecutionId string           `protobuf:"bytes,1,opt,name=job_execution_id,json=jobExecutionId,proto3" json:"job_execution_id,omitempty"`
	Job            *v1.ConfigObject `protobuf:"bytes,5,opt,name=job,proto3" json:"job,omitempty"`
	Seed           *v1.ConfigObject `protobuf:"bytes,6,opt,name=seed,proto3" json:"seed,omitempty"`
	// When this seed should stop crawling. Absence of this value indicates no timeout
	Timeout *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=timeout,proto3" json:"timeout,omitempty"`
	// contains filtered or unexported fields
}

type FrontierClient interface {
	// Start crawling seed
	CrawlSeed(ctx context.Context, in *CrawlSeedRequest, opts ...grpc.CallOption) (*CrawlExecutionId, error)
	// Request a URI from the Frontiers queue.
	// Used by a Harvester to fetch a new page. If no URI is ready for harvesting, Frontier should return
	// gRPC status NOT_FOUND. Harvester should then retry the request after a reasonable backoff time.
	GetNextPage(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*PageHarvestSpec, error)
	// Inform Frontier that a page fetch was finished.
	// Contains metrics, outlinks and error as a stream of messages. Client closes stream when finished.
	PageCompleted(ctx context.Context, opts ...grpc.CallOption) (Frontier_PageCompletedClient, error)
	// The number of busy CrawlHostGroups which essentially is the number of web pages currently downloading
	BusyCrawlHostGroupCount(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*CountResponse, error)
	// Total number of queued URI's
	QueueCountTotal(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*CountResponse, error)
	// Number of queued URI's for a CrawlExecution
	QueueCountForCrawlExecution(ctx context.Context, in *CrawlExecutionId, opts ...grpc.CallOption) (*CountResponse, error)
	// Number of queued URI's for a CrawlHostGroup
	QueueCountForCrawlHostGroup(ctx context.Context, in *CrawlHostGroup, opts ...grpc.CallOption) (*CountResponse, error)
}

type FrontierServer interface {
	// Start crawling seed
	CrawlSeed(context.Context, *CrawlSeedRequest) (*CrawlExecutionId, error)
	// Request a URI from the Frontiers queue.
	// Used by a Harvester to fetch a new page. If no URI is ready for harvesting, Frontier should return
	// gRPC status NOT_FOUND. Harvester should then retry the request after a reasonable backoff time.
	GetNextPage(context.Context, *emptypb.Empty) (*PageHarvestSpec, error)
	// Inform Frontier that a page fetch was finished.
	// Contains metrics, outlinks and error as a stream of messages. Client closes stream when finished.
	PageCompleted(Frontier_PageCompletedServer) error
	// The number of busy CrawlHostGroups which essentially is the number of web pages currently downloading
	BusyCrawlHostGroupCount(context.Context, *emptypb.Empty) (*CountResponse, error)
	// Total number of queued URI's
	QueueCountTotal(context.Context, *emptypb.Empty) (*CountResponse, error)
	// Number of queued URI's for a CrawlExecution
	QueueCountForCrawlExecution(context.Context, *CrawlExecutionId) (*CountResponse, error)
	// Number of queued URI's for a CrawlHostGroup
	QueueCountForCrawlHostGroup(context.Context, *CrawlHostGroup) (*CountResponse, error)
	// contains filtered or unexported methods
}

type Frontier_PageCompletedClient interface {
	Send(*PageHarvest) error
	CloseAndRecv() (*emptypb.Empty, error)
	grpc.ClientStream
}

type Frontier_PageCompletedServer interface {
	SendAndClose(*emptypb.Empty) error
	Recv() (*PageHarvest, error)
	grpc.ServerStream
}

type JobExecutionStatus struct {
	Id                  string                   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	JobId               string                   `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	State               JobExecutionStatus_State `protobuf:"varint,3,opt,name=state,proto3,enum=veidemann.api.frontier.v1.JobExecutionStatus_State" json:"state,omitempty"`
	ExecutionsState     map[string]int32         `` /* 195-byte string literal not displayed */
	StartTime           *timestamppb.Timestamp   `protobuf:"bytes,6,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
	EndTime             *timestamppb.Timestamp   `protobuf:"bytes,7,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"`
	DocumentsCrawled    int64                    `protobuf:"varint,8,opt,name=documents_crawled,json=documentsCrawled,proto3" json:"documents_crawled,omitempty"`
	BytesCrawled        int64                    `protobuf:"varint,9,opt,name=bytes_crawled,json=bytesCrawled,proto3" json:"bytes_crawled,omitempty"`
	UrisCrawled         int64                    `protobuf:"varint,10,opt,name=uris_crawled,json=urisCrawled,proto3" json:"uris_crawled,omitempty"`
	DocumentsFailed     int64                    `protobuf:"varint,11,opt,name=documents_failed,json=documentsFailed,proto3" json:"documents_failed,omitempty"`
	DocumentsOutOfScope int64                    `protobuf:"varint,12,opt,name=documents_out_of_scope,json=documentsOutOfScope,proto3" json:"documents_out_of_scope,omitempty"`
	DocumentsRetried    int64                    `protobuf:"varint,13,opt,name=documents_retried,json=documentsRetried,proto3" json:"documents_retried,omitempty"`
	DocumentsDenied     int64                    `protobuf:"varint,14,opt,name=documents_denied,json=documentsDenied,proto3" json:"documents_denied,omitempty"`
	Error               *v11.Error               `protobuf:"bytes,15,opt,name=error,proto3" json:"error,omitempty"` // Extra description of error state
	DesiredState        JobExecutionStatus_State ``                                                                 // Used when external process want to abort execution
	/* 155-byte string literal not displayed */
	// contains filtered or unexported fields
}

type JobExecutionStatus_State int32

type PageHarvest struct {

	// Types that are assignable to Msg:
	//
	//	*PageHarvest_Metrics_
	//	*PageHarvest_Outlink
	//	*PageHarvest_Error
	Msg isPageHarvest_Msg `protobuf_oneof:"msg"`
	// Session token from the PageHarvestSpec.
	SessionToken string `protobuf:"bytes,5,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"`
	// contains filtered or unexported fields
}

type PageHarvestSpec struct {

	// The URI to fetch
	QueuedUri *QueuedUri `protobuf:"bytes,1,opt,name=queued_uri,json=queuedUri,proto3" json:"queued_uri,omitempty"`
	// The configuration for the fetch
	CrawlConfig *v1.ConfigObject `protobuf:"bytes,2,opt,name=crawl_config,json=crawlConfig,proto3" json:"crawl_config,omitempty"`
	// Session token for this request.
	// The Harvester is responsible for setting the same session token in all responses to this request.
	SessionToken string `protobuf:"bytes,3,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"`
	// contains filtered or unexported fields
}

type PageHarvest_Error struct {
	// If the overall page fetch failed. Should not be used for a singel uri failure
	Error *v11.Error `protobuf:"bytes,4,opt,name=error,proto3,oneof"`
}

type PageHarvest_Metrics struct {

	// The number of uri's downloaded. The requested uri + embedded resources
	UriCount int32 `protobuf:"varint,1,opt,name=uri_count,json=uriCount,proto3" json:"uri_count,omitempty"`
	// Byte count for the resources downloaded. Includes embedded resources
	BytesDownloaded int64 `protobuf:"varint,2,opt,name=bytes_downloaded,json=bytesDownloaded,proto3" json:"bytes_downloaded,omitempty"`
	// contains filtered or unexported fields
}

type PageHarvest_Metrics_ struct {
	// Collected metrics for the page fetched
	Metrics *PageHarvest_Metrics `protobuf:"bytes,2,opt,name=metrics,proto3,oneof"`
}

type PageHarvest_Outlink struct {
	// The outlinks found in the harvested page
	Outlink *QueuedUri `protobuf:"bytes,3,opt,name=outlink,proto3,oneof"`
}

type QueuedUri struct {
	Id                  string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	ExecutionId         string                 `protobuf:"bytes,2,opt,name=execution_id,json=executionId,proto3" json:"execution_id,omitempty"`
	DiscoveredTimeStamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=discovered_time_stamp,json=discoveredTimeStamp,proto3" json:"discovered_time_stamp,omitempty"`
	// Sequence number to order the fetch of uris from a seed
	Sequence int64  `protobuf:"varint,4,opt,name=sequence,proto3" json:"sequence,omitempty"`
	Uri      string `protobuf:"bytes,5,opt,name=uri,proto3" json:"uri,omitempty"`
	Ip       string `protobuf:"bytes,7,opt,name=ip,proto3" json:"ip,omitempty"`
	// *
	// Get the discoveryPath,
	//
	//	R - Redirect
	//	E - Embed
	//	X - Speculative embed (aggressive/Javascript link extraction)
	//	L - Link
	//	P - Prerequisite (as for DNS or robots.txt before another URI)
	DiscoveryPath          string                 `protobuf:"bytes,8,opt,name=discovery_path,json=discoveryPath,proto3" json:"discovery_path,omitempty"`
	Referrer               string                 `protobuf:"bytes,9,opt,name=referrer,proto3" json:"referrer,omitempty"`
	Cookies                []*Cookie              `protobuf:"bytes,10,rep,name=cookies,proto3" json:"cookies,omitempty"`
	PageFetchTimeMs        int64                  `protobuf:"varint,11,opt,name=page_fetch_time_ms,json=pageFetchTimeMs,proto3" json:"page_fetch_time_ms,omitempty"` // The time used to fetch and render the the uri including dependencies
	Retries                int32                  `protobuf:"varint,12,opt,name=retries,proto3" json:"retries,omitempty"`                                            // Number of times this uri has been scheduled for retry.
	EarliestFetchTimeStamp *timestamppb.Timestamp ``                                                                                                                 // Do not fetch this uri before this time
	/* 132-byte string literal not displayed */
	CrawlHostGroupId string        `protobuf:"bytes,14,opt,name=crawl_host_group_id,json=crawlHostGroupId,proto3" json:"crawl_host_group_id,omitempty"` // The Crawl Host Group calculated for this uri
	PolitenessRef    *v1.ConfigRef `protobuf:"bytes,15,opt,name=politeness_ref,json=politenessRef,proto3" json:"politeness_ref,omitempty"`              // Ref to the politeness config used when discovering this uri
	Error            *v11.Error    `protobuf:"bytes,16,opt,name=error,proto3" json:"error,omitempty"`                                                   // Contains the error reason if fetch failed
	// Id of the job execution for this uri.
	// A job execution is an instance of a job
	JobExecutionId      string                 `protobuf:"bytes,17,opt,name=job_execution_id,json=jobExecutionId,proto3" json:"job_execution_id,omitempty"`
	Unresolved          bool                   `protobuf:"varint,18,opt,name=unresolved,proto3" json:"unresolved,omitempty"` // If true, then this uri is just added to the queue and no resolution of ip or robots.txt checks are done yet.
	FetchStartTimeStamp *timestamppb.Timestamp `protobuf:"bytes,19,opt,name=fetch_start_time_stamp,json=fetchStartTimeStamp,proto3" json:"fetch_start_time_stamp,omitempty"`
	// The weighting between jobs when two jobs compete on fetching resources from the same hosts.
	// Copied from CrawlConfig for efficiency.
	PriorityWeight float64 `protobuf:"fixed64,20,opt,name=priority_weight,json=priorityWeight,proto3" json:"priority_weight,omitempty"`
	// The seed uri which was the starting point for this uri.
	SeedUri string `protobuf:"bytes,21,opt,name=seed_uri,json=seedUri,proto3" json:"seed_uri,omitempty"`
	// Annotations used as parameters to scripts.
	Annotation []*v1.Annotation `protobuf:"bytes,22,rep,name=annotation,proto3" json:"annotation,omitempty"`
	// contains filtered or unexported fields
}

type UnimplementedFrontierServer struct {
}

type UnsafeFrontierServer interface {
	// contains filtered or unexported methods
}