crawler

type Config struct {
	PrivateNetworkDetector PrivateNetworkDetector
	URLGetter              URLGetter
	Graph                  Graph
	Indexer                Indexer
	FetchWorkers           int
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Graph interface {
	UpsertLink(link *graph.Link) error
	UpsertEdge(edge *graph.Edge) error
	RemoveStaleEdges(fromID uuid.UUID, updateBefore time.Time) error
}

type Indexer interface {
	Index(doc *index.Document) error
}

type PrivateNetworkDetector interface {
	IsPrivate(host string) (bool, error)
}

type URLGetter interface {
	Get(url string) (*http.Response, error)
}