crawler

type Anchor ¶

type Anchor struct {
	Href  string
	Title string
}

Anchor represent an HTML anchor tag.

type Collector ¶

type Collector interface {
	Collect(url *url.URL) (*http.Response, []Anchor, error)
}

type Crawler ¶

type Crawler struct {
	URL *url.URL

	Collector
	Processor
	Filter
	Executor *executor.Executor
}

type CrossDomainFilter ¶

type CrossDomainFilter struct {
	Domain string
}

func (filter CrossDomainFilter) Filter(urls []string) []string

type DefaultProcessor ¶

type DefaultProcessor struct{}

DefaultProcessor is the default implementation of the crawler.Processor interface. This just creates a appropriate CrawlReport instance from the results of the collector.

func (processor DefaultProcessor) Process(URL *url.URL, res *http.Response, anchors []Anchor,
	err error) executor.Report

type Filter ¶

type Filter interface {
	Filter(urls []string) []string
}

type NoneFilter ¶

type NoneFilter struct{}

func (filter NoneFilter) Filter(urls []string) []string

type Processor ¶

type Processor interface {
	Process(url *url.URL, response *http.Response, connectedURLs []Anchor, err error) executor.Report
}

type Report ¶

type Report struct {
	Url        string
	HTTPStatus int
	Err        string
	Anchors    []Anchor
}

Report represents the results of the crawling task of a single URL.

func (report Report) Status() int

func (report Report) String() string

type Task ¶

type Task struct {
	Crawler Crawler
}

func (task Task) Execute() executor.Report

func (task Task) String() string

type URLCollector ¶

type URLCollector struct {
	URLMap      map[uint64]bool
	AnchorRegex *regexp.Regexp
	*sync.Mutex
}

URLCollector is an implementation of the collector interface that collects Anchors for the pages that the crawler visits.

func (collector *URLCollector) Collect(URL *url.URL) (*http.Response, []Anchor, error)