crawler

type Crawler struct {
	Client        *http.Client
	Domain        string
	DomainID      int
	Authoritative bool
	Exclude       []*regexp.Regexp
	Delay         time.Duration
	RetryAfter    time.Duration
	Robots        *robotstxt.Group
	UserAgent     string
	Start         time.Time
	// contains filtered or unexported fields
}

type Metadata struct {
	Title       *string
	Robots      []string
	Author      *string
	Description *string
	Canonical   *url.URL
	JavaScript  bool
}