crawler

type CrawlItem struct {
	Url     string
	Retries int
	Result  Result
}

type Crawler struct {
	sync.Mutex

	ConcurrentRequests int
	MaxRetries         int

	AllowCrawlPatterns []*regexp.Regexp

	Logger *logrus.Logger

	Client *http.Client
	// contains filtered or unexported fields
}

type Result struct {
	// The final (possibly redirected) url.
	Url string

	Error    error
	CanRetry bool

	Urls []string
}