crawler

type BasicAuthClient struct {
	Options *ClientOptions
	// contains filtered or unexported fields
}

type Client interface {
	Get(u string) (*http.Response, error)
	Head(u string) (*http.Response, error)
	Do(req *http.Request) (*http.Response, error)
}

type ClientOptions struct {
	UserAgent        string
	BasicAuth        bool
	BasicAuthDomains []string
	AuthUser         string
	AuthPass         string
}

type Crawler struct {
	// contains filtered or unexported fields
}

type CrawlerClient interface {
	Get(u string) (*http.Response, error)
	Head(u string) (*http.Response, error)
	Do(req *http.Request) (*http.Response, error)
}

type HttpCrawler struct {
	// contains filtered or unexported fields
}

type Options struct {
	MaxPageReports     int
	IgnoreRobotsTxt    bool
	FollowNofollow     bool
	IncludeNoindex     bool
	UserAgent          string
	CrawlSitemap       bool
	AllowSubdomains    bool
	BasicAuth          bool
	AuthUser           string
	AuthPass           string
	CheckExternalLinks bool
}

type Parser struct {
	ParsedURL *url.URL
	Headers   *http.Header
	// contains filtered or unexported fields
}

type Queue struct {
	// contains filtered or unexported fields
}

type RequestMessage struct {
	URL   string
	Depth int
}

type ResponseMessage struct {
	URL      string
	Response *http.Response
	Error    error
	Depth    int
}

type RobotsChecker struct {
	// contains filtered or unexported fields
}

type SitemapChecker struct {
	// contains filtered or unexported fields
}

type URLStorage struct {
	// contains filtered or unexported fields
}