Documentation ¶
Index ¶
Constants ¶
View Source
const ( DefaultCrawlDelay = 5 * time.Second DefaultCrawlPoliteness = false DefaultLogFormat = "text" DefaultLogLevel = "info" DefaultUserAgent = fetchbot.DefaultUserAgent )
Variables ¶
View Source
var ( DefaultHTTPClient = http.DefaultClient DefaultCache = NewMemoryCache() )
View Source
var (
ErrNotFound = errors.New("not found")
)
Functions ¶
This section is empty.
Types ¶
type CmdBasicAuth ¶
type CmdBasicAuth struct { *fetchbot.Cmd S *url.URL D int // contains filtered or unexported fields }
func (*CmdBasicAuth) BasicAuth ¶
func (c *CmdBasicAuth) BasicAuth() (string, string)
func (*CmdBasicAuth) Depth ¶
func (c *CmdBasicAuth) Depth() int
func (*CmdBasicAuth) SourceURL ¶
func (c *CmdBasicAuth) SourceURL() *url.URL
type Crawler ¶
type Crawler struct { CrawlDelay time.Duration CrawlDuration time.Duration CrawlPoliteness bool LogFormat string LogLevel string Logger *logrus.Logger UserAgent string HTTPClient fetchbot.Doer Cache Cache // contains filtered or unexported fields }
func (*Crawler) Match ¶
func (c *Crawler) Match(r *ResponseCriteria) *fetchbot.ResponseMatcher
func (*Crawler) Register ¶
func (c *Crawler) Register(rc *ResponseCriteria, procs ...Processor)
type MemoryCache ¶
func NewMemoryCache ¶
func NewMemoryCache() *MemoryCache
func (*MemoryCache) Delete ¶
func (r *MemoryCache) Delete(key string) error
func (*MemoryCache) Get ¶
func (r *MemoryCache) Get(key string) (interface{}, error)
func (*MemoryCache) Set ¶
func (r *MemoryCache) Set(key string, value interface{}) error
type ProcessorFunc ¶
Source Files ¶
Click to show internal directories.
Click to hide internal directories.