Documentation
¶
Index ¶
Constants ¶
const ( CRAWLER_VB_ATTACHMENTS = "vb-attachments" CRAWLER_IMAGE = "img" CRAWLER_FILE = "file" )
const ( PAGER_VB4 = "vb4" PAGER_QUERY = "query" PAGER_URLCUT = "cutter" )
const DEFAULT_DL_JOBS = 5
const DEFAULT_REDIRECTS = 10
Variables ¶
This section is empty.
Functions ¶
func Crawl ¶
func Crawl(cc *CrawlContext) error
Types ¶
type CrawlContext ¶
type CrawlContext struct {
Cookies []*http.Cookie
Pager PagerInterface
Crawler CrawlerInterface
// contains filtered or unexported fields
}
func NewCrawlContext ¶
func NewCrawlContext(pager string, crawler string, defaultDir string) (*CrawlContext, error)
func (*CrawlContext) SetOptions ¶
func (cc *CrawlContext) SetOptions(args []string) error
Parse global options and attach them to the CrawlContext
type CrawlerInterface ¶
type CrawlerInterface interface {
Crawl(*url.URL) error
Finish()
SetOptions([]string) error
Setup()
}
func NewFileCrawler ¶
func NewFileCrawler(cc *CrawlContext) (CrawlerInterface, error)
func NewImageCrawler ¶
func NewImageCrawler(cc *CrawlContext) (CrawlerInterface, error)
func NewVBAttachmentCrawler ¶
func NewVBAttachmentCrawler(cc *CrawlContext) (CrawlerInterface, error)
type FileCrawler ¶
type FileCrawler struct {
// contains filtered or unexported fields
}
FileCrawler is a crawler that treats every input from the pager as a file that needs to be downloaded.
func (FileCrawler) Finish ¶
func (c FileCrawler) Finish()
Finish() is a default cleanup function for crawlers, If baseCrawler's Setup() or setup() method was used Finish() closes baseCrawler's DownloadDispatcher and yields until all Downloads have been finished. Otherwise it does nothing.
func (FileCrawler) SetOptions ¶
type ImageCrawler ¶
type ImageCrawler struct {
// contains filtered or unexported fields
}
func (ImageCrawler) Finish ¶
func (c ImageCrawler) Finish()
Finish() is a default cleanup function for crawlers, If baseCrawler's Setup() or setup() method was used Finish() closes baseCrawler's DownloadDispatcher and yields until all Downloads have been finished. Otherwise it does nothing.
func (*ImageCrawler) SetOptions ¶
func (r *ImageCrawler) SetOptions(args []string) error
type PagerInterface ¶
type PagerInterface interface {
Next() (*url.URL, error)
PageNum() int
SetOptions([]string) error
SetUrl(string) error
}
func NewQueryPager ¶
func NewQueryPager(cc *CrawlContext) PagerInterface
func NewURLCuttingPager ¶
func NewURLCuttingPager(cc *CrawlContext) PagerInterface
func NewVB4Pager ¶
func NewVB4Pager(cc *CrawlContext) PagerInterface
type QueryPager ¶
type QueryPager struct {
// contains filtered or unexported fields
}
func (*QueryPager) PageNum ¶
func (r *QueryPager) PageNum() int
func (*QueryPager) SetOptions ¶
func (r *QueryPager) SetOptions(args []string) error
func (*QueryPager) SetUrl ¶
func (r *QueryPager) SetUrl(addr string) error
type URLCuttingPager ¶
type URLCuttingPager struct {
// contains filtered or unexported fields
}
URLCuttingPager browses through the pages by cutting out a part of itself and replacing that with an increasing number.
func (*URLCuttingPager) PageNum ¶
func (r *URLCuttingPager) PageNum() int
func (*URLCuttingPager) SetOptions ¶
func (r *URLCuttingPager) SetOptions(args []string) error
func (*URLCuttingPager) SetUrl ¶
func (r *URLCuttingPager) SetUrl(addr string) error
type VB4Pager ¶
type VB4Pager struct {
Start int
End int
Thread *url.URL
// contains filtered or unexported fields
}
func (*VB4Pager) SetOptions ¶
type VBAttachmentCrawler ¶
type VBAttachmentCrawler struct {
// contains filtered or unexported fields
}
func (VBAttachmentCrawler) Finish ¶
func (c VBAttachmentCrawler) Finish()
Finish() is a default cleanup function for crawlers, If baseCrawler's Setup() or setup() method was used Finish() closes baseCrawler's DownloadDispatcher and yields until all Downloads have been finished. Otherwise it does nothing.
func (*VBAttachmentCrawler) SetOptions ¶
func (r *VBAttachmentCrawler) SetOptions(args []string) error
Notes ¶
Bugs ¶
Password needs to be filtered out of the url before printing it.
Source Files
Directories