Documentation ¶
Index ¶
Constants ¶
View Source
const ( // NameOfXici 西刺代理 `https://www.xicidaili.com/nn/` NameOfXici = "xici" // NameOfKuai 快代理 `https://www.kuaidaili.com/ops/`, `https://www.kuaidaili.com/free/` NameOfKuai = "kuai" // NameOfYun 云代理,质量较高. `http://www.ip3366.net/free/` NameOfYun = "yun" // NameOfIphai ip海代理,`http://www.iphai.com/free/ng` NameOfIphai = "iphai" // NameOfXila 西拉免费代理,`http://www.xiladaili.com/` NameOfXila = "xila" // NameOfNima 泥马代理,量较大,`http://www.nimadaili.com/` NameOfNima = "nima" // NameOfEightnine 89免费代理,`http://www.89ip.cn/` NameOfEightnine = "eightnine" // NameOfHappy 开心代理,`http://ip.kxdaili.com/` NameOfHappy = "kaixin" )
View Source
const ( Idle = iota Crawling CoolDown )
Variables ¶
This section is empty.
Functions ¶
func CoolDownTime ¶
CoolDownTime sets the sleep time after crawlOnce, the purpose is to reduce the risk being banned of ip by the website.
Types ¶
type Spider ¶
type Spider struct {
// contains filtered or unexported fields
}
Spider provides the instance for crawling jobs.
func BuildAndInitAll ¶
func BuildAndInitAll() (spiders []*Spider)
BuildAndInitAll returns all of the enable spider.
func (*Spider) Start ¶
func (s *Spider) Start(ch proxy.CachedChan)
Start calls crawlOnce after sleeping the period duration or when receiving a re-crawl chan.
Click to show internal directories.
Click to hide internal directories.