Documentation
¶
Index ¶
- Variables
- func DecodeProxy(vm *otto.Otto, js string) (proxy string, err error)
- func DecodeString(vm *otto.Otto, js string) (proxy string, err error)
- func ParsePort(vm *otto.Otto, PortJs string) (port string, err error)
- type Crawler
- type IpHai
- type Spider
- func (s *Spider) Cron() string
- func (s *Spider) Enabled() bool
- func (s *Spider) Fetch(proxyURL string, useProxy bool) (body string, err error)
- func (s *Spider) GetProxyChan() chan<- *model.HttpProxy
- func (s *Spider) GetReferer() string
- func (s *Spider) Name() string
- func (s *Spider) NeedRetry() bool
- func (s *Spider) Parse(string) ([]*model.HttpProxy, error)
- func (s *Spider) RandomDelay() bool
- func (s *Spider) Retry() uint
- func (s *Spider) SetProxyChan(ch chan<- *model.HttpProxy)
- func (s *Spider) StartUrl() []string
- func (s *Spider) TimeOut() int
Constants ¶
This section is empty.
Variables ¶
View Source
var ListOfSpider = []Crawler{ &IpHai{}, &rudnkh{}, &coolProxy{}, &xici{}, &spys{}, &pubProxy{}, &kuaiProxy{}, &cn66{}, &feiyi{}, &ip89{}, &goubanjia{}, &freeip{}, &ab57{}, &clarketm{}, &httptunnel{}, &proxylist{}, &proxylistplus{}, &aliveProxy{}, &proxyDb{}, &usProxy{}, &siteDigger{}, &dogdev{}, &newProxy{}, &xseo{}, &ultraProxies{}, &premProxy{}, &nntime{}, &proxyListsLine{}, &myProxy{}, &proxyIpList{}, &blackHat{}, &proxyLists{}, &ip3366{}, &xiladaili{}, &nimadaili{}, &zdy{}, }
View Source
var (
MaxProxyReachedErr = errors.New("max proxy reached")
)
Functions ¶
Types ¶
type Crawler ¶
type Crawler interface {
Run()
StartUrl() []string
Cron() string
Name() string
Retry() uint
NeedRetry() bool
Enabled() bool
// url , if use proxy
Fetch(string, bool) (string, error)
SetProxyChan(chan<- *model.HttpProxy)
GetProxyChan() chan<- *model.HttpProxy
Parse(string) ([]*model.HttpProxy, error)
}
func GetSpiders ¶
type Spider ¶
type Spider struct {
// contains filtered or unexported fields
}
func (*Spider) GetProxyChan ¶
func (*Spider) GetReferer ¶
func (*Spider) RandomDelay ¶
func (*Spider) SetProxyChan ¶
Source Files
¶
- base.go
- enter_ab57.go
- enter_proxylists.go
- enter_pubproxy.go
- enter_rudnkh.go
- html_5u.go
- html_cn_proxy.go
- html_cz.go
- html_feiyi.go
- html_gatherproxy.go
- html_goubanjia.go
- html_hai.go
- html_ihuan.go
- html_ip3366.go
- html_kuaidaili.go
- html_kxdaili.go
- html_nimadaili.go
- html_nntime.go
- html_premproxy.go
- html_proxydb.go
- html_proxylistplus.go
- html_proxylists.go
- html_site_digger.go
- html_spys.go
- html_ultraproxies.go
- html_us_proxy.go
- html_xici.go
- html_xila.go
- html_zdy.go
- json_cool_proxy.go
- list.go
- re_66cn.go
- re_aliveproxy.go
- re_blackhat.go
- re_clarketm.go
- re_dogdev.go
- re_freeip.go
- re_httptunnel.go
- re_ip89.go
- re_my_proxy.go
- re_newproxy.go
- re_proxy_ip_list.go
- re_proxylist.go
- re_xseo.go
Click to show internal directories.
Click to hide internal directories.