Documentation ¶
Index ¶
- type AutoProxyGetter
- type DefaultAutoProxyGetter
- type DefaultProxyPool
- func (p *DefaultProxyPool) CheckExist(proxyAddr string) bool
- func (p *DefaultProxyPool) CheckProxy(proxyAddr string) bool
- func (p *DefaultProxyPool) EraseProxy(proxy string) int
- func (p *DefaultProxyPool) GetProxy() (string, error)
- func (p *DefaultProxyPool) LenOfProxies() int
- func (p *DefaultProxyPool) PushProxy(proxy ...string)
- type ProxyPool
- type WithThresholdDecorator
- type WithTimeDecorator
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AutoProxyGetter ¶
type AutoProxyGetter interface { ProxyPool proxy_crawler.Crawler }
type DefaultAutoProxyGetter ¶
type DefaultAutoProxyGetter struct { ProxyPool proxy_crawler.Crawler }
type DefaultProxyPool ¶
type DefaultProxyPool struct { CheckBeforeGetProxy bool // if true, proxy will be checked for availability before returning; otherwise, it will be returned directly // contains filtered or unexported fields }
func (*DefaultProxyPool) CheckExist ¶
func (p *DefaultProxyPool) CheckExist(proxyAddr string) bool
func (*DefaultProxyPool) CheckProxy ¶
func (p *DefaultProxyPool) CheckProxy(proxyAddr string) bool
func (*DefaultProxyPool) EraseProxy ¶
func (p *DefaultProxyPool) EraseProxy(proxy string) int
The efficiency is not high when the number of proxies is large
func (*DefaultProxyPool) GetProxy ¶
func (p *DefaultProxyPool) GetProxy() (string, error)
func (*DefaultProxyPool) LenOfProxies ¶
func (p *DefaultProxyPool) LenOfProxies() int
func (*DefaultProxyPool) PushProxy ¶
func (p *DefaultProxyPool) PushProxy(proxy ...string)
type ProxyPool ¶
type ProxyPool interface { GetProxy() (string, error) // return an usable proxy. if there is not an usable proxy, return "", error CheckProxy(proxy string) bool // check if the proxy is usable EraseProxy(proxy string) int // erase the proxy from the proxy list PushProxy(proxy ...string) // push the proxy into the proxy list LenOfProxies() int }
type WithThresholdDecorator ¶
type WithThresholdDecorator struct { AutoProxyGetter // contains filtered or unexported fields }
func WrapWithThresholdDecorator ¶
func WrapWithThresholdDecorator(a AutoProxyGetter, threshold int) *WithThresholdDecorator
WrapWithThresholdDecorator 达到阈值时爬取代理ip
type WithTimeDecorator ¶
type WithTimeDecorator struct { AutoProxyGetter // contains filtered or unexported fields }
func WrapWithTimeDecorator ¶
func WrapWithTimeDecorator(a AutoProxyGetter, interval int) *WithTimeDecorator
WrapWithTimeDecorator 周期性爬取代理ip
Click to show internal directories.
Click to hide internal directories.