xspider

type BaseItemPipeline struct {
	ModuleName string
	Logger     *zap.SugaredLogger
	Stats      StatsCollector
}

type BaseMiddleware struct {
	//中间件模块名称
	ModuleName string
	Logger     *zap.SugaredLogger
	Stats      StatsCollector
}

type CloseSpiderFunc func(*Spider)

type Context struct {
	// contains filtered or unexported fields
}

type DefaultDownloader struct {
	// contains filtered or unexported fields
}

type DefaultDupeFilter struct {
	// contains filtered or unexported fields
}

type DefaultScheduler struct {
	// contains filtered or unexported fields
}

type DepthMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type DownloadStatsMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type DownloadTimeoutMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type Downloader interface {
	FromSpider(spider *Spider)
	Fetch(request *Request) (*Response, error)
	IsFree() bool
	IsEmpty() bool
	Close()
	NextRequestCircle(heartbeat time.Duration)
	ProcessDownloader(signal *Signal, spider *Spider)
}

type DownloaderMiddlewareManager struct {
	ModuleName string
	// contains filtered or unexported fields
}

type DownloaderMiddlewarer interface {
	GetModuleName() string

	FromSpider(spider *Spider)

	ProcessRequest(request *Request, spider *Spider) RequestResponse

	ProcessResponse(request *Request, response *Response, spider *Spider) RequestResponse

	ProcessException(request *Request, err error, spider *Spider) RequestResponse
}

type DownloaderSlot struct {
	// contains filtered or unexported fields
}

type DupeFilter interface {
	FromSpider(spider *Spider)
	RequestSeen(request *Request) bool
	RequestFingerprint(request *Request) string
	Close()
	Log(request *Request)
}

type ErrBase struct {
	Reason string
}

type ErrDropItem struct {
	ErrBase
}

type ErrHttpError struct {
	ErrBase
}

type ErrIgnoreRequest struct {
	ErrBase
}

type ErrorbackFunc func(*Failure) RequestItems

type Failure struct {
	Request  *Request
	Response *Response
	Spider   *Spider
	Error    error
}

type HttpAuthMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type HttpErrorMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type Item map[string]interface{}

type ItemPipelineManager struct {
	ModuleName string
	// contains filtered or unexported fields
}

type ItemPipeliner interface {
	GetModuleName() string
	ProcessItem(Item, *Spider) Item
	OpenSpider(*Spider)
	CloseSpider(*Spider)
}

type ItemSlot struct {
	// contains filtered or unexported fields
}

type LifoPriorityQueue struct {
	// contains filtered or unexported fields
}

type OpenSpiderFunc func(*Spider)

type ParseFunc func(*Response, *Spider) RequestItems

type PriorityQueuer interface {
	FromSpider(*Spider)
	Push(item interface{}, priority int)
	Pop() interface{}
	Peek() interface{}
	Size() int
	IsEmpty() bool
	Close()
}

type ProcessExceptionFunc func(request *Request, err error, spider *Spider) RequestResponse

type ProcessItemFunc func(Item, *Spider) Item

type ProcessRequestFunc func(request *Request, spider *Spider) RequestResponse

type ProcessResponseFunc func(request *Request, response *Response, spider *Spider) RequestResponse

type ProcessSpiderExceptionFunc func(response *Response, err error, spider *Spider) RequestItems

type ProcessSpiderInputFunc func(response *Response, spider *Spider)

type ProcessSpiderOutputFunc func(response *Response, result RequestItems, spider *Spider) RequestItems

type ProcessStartRequestsFunc func(result []*Request, spider *Spider) []*Request

type Request struct {
	Url        *url.URL
	Method     string
	Headers    *http.Header
	Body       io.Reader
	Cookies    []http.Cookie
	Encoding   string
	Priority   int
	DontFilter bool
	Ctx        *Context
	//请求抛出错误时的回调函数，错误包括404、请求超时等
	Errback ErrorbackFunc
	//request请求下载完成后处理其response的回调函数
	//默认调用Parse()
	Callback ParseFunc
}

type RequestItems []interface{}

type RequestResponse interface{}

type Response struct {
	StatusCode int
	Body       []byte
	Ctx        *Context
	Request    *Request
	Headers    *http.Header
}

type ResponseSlot struct {
	// contains filtered or unexported fields
}

type RetryMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type Scheduler interface {
	FromSpider(spider *Spider)
	HasPendingRequests() bool
	Close()
	EnqueueRequest(request *Request) bool
	NextRequest() *Request
}

type Setter interface {
	Add(elems ...interface{})
	Clear()
	Pop() interface{}
	Remove(elems ...interface{}) error
	Discard(elems ...interface{})
	Has(elems ...interface{}) bool
	Len() int
}

type Setting struct {
	// 爬虫机器人名称
	// BotName string
	// 是否启用下载状态记录功能
	DownloadStats bool
	// 下载超时时间
	DownloadTimeout time.Duration
	// 单个域名下每个Request的请求间隔
	DownloadDelay time.Duration
	//启用后实际请求间隔会在0.5到1.5倍的DownloadDelay之间随机选取
	RandomizeDownloadDelay bool
	//单个Request最大下载数据量
	DownloadMaxSize int
	//单个Request开始在日志中输出警报信息的下载数据量
	DownloadWarnSize int
	//单个页面允许爬取的最大深度，为0代表无限制
	DepthLimit int
	//用来根据请求深度调整Request中Priority值的整数
	//调整计算公式如下：
	//Request.Priority = Request.Priority - ( Request.Depth * DepthPriority )
	DepthPriority int
	// 是否启用request深度相关记录功能
	DepthStatsVerbose bool
	//是否自动重试
	RetryEnabled bool
	//除第一次外最大重试次数
	MaxRetryTimes int
	//自动重试的状态码
	RetryHttpCodes []int
	// 重试请求优先级的调整参数
	// request.Priority += priorityAdjust
	RetryPriorityAdjust int

	//是否对Request进行去重过滤
	FilterEnabled bool
	//同时处理的最大Item个数
	ConcurrentItems int
	//爬虫下载器同时下载的最大Request个数
	ConcurrentRequests int
	//单个域名允许同时访问的请求个数
	ConcurrentRequestsPerDomain int
	// 允许下载数据占用的最大内存
	ResponseMaxActiveSize int
	// 是否启用日志功能
	LogEnabled bool
	// 除终端外额外将日志内容保存到指定文件
	// 指定ErrFile时仅将Warning以下的日志信息保存到该文件
	LogFile string
	// 指定时会将Warning及以上的日志信息保存到指定文件
	// 仅指定ErrFile时，Warning以下的日志信息仅在终端显示而不保存
	ErrFile string
	// 日志显示及保存的最低日志等级
	LogLevel zapcore.Level
	// url最大长度限制
	UrlLengthLimit int

	HttpErrorAllowAll     bool
	HttpErrorAllowedCodes []int
	// 默认请求头
	DefaultRequestHeaders http.Header
	// 默认UserAgent
	UserAgent string
	// 请求过滤器
	FilterClass DupeFilter
	// 调度器优先级队列
	SchedulerPriorityQueue PriorityQueuer
	// 用户自定义的设置项
	ExtensionSettings *Context

	DownloaderMiddlewaresBase map[int]DownloaderMiddlewarer

	DownloaderMiddlewares map[int]DownloaderMiddlewarer

	SpiderMiddlewaresBase map[int]SpiderMiddlewarer

	SpiderMiddlewares map[int]SpiderMiddlewarer

	ItemPipelinesBase map[int]ItemPipeliner

	ItemPipelines map[int]ItemPipeliner

	SchedulerClass Scheduler

	DownloaderClass Downloader

	HttpUser       string
	HttpPass       string
	HttpAuthDomain []string
}

type Signal struct {
	From  int
	To    int
	Index int
	Body  interface{}
}

type Spider struct {
	// 该spider的名称，用于日志记录
	Name string
	// 可选。包含了spider允许爬取的域名(domain)列表(list)。
	// 当OffsiteMiddleware启用时，域名不在列表中的URL不会被跟进。
	AllowedDomains []string
	// URL列表。当没有制定特定的URL时，spider将从该列表中开始进行爬取。
	// 因此，第一个被获取到的页面的URL将是该列表之一。
	// 后续的URL将会从获取到的数据中提取。
	StartUrls []string
	// 用于生成该爬虫爬取的起始Request，默认使用StartUrls中的链接生成Request
	StartRequestsFunc func(*Spider) []*Request
	// 当response没有指定回调函数时，该方法是xspider处理下载的response的默认方法。
	DefaultParseFunc ParseFunc
	// 爬取结束后执行的自定义函数
	CloseFunc func(*Spider)
	// 爬虫的设置参数，多个爬虫可共用一个相同的设置
	Settings *Setting
	// 日志记录器
	Log *zap.SugaredLogger
	// 爬虫状态记录器
	Stats StatsCollector
	// contains filtered or unexported fields
}

type SpiderMiddlewareManager struct {
	ModuleName string
	// contains filtered or unexported fields
}

type SpiderMiddlewarer interface {
	GetModuleName() string

	FromSpider(spider *Spider)

	ProcessSpiderInput(response *Response, spider *Spider)

	ProcessSpiderOutput(response *Response, result RequestItems, spider *Spider) RequestItems

	ProcessSpiderException(response *Response, err error, spider *Spider) RequestItems

	ProcessStartRequests(result []*Request, spider *Spider) []*Request
}

type SpiderOutputData struct {
	Response *Response
	Result   RequestItems
}

type StatsCollect struct {
	// contains filtered or unexported fields
}

type StatsCollector interface {
	GetValue(key string, dft int) int
	GetStats() StatsMap
	SetValue(key string, value int)
	SetStats(stats StatsMap)
	IncValue(key string, count int, start int)
	MaxValue(key string, value int)
	MinValue(key string, value int)
	ClearStats()
}

type StatsMap map[string]int

type UrlLengthMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}

type UserAgentMiddleware struct {
	BaseMiddleware
	// contains filtered or unexported fields
}