spider

type Context struct {
	Ctx      context.Context
	Cancel   context.CancelFunc
	Response *http.Response

	Temp map[string]interface{}
	// contains filtered or unexported fields
}

type MiddlewareHandler func(ctx *Context)

type MiddlewareHandlerErr func(ctx *Context) error

type Resource struct {
	SpiderUniqueKey string
	*downloader.Request
}

type Schedule struct {
	ResourcePoolList chan Resource
	ConcurrentNum    int // 并发数量
	// contains filtered or unexported fields
}

type Spider struct {
	UniqueKey string // 唯一标识符
	STATUS    uint   // 状态

	Downloader    downloader.Downloader          // 下载器
	RuleHandlers  map[string][]MiddlewareHandler // 规则中间件
	CloseCallback func(s *Spider)                // 回调关闭
	// contains filtered or unexported fields
}