talpa

type Crawler struct {
	// contains filtered or unexported fields
}

type Downloader interface {
	Open()
	Close()
	Fetch(req *gen.Request, h Helper)
	NumWaitingJobs() int
	NumWorkers() int
}

type Helper interface {
	PutRequest(reqs ...*gen.Request)
	PutJob(jobs ...func())
}

type JobScheduler interface {
	Put(job ...func())
	Get(number int64) []func()
	// contains filtered or unexported methods
}

type RequestScheduler interface {
	Put(reqs ...*gen.Request)
	Get(number int64) []*gen.Request
	// contains filtered or unexported methods
}

type Scraper interface {
	Open()
	Close()
	Send(job func())
	NumWaitingJobs() int
	NumWorkers() int
}

type Spider interface {
	// 生成初始的请求
	// 所有生成的请求都需要在Context设置一个"CallBack"用于对响应的解析
	// 可选的, 可以设置一个"ErrBack"用于处理发送请求时可能产生的错误
	StartRequests() []*gen.Request
}