goscrapy

type Context struct {
	context.Context
	// contains filtered or unexported fields
}

type DefaultDownloader struct {
	// contains filtered or unexported fields
}

type Downloader interface {
	Download(*Request) (*Response, error)
}

type Engine struct {
	// contains filtered or unexported fields
}

type FIFOScheduler struct {
	// contains filtered or unexported fields
}

type Items struct {
	sync.Map
	// contains filtered or unexported fields
}

type Option func(e *Engine)

type Pipeline interface {
	Name() string       // returns pipeline's name
	ItemList() []string // returns all items' name that this pipeline cares about
	Handle(items *Items) error
}

type Request struct {
	Method string      `json:"method,omitempty"`
	URL    string      `json:"url,omitempty"`
	Header http.Header `json:"header,omitempty"`
	Query  url.Values  `json:"query,omitempty"`
	// using to decide scheduling sequence. It only means something when using a
	// scheduler that schedules requests based on request weight.
	Weight int
	// contains filtered or unexported fields
}

type RequestHandleFunc func(*Request) error

type Response struct {
	Status     string `json:"status,omitempty"`      // e.g. "200 OK"
	StatusCode int    `json:"status_code,omitempty"` // e.g. 200
	// Request represents request that was send to obtain this response.
	Request *Request `json:"request,omitempty"`
	// Document represents an HTML document to be manipulated.
	Document *goquery.Document `json:"-"`
	// Body represents the response body.
	Body []byte `json:"-"`
	// ContentLength records the length of the associated content. more details see http.Response.
	ContentLength int64 `json:"content_length,omitempty"`
	// Header represents response header, maps header keys to values.
	Header http.Header `json:"header,omitempty"`
}

type ResponseHandleFunc func(*Response) error

type Scheduler interface {
	Start() error                             // start scheduler
	Stop() error                              // stop scheduler
	PopRequest() (req *Request, hasMore bool) // return next request from scheduler
	PushRequest(req *Request) (ok bool)       // add new request into scheduler
	HasMore() bool                            // returns true if there are more request to be scheduled
}

type Spider interface {
	Name() string
	StartRequests() []*Request
	URLMatcher() URLMatcher
	Parse(ctx *Context) (*Items, []*Request, error)
}

type StringMatcher struct {
	// contains filtered or unexported fields
}

type URLMatcher interface {
	// Match returns true if url is matched
	Match(url string) bool
}

type URLRegExpMatcher struct {
	// contains filtered or unexported fields
}

type WeightedScheduler struct {
	// contains filtered or unexported fields
}