README
¶
mellivora
程序设计
- spider 产生请求发送给 engine
- engine 把请求发给 scheduler 调度
- engine 从 scheduler 取到待执行的请求
- engine 把请求发给 middleware 进行包装
- middleware 把请求发给 downloader 下载
- downloader 下载
- middleware 对 response 进行处理
- middleware 把 response 发送给 engine
- engine 把返回给 spider,回到开头
TODO
- Context 序列化, 支持远程任务
- 优化 Spider 接口
- 增加扩展功能
- 添加更多常用中间件
Documentation
¶
Index ¶
- func NewMiddleware(next func(MiddlewareFunc) MiddlewareFunc) *middleware
- type BloomFilter
- type Closable
- type Context
- func (c *Context) Engine() *Engine
- func (c *Context) GetDepth() int64
- func (c *Context) GetDontFilter() bool
- func (c *Context) GetRequest() *http.Request
- func (c *Context) MarshalText() ([]byte, error)
- func (c *Context) MustValue(k string, v interface{})
- func (c *Context) Set(k string, v interface{})
- func (c *Context) SetDepth(depth int64)
- func (c *Context) SetDontFilter(dontFilter bool)
- func (c *Context) SetRequest(req *http.Request)
- func (c *Context) SetResponse(response *Response)
- func (c *Context) SetRoundTripper(rt http.RoundTripper)
- func (c *Context) UnmarshalText(text []byte) error
- func (c *Context) Value(k string, v interface{}) error
- type ContextSerializable
- type ContextSerializer
- type Downloader
- type Engine
- type Extension
- type Filter
- type HandleFunc
- type LifoScheduler
- type Middleware
- type MiddlewareFunc
- type Pipeline
- type RequestOptions
- func (c *RequestOptions) GetDepth() int64
- func (c *RequestOptions) GetDontFilter() bool
- func (c *RequestOptions) MarshalText() ([]byte, error)
- func (c *RequestOptions) MustValue(k string, v interface{})
- func (c *RequestOptions) Set(k string, v interface{})
- func (c *RequestOptions) SetDepth(depth int64)
- func (c *RequestOptions) SetDontFilter(dontFilter bool)
- func (c *RequestOptions) UnmarshalText(text []byte) error
- func (c *RequestOptions) Value(k string, v interface{}) error
- type RequestOptionsFunc
- type Response
- type Runnable
- type Scheduler
- type Spider
- type Task
- func Get(url string, handler HandleFunc, options ...RequestOptionsFunc) (Task, error)
- func Gets(urls []string, handler HandleFunc, options ...RequestOptionsFunc) (t Task, err error)
- func MustGet(url string, handler HandleFunc, options ...RequestOptionsFunc) Task
- func NewItems(items ...interface{}) Task
- func Request(request *http.Request, handler HandleFunc, options ...RequestOptionsFunc) Task
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewMiddleware ¶
func NewMiddleware(next func(MiddlewareFunc) MiddlewareFunc) *middleware
NewMiddleware create a middleware instance nolint
Types ¶
type BloomFilter ¶
type BloomFilter struct {
*bloom.BloomFilter
}
func NewBloomFilter ¶
func NewBloomFilter() *BloomFilter
func (*BloomFilter) Add ¶
func (b *BloomFilter) Add(c *Context)
func (*BloomFilter) Exist ¶
func (b *BloomFilter) Exist(c *Context) bool
type Closable ¶
type Closable interface {
// Close release all resources used by this object, including goroutines.
Close() error
}
Closable is the interface for objects that can release its resources.
type Context ¶
type Context struct {
*Response
// contains filtered or unexported fields
}
Context represents the context of the current HTTP request. It holds request and response objects, path, path parameters, data and registered handler.
func NewContext ¶
func NewContext(core *Engine, request *http.Request, handler HandleFunc) *Context
NewContext returns a Context instance.
func (*Context) GetDontFilter ¶
func (c *Context) GetDontFilter() bool
GetDontFilter returns `depth`.
func (*Context) GetRequest ¶
GetRequest returns `*http.Request`.
func (*Context) MarshalText ¶
func (*Context) SetDontFilter ¶
func (c *Context) SetDontFilter(dontFilter bool)
SetDontFilter sets `depth`.
func (*Context) SetRequest ¶
SetRequest sets `*http.Request`.
func (*Context) SetResponse ¶
SetResponse sets `*Response`.
func (*Context) SetRoundTripper ¶
func (c *Context) SetRoundTripper(rt http.RoundTripper)
SetRoundTripper sets `http.RoundTripper`.
func (*Context) UnmarshalText ¶
type ContextSerializable ¶
type ContextSerializer ¶
type ContextSerializer struct {
// contains filtered or unexported fields
}
func NewContextSerializer ¶
func NewContextSerializer() *ContextSerializer
type Downloader ¶
type Downloader struct{}
func NewDownloader ¶
func NewDownloader() *Downloader
func (*Downloader) Next ¶
func (d *Downloader) Next(handler MiddlewareFunc) MiddlewareFunc
type Engine ¶
type Engine struct {
// contains filtered or unexported fields
}
Engine is the top-level framework instance.
func (*Engine) Use ¶
func (e *Engine) Use(middlewares ...Middleware)
Use adds middleware to the chain which is run before spider.
type HandleFunc ¶
MiddlewareFunc defines a function to serve *Context.
type LifoScheduler ¶
type LifoScheduler struct {
// contains filtered or unexported fields
}
func NewLifoScheduler ¶
func NewLifoScheduler() *LifoScheduler
func (*LifoScheduler) Close ¶
func (l *LifoScheduler) Close()
func (*LifoScheduler) Pop ¶
func (l *LifoScheduler) Pop() (c []byte)
func (*LifoScheduler) Push ¶
func (l *LifoScheduler) Push(c []byte)
type Middleware ¶
type Middleware interface {
Next(MiddlewareFunc) MiddlewareFunc
}
Middleware defines a interface to process middleware.
type MiddlewareFunc ¶
MiddlewareFunc defines a function to serve *Context.
type RequestOptions ¶
type RequestOptions struct {
// contains filtered or unexported fields
}
func NewRequestOptions ¶
func NewRequestOptions() *RequestOptions
func (*RequestOptions) GetDepth ¶
func (c *RequestOptions) GetDepth() int64
GetDepth returns `depth`.
func (*RequestOptions) GetDontFilter ¶
func (c *RequestOptions) GetDontFilter() bool
GetDontFilter returns `depth`.
func (*RequestOptions) MarshalText ¶
func (*RequestOptions) SetDepth ¶
func (c *RequestOptions) SetDepth(depth int64)
SetDepth sets `depth`.
func (*RequestOptions) SetDontFilter ¶
func (c *RequestOptions) SetDontFilter(dontFilter bool)
SetDontFilter sets `depth`.
func (*RequestOptions) UnmarshalText ¶
type RequestOptionsFunc ¶
type RequestOptionsFunc func(options *RequestOptions)
func DontFilter ¶
func DontFilter() RequestOptionsFunc
DontFilter returns a RequestOptionsFunc which sets the dontFilter
func WithValue ¶
func WithValue(k string, v interface{}) RequestOptionsFunc
WithValue returns a RequestOptionsFunc which sets k,v in setter The provided value must be serializable
type Response ¶
func NewResponse ¶
type Runnable ¶
type Runnable interface {
// Start starts the runnable object. Upon the method returning nil, the object begins to function properly.
Start() error
Closable
}
Runnable is the interface for objects that can start to work and stop on demand.
type Task ¶
type Task interface {
Requests() []*http.Request
Items() []interface{}
RequestOptions() []RequestOptionsFunc
Handler() HandleFunc
}
func Get ¶
func Get(url string, handler HandleFunc, options ...RequestOptionsFunc) (Task, error)
func Gets ¶
func Gets(urls []string, handler HandleFunc, options ...RequestOptionsFunc) (t Task, err error)
func MustGet ¶
func MustGet(url string, handler HandleFunc, options ...RequestOptionsFunc) Task
func Request ¶
func Request(request *http.Request, handler HandleFunc, options ...RequestOptionsFunc) Task
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
example
|
|
|
library
|
|
|
testing
|
|
|
core
Package core is a generated GoMock package.
|
Package core is a generated GoMock package. |
|
roundtripper
Package roundtripper is a generated GoMock package.
|
Package roundtripper is a generated GoMock package. |
Click to show internal directories.
Click to hide internal directories.