crawl

package module
v0.0.0-...-1954725 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 11, 2021 License: MIT Imports: 14 Imported by: 0

README

crawl

http 请求的封装, 链式API

示例

Documentation

Index

Constants

View Source 
const (
	CharsetUTF8            = "utf-8"
	MimeNone               = ""
	MimeFormUrlencoded     = "application/x-www-form-urlencoded"
	MimeFormUrlencodedUTF8 = MimeFormUrlencoded + "; " + CharsetUTF8
	MimeJSON               = "application/json"
	MimeJSONUTF8           = MimeJSON + "; " + CharsetUTF8
	MimeXML                = "text/xml"
	MimeXMLUTF8            = MimeXML + "; " + CharsetUTF8
)
View Source 
const HeaderContentType = "Content-Type"

Variables

This section is empty.

Functions

func FormInput 

func FormInput(form url.Values) (io.Reader, string)

func JSONInput 

func JSONInput(input interface{}) (io.Reader, string)

func Sleep 

func Sleep(ctx context.Context, sleep time.Duration) error

func XMLInput 

func XMLInput(xmlObj interface{}) (io.Reader, string)

Types

type ClientOption 

type ClientOption interface{ Apply(client *http.Client) }

func Proxy 

func Proxy(proxy ProxyFunc) ClientOption

Proxy 设置代理

type ClientOptionFunc 

type ClientOptionFunc func(client *http.Client)

func (ClientOptionFunc) Apply 

func (fn ClientOptionFunc) Apply(client *http.Client)

type ClientOptions 

type ClientOptions []ClientOption

func (ClientOptions) Apply 

func (opts ClientOptions) Apply(client *http.Client)

type Crawl 

type Crawl struct {
	// contains filtered or unexported fields
}

func NewCrawl 

func NewCrawl() *Crawl

func With 

func With(crawler *Crawler) *Crawl

func (*Crawl) ClientOpt 

func (c *Crawl) ClientOpt(options ...ClientOption) *Crawl

ClientOpt 客户端配置

func (*Crawl) Enqueue 

func (c *Crawl) Enqueue() *Crawler

Enqueue 将请求添加到队列

func (*Crawl) EnqueueTo 

func (c *Crawl) EnqueueTo(crawler *Crawler) *Crawler

EnqueueTo 将请求添加到队列

func (*Crawl) Get 

func (c *Crawl) Get(url string) *Crawl

Get 请求

func (*Crawl) GetBytes 

func (c *Crawl) GetBytes(url string, data *[]byte) *Crawl

GetBytes 请求并将结果读取为字节组

func (*Crawl) GetJSON 

func (c *Crawl) GetJSON(url string, out interface{}) *Crawl

GetJSON 请求并将结果以JSON方式解析到对象

func (*Crawl) Method 

func (c *Crawl) Method(method string) *Crawl

Method 请求方法

func (*Crawl) PostForm 

func (c *Crawl) PostForm(url string, data url.Values) *Crawl

PostForm 表单提交请求

func (*Crawl) PostJSON 

func (c *Crawl) PostJSON(url string, data interface{}) *Crawl

PostJSON JSON提交请求

func (*Crawl) PostXML 

func (c *Crawl) PostXML(url string, data interface{}) *Crawl

PostXML XML提交请求

func (*Crawl) Process 

func (c *Crawl) Process(processor Processor) *Crawl

Process 设置处理方法

func (*Crawl) Query 

func (c *Crawl) Query(query url.Values) *Crawl

Query 设置Query参数

func (*Crawl) QueryAdd 

func (c *Crawl) QueryAdd(key, value string) *Crawl

QueryAdd 添加Query参数

func (*Crawl) QuerySet 

func (c *Crawl) QuerySet(key, value string) *Crawl

QuerySet 设置

func (*Crawl) RequestOpt 

func (c *Crawl) RequestOpt(options ...RequestOption) *Crawl

RequestOpt 请求配置

func (*Crawl) Run 

func (c *Crawl) Run(ctx context.Context) (err error)

Run 执行请求

func (*Crawl) SetBody 

func (c *Crawl) SetBody(data io.Reader, contentType string) *Crawl

SetBody 设置请求数据

func (*Crawl) Url 

func (c *Crawl) Url(url string) *Crawl

Url 请求地址

func (*Crawl) With 

func (c *Crawl) With(crawler *Crawler) *Crawl

type Crawler 

type Crawler struct {
	// contains filtered or unexported fields
}

func New 

func New() *Crawler

func (*Crawler) ClientOpt 

func (c *Crawler) ClientOpt(options ...ClientOption) *Crawler

func (*Crawler) CookieEnabled 

func (c *Crawler) CookieEnabled(enabled ...bool) *Crawler

func (*Crawler) Crawl 

func (c *Crawler) Crawl() *Crawl

func (*Crawler) Enqueue 

func (c *Crawler) Enqueue(crawl *Crawl) *Crawler

func (*Crawler) ErrTry 

func (c *Crawler) ErrTry(trySleep func(i int) time.Duration) *Crawler

func (*Crawler) GetCookie 

func (c *Crawler) GetCookie(u *url.URL, key string) string

func (*Crawler) MaxTry 

func (c *Crawler) MaxTry(maxTry int) *Crawler

func (*Crawler) RequestOpt 

func (c *Crawler) RequestOpt(options ...RequestOption) *Crawler

func (*Crawler) SetCookie 

func (c *Crawler) SetCookie(u *url.URL, cookie *http.Cookie)

func (*Crawler) Start 

func (c *Crawler) Start(ctx context.Context) error

type Processor 

type Processor = func(resp *http.Response) error

func ProcessBytes 

func ProcessBytes(readBytes func(data []byte) error) Processor

func ProcessDownload 

func ProcessDownload(fn string, report func(cur, total int64)) Processor

func ProcessJSON 

func ProcessJSON(out interface{}) Processor

func ProcessProgress 

func ProcessProgress(readBody func(body io.Reader) error, report func(cur, total int64)) Processor

func ProcessXML 

func ProcessXML(out interface{}) Processor

type ProxyFunc 

type ProxyFunc func(*http.Request) (*url.URL, error)

type RequestOption 

type RequestOption interface{ Apply(req *http.Request) }

func Accept 

func Accept(accept string) RequestOption

func AcceptEncoding 

func AcceptEncoding(acceptEncoding string) RequestOption

func AcceptLanguage 

func AcceptLanguage(acceptLanguage string) RequestOption

func ContentType 

func ContentType(contentType string) RequestOption

func DefaultOption 

func DefaultOption() RequestOption

func FormUrlencoded 

func FormUrlencoded() RequestOption

func HeaderDel 

func HeaderDel(keys ...string) RequestOption

func HeaderSet 

func HeaderSet(key, value string) RequestOption

func KeepAlive 

func KeepAlive(keepAlive ...bool) RequestOption

func Referer 

func Referer(referer string) RequestOption

func UserAgent 

func UserAgent(userAgent string) RequestOption

type RequestOptionFunc 

type RequestOptionFunc func(req *http.Request)

func (RequestOptionFunc) Apply 

func (fn RequestOptionFunc) Apply(req *http.Request)

type RequestOptions 

type RequestOptions []RequestOption

func (RequestOptions) Apply 

func (opts RequestOptions) Apply(req *http.Request)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.