requests

package module
v0.0.0-...-61ad528 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 23, 2024 License: GPL-3.0 Imports: 35 Imported by: 9

README

Requests - A next-generation HTTP client for Golang.

Features

Benchmark

gospider007/requests > imroc/req > go-resty > wangluozhe/requests > curl_cffi > httpx > psf/requests

Supported Go Versions

Recommended to use go1.21.3 and above. Initially Requests started supporting go modules

Installation

go get github.com/gospider007/requests

Usage

import "github.com/gospider007/requests"

Quickly Send Requests

package main

import (
	"log"
	"time"
	"github.com/gospider007/requests"
)

func main() {
    resp, err := requests.Get(nil, "http://httpbin.org/anything")
    if err != nil {
      log.Panic(err)
    }
    log.Print(resp.Text())    // Get content and parse as string
    log.Print(resp.Content()) // Get content as bytes
    log.Print(resp.Json())    // Get content and parse as gjson JSON
    log.Print(resp.Html())    // Get content and parse as goquery DOM
    log.Print(resp.Cookies()) // Get cookies
}

Contributing

If you have a bug report or feature request, you can open an issue

Contact

If you have questions, feel free to reach out to us in the following ways:

  • QQ Group (Chinese): 939111384 -
  • WeChat (Chinese): gospider007

Sponsors

If you like and it really helps you, feel free to reward me with a cup of coffee, and don't forget to mention your github id.


Wechat

Alipay

Documentation

Index

Constants

View Source
const (
	UserAgent      = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + chromeV + ".0.0.0 Safari/537.36 Edg/" + chromeV + chromeE
	SecChUa        = `"Chromium";v="` + chromeV + `", "Microsoft Edge";v="` + chromeV + `", "Not=A?Brand";v="99"`
	AcceptLanguage = "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6"
)

Variables

View Source
var ErrgospiderBodyClose = errors.New("gospider body close error")

Functions

func CreateReqCtx

func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context

func GetReqCtxData

func GetReqCtxData(ctx context.Context) *reqCtxData

func NewDialer

func NewDialer(option DialOption) *net.Dialer

func NewReqCtxData

func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error)

func ReadRequest

func ReadRequest(b *bufio.Reader) (*http.Request, error)

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

Connection Management

func NewClient

func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)

New Connection Management

func (*Client) ClearCookies

func (obj *Client) ClearCookies()

clear cookies

func (*Client) Close

func (obj *Client) Close()

Close the client and cannot be used again after shutdown

func (*Client) CloseConns

func (obj *Client) CloseConns()

Close idle connections. If the connection is in use, wait until it ends before closing

func (*Client) Connect

func (obj *Client) Connect(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Connect request and returns the response.

func (*Client) Delete

func (obj *Client) Delete(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Delete request and returns the response.

func (*Client) ForceCloseConns

func (obj *Client) ForceCloseConns()

Close the connection, even if it is in use, it will be closed

func (*Client) Get

func (obj *Client) Get(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Get request and returns the response.

func (*Client) GetCookies

func (obj *Client) GetCookies(href *url.URL) Cookies

get cookies

func (*Client) Head

func (obj *Client) Head(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Head request and returns the response.

func (*Client) Options

func (obj *Client) Options(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Options request and returns the response.

func (*Client) Patch

func (obj *Client) Patch(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Patch request and returns the response.

func (*Client) Post

func (obj *Client) Post(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Post request and returns the response.

func (*Client) Put

func (obj *Client) Put(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Put request and returns the response.

func (*Client) Request

func (obj *Client) Request(ctx context.Context, method string, href string, options ...RequestOption) (response *Response, err error)

Define a function named Request that takes in four parameters:

func (*Client) SetCookies

func (obj *Client) SetCookies(href *url.URL, cookies ...any) error

set cookies

func (*Client) SetGetProxy

func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error))

Modify the proxy method of the client

func (*Client) SetProxy

func (obj *Client) SetProxy(proxyUrl string) (err error)

Modifying the client's proxy

func (*Client) Trace

func (obj *Client) Trace(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Trace request and returns the response.

type ClientOption

type ClientOption struct {
	Ja3Spec               ja3.Ja3Spec                                                                           //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
	H2Ja3Spec             ja3.H2Ja3Spec                                                                         //h2 fingerprint
	Proxy                 string                                                                                //proxy,support https,http,socks5
	ForceHttp1            bool                                                                                  //force  use http1 send requests
	Ja3                   bool                                                                                  //enable ja3 fingerprint
	DisCookie             bool                                                                                  //disable cookies
	DisDecode             bool                                                                                  //disable auto decode
	DisUnZip              bool                                                                                  //disable auto zip decode
	DisAlive              bool                                                                                  //disable  keepalive
	Bar                   bool                                                                                  ////enable bar display
	OptionCallBack        func(ctx context.Context, option *RequestOption) error                                //option callback,if error is returnd, break request
	ResultCallBack        func(ctx context.Context, option *RequestOption, response *Response) error            //result callback,if error is returnd,next errCallback
	ErrCallBack           func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
	RequestCallBack       func(ctx context.Context, request *http.Request, response *http.Response) error       //request and response callback,if error is returnd,reponse is error
	MaxRetries            int                                                                                   //try num
	MaxRedirect           int                                                                                   //redirect num ,<0 no redirect,==0 no limit
	Headers               any                                                                                   //default headers
	Timeout               time.Duration                                                                         //request timeout
	ResponseHeaderTimeout time.Duration                                                                         //ResponseHeaderTimeout ,default:300
	TlsHandshakeTimeout   time.Duration                                                                         //tls timeout,default:15

	//network card ip
	DialTimeout time.Duration //dial tcp timeout,default:15
	KeepAlive   time.Duration //keepalive,default:30
	LocalAddr   *net.TCPAddr
	Dns         *net.UDPAddr  //dns
	AddrType    gtls.AddrType //dns parse addr type
	Jar         *Jar          //custom cookies

	//other option
	GetProxy    func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
	GetAddrType func(host string) gtls.AddrType
}

Connection Management Options

type Cookies

type Cookies []*http.Cookie

cookies

func ReadCookies

func ReadCookies(val any) (Cookies, error)

read cookies or parse cookies,support json,map,[]string,http.Header,string

func ReadSetCookies

func ReadSetCookies(val any) (Cookies, error)

read set cookies or parse set cookies,support json,map,[]string,http.Header,string

func (Cookies) Get

func (obj Cookies) Get(name string) *http.Cookie

get cookie by name

func (Cookies) GetVal

func (obj Cookies) GetVal(name string) string

get cookie value by name,return string

func (Cookies) GetVals

func (obj Cookies) GetVals(name string) []string

get cookie values by name, return []string

func (Cookies) Gets

func (obj Cookies) Gets(name string) Cookies

get cookies by name

func (Cookies) String

func (obj Cookies) String() string

return cookies with string,join with '; '

type DialClient

type DialClient struct {
	// contains filtered or unexported fields
}

func NewDail

func NewDail(option DialOption) *DialClient

func (*DialClient) DialContext

func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error)

func (*DialClient) DialContextWithProxy

func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error)

func (*DialClient) Socks5Proxy

func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error)

type DialOption

type DialOption struct {
	DialTimeout time.Duration
	KeepAlive   time.Duration
	LocalAddr   *net.TCPAddr  //network card ip
	AddrType    gtls.AddrType //first ip type
	Dns         *net.UDPAddr
	GetAddrType func(host string) gtls.AddrType
}

type Event

type Event struct {
	Data    string //data
	Event   string //event
	Id      string //id
	Retry   int    //retry num
	Comment string //comment info
}

type File

type File struct {
	FileName    string
	ContentType string
	Content     any
}

Upload files with form-data,

type Jar

type Jar struct {
	// contains filtered or unexported fields
}

cookies jar

func NewJar

func NewJar() *Jar

new cookies jar

func (*Jar) ClearCookies

func (obj *Jar) ClearCookies()

Clear cookies

func (*Jar) GetCookies

func (obj *Jar) GetCookies(u *url.URL) Cookies

Get cookies

func (*Jar) SetCookies

func (obj *Jar) SetCookies(u *url.URL, cookies ...any) error

Set cookies

type OrderMap

type OrderMap struct {
	// contains filtered or unexported fields
}

func NewOrderMap

func NewOrderMap() *OrderMap

func (*OrderMap) Del

func (obj *OrderMap) Del(key string)

func (*OrderMap) Keys

func (obj *OrderMap) Keys() []string

func (*OrderMap) MarshalJSON

func (obj *OrderMap) MarshalJSON() ([]byte, error)

func (*OrderMap) Set

func (obj *OrderMap) Set(key string, val any)

type RequestOption

type RequestOption struct {
	Ja3Spec         ja3.Ja3Spec                                                                           //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
	H2Ja3Spec       ja3.H2Ja3Spec                                                                         //custom h2 fingerprint
	Proxy           string                                                                                //proxy,support http,https,socks5,example:http://127.0.0.1:7005
	ForceHttp1      bool                                                                                  //force  use http1 send requests
	Ja3             bool                                                                                  //enable ja3 fingerprint
	DisCookie       bool                                                                                  //disable cookies,not use cookies
	DisDecode       bool                                                                                  //disable auto decode
	DisUnZip        bool                                                                                  //disable auto zip decode
	DisAlive        bool                                                                                  //disable  keepalive
	Bar             bool                                                                                  //enable bar display
	OptionCallBack  func(ctx context.Context, option *RequestOption) error                                //option callback,if error is returnd, break request
	ResultCallBack  func(ctx context.Context, option *RequestOption, response *Response) error            //result callback,if error is returnd,next errCallback
	ErrCallBack     func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
	RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error       //request and response callback,if error is returnd,reponse is error

	MaxRetries            int           //try num
	MaxRedirect           int           //redirect num ,<0 no redirect,==0 no limit
	Headers               any           //request headers:json,map,header
	Timeout               time.Duration //request timeout
	ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
	TlsHandshakeTimeout   time.Duration

	//network card ip
	DialTimeout time.Duration //dial tcp timeout,default:15
	KeepAlive   time.Duration //keepalive,default:30
	LocalAddr   *net.TCPAddr
	Dns         *net.UDPAddr  //dns
	AddrType    gtls.AddrType //dns parse addr type                                             //tls timeout,default:15
	Jar         *Jar          //custom cookies

	// other option
	Method      string //method
	Url         *url.URL
	Host        string
	Referer     string //set headers referer value
	ContentType string //headers Content-Type value
	Cookies     any    // cookies,support :json,map,str,http.Header

	Params any //url params,join url query,json,map
	Json   any //send application/json,support io.Reader,:string,[]bytes,json,map
	Data   any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
	Form   any //send multipart/form-data,file upload,support io.Reader, json,map
	Text   any //send text/xml,support: io.Reader, string,[]bytes,json,map
	Body   any //not setting context-type,support io.Reader, string,[]bytes,json,map

	Stream   bool             //disable auto read
	WsOption websocket.Option //websocket option
	DisProxy bool             //force disable proxy
	// contains filtered or unexported fields
}

Options for sending requests

func (*RequestOption) Client

func (obj *RequestOption) Client() *Client

type Response

type Response struct {
	// contains filtered or unexported fields
}

func Connect

func Connect(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Connect request and returns the response.

func Delete

func Delete(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Delete request and returns the response.

func Get

func Get(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a GET request and returns the response.

func Head(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Head request and returns the response.

func Options

func Options(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Options request and returns the response.

func Patch

func Patch(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Patch request and returns the response.

func Post

func Post(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Post request and returns the response.

func Put

func Put(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Put request and returns the response.

func Request

func Request(ctx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error)

Define a function named Request that takes in four parameters:

func Trace

func Trace(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Trace request and returns the response.

func (*Response) CloseBody

func (obj *Response) CloseBody()

close body

func (*Response) CloseConn

func (obj *Response) CloseConn()

safe close conn

func (*Response) Conn

func (obj *Response) Conn() *connecotr

must stream=true

func (*Response) Content

func (obj *Response) Content() []byte

return content with []byte

func (*Response) ContentEncoding

func (obj *Response) ContentEncoding() string

return content encoding

func (*Response) ContentLength

func (obj *Response) ContentLength() int64

return content length

func (*Response) ContentType

func (obj *Response) ContentType() string

return content type

func (*Response) Cookies

func (obj *Response) Cookies() Cookies

return response cookies

func (*Response) Decode

func (obj *Response) Decode(encoding string)

change decoding with content

func (*Response) ForceCloseConn

func (obj *Response) ForceCloseConn()

force close conn

func (*Response) Headers

func (obj *Response) Headers() http.Header

return response headers

func (*Response) Html

func (obj *Response) Html() *bs4.Client

return content with parse html

func (*Response) InPool

func (obj *Response) InPool() bool

conn is in pool ?

func (*Response) IsNewConn

func (obj *Response) IsNewConn() bool

conn is new conn

func (*Response) IsStream

func (obj *Response) IsStream() bool

return true if response is stream

func (*Response) Json

func (obj *Response) Json(vals ...any) (*gson.Client, error)

return content with json and you can parse struct

func (*Response) Location

func (obj *Response) Location() (*url.URL, error)

return URL redirected address

func (*Response) Map

func (obj *Response) Map() (data map[string]any, err error)

return content with map[string]any

func (*Response) Proto

func (obj *Response) Proto() string

return response Proto

func (*Response) Proxy

func (obj *Response) Proxy() string

conn proxy

func (*Response) ReadBody

func (obj *Response) ReadBody() (err error)

read body

func (*Response) SetContent

func (obj *Response) SetContent(val []byte)

set response content with []byte

func (*Response) Sse

func (obj *Response) Sse() *Sse

return sse client

func (*Response) Status

func (obj *Response) Status() string

return response status

func (*Response) StatusCode

func (obj *Response) StatusCode() int

return response status code

func (*Response) Text

func (obj *Response) Text() string

return content with string

func (*Response) Url

func (obj *Response) Url() *url.URL

return response url

func (*Response) WebSocket

func (obj *Response) WebSocket() *websocket.Conn

return websocket client

type Sse

type Sse struct {
	// contains filtered or unexported fields
}

func (*Sse) Close

func (obj *Sse) Close() error

close sse

func (*Sse) Recv

func (obj *Sse) Recv() (Event, error)

recv sse envent data

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL