README
¶
esme
一个go实现的多任务、多线程的网络请求SDK
1. 特性
- 基本的网络请求实现
- 自定义请求参数
- http 代理的使用
- http 请求 debug 模式
- http 请求的回调，包括成功回调、失败回调和重试回调
- 支持自己设置 http.Transport
- 内存中的任务队列实现
- redis 任务队列实现
2. 安装
go get -u github.com/zituocn/esme
3. 简单的HTTP请求
一个 http get 请求的演示代码
demo code
package main
import (
"fmt"
"github.com/zituocn/esme"
)
func main() {
// 请求一个天气预报的接口
ctx := esme.HttpGet("https://tenapi.cn/wether/?city=%E6%88%90%E9%83%BD")
// 成功的回调
ctx.SetSucceedFunc(func(c *esme.Context) {
fmt.Println("请求成功了:")
fmt.Println("返回值 :", c.ToString())
})
// 失败的回调
ctx.SetFailedFunc(func(c *esme.Context) {
fmt.Println("请求出错了...")
fmt.Println("返回状态值 :", c.Response.StatusCode)
fmt.Println("返回值 :", c.ToString())
})
// 设置http代理
ctx.SetProxy("http://10.10.10.10:8888")
// 执行请求
ctx.Do()
}
返回值
2022/04/26 19:42:57.625 [I] context.go:120: [success] callback -> main.main.func1
请求成功了:
返回值 : ....
4. 使用任务队列
使用内存任务队列的演示
demo code
package main
import (
"fmt"
"net/http"
"github.com/zituocn/esme"
)
var (
// queue 一个内存中的任务队列
queue = esme.NewMemQueue()
)
// AddTask 添加任务
func AddTask() {
city := []string{"北京", "上海", "成都", "深圳", "西安"}
header := &http.Header{}
header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
for _, item := range city {
queue.Add(&esme.Task{
Url: fmt.Sprintf("%s%s", "https://tenapi.cn/wether/?city=", item),
Method: "GET",
Header: header,
})
}
}
func main() {
// 生成任务队列
AddTask()
// 设置 任务参数
job := esme.NewJob("wether", 1, queue, esme.JobOptions{
SucceedFunc: func(ctx *esme.Context) {
fmt.Println("成功的回调")
fmt.Println("返回信息 :", ctx.ToString())
},
FailedFunc: func(ctx *esme.Context) {
fmt.Println("失败的回调")
fmt.Println("返回状态 :", ctx.Response.StatusCode)
},
})
// 执行
job.Do()
}
5. 更多文档
- http请求的参数设置&&响应处理
- 使用
redis任务队列 - 在任务队列中，使用
代理IP池(多个代理IP使用) - 和
goquery库的配合使用 - 和
gjson库的配合使用 - 把数据存储到
mysql中
6. 感谢
Documentation
¶
Index ¶
- func GetByte(obj interface{}) (data []byte)
- func GetFuncName(i interface{}) string
- func GetRandSleepTime(min, max int) int
- func GetStatusCodeString(code int) string
- func RandomUserAgent(userAgentType UserAgentType) string
- func Str2Int64(s string) int64
- type CallbackFunc
- type Context
- func DoRequest(url, method string, vs ...interface{}) *Context
- func HttpGet(url string, vs ...interface{}) *Context
- func HttpPost(url string, vs ...interface{}) *Context
- func HttpPut(url string, data []byte, vs ...interface{}) *Context
- func NewContext(req *http.Request, vs ...interface{}) *Context
- func NewRequest(url, method string, vs ...interface{}) (*Context, error)
- func (c *Context) Do()
- func (c *Context) GetExecTime() time.Duration
- func (c *Context) SetCompleteFunc(fn CallbackFunc) *Context
- func (c *Context) SetFailedFunc(fn CallbackFunc) *Context
- func (c *Context) SetIsDebug(isDebug bool) *Context
- func (c *Context) SetProxy(httpProxy string) *Context
- func (c *Context) SetProxyLib(lib *ProxyLib) *Context
- func (c *Context) SetRetryFunc(fn CallbackFunc) *Context
- func (c *Context) SetSleepTime(sleepTime int) *Context
- func (c *Context) SetStartFunc(fn CallbackFunc) *Context
- func (c *Context) SetSucceedFunc(fn CallbackFunc) *Context
- func (c *Context) SetTimeOut(timeout int) *Context
- func (c *Context) SetTransport(f func() *http.Transport) *Context
- func (c *Context) ToByte() []byte
- func (c *Context) ToHTML() string
- func (c *Context) ToJSON(v interface{}) error
- func (c *Context) ToSection(path string) string
- func (c *Context) ToString() string
- type Cookie
- type FormData
- type Header
- type Job
- type JobOptions
- type MemQueue
- type ProxyIP
- type ProxyLib
- type RedisQueue
- type StatusCode
- type Task
- type TodoQueue
- type UserAgentType
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetFuncName ¶
func GetFuncName(i interface{}) string
GetFuncName return the function name by reflection
func GetStatusCodeString ¶
func RandomUserAgent ¶ added in v0.0.2
func RandomUserAgent(userAgentType UserAgentType) string
RandomUserAgent 取一个随机的user-agent
Types ¶
type Context ¶
type Context struct {
// http request
Request *http.Request
// http response
Response *http.Response
// error
Err error
// reqeusted task
Task *Task
// RespBody []byte returned by the request
RespBody []byte
// Param context parameter
Data map[string]interface{}
// contains filtered or unexported fields
}
Context request and response context
func NewContext ¶
NewContext returns new Context
func NewRequest ¶
NewRequest returns esme.context and error
func (*Context) GetExecTime ¶ added in v0.0.2
GetExecTime get request execution time
func (*Context) SetCompleteFunc ¶
func (c *Context) SetCompleteFunc(fn CallbackFunc) *Context
SetCompleteFunc Set the callback for request completion
func (*Context) SetFailedFunc ¶
func (c *Context) SetFailedFunc(fn CallbackFunc) *Context
SetFailedFunc Callback after setting failure
func (*Context) SetIsDebug ¶
SetIsDebug set debug
func (*Context) SetProxyLib ¶
SetProxyLib set proxy lib
func (*Context) SetRetryFunc ¶
func (c *Context) SetRetryFunc(fn CallbackFunc) *Context
SetRetryFunc Set callback for retry
func (*Context) SetSleepTime ¶
SetSleepTime Set sleep time for http requests
func (*Context) SetStartFunc ¶
func (c *Context) SetStartFunc(fn CallbackFunc) *Context
SetStartFunc Set the callback for the start of the request
func (*Context) SetSucceedFunc ¶
func (c *Context) SetSucceedFunc(fn CallbackFunc) *Context
SetSucceedFunc Callback after successful setting
func (*Context) SetTransport ¶ added in v0.0.7
SetTransport Set the client's Transport
type Job ¶
type Job struct {
// contains filtered or unexported fields
}
Job job struct
type JobOptions ¶
type JobOptions struct {
// StartFunc Callback at start
StartFunc CallbackFunc
// SucceedFunc Callback after success
SucceedFunc CallbackFunc
// RetryFunc retry callback
RetryFunc CallbackFunc
// FailedFunc callback after failure
FailedFunc CallbackFunc
// CompleteFunc Callback for request completion
CompleteFunc CallbackFunc
// ProxyIP proxy ip
ProxyIP string
// ProxyLib proxy ip library
ProxyLib *ProxyLib
// SheepTime Sleep time for http request execution
// millisecond
SheepTime int
// TimeOut http request timeout
// millisecond
TimeOut int
// 是否打印调试
IsDebug bool
}
JobOptions 任务参数
type MemQueue ¶
type MemQueue struct {
// contains filtered or unexported fields
}
MemQueue in-memory queue
type ProxyLib ¶
type ProxyLib struct {
// contains filtered or unexported fields
}
ProxyLib http proxy lib
type RedisQueue ¶
type RedisQueue struct {
// contains filtered or unexported fields
}
RedisQueue task queue in redis
func (*RedisQueue) AddTasks ¶
func (q *RedisQueue) AddTasks(list []*Task)
AddTasks add multiple tasks to the queue
func (*RedisQueue) IsEmpty ¶
func (q *RedisQueue) IsEmpty() bool
IsEmpty returns whether the queue is empty
func (*RedisQueue) Pop ¶
func (q *RedisQueue) Pop() *Task
Pop get a task while removing it from the queue
from left
func (*RedisQueue) Print ¶
func (q *RedisQueue) Print()
type StatusCode ¶
type Task ¶
type Task struct {
// Url request address
Url string `json:"url"`
// Method request method
Method string `json:"method"`
// Payload request payload
Payload []byte `json:"payload"`
// FormData request formData
FormData FormData `json:"form_data"`
// header *http.Header
Header *http.Header `json:"header"`
// Data Contextual data passing
Data map[string]interface{}
}
Task http Task
type TodoQueue ¶
type TodoQueue interface {
// Add add an element to the queue
Add(task *Task)
// AddTasks Add multiple elements to the queue at once
AddTasks(list []*Task)
// Pop pop and return an element
Pop() *Task
// Clear empty the queue
Clear() bool
// Size queue length
Size() int
// IsEmpty is empty
IsEmpty() bool
// Print print
Print()
}
TodoQueue interface
func NewRedisQueue ¶
func NewRedisQueue(key string, rc *goredis.RedisConfig) TodoQueue
NewRedisQueue use redis configuration
type UserAgentType ¶ added in v0.0.2
type UserAgentType int
UserAgentType user-agent类型
const ( // PCUserAgent pc版本的 PCUserAgent UserAgentType = iota + 1 // MobileUserAgent 移版本的 MobileUserAgent )
Source Files
¶
Directories
Click to show internal directories.
Click to hide internal directories.