tools

package

v1.1.3 Latest Latest Go to latest Published: Apr 13, 2023 License: GPL-3.0 Imports: 29 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/cvelab/requests

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func ActionFuncSetCookies(domain *string, cookies *types.Dict) chromedp.ActionFunc
func ConvertGbkToUtf8(str string) string
func ConvertStructToDict(obj any) types.Dict
func ConvertUtf8ToGbk(str string) string
func ConvertValueToString(obj reflect.Value) string
func CovertStructToJson(obj any) map[string]any
func CustomDownloader(response *requests.Response, fileTypes []string, savePath string, ...) string
func Download(session *requests.Session, rawUrl *string, savePath *string)
func EventListenerConsoleAPICalled(result *[]ConsoleLog) func(ev interface{})
func EventListenerNetwork(networkLogs *map[string]NetworkLog) func(ev interface{})
func EventListenerParserToDownload(session *requests.Session, typeDict *map[string]bool, savePath *string) func(ev interface{})
func FileDownload(response *requests.Response, fileTypes []string, savePath string) string
func HmacSha256(data []byte, secret []byte) []byte
func HmacSha256Base64Encode(data []byte, secret []byte) string
func HookClientJA3Func(fingerprint string) types.Hook
func HookClientMitmFunc(fingerprint string) types.Hook
func HookCloudFlareWorkerFunc(workerHost string, headers types.Dict) types.Hook
func HookRandomUserAgent(request any) (error, any)
func HookResponseGbkToUtf8(response any) (error, any)
func HookResponseUtf8ToGbk(response any) (error, any)
func Md5(data []byte) string
func RandomIPv4() string
func RandomIPv6() string
func RandomUserAgent() string
func ReportHTTPResponse(response *requests.Response, savePath *string) string
type ConsoleLog
type NetworkLog

Constants ¶

This section is empty.

Variables ¶

View Source

var UserAgentList = map[string][]string{
	"windows": {
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36",
		"Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.3729.169 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.5195.52 Safari/537.36",
	},
	"darwin": {
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
	},
	"linux": {
		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.40 Safari/537.36",
		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
	},
}

Functions ¶

func ActionFuncSetCookies ¶

func ActionFuncSetCookies(domain *string, cookies *types.Dict) chromedp.ActionFunc

func ConvertGbkToUtf8 ¶

func ConvertGbkToUtf8(str string) string

func ConvertStructToDict ¶

func ConvertStructToDict(obj any) types.Dict

ConvertStructToDict 需要结构体字段Tag设置为 `dict:"目标键名"` 或 `dict:"目标键名,omitempty"` 如 Text string `dict:"text,omitempty"`

func ConvertUtf8ToGbk ¶

func ConvertUtf8ToGbk(str string) string

func ConvertValueToString ¶

func ConvertValueToString(obj reflect.Value) string

func CovertStructToJson ¶

func CovertStructToJson(obj any) map[string]any

CovertStructToJson 需要结构体字段Tag设置为 `json:"目标键名"` 或 `json:"目标键名,omitempty"` 如 Text string `json:"text,omitempty"`

func CustomDownloader ¶

func CustomDownloader(response *requests.Response, fileTypes []string, savePath string, flags []chromedp.ExecAllocatorOption, actions ...chromedp.Action) string

func Download ¶

func Download(session *requests.Session, rawUrl *string, savePath *string)

func EventListenerConsoleAPICalled ¶

func EventListenerConsoleAPICalled(result *[]ConsoleLog) func(ev interface{})

func EventListenerNetwork ¶

func EventListenerNetwork(networkLogs *map[string]NetworkLog) func(ev interface{})

func EventListenerParserToDownload ¶

func EventListenerParserToDownload(session *requests.Session, typeDict *map[string]bool, savePath *string) func(ev interface{})

func FileDownload ¶

func FileDownload(response *requests.Response, fileTypes []string, savePath string) string

FileDownload 文件下载器，参考 https://github.com/chromedp/examples/blob/master/download_image/main.go

func HmacSha256 ¶

func HmacSha256(data []byte, secret []byte) []byte

func HmacSha256Base64Encode ¶

func HmacSha256Base64Encode(data []byte, secret []byte) string

func HookClientJA3Func ¶

func HookClientJA3Func(fingerprint string) types.Hook

HookClientJA3Func 传入JA3 Client指纹例如 "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0"

func HookClientMitmFunc ¶

func HookClientMitmFunc(fingerprint string) types.Hook

HookClientMitmFunc 传入MitmEngine指纹，自动转换例如 "303:4,5,a,13,2f,32,33,35,38,39,c009,c00a,c013,c014:0,a,b,17,ff01:17,18:0" 目前不是很优雅，封装了JA3的函数，后面自己写个TP生成器

func HookCloudFlareWorkerFunc ¶

func HookCloudFlareWorkerFunc(workerHost string, headers types.Dict) types.Hook

func HookRandomUserAgent ¶

func HookRandomUserAgent(request any) (error, any)

func HookResponseGbkToUtf8 ¶

func HookResponseGbkToUtf8(response any) (error, any)

func HookResponseUtf8ToGbk ¶

func HookResponseUtf8ToGbk(response any) (error, any)

func Md5 ¶

func Md5(data []byte) string

func RandomIPv4 ¶

func RandomIPv4() string

RandomIPv4 不完全合法，包含了 0/24 127/8 等地址，可能造成被WAF拦截

func RandomIPv6 ¶

func RandomIPv6() string

RandomIPv6 不确定合法程度，没有细看相关私有地址等内容，可能造成被WAF拦截

func RandomUserAgent ¶

func RandomUserAgent() string

func ReportHTTPResponse ¶

func ReportHTTPResponse(response *requests.Response, savePath *string) string

Types ¶

type ConsoleLog ¶

type ConsoleLog struct {
	Type  string
	Value string
}

type NetworkLog ¶

type NetworkLog struct {
	RequestID  string
	Time       time.Time
	URL        string
	Type       string
	StatusCode int64
	EndURL     string
	IP         string
	Error      string
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL