miner

package
v0.0.0-...-2170d9f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2020 License: Apache-2.0 Imports: 23 Imported by: 50

Documentation

Overview

Package miner is the core of this project, use to request for http api.

Example: 

package main

import (
"fmt"

"github.com/admpub/marmot/miner"
)

func main() {
	// Use Default Worker, You can Also New One:
	// worker:=miner.New(nil)
	miner.SetLogLevel(miner.DEBUG)
	_, err := miner.SetURL("https://www.whitehouse.gov").Go()
	if err != nil {
		fmt.Println(err.Error())
	} else {
		fmt.Println(miner.String())
	}
}

Index

Constants

View Source 
const (
	// WaitTime Default wait time
	WaitTime = 5

	GET      = "GET"
	POST     = "POST"
	POSTJSON = "POSTJSON"
	POSTXML  = "POSTXML"
	POSTFILE = "POSTFILE"
	PUT      = "PUT"
	PUTJSON  = "PUTJSON"
	PUTXML   = "PUTXML"
	PUTFILE  = "PUTFILE"
	DELETE   = "DELETE"
	OTHER    = "OTHER" // this stand for you can use other method this lib not own.

	HTTPFORMContentType = "application/x-www-form-urlencoded"
	HTTPJSONContentType = "application/json"
	HTTPXMLContentType  = "text/xml"
	HTTPFILEContentType = "multipart/form-data"
)

Variables

View Source 
var (
	Debugf        = log.Debugf
	CheckRedirect = func(req *http.Request, via []*http.Request) error {
		Debugf("[GoWorker] Redirect:%v", req.URL)
		return nil
	}
)
View Source 
var (
	// Save Cookie, No timeout!
	Client = NewClient()

	// Not Save Cookie
	NoCookieClient = NewHTTPClient(
		httpClientOptions.Timeout(time.Second*time.Duration(DefaultTimeOut)),
		httpClientOptions.CheckRedirect(CheckRedirect),
	)
)

Default Client

View Source 
var (
	DefaultHeader = map[string][]string{
		"User-Agent": {
			defaultUserAgent,
		},
	}

	// DefaultTimeOut http get and post No timeout
	DefaultTimeOut = 30
)
View Source 
var Pool = &_Workers{ws: make(map[string]*Worker)}

Pool for many Worker, every Worker can only serial execution

View Source 
var UserAgentf http.FileSystem
View Source 
var UserAgents = map[int]string{}

UserAgents Global User-Agent provide

Functions

func CloneHeader 

func CloneHeader(h map[string][]string) map[string][]string

CloneHeader Clone a header, If not exist Ua, Set our Ua!

func CopyM 

func CopyM(h http.Header) http.Header

CopyM Header map[string][]string ,can use to copy a http header, so that they are not effect each other

func Delete 

func Delete() (body []byte, e error)

func FixCharset 

func FixCharset(body []byte, extra interface{}, detectCharset bool, defaultEncoding ...string) ([]byte, error)

func Get 

func Get() (body []byte, e error)

func GetCookies 

func GetCookies() []*http.Cookie

func Go 

func Go() (body []byte, e error)

func GoByMethod 

func GoByMethod(method string) (body []byte, e error)

func JSONToString 

func JSONToString() (string, error)

JSONToString This make effect only your Worker exec serial! Attention! Change Your JSON'like Raw data to string

func MergeCookie 

func MergeCookie(before []*http.Cookie, after []*http.Cookie) []*http.Cookie

MergeCookie Merge Cookie, not use

func NewClient 

func NewClient(timeouts ...time.Duration) *http.Client

NewClient New a client, diff from proxy client

func NewHTTPClient 

func NewHTTPClient(options ...com.HTTPClientOptions) *http.Client

NewHTTPClient New a client

func NewJar 

func NewJar() *cookiejar.Jar

NewJar Cookie record Jar

func NewProxyClient 

func NewProxyClient(proxystring string, timeouts ...time.Duration) (*http.Client, error)

NewProxyClient New a Proxy client, Default save cookie, Can timeout We should support some proxy way such as http(s) or socks

func OtherGo 

func OtherGo(method, contenttype string) (body []byte, e error)

func OutputMaps 

func OutputMaps(info string, args map[string][]string)

OutputMaps Just debug a map

func Post 

func Post() (body []byte, e error)

func PostFile 

func PostFile() (body []byte, e error)

func PostJSON 

func PostJSON() (body []byte, e error)

func PostXML 

func PostXML() (body []byte, e error)

func Put 

func Put() (body []byte, e error)

func PutFile 

func PutFile() (body []byte, e error)

func PutJSON 

func PutJSON() (body []byte, e error)

func PutXML 

func PutXML() (body []byte, e error)

func RandomUserAgent 

func RandomUserAgent() string

RandomUserAgent Reback random User-Agent

func SetGlobalTimeout 

func SetGlobalTimeout(num int)

SetGlobalTimeout Set global timeout, it can only by this way!

func String 

func String() string

ToString This make effect only your Worker exec serial! Attention! Change Your Raw data To string

func TooShortSizes 

func TooShortSizes(data []byte, sizes float64) error

TooShortSizes if a file size small than sizes(KB) ,it will be throw a error

func UserAgentInit 

func UserAgentInit()

UserAgentInit User-Agent init

func Wait 

func Wait(waittime int)

Wait some secord

Types

type Worker 

type Worker struct {
	// In order fast chain func call I put the basic config below
	URL          string         // Which url we want
	Method       string         // Get/Post method
	Header       http.Header    // Http header
	Data         url.Values     // Sent by form data
	FileName     string         // FileName which sent to remote
	FileFormName string         // File Form Name which sent to remote
	BinaryData   []byte         // Sent by binary data, can together with File
	Wait         int            // Wait Time
	Client       *http.Client   // Our Client
	Request      *http.Request  // Debug
	Response     *http.Response // Debug
	Raw          []byte         // Raw data we get

	// The name below is not so good but has already been used in many project, so bear it.
	PreURL     string // Pre url
	StatusCode int    // the last url response code, such as 404
	FetchTimes int    // Url fetch number times
	ErrorTimes int    // Url fetch error times
	IP         string // worker proxy ip, just for user to record their proxy ip, default: localhost

	// AOP like Java
	Ctx           context.Context
	BeforeAction  func(context.Context, *Worker)
	AfterAction   func(context.Context, *Worker)
	PesterOptions []pester.ApplyOptions

	// ResponseCharset is the character encoding of the response body.
	// Leave it blank to allow automatic character encoding of the response body.
	ResponseCharset string

	// DetectCharset can enable character encoding detection for non-utf8 response bodies
	// without explicit charset declaration. This feature uses https://github.com/webx-top/chardet
	DetectCharset bool

	MaxRetries int
	// contains filtered or unexported fields
}

Worker is the main object to sent http request and return result of response 

var DefaultWorker *Worker

DefaultWorker Global Worker

func Clear 

func Clear() *Worker

func ClearAll 

func ClearAll() *Worker

func ClearCookie 

func ClearCookie() *Worker

func New 

func New(ipstring interface{}, timeout ...time.Duration) (*Worker, error)

New Alias Name for NewWorker

func NewAPI 

func NewAPI(timeout ...time.Duration) *Worker

NewAPI New API Worker, No Cookie Keep.

func NewWorker 

func NewWorker(ipstring interface{}, timeout ...time.Duration) (*Worker, error)

NewWorker New a worker, if ipstring is a proxy address, New a proxy client. Proxy address such as: 

http://[user]:[password@]ip:port, [] stand it can choose or not. case: socks5://127.0.0.1:1080

func NewWorkerByClient 

func NewWorkerByClient(client *http.Client) *Worker

NewWorkerByClient New Worker by Your Client

func SetAfterAction 

func SetAfterAction(fc func(context.Context, *Worker)) *Worker

func SetBeforeAction 

func SetBeforeAction(fc func(context.Context, *Worker)) *Worker

func SetBinary 

func SetBinary(data []byte) *Worker

func SetContext 

func SetContext(ctx context.Context) *Worker

func SetCookie 

func SetCookie(v string) *Worker

func SetCookieByFile 

func SetCookieByFile(file string) (*Worker, error)

func SetDetectCharset 

func SetDetectCharset(on bool) *Worker

func SetFileInfo 

func SetFileInfo(fileName, fileFormName string) *Worker

func SetForm 

func SetForm(form url.Values) *Worker

func SetFormParm 

func SetFormParm(k, v string) *Worker

func SetHeader 

func SetHeader(header http.Header) *Worker

SetHeader Default Worker SetHeader!

func SetHeaderParm 

func SetHeaderParm(k, v string) *Worker

func SetMaxRetries 

func SetMaxRetries(maxRetries int) *Worker

func SetMethod 

func SetMethod(method string) *Worker

func SetPesterOptions 

func SetPesterOptions(options ...pester.ApplyOptions) *Worker

func SetRefer 

func SetRefer(refer string) *Worker

func SetResponseCharset 

func SetResponseCharset(charset string) *Worker

func SetURL 

func SetURL(url string) *Worker

func SetUserAgent 

func SetUserAgent(ua string) *Worker

func SetWaitTime 

func SetWaitTime(num int) *Worker

func (*Worker) Clear 

func (worker *Worker) Clear() *Worker

Clear data we sent

func (*Worker) ClearAll 

func (worker *Worker) ClearAll() *Worker

All clear include header

func (*Worker) ClearCookie 

func (worker *Worker) ClearCookie() *Worker

ClearCookie Clear Cookie

func (*Worker) Delete 

func (worker *Worker) Delete() (body []byte, e error)

func (*Worker) Get 

func (worker *Worker) Get() (body []byte, e error)

Get method

func (*Worker) GetCookies 

func (worker *Worker) GetCookies() []*http.Cookie

GetCookies Get Cookies

func (*Worker) Go 

func (worker *Worker) Go() (body []byte, e error)

Go Auto decide which method, Default Get.

func (*Worker) GoByMethod 

func (worker *Worker) GoByMethod(method string) (body []byte, e error)

func (*Worker) JSONToString 

func (worker *Worker) JSONToString() (string, error)

JSONToString This make effect only your worker exec serial! Attention! Change Your JSON'like Raw data to string

func (*Worker) OtherGo 

func (worker *Worker) OtherGo(method, contentType string) (body []byte, e error)

OtherGo Other Method 

  Method         = "OPTIONS"                ; Section 9.2
                 | "GET"                    ; Section 9.3
                 | "HEAD"                   ; Section 9.4
                 | "POST"                   ; Section 9.5
                 | "PUT"                    ; Section 9.6
                 | "DELETE"                 ; Section 9.7
                 | "TRACE"                  ; Section 9.8
                 | "CONNECT"                ; Section 9.9
                 | extension-method
extension-method = token
  token          = 1*<any CHAR except CTLs or separators>

Content Type 

"application/x-www-form-urlencoded"
"application/json"
"text/xml"
"multipart/form-data"

func (*Worker) OtherGoBinary 

func (worker *Worker) OtherGoBinary(method, contentType string) (body []byte, e error)

func (*Worker) Post 

func (worker *Worker) Post() (body []byte, e error)

Post Almost include bellow: 

"application/x-www-form-urlencoded"
"application/json"
"text/xml"
"multipart/form-data"

func (*Worker) PostFile 

func (worker *Worker) PostFile() (body []byte, e error)

func (*Worker) PostJSON 

func (worker *Worker) PostJSON() (body []byte, e error)

func (*Worker) PostXML 

func (worker *Worker) PostXML() (body []byte, e error)

func (*Worker) Put 

func (worker *Worker) Put() (body []byte, e error)

Put .

func (*Worker) PutFile 

func (worker *Worker) PutFile() (body []byte, e error)

func (*Worker) PutJSON 

func (worker *Worker) PutJSON() (body []byte, e error)

func (*Worker) PutXML 

func (worker *Worker) PutXML() (body []byte, e error)

func (*Worker) SetAfterAction 

func (worker *Worker) SetAfterAction(fc func(context.Context, *Worker)) *Worker

func (*Worker) SetBeforeAction 

func (worker *Worker) SetBeforeAction(fc func(context.Context, *Worker)) *Worker

func (*Worker) SetBinary 

func (worker *Worker) SetBinary(data []byte) *Worker

func (*Worker) SetContext 

func (worker *Worker) SetContext(ctx context.Context) *Worker

Set Context so Action can soft

func (*Worker) SetCookie 

func (worker *Worker) SetCookie(v string) *Worker

func (*Worker) SetCookieByFile 

func (worker *Worker) SetCookieByFile(file string) (*Worker, error)

SetCookieByFile Set Cookie by file.

func (*Worker) SetDetectCharset 

func (worker *Worker) SetDetectCharset(on bool) *Worker

func (*Worker) SetFileInfo 

func (worker *Worker) SetFileInfo(fileName, fileFormName string) *Worker

func (*Worker) SetForm 

func (worker *Worker) SetForm(form url.Values) *Worker

func (*Worker) SetFormParm 

func (worker *Worker) SetFormParm(k, v string) *Worker

func (*Worker) SetHeader 

func (worker *Worker) SetHeader(header http.Header) *Worker

SetHeader Java Bean Chain pattern

func (*Worker) SetHeaderParm 

func (worker *Worker) SetHeaderParm(k, v string) *Worker

func (*Worker) SetHost 

func (worker *Worker) SetHost(host string) *Worker

func (*Worker) SetMaxRetries 

func (worker *Worker) SetMaxRetries(maxRetries int) *Worker

func (*Worker) SetMethod 

func (worker *Worker) SetMethod(method string) *Worker

func (*Worker) SetPesterOptions 

func (worker *Worker) SetPesterOptions(options ...pester.ApplyOptions) *Worker

func (*Worker) SetRefer 

func (worker *Worker) SetRefer(refer string) *Worker

func (*Worker) SetResponseCharset 

func (worker *Worker) SetResponseCharset(charset string) *Worker

func (*Worker) SetURL 

func (worker *Worker) SetURL(url string) *Worker

SetURL at the same time SetHost

func (*Worker) SetUserAgent 

func (worker *Worker) SetUserAgent(ua string) *Worker

func (*Worker) SetWaitTime 

func (worker *Worker) SetWaitTime(num int) *Worker

func (*Worker) String 

func (worker *Worker) String() string

ToString This make effect only your worker exec serial! Attention! Change Your Raw data To string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.