http

package
v0.0.0-...-f2a16fb Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 5, 2016 License: MIT Imports: 10 Imported by: 174

Documentation

Index

Constants

View Source
const (
	CONTENT_TYPE      = "Content-Type"
	JSON_CONTENT_TYPE = "application/json"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type Client

type Client struct {
	GoClient *Http.Client
}

func NewClient

func NewClient() *Client

func (*Client) ClearProxy

func (this *Client) ClearProxy()

func (*Client) SetProxy

func (this *Client) SetProxy(urlString string)

type HttpServer

type HttpServer struct {
	Http.Server
}

func NewHttpServer

func NewHttpServer(setting *util.Settings, handler Http.Handler) *HttpServer

func (*HttpServer) Start

func (this *HttpServer) Start(wg sync.WaitGroup)

type Request

type Request struct {
	GoRequest  *Http.Request
	CookieJar  int // if the web site store message in cookie which show diffrent result,you shoud use it
	UniqueName string
	SpiderName string
	ParserName string // crawler will send the response to named parseFunction
	NodeName   string
	Depth      int
	Proxy      string // proxy settings,just setting the proxy,downloader will take care of rest thing
	Retry      int    // fail and retry time
}

what a request would do * basic request * global unique key * cookie jar index * spider belong to * parse belong to

func NewRequest

func NewRequest(method, url, spiderName, parserName string, body io.Reader, cookieJar int) (*Request, error)

func (*Request) SetNodeName

func (this *Request) SetNodeName(nodeName string)

type Response

type Response struct {
	GoResponse *Http.Response
	Request    *Request
	SpiderName string
	ParserName string
	NodeName   string
	Body       string
}

func NewResponse

func NewResponse(response *Http.Response, request *Request, spiderName, parserName, nodeName string) *Response

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL