collect

package
v0.0.0-...-b3f7bda Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 21, 2023 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DeterminEncoding

func DeterminEncoding(r *bufio.Reader) encoding.Encoding

Types

type BaseFetch

type BaseFetch struct {
}

func (*BaseFetch) Get

func (b *BaseFetch) Get(req *Request) ([]byte, error)

type BrowserFetch

type BrowserFetch struct {
	Timeout time.Duration
	Proxy   proxy.ProxyFunc
	Logger  *zap.Logger
}

func (*BrowserFetch) Get

func (b *BrowserFetch) Get(request *Request) ([]byte, error)

type Fetcher

type Fetcher interface {
	Get(url *Request) ([]byte, error)
}

type ParseResult

type ParseResult struct {
	Requesrts []*Request
	Items     []interface{}
}

type Request

type Request struct {
	Url       string // 访问的防战
	Cookie    string
	WaitTime  time.Duration
	Depth     int
	MaxDepth  int
	ParseFunc func([]byte, *Request) ParseResult // 解析从网站获取到的网站信息
}

func (*Request) Check

func (r *Request) Check() error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL