spider

package module
v0.0.0-...-62ddf5d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 7, 2020 License: MIT Imports: 12 Imported by: 0

README

simpleSpider

a very simple go spider

Documentation

Index

Constants

View Source 
const (
	TextAll   = "text/*"
	HtmlType  = "text/html"
	CssType   = "text/css"
	PlainType = "text/plain"

	ApplicationAll = "application/*"
	JsType         = "application/javascript"

	ImageAll = "image/*"
	PngType  = "image/png"
	IconType = "image/x-icon"

	VideoAll = "video/*"
	Mp4Type  = "video/mp4"
)

Variables

This section is empty.

Functions

func AppendSlice 

func AppendSlice(m []string, check func(ele string) bool, arg ...string) []string

Types

type ContentType 

type ContentType []string

type Domain 

type Domain []string

type Spider 

type Spider struct {
	URL                URL
	SpiderDepth        int
	Goroutines         int
	ActionTimeout      time.Duration
	AllowedDomain      Domain
	AllowedContentType ContentType

	PreAction      func(req *http.Request) bool
	PostAction     func(req *http.Request, resp *http.Response, body *bytes.Buffer) bool
	ExpandPolicy   func(req *http.Request, body *bytes.Buffer) URL
	RedirectPolicy func(req *http.Request, via []*http.Request) error
	OnError        func(req *http.Request, err error)
	// contains filtered or unexported fields
}

func (*Spider) Run 

func (s *Spider) Run() error

type URL 

type URL []string

func DefaultExpandPolicy 

func DefaultExpandPolicy(req *http.Request, body *bytes.Buffer) URL

func (*URL) Add 

func (u *URL) Add(urls ...string)

Source Files

View all Source files

Directories

Path Synopsis
example
baiduNews
basicZhihu

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.