crawler

type Context struct {
	Crawler *Crawler

	Request  *http.Request
	Response *http.Response
	Document *goquery.Document
	Param    map[string]interface{}
	// contains filtered or unexported fields
}

type Crawler struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type Option struct {
	Name            string
	LogDisable      bool
	AutoStopDisable bool
	PauseTime       []int

	DefaultMethod string
	StorerWork    func(map[string][]interface{}) map[string]bool
	// contains filtered or unexported fields
}

type Queue struct {
	URL    string
	Method string
	Rule   string
	Param  map[string]interface{}
}

type Rule struct {
	Timeout       time.Duration
	BeforeRequest func(*http.Request)
	Parse         func(*Context) bool
	// contains filtered or unexported fields
}