spider

type CSVConf struct {
	CSVFilePath string
	CSVFileName string
}

type Context struct {
	// contains filtered or unexported fields
}

type HTMLElement struct {
	Name     string
	Text     string
	Request  *Request
	Response *Response
	DOM      *goquery.Selection
	// contains filtered or unexported fields
}

type Limit struct {
	Enable bool
	// DomainRegexp is a regular expression to match against domains
	DomainRegexp string
	// DomainRegexp is a glob pattern to match against domains
	DomainGlob string
	// Delay is the duration to wait before creating a new request to the matching domains
	Delay time.Duration
	// RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
	RandomDelay time.Duration
	// Parallelism is the number of the maximum allowed concurrent requests of the matching domains
	Parallelism int
}

type MySQLConf struct {
	Host     string
	Port     int
	User     string
	Password string
	DBName   string
}

type Node struct {
	OnRequest  func(ctx *Context, req *Request)
	OnError    func(ctx *Context, res *Response, err error) error
	OnResponse func(ctx *Context, res *Response) error
	OnHTML     map[string]func(ctx *Context, el *HTMLElement) error
	OnXML      map[string]func(ctx *Context, el *XMLElement) error
	OnScraped  func(ctx *Context, res *Response) error
}

type Option struct {
	UserAgent              string
	MaxDepth               int
	AllowedDomains         []string
	URLFilters             []*regexp.Regexp
	AllowURLRevisit        bool
	MaxBodySize            int
	IgnoreRobotsTxt        bool
	ParseHTTPErrorResponse bool
	DisableCookies         bool
}

type OutputConfig struct {
	Type      string
	CSVConf   CSVConf
	MySQLConf MySQLConf
}

type Request struct {
	URL     *url.URL
	Headers *http.Header
	Method  string
	Body    io.Reader
	// contains filtered or unexported fields
}

type Response struct {
	StatusCode int
	Body       []byte
	Request    *Request
	Headers    *http.Header
	// contains filtered or unexported fields
}

type Rule struct {
	Head  func(ctx *Context) error
	Nodes map[int]*Node
}

type Task struct {
	ID uint64
	TaskRule
	TaskConfig
}

type TaskConfig struct {
	CronSpec     string
	Option       Option
	Limit        Limit
	ProxyURLs    []string
	OutputConfig OutputConfig
}

type TaskRule struct {
	Name                   string
	Description            string
	Namespace              string
	OutputFields           []string
	DisableCookies         bool
	AllowURLRevisit        bool
	IgnoreRobotsTxt        bool
	ParseHTTPErrorResponse bool
	Rule                   *Rule
}

type XMLElement struct {
	Name     string
	Text     string
	Request  *Request
	Response *Response
	DOM      interface{}
	// contains filtered or unexported fields
}