fetcher

type BaseFetcher struct{}

type BrowserFetcher struct {
	Timeout time.Duration
	Proxy   proxy.Func
	Logger  *zap.Logger
}

type Context struct {
	Body []byte
	Req  *Request
}

type Fetcher interface {

	// Get Fetch the html content according to url
	Get(url *Request) ([]byte, error)
}

type ParseResult struct {
	Requests []*Request
	Items    []interface{}
}

type Property struct {
	// The unique signature of the Task
	Name     string        `json:"name"`
	Url      string        `json:"url"`
	Cookie   string        `json:"cookie"`
	WaitTime time.Duration `json:"wait_time"`
	// Mark whether the site can be crawled repeated
	Reload   bool  `json:"reload"`
	MaxDepth int64 `json:"max_depth"`
	// Headers needs to be added to http headers
	Headers map[string]string `json:"headers"`
}

type RedirectFetcher struct {
	Timeout time.Duration
	Proxy   proxy.Func
	Logger  *zap.Logger
}

type Request struct {
	Task     *Task
	Url      string
	Method   string
	Depth    int64
	Priority int64
	RuleName string
	TempData *Temp
	// contains filtered or unexported fields
}

type Rule struct {
	ItemFields []string
	ParseFunc  func(*Context) (ParseResult, error)
}

type RuleModel struct {
	Name      string `json:"name"`
	ParseFunc string `json:"parse_script"`
}

type RuleTree struct {

	// the entry of crawling rules
	Root func() ([]*Request, error)

	// the hashmap of rules
	// key: rule's name
	// value: the specific rule
	Trunk map[string]*Rule
}

type Task struct {
	Property

	Visited     map[string]bool
	VisitedLock sync.Mutex

	RootReq *Request
	Fetcher Fetcher
	Rule    RuleTree

	Logger  *log.Logger
	Storage collector.Store
	Limiter limiter.MultiLimiter
}

type TaskModel struct {
	Property
	Root  string      `json:"root_script"`
	Rules []RuleModel `json:"rules"`
}

type Temp struct {
	// contains filtered or unexported fields
}