spider

type Config struct {
	Threadnum    int
	Buffersize   int
	Deps         int
	FocusSpider  bool
	Crawlfunc    string // simple,puppeteer
	CrawlTimeout int
	CrawlRetry   int
}

type Content struct {
	Dsn   string
	Conn  int
	Crawl func(pg *PageInfo, doc *goquery.Document) *PageInfo
	Parse func(hosturl string, pg *PageInfo, save chan<- interface{}) bool
	Save  func(result interface{})
}

type DB struct {
	// contains filtered or unexported fields
}

type DBLinkInfo struct {
	Host  string
	Title string
	Name  string
	Url   string
}

type DoneDB struct {
	// contains filtered or unexported fields
}

type FindData struct {
	Title string
	Name  string
	URL   string
}

type JobDB struct {
	// contains filtered or unexported fields
}

type LoopSpider struct {
	Thread int
	Buffer int
	Cur    string
	// contains filtered or unexported fields
}

type LoopSpiderSlot interface {
	Name() string
	DefaultCur() string
	Crawl(cur string) bool
	NextCur(cur string) string
}

type LoopSpiderStatus struct {
	Cur      string
	CurInt   int
	UsedTime string
	Done     int64
	Speed    string
	Fail     int64
	OK       int64
}

type PageInfo struct {
	UI    URLInfo
	Title string
	Son   []PageLinkInfo
}

type PageLinkInfo struct {
	UI   URLInfo
	Name string
}

type SpiderData struct {
	// contains filtered or unexported fields
}

type Stat struct {
	CrawBePushJobNum int

	CrawChannelNum  int
	CrawFunc        string
	CrawNum         int
	CrawRetrtyNum   int
	CrawOKNum       int64
	CrawFailNum     int
	CrawOKTotalTime int64
	CrawOKAvgTime   int64

	ParseChannelNum int
	ParseNum        int
	ParseValidNum   int
	ParseSpawnNum   int
	ParseFinishNum  int
	ParseTooDeepNum int
	ParseJobNum     int

	SaveChannelNum int
	SaveNum        int

	InsertNum       int64
	InsertTotalTime int64
	InsertAvgTime   int64

	JobInsertNum       int64
	JobInsertTotalTime int64
	JobInsertAvgTime   int64
	JobPopNum          int64
	JobPopTotalTime    int64
	JobPopAvgTime      int64
	JobHasNum          int64
	JobHasTotalTime    int64
	JobHasAvgTime      int64

	DoneInsertNum       int64
	DoneInsertTotalTime int64
	DoneInsertAvgTime   int64
	DoneHasNum          int64
	DoneHasTotalTime    int64
	DoneHasAvgTime      int64
}

type URLInfo struct {
	Url  string
	Deps int
}