spider

type NewsItem struct {
	Title       string    `json:title`
	Description string    `json:description`
	Source      string    `json:source`
	Url         string    `json:url`
	CreatedAt   time.Time `json:created_at`
}

type SearchEngine interface {
	EngineName() string
	UrlFormat() string
	Keyword() string
	ParsePage(document *goquery.Document) ([]NewsItem, error)
}

type Spider struct {
	Engines map[string]SearchEngine
	// contains filtered or unexported fields
}

type SubscribeJob struct {
	SpiderFrequency    time.Duration
	SubscribeFrequency time.Duration
	Keyword            string
	SelectedEngine     []string
}