crawler

package
v0.0.0-...-fd15a86 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 21, 2020 License: GPL-3.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ScoreByBattle

func ScoreByBattle(p Post) float64

Types

type Loader

type Loader interface {
	Load(string) (*goquery.Document, error)
}

type PageCrawler

type PageCrawler struct {
	Loader
}

func NewPageCrawler

func NewPageCrawler() *PageCrawler

func (*PageCrawler) ScanBoard

func (c *PageCrawler) ScanBoard(ctx context.Context, board string, from, to time.Time) <-chan ScanResult

func (*PageCrawler) Trending

func (s *PageCrawler) Trending(ctx context.Context, opts ...TrendingOption) ([]Trending, error)

func (*PageCrawler) VisitPost

func (p *PageCrawler) VisitPost(page string) (Post, error)

type Post

type Post struct {
	ID          string    `json:"id"`
	URL         string    `json:"url"`
	CreatedAt   time.Time `json:"createdAt"`
	Title       string    `json:"title"`
	Re          bool      `json:"reply"`
	Author      string    `json:"author"`
	TextContent string    `json:"textContent"`
	PushTotal   int       `json:"pushTotal"`
	BattlePush  int       `json:"battlePush"`
	UniquePush  int       `json:"uniquePush"`
	PushUp      int       `json:"pushUp"`
	PushDown    int       `json:"pushDown"`
}

func (*Post) MarshalJSON

func (p *Post) MarshalJSON() ([]byte, error)

type ScanResult

type ScanResult struct {
	Post Post
	Err  error
}

type Thread

type Thread struct {
	Score   float64 `json:"score"`
	Deviate float64 `json:"deviate"`
	Posts   []Post  `json:"posts"`
}
type Trending struct {
	TrendingOption
	StartAt time.Time `json:"jobStartAt"`
	EndAt   time.Time `json:"jobEndAt"`
	Threads []Thread  `json:"threads"`
}

type TrendingOption

type TrendingOption struct {
	Board   string        `json:"board"`
	From    time.Time     `json:"trendingFrom"`
	To      time.Time     `json:"trendingTo"`
	Timeout time.Duration `json:"timeout"`
	Deviate float64       `json:"deviate"`
}

func (TrendingOption) IsValid

func (opt TrendingOption) IsValid() bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL