scraper

type DoOption interface {
	// contains filtered or unexported methods
}

type FetchStatusNotOKError struct {
	Page *Page
}

type FetchThrottledError struct{}

type Limiter interface {
	Take() time.Time
}

type OptDoBrowser struct{}

type OptDoLimiter struct {
	Limiter Limiter
}

type OptDoReplace struct{}

type OptDoSilentThrottle struct {
	PageBytesRegexp *regexp.Regexp
}

type Option interface {
	// contains filtered or unexported methods
}

type Page struct {
	Meta     PageMeta     `json:"meta"`
	Request  PageRequest  `json:"request"`
	Response PageResponse `json:"response"`
}

type PageMeta struct {
	Version     uint16        `json:"version"`
	Source      string        `json:"-"`
	RetrieveDur time.Duration `json:"-"`
	ScrapedAt   time.Time     `json:"scraped_at"`
	FetchDur    time.Duration `json:"fetch_dur"`
}

type PageRequest struct {
	URL           string      `json:"url"`
	RedirectedURL string      `json:"redirected_url,omitempty"`
	Method        string      `json:"method"`
	Header        http.Header `json:"header,omitempty"`
	Body          []byte      `json:"body,omitempty"`
}

type PageResponse struct {
	StatusCode int         `json:"status_code"`
	Header     http.Header `json:"header,omitempty"`
	Body       []byte      `json:"body,omitempty"`
}

type Scraper struct {
	// contains filtered or unexported fields
}