scraper

package
v0.0.0-...-0ee86d9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 24, 2022 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Job

type Job struct {
	ProcessorID        int
	JobID              int
	URL                string
	Proxy              proxies.Proxy
	UserAgent          string
	Request            *http.Request
	ResponseBodyBuffer *bytes.Buffer
	PropertyRecord     tax.PropertyRecord
	Duplicate          bool
	Error              error
	Scraper            *Scraper
}

func (*Job) Process

func (j *Job) Process()

func (*Job) ProcessError

func (j *Job) ProcessError(removeURL bool, fun string, nerr error) error

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

TODO: Add flags to pull in db creds, worker count,

func NewScraper

func NewScraper(proxyClient *proxies.ProxyClient, uac *useragents.UserAgentClient, db *sql.DB, httpClient *http.Client) *Scraper

func (*Scraper) AddPropertyRecordToDB

func (s *Scraper) AddPropertyRecordToDB(workerID, jobID int, pUrl string, pr tax.PropertyRecord) error

func (*Scraper) PropertyExists

func (s *Scraper) PropertyExists(url string) (bool, error)

func (*Scraper) Scrape

func (s *Scraper) Scrape(urls []string)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL