common

package
v0.0.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 19, 2023 License: MIT Imports: 27 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ErrOutOfScope = errors.New("out of scope")

Functions

func BuildHttpClient

func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirectCallback RedirectCallback) (*retryablehttp.Client, *fastdialer.Dialer, error)

BuildClient builds a http client based on a profile

Types

type CrawlSession

type CrawlSession struct {
	Ctx        context.Context
	CancelFunc context.CancelFunc
	URL        *url.URL
	Hostname   string
	Queue      *queue.Queue
	HttpClient *retryablehttp.Client
	Browser    *rod.Browser
}

type DoRequestFunc

type DoRequestFunc func(crawlSession *CrawlSession, req *navigation.Request) (*navigation.Response, error)

type RedirectCallback

type RedirectCallback func(resp *http.Response, depth int)

type Shared

type Shared struct {
	Headers    map[string]string
	KnownFiles *files.KnownFiles
	Options    *types.CrawlerOptions
}

func NewShared

func NewShared(options *types.CrawlerOptions) (*Shared, error)

func (*Shared) Do

func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error

func (*Shared) Enqueue

func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.Request)

func (*Shared) NewCrawlSessionWithURL

func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error)

func (*Shared) Output

func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, err error)

func (*Shared) ValidateScope

func (s *Shared) ValidateScope(URL string, root string) bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL