crawler

package
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 31, 2023 License: MIT Imports: 10 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Crawler 

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler 

func NewCrawler(opts CrawlerOptions) *Crawler

func (*Crawler) CanCrawl 

func (c *Crawler) CanCrawl(ctx context.Context, urlString string) bool

func (*Crawler) Crawl 

func (c *Crawler) Crawl(ctx context.Context, urlString string) (*http.Header, *html.Node, error)

func (*Crawler) HasDNSARecord 

func (c *Crawler) HasDNSARecord(ctx context.Context, domainName string) bool

type CrawlerOptions 

type CrawlerOptions struct {
	TimeOutInSeconds        int
	MinContentLengthInBytes int64
	MaxContentLengthInBytes int64
	ContentReadLimitInBytes int64
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.