gopdynamiccrawler

package

v0.0.0-...-37b0e48 Latest Latest Go to latest Published: Apr 27, 2024 License: MIT Imports: 15 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/hophouse/gop

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func InitCrawler()
func NewOptions(url *string, logFileOption *os.File, report *bool, recursive *bool, ...)
func PrintOptions(options *Options)
func RunCrawlerCmd()
func TreatA(doc *goquery.Document) []string
func TreatLinkHref(doc *goquery.Document) []string
func TreatScriptSrc(doc *goquery.Document) []string
type Options
type URLVisitedStruct

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	Internal_ressources []*gopstaticcrawler.Ressource
	External_ressources []*gopstaticcrawler.Ressource
	URLVisited          URLVisitedStruct
	ScreenshotList      []gopchromedp.Item
	ScreenshotChan      chan struct{}
	UrlChan             chan string
)

Functions ¶

func InitCrawler ¶

func InitCrawler()

func NewOptions ¶

func NewOptions(url *string, logFileOption *os.File, report *bool, recursive *bool, screenshot *bool, cookie *string, proxy *string, delay *int, concurrency *int)

func PrintOptions ¶

func PrintOptions(options *Options)

func RunCrawlerCmd ¶

func RunCrawlerCmd()

RunCrawlerCmd Run the crawler

func TreatA ¶

func TreatA(doc *goquery.Document) []string

func TreatLinkHref ¶

func TreatLinkHref(doc *goquery.Document) []string

func TreatScriptSrc ¶

func TreatScriptSrc(doc *goquery.Document) []string

Types ¶

type Options ¶

type Options struct {
	UrlPtr         *string
	LogFile        *os.File
	ReportPtr      *bool
	RecursivePtr   *bool
	ScreenshotPtr  *bool
	CookiePtr      *string
	ProxyPtr       *string
	DelayPtr       *int
	ConcurrencyPtr *int
}

var GoCrawlerOptions Options

type URLVisitedStruct ¶

type URLVisitedStruct struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL