core

package

v2.0.18 Latest Latest Go to latest Published: Feb 23, 2024 License: GPL-3.0 Imports: 15 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/Malwarize/webpalm

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
type Cache
type Crawler
- func NewCrawler(options *shared.Options) *Crawler

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	GeneralRegex = `((?:https?)://[\w\-]+(?:\.[\w\-]+)+[\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])`
	HrefRegex    = `href=["']([^"']+)["']`
)

View Source

var UnreadableExtensions = []string{
	".png",
	".jpg",
	".jpeg",
	".gif",
	".pdf",
	".doc",
	".docx",
	".xls",
	".xlsx",
	".ppt",
	".pptx",
	".zip",
	".rar",
	".tar",
	".gz",
	".exe",
	".mp3",
	".mp4",
	".avi",
	".mov",
	".wmv",
	".flv",
	".wav",
	".mpeg",
	".mpg",
	".m4v",
	".swf",
	".svg",
	".ico",
	".ttf",
	".woff",
	".woff2",
	".eot",
	".otf",
	".psd",
	".ai",
	".eps",
	".indd",
	".raw",
	".webm",
	".m4a",
	".m4p",
	".m4b",
	".m4r",
}

Functions ¶

This section is empty.

Types ¶

type Cache ¶

type Cache struct {
	Visited map[string]bool
	Lock    sync.Mutex
}

func (*Cache) AddVisited ¶

func (c *Cache) AddVisited(url string)

func (*Cache) Flush ¶

func (c *Cache) Flush()

func (*Cache) IsVisited ¶

func (c *Cache) IsVisited(url string) bool

type Crawler ¶

type Crawler struct {
	RootURL        string
	Level          int
	ExportFile     string
	RegexMap       map[string]string
	ExcludedStatus []int
	IncludedUrls   []string
	Client         *http.Client
	UserAgent      string
	Cache          Cache
	Workers        int
	Delay          int
}

func NewCrawler ¶

func NewCrawler(options *shared.Options) *Crawler

func (*Crawler) AddMatches ¶

func (c *Crawler) AddMatches(page *webtree.Page)

func (*Crawler) Crawl ¶

func (c *Crawler) Crawl()

func (*Crawler) CrawlNodeBlock ¶

func (c *Crawler) CrawlNodeBlock(w *webtree.Node, levelChangedChan chan int)

func (*Crawler) CrawlNodeLive ¶

func (c *Crawler) CrawlNodeLive(w *webtree.Node)

func (*Crawler) Export ¶

func (c *Crawler) Export(tree *webtree.Node, format string, filename string) error

func (*Crawler) ExportJSON ¶

func (c *Crawler) ExportJSON(root *webtree.Node, filename string) error

func (*Crawler) ExportTXT ¶

func (c *Crawler) ExportTXT(root *webtree.Node, filename string) error

func (*Crawler) ExportXML ¶

func (c *Crawler) ExportXML(tree *webtree.Node, filename string) error

func (*Crawler) ExtractLinks ¶

func (c *Crawler) ExtractLinks(page *webtree.Page) (links []string)

func (*Crawler) Fetch ¶

func (c *Crawler) Fetch(page *webtree.Page)

func (*Crawler) IsSkipablePage ¶

func (c *Crawler) IsSkipablePage(page *webtree.Page) bool

func (*Crawler) ProcessANode ¶ added in v2.0.14

func (c *Crawler) ProcessANode(node *webtree.Node)

func (*Crawler) SaveResults ¶

func (c *Crawler) SaveResults(root *webtree.Node)

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL