xcrawl3r

type Crawler struct {
	Domain            string
	IncludeSubdomains bool
	Seeds             []string

	Depth     int
	Headless  bool
	Headers   []string
	Proxies   []string
	Render    bool
	Timeout   int
	UserAgent string

	Concurrency    int
	Delay          int
	MaxRandomDelay int
	Parallelism    int

	Debug bool

	PageCollector         *colly.Collector
	FileURLsRegex         *regexp.Regexp
	FileCollector         *colly.Collector
	URLsNotToRequestRegex *regexp.Regexp
	URLsRegex             *regexp.Regexp
}

type Options struct {
	Domain            string
	IncludeSubdomains bool
	Seeds             []string

	Depth     int
	Headless  bool
	Headers   []string
	Proxies   []string
	Render    bool
	Timeout   int // seconds
	UserAgent string

	Concurrency    int
	Delay          int // seconds
	MaxRandomDelay int // seconds
	Parallelism    int

	Debug bool
}

type URL struct {
	Source string
	Value  string
}