utils

func DomainToOutputFolder ¶

func DomainToOutputFolder(urlObject *url.URL) string

DomainToOutputFolder returns the name of the folder for a given domain; this folder will hold all the files crawled from this domain, and our system files. That allows to have multiple crawlers working in parallel, given they crawl different sites.

func NormalizeUrlObject ¶

func NormalizeUrlObject(urlObject *url.URL) (*url.URL, error)

func UrlToFileStructure ¶

func UrlToFileStructure(urlObject *url.URL) (path, filename string)

UrlToFileStructure converts URL path to a file path and name

func UrlToHost ¶

func UrlToHost(urlObject *url.URL) (string, error)