crawler

type CrawlPsParams struct {
	SiteToCrawl      string
	LaunchThreadsNum int
	VerboseMode      bool
	ReqThrottling    time.Duration
	TimeLimit        time.Duration
}

type CrawledUrlMap map[string]int

type ProcessorLinks struct {
	Pending   int
	Requested int
	Processed int
}

type SpendTime struct {
	Total      int64
	RequestAvg int64
	ParseAvg   int64
}

type StatisticsMap struct {
	CrawledUrlMap  CrawledUrlMap
	ProcessorLinks ProcessorLinks
	SpendTime      SpendTime
}