frontier

type Frontier struct {
	Paused *utils.TAtomBool

	FinishingQueueWriter *utils.TAtomBool
	FinishingQueueReader *utils.TAtomBool
	IsQueueWriterActive  *utils.TAtomBool
	IsQueueReaderActive  *utils.TAtomBool
	JobPath              string

	// PullChan and PushChan are respectively the channels used for workers
	// to get new URLs to archive, and the channel to push the discovered URLs
	// to the frontier
	PullChan chan *Item
	PushChan chan *Item

	// Queue is a local queue storing all the URLs to crawl
	// it's a prefixed queue, basically one sub-queue per host
	Queue *goque.PrefixQueue
	// QueueCount store the number of URLs currently queued
	QueueCount *ratecounter.Counter

	// HostPool is an struct that contains a map and a Mutex.
	// the map contains all the different hosts that Zeno crawled,
	// with a counter for each, going through that map gives us
	// the prefix to query from the queue
	HostPool *sync.Map

	UseSeencheck bool
	Seencheck    *Seencheck
	LoggingChan  chan *FrontierLogMessage
}

type FrontierLogMessage struct {
	Fields  map[string]interface{}
	Message string
	Level   logrus.Level
}

type Item struct {
	ID             string
	Hash           uint64
	Hop            uint8
	Host           string
	Type           string
	Redirect       int
	URL            *url.URL
	ParentItem     *Item
	LocallyCrawled uint64
}

type Pair struct {
	Key, Value interface{}
}

type PoolItem struct {
	TotalCount  uint64
	ActiveCount uint64
}

type Seencheck struct {
	SeenCount *ratecounter.Counter
	SeenDB    leveldb.Store
}