crawler

type CrawlQueue struct {
	sync.Mutex
	// contains filtered or unexported fields
}

type Crawler struct {
	// contains filtered or unexported fields
}

type CrawlingState struct {
	LastStatusCode int       `riak:"lastStatusCode"`
	LastDownload   time.Time `riak:"lastDownload"`
	Deleted        bool      `riak:"deleted"`
}

type Exchange struct {
	IPAddr string
	Port   int
}

type Page struct {
	URL         string        `riak:"url"`
	ContentType string        `riak:"contentType"`
	Body        []byte        `riak:"body"`
	RedirectTo  string        `riak:"redirectTo"`
	State       CrawlingState `riak:"state"`
	riak.Model
}

type PageStore struct {
	// contains filtered or unexported fields
}

type QueueElement struct {
	// contains filtered or unexported fields
}