walker

type ConfigStruct struct {
	Fetcher struct {
		MaxDNSCacheEntries       int      `yaml:"max_dns_cache_entries"`
		UserAgent                string   `yaml:"user_agent"`
		AcceptFormats            []string `yaml:"accept_formats"`
		AcceptProtocols          []string `yaml:"accept_protocols"`
		MaxHTTPContentSizeBytes  int64    `yaml:"max_http_content_size_bytes"`
		IgnoreTags               []string `yaml:"ignore_tags"`
		MaxLinksPerPage          int      `yaml:"max_links_per_page"`
		NumSimultaneousFetchers  int      `yaml:"num_simultaneous_fetchers"`
		BlacklistPrivateIPs      bool     `yaml:"blacklist_private_ips"`
		HTTPTimeout              string   `yaml:"http_timeout"`
		HonorMetaNoindex         bool     `yaml:"honor_meta_noindex"`
		HonorMetaNofollow        bool     `yaml:"honor_meta_nofollow"`
		ExcludeLinkPatterns      []string `yaml:"exclude_link_patterns"`
		IncludeLinkPatterns      []string `yaml:"include_link_patterns"`
		DefaultCrawlDelay        string   `yaml:"default_crawl_delay"`
		MaxCrawlDelay            string   `yaml:"max_crawl_delay"`
		PurgeSidList             []string `yaml:"purge_sid_list"`
		ActiveFetchersTTL        string   `yaml:"active_fetchers_ttl"`
		ActiveFetchersCacheratio float32  `yaml:"active_fetchers_cacheratio"`
		ActiveFetchersKeepratio  float32  `yaml:"active_fetchers_keepratio"`
		HTTPKeepAlive            string   `yaml:"http_keep_alive"`
		HTTPKeepAliveThreshold   string   `yaml:"http_keep_alive_threshold"`
		MaxPathLength            int      `yaml:"max_path_length"`
	} `yaml:"fetcher"`

	Dispatcher struct {
		MaxLinksPerSegment         int     `yaml:"num_links_per_segment"`
		RefreshPercentage          float64 `yaml:"refresh_percentage"`
		NumConcurrentDomains       int     `yaml:"num_concurrent_domains"`
		MinLinkRefreshTime         string  `yaml:"min_link_refresh_time"`
		DispatchInterval           string  `yaml:"dispatch_interval"`
		CorrectLinkNormalization   bool    `yaml:"correct_link_normalization"`
		EmptyDispatchRetryInterval string  `yaml:"empty_dispatch_retry_interval"`
	} `yaml:"dispatcher"`

	Cassandra struct {
		Hosts                 []string `yaml:"hosts"`
		Keyspace              string   `yaml:"keyspace"`
		ReplicationFactor     int      `yaml:"replication_factor"`
		Timeout               string   `yaml:"timeout"`
		CQLVersion            string   `yaml:"cql_version"`
		ProtoVersion          int      `yaml:"proto_version"`
		Port                  int      `yaml:"port"`
		NumConns              int      `yaml:"num_conns"`
		NumStreams            int      `yaml:"num_streams"`
		DiscoverHosts         bool     `yaml:"discover_hosts"`
		MaxPreparedStmts      int      `yaml:"max_prepared_stmts"`
		AddNewDomains         bool     `yaml:"add_new_domains"`
		AddedDomainsCacheSize int      `yaml:"added_domains_cache_size"`
		StoreResponseBody     bool     `yaml:"store_response_body"`
		StoreResponseHeaders  bool     `yaml:"store_response_headers"`
		NumQueryRetries       int      `yaml:"num_query_retries"`
		DefaultDomainPriority int      `yaml:"default_domain_priority"`
	} `yaml:"cassandra"`

	Console struct {
		Port                     int    `yaml:"port"`
		TemplateDirectory        string `yaml:"template_directory"`
		PublicFolder             string `yaml:"public_folder"`
		MaxAllowedDomainPriority int    `yaml:"max_allowed_domain_priority"`
	} `yaml:"console"`
}

type Datastore interface {
	// ClaimNewHost returns a hostname that is now claimed for this crawler to
	// crawl. A segment of links for this host is assumed to be available.
	// Returns the domain of the segment it claimed, or "" if there are none
	// available.
	ClaimNewHost() string

	// UnclaimHost indicates that all links from `LinksForHost` have been
	// processed, so other work may be done with this host. For example the
	// dispatcher will be free analyze the links and generate a new segment.
	UnclaimHost(host string)

	// LinksForHost returns a channel that will feed URLs for a given host.
	LinksForHost(host string) <-chan *URL

	// StoreURLFetchResults takes the return data/metadata from a fetch and
	// stores the visit. Fetchers will call this once for each link in the
	// segment being crawled.
	StoreURLFetchResults(fr *FetchResults)

	// StoreParsedURL stores a URL parsed out of a page (i.e. a URL we may not
	// have crawled yet). `u` is the URL to store. `fr` is the FetchResults
	// object for the fetch from which we got the URL, for any context the
	// datastore may want. A datastore implementation should handle `fr` being
	// nil, so links can be seeded without a fetch having occurred.
	//
	// URLs passed to StoreParsedURL should be absolute.
	//
	// This layer should handle efficiently deduplicating
	// links (i.e. a fetcher should be safe feeding the same URL many times.
	StoreParsedURL(u *URL, fr *FetchResults)

	// KeepAlive will be called periodically in fetcher. This method should
	// notify the datastore that this fetcher is still alive.
	KeepAlive() error

	// Close will be called when no more Datastore calls will be made, allowing
	// any necessary cleanup to take place.
	Close()
}

type Dispatcher interface {
	// StartDispatcher should be a blocking call that starts the dispatcher. It
	// should return an error if it could not start or stop properly and nil
	// when it has safely shut down and stopped all internal processing.
	StartDispatcher() error

	// Stop signals the dispatcher to stop. It should block until all internal
	// goroutines have stopped.
	StopDispatcher() error
}

type FetchManager struct {
	// Handler must be set to handle fetch responses.
	Handler Handler

	// Datastore must be set to drive the fetching.
	Datastore Datastore

	// Transport can be set to override the default network transport the
	// FetchManager is going to use. Good for faking remote servers for
	// testing.
	Transport http.RoundTripper

	// TransNoKeepAlive stores a RoundTripper with Keep-Alive set to 0 IF
	// http_keep_alive == "threshold". Otherwise it's nil.
	TransNoKeepAlive http.RoundTripper

	// Parsed duration of the string Config.Fetcher.HTTPKeepAliveThreshold
	KeepAliveThreshold time.Duration
	// contains filtered or unexported fields
}

type FetchResults struct {

	// URL that was requested; will always be populated. If this URL redirects,
	// RedirectedFrom will contain a list of all requested URLS.
	URL *URL

	// A list of redirects. During this request cycle, the first request URL is stored
	// in URL. The second request (first redirect) is stored in RedirectedFrom[0]. And
	// the Nth request (N-1 th redirect) will be stored in RedirectedFrom[N-2],
	// and this is the URL that furnished the http.Response.
	RedirectedFrom []*URL

	// Response object; nil if there was a FetchError or ExcludedByRobots is
	// true. Response.Body may not be the same object the HTTP request actually
	// returns; the fetcher may have read in the response to parse out links,
	// replacing Response.Body with an alternate reader.
	Response *http.Response

	// If the user has set cassandra.store_response_body to true in the config file,
	// then the content of the link will be stored in Body (and consequently stored in the
	// body column of the links table). Otherwise Body is the empty string.
	Body string

	// FetchError if the net/http request had an error (non-2XX HTTP response
	// codes are not considered errors)
	FetchError error

	// Time at the beginning of the request (if a request was made)
	FetchTime time.Time

	// True if we did not request this link because it is excluded by
	// robots.txt rules
	ExcludedByRobots bool

	// True if the page was marked as 'noindex' via a <meta> tag. Whether it
	// was crawled depends on the honor_meta_noindex configuration parameter
	MetaNoIndex bool

	// True if the page was marked as 'nofollow' via a <meta> tag. Whether it
	// was crawled depends on the honor_meta_nofollow configuration parameter
	MetaNoFollow bool

	// The Content-Type of the fetched page.
	MimeType string

	// Fingerprint of the reponse body computed with fnv algorithm (see
	// hash/fnv in standard library)
	FnvFingerprint int64

	// Fingerprint of the text parsed out of the response body, also computed
	// with fnv
	FnvTextFingerprint int64
}

type HTMLParser struct {
	// A concatenation of all text, excluding content from script/style tags
	Text []byte
	// A list of links found on the parsed page
	Links []*URL
	// true if <meta name="ROBOTS" content="noindex"> was found
	HasMetaNoIndex bool
	// true if <meta name="ROBOTS" content="nofollow"> was found
	HasMetaNoFollow bool
}

type Handler interface {
	// HandleResponse will be called by fetchers as they make requests.
	// Handlers can do whatever they want with responses. HandleResponse will
	// be called as long as the request successfully reached the remote server
	// and got an HTTP code. This means there should never be a FetchError set
	// on the FetchResults.
	HandleResponse(res *FetchResults)
}

type MockDatastore struct {
	mock.Mock
}

type MockDispatcher struct {
	mock.Mock
}

type MockHTTPHandler struct {
	// contains filtered or unexported fields
}

type MockHandler struct {
	mock.Mock
}

type MockRemoteServer struct {
	*MockHTTPHandler
	// contains filtered or unexported fields
}

type MockResponse struct {
	// Status defaults to 200
	Status int

	// Status defaults to "GET"
	Method string

	// Body defaults to nil (no response body)
	Body string

	// Headers of response
	Headers http.Header

	//ContentType defaults to "text/html"
	ContentType string

	// How long is the content
	ContentLength int
}

type URL struct {
	*url.URL

	// LastCrawled is the last time we crawled this URL, for example to use a
	// Last-Modified header.
	LastCrawled time.Time
}