Documentation ¶
Index ¶
- func AppendFileLines(lines []string, path string) error
- func PrintConfig(config Config)
- func ReadFileLines(path string) ([]string, error)
- func SearchAllKeywords(keywords []string, config Config) []string
- type CSVWriter
- type Config
- type Crawler
- type CrawlerStats
- type SQLiteWriter
- type SearchResults
- type Writer
- type WriterInit
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func AppendFileLines ¶
func PrintConfig ¶
func PrintConfig(config Config)
func ReadFileLines ¶
func SearchAllKeywords ¶
SearchAllKeywords take each keyword (can be multi word) and perform GoogleSearch retrieving all results that Google Custom Search returns
Types ¶
type Config ¶
type Config struct { ApiKey string Cx string OutputDriver string KeywordsFile string CrawlIgnoreDomains []string CrawlAllowedUrlsRegex string CrawlUserAgent string CrawlTag string CrawlMatchRegex string CrawlUrlsFile string CrawlCacheDir string CrawlDepth int CrawlThreads int CrawlLog bool MatchOutputChunks int MaxSearchResults int }
func ReadEnvConfig ¶
type Crawler ¶
type Crawler struct {
// contains filtered or unexported fields
}
func (*Crawler) GetStats ¶
func (crawler *Crawler) GetStats() CrawlerStats
type CrawlerStats ¶
type CrawlerStats struct {
// contains filtered or unexported fields
}
type SQLiteWriter ¶
type SQLiteWriter struct {
// contains filtered or unexported fields
}
func (*SQLiteWriter) Close ¶
func (w *SQLiteWriter) Close() error
func (*SQLiteWriter) Init ¶
func (w *SQLiteWriter) Init(cacheMaxSize int) error
func (*SQLiteWriter) WriteAll ¶
func (w *SQLiteWriter) WriteAll(records [][]string) error
func (*SQLiteWriter) WriteWithCache ¶
func (w *SQLiteWriter) WriteWithCache(key string, value string, forceWrite bool) error
type SearchResults ¶
type SearchResults struct { Links []string // contains filtered or unexported fields }
func GoogleSearch ¶
GoogleSearch returns a SearchResults struct with Links as a list of links found
type Writer ¶
type WriterInit ¶
Click to show internal directories.
Click to hide internal directories.