Documentation ¶
Index ¶
- Constants
- Variables
- func GetAvailablePort() uint
- func LinksFromBodies(host *url.URL, bodies ...*ResponseBody) []*url.URL
- func NewWorker(conf WorkerConfig) (*worker, error)
- func PullImage(c *docker.Client, img string) error
- func ReadDomainsFromFile(path string) (<-chan Domain, error)
- func RetrieveLinks(host *url.URL, body []byte) ([]*url.URL, error)
- func ScanForServers(ctx context.Context, domains <-chan Domain) <-chan *url.URL
- func WaitForEndpoint(ctx context.Context, endpoint string) error
- type BrowserEvents
- type BrowserRequest
- type BrowserResponse
- type BrowserScreenshot
- type BrowserTimes
- type CTXLOGGER
- type CallFrame
- type Causer
- type ChromeEventParam
- type CrawlAction
- type CrawlRequest
- type CrawlResponse
- type Domain
- type DomainFileProvider
- type DomainFileProviderConfig
- type Host
- type Initiator
- type JavaScriptConsole
- type NoParamErr
- type NotOfTypeErr
- type Page
- type PageHandleFunc
- type PageMiddleware
- type PageStore
- type PhishTankProvider
- type PhishTankProviderConfig
- type Resolution
- type ResponseBody
- type URLChanProvider
- type URLHandleFunc
- type URLMiddleware
- type URLProvider
- type URLStore
- type Worker
- type WorkerConfig
- type WorkerController
- type WorkerControllerConfig
Constants ¶
View Source
const ( CHROME_REQ_WILL_BE_SENT = "Network.requestWillBeSent" CHROME_RESP_RECEIVED = "Network.responseReceived" CHROME_LOADING_FAILED = "Network.loadingFailed" CHROME_LOADING_FINISHED = "Network.loadingFinished" CUSTOM_GOT_BODY = "Custom.body" )
Variables ¶
View Source
var ( ErrFuncTimeout = errors.New("timeout") ErrNameServer = errors.New("unable to get name servers") ErrDockerConn = errors.New("docker connection not responding") ErrTimeoutDOM = errors.New("timeout loading document object model") )
View Source
var DefaultResolution = &Resolution{
Width: 1366,
Height: 768,
}
Functions ¶
func GetAvailablePort ¶
func GetAvailablePort() uint
func LinksFromBodies ¶
func LinksFromBodies(host *url.URL, bodies ...*ResponseBody) []*url.URL
func NewWorker ¶
func NewWorker(conf WorkerConfig) (*worker, error)
func ReadDomainsFromFile ¶
func ScanForServers ¶
Types ¶
type BrowserEvents ¶
type BrowserEvents struct {
// contains filtered or unexported fields
}
type BrowserRequest ¶
type BrowserResponse ¶
type BrowserScreenshot ¶
type BrowserScreenshot struct { Screenshot []byte Resolution Resolution Kind string Taken time.Time }
type BrowserTimes ¶
type BrowserTimes struct { StartTime float64 EndTime float64 ConnectStartTime *float64 ConnectEndTime *float64 SendStartTime *float64 SendEndTime *float64 }
func (*BrowserTimes) Align ¶
func (bt *BrowserTimes) Align()
type ChromeEventParam ¶
type ChromeEventParam struct {
// contains filtered or unexported fields
}
type CrawlAction ¶
type CrawlAction struct { Parent *CrawlAction Initiator Initiator Host Host Request network.Request Response *network.Response Error *string Body *ResponseBody Timings BrowserTimes }
func ActionsFromEvents ¶
func ActionsFromEvents(events *BrowserEvents) []*CrawlAction
func (*CrawlAction) Finished ¶
func (ca *CrawlAction) Finished() bool
type CrawlResponse ¶
type CrawlResponse struct { Primary *CrawlAction Secondaries []CrawlAction Error error }
type DomainFileProvider ¶
type DomainFileProvider struct {
// contains filtered or unexported fields
}
func NewDomainFileProvider ¶
func NewDomainFileProvider(path string, conf *DomainFileProviderConfig) (*DomainFileProvider, error)
func (*DomainFileProvider) Close ¶
func (dfp *DomainFileProvider) Close()
func (*DomainFileProvider) UrlsC ¶
func (dfp *DomainFileProvider) UrlsC() <-chan *url.URL
type Host ¶
func GetHostInfo ¶
type JavaScriptConsole ¶
type NoParamErr ¶
type NoParamErr struct {
// contains filtered or unexported fields
}
func (*NoParamErr) Error ¶
func (npe *NoParamErr) Error() string
type NotOfTypeErr ¶
type NotOfTypeErr struct {
// contains filtered or unexported fields
}
func (*NotOfTypeErr) Error ¶
func (note *NotOfTypeErr) Error() string
type Page ¶
type Page struct { InitialURL *url.URL Actions []*CrawlAction Resolution string Console []*JavaScriptConsole Screenshots []*BrowserScreenshot Error error DocumentURLs []*url.URL InitiatedTime time.Time LoadedTime time.Time TerminatedTime time.Time }
type PageHandleFunc ¶
type PageHandleFunc func(Page)
type PageMiddleware ¶
type PageMiddleware func(PageHandleFunc) PageHandleFunc
type PhishTankProvider ¶
type PhishTankProvider struct {
// contains filtered or unexported fields
}
func NewPhishTankProvider ¶
func NewPhishTankProvider() *PhishTankProvider
func NewPhishTankProviderWithConfig ¶
func NewPhishTankProviderWithConfig(conf PhishTankProviderConfig) *PhishTankProvider
func (*PhishTankProvider) Close ¶
func (ptr *PhishTankProvider) Close()
func (*PhishTankProvider) UrlsC ¶
func (ptr *PhishTankProvider) UrlsC() <-chan *url.URL
type PhishTankProviderConfig ¶
type Resolution ¶
func (Resolution) String ¶
func (r Resolution) String() string
type ResponseBody ¶
type URLChanProvider ¶
func (URLChanProvider) UrlsC ¶
func (ucp URLChanProvider) UrlsC() <-chan *url.URL
type URLHandleFunc ¶
func SkipURLsMiddleware ¶
func SkipURLsMiddleware(URLHandleFunc) URLHandleFunc
type URLMiddleware ¶
type URLMiddleware func(URLHandleFunc) URLHandleFunc
type URLProvider ¶
type Worker ¶
type Worker interface { io.Closer Run(queue <-chan CrawlRequest, results chan<- Page) error }
type WorkerConfig ¶
type WorkerController ¶
type WorkerController struct {
// contains filtered or unexported fields
}
func NewWorkerController ¶
func NewWorkerController(ctx context.Context, conf WorkerControllerConfig) (*WorkerController, error)
func (*WorkerController) AddWorker ¶
func (wc *WorkerController) AddWorker() error
func (*WorkerController) Close ¶
func (wc *WorkerController) Close() error
type WorkerControllerConfig ¶
type WorkerControllerConfig struct { URLStore URLStore PageStore PageStore Logger *zap.Logger WorkerProducer func() (Worker, error) PageMiddleware []PageMiddleware URLMiddleware []URLMiddleware }
Click to show internal directories.
Click to hide internal directories.