Documentation ¶
Index ¶
Constants ¶
const BrowserEventWaitTime time.Duration = 15
BrowserEventWaitTime is the seconds we wait while attempting to fetch some events from DevTools, before failing.
const BrowserTimeout time.Duration = 1
BrowserTimeout is the minutes we will wait before declaring failed the connection to our debugged browser or to the URL failed.
const BrowserWaitTime time.Duration = 5
BrowserWaitTime is the seconds we will wait before fetching navigation results.
const TorSocksProxy string = "socks5://127.0.0.1:9050"
TorSocksProxy defines the default SOCKS5 conection string for Tor.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Browser ¶
type Browser struct { Datetime time.Time `json:"datetime" bson:"datetime"` // Current UTC date and time. Proxy string `json:"proxy" bson:"proxy"` // Proxy connection string specified to the browser (if any). DebugPort int `json:"debug_port" bson:"debug_port"` // Randomly picked port to use for DevTools debug. DebugURL string `json:"debug_url" bson:"debug_url"` // DevTools debug URL. LogEvents bool `json:"log_events" bson:"log_events"` // Flag to indicate whether to log all DevTools events. UserAgent string `json:"user_agent" bson:"user_agent"` // Randomly picked User Agent specified to the browser. ImageName string `json:"image_name" bson:"image_name"` // Name of the Docker image used. NetworkID string `json:"network_id" bson:"network_id"` // ID of Docker network created for the execution. ContainerID string `json:"container_id" bson:"container_id",` // ID of the Docker contained used. FrameID string `json:"frame_id" bson:"frame_id"` // ID of the primary frame. URL string `json:"url" bson:"url"` // URL analyzed. FinalURL string `json:"final_url" bson:"final_url"` // Last detected URL visited, from navigation history. FinalURLError string `json:"final_url_error" bson:"final_url_error"` // Error text for the last detected visit. RequestEvents []*network.RequestWillBeSentReply `json:"request_events" bson:"request_events"` // All requests performed (list of DevTools RequestWillBeSentReply events). ResponseEvents []*network.ResponseReceivedReply `json:"response_events" bson:"response_events"` // All responses recorded during execution (list of DevTools ResponseReceivedReply events). ErrorEvents []*network.LoadingFailedReply `json:"error_events" bson:"error_events",` // All errors recorded during execution (list of DevTools LoadingFailedReply events). Visits []Visit `json:"visits" bson:"visits"` // List of visits, inclusive of requests, responses and errors. ResourcesData ResourcesData `json:"resources_data" bson:"resources_data"` // List of resources loaded by all visits. Downloads []Download `json:"downloads" bson:"downloads"` // List of file downloads offered during execution. Dialogs []Dialog `json:"dialogs" bson:"dialogs"` // List of JavaScript dialogs that were prompted during execution. HTML string `json:"html" bson:"html"` // DOM HTML of the last open frame. HTMLSHA256 string `json:"html_sha256" bson:"html_sha256"` // SHA256 hash of the last DOM HTML. ScreenshotPath string `json:"screenshot_path" bson:"screenshot_path"` // Path to the screenshot if saved to disk. ScreenshotData string `json:"screenshot_data" bson:"screenshot_data"` // Base64-encoded binary data of the screenshot. }
Browser is the main struct containing information about the results of the analysis of the given URL.
type ByChronologicalOrder ¶
type ByChronologicalOrder []*network.RequestWillBeSentReply
ByChronologicalOrder is used to order requests by timestamp.
func (ByChronologicalOrder) Len ¶
func (r ByChronologicalOrder) Len() int
func (ByChronologicalOrder) Less ¶
func (r ByChronologicalOrder) Less(i, j int) bool
func (ByChronologicalOrder) Swap ¶
func (r ByChronologicalOrder) Swap(i, j int)
type Dialog ¶
type Dialog struct { URL string `json:"url" bson:"url"` // URL which prompted the JavaScript dialog. Type string `json:"type" bson:"type"` // Type of dialog. Message string `json:"message" bson:"message"` // Message contained in the dialog. }
Dialog contains details of JavaScript dialogs opened.
type Download ¶
type Download struct { URL string `json:"url" bson:"url"` // URL of the file download offered. FileName string `json:"file_name" bson:"file_name"` // File name offered for the download. }
Download contains details of a file download offered by the analyzed URL.
type LogCodec ¶
type LogCodec struct {
// contains filtered or unexported fields
}
LogCodec captures the output from writing RPC requests and reading responses on the connection. It implements rpcc.Codec via WriteRequest and ReadResponse. Adapted from: https://pkg.go.dev/github.com/mafredri/cdp#example-package-Logging
func (*LogCodec) ReadResponse ¶
ReadResponse unmarshals from the connection into v whilst echoing what is read into a buffer for logging.
type NavigationHistory ¶
type NavigationHistory []page.NavigationEntry
NavigationHistory is a list of NavigationEntry from DevTools. It should be an accurate representation of all main navigation events performed while analyzing the URL (opposite to a full list of visits or requests/responses which might have been executed in an iframe or through JavaScript).
type RequestResponse ¶
type RequestResponse struct { Request *network.RequestWillBeSentReply `json:"request"` Response *network.ResponseReceivedReply `json:"response"` }
RequestResponse contains the combination of an HTTP request and its relevant response. It should only be used to mark resources loaded by single visits.
type ResourceDataEntry ¶
type ResourceDataEntry struct { VisitID string `json:"visit_id" bson:"visit_id"` // ID of the visit which loaded this resource. RequestID string `json:"request_id" bson:"request_id"` // ID of the specific request from the visit. Type string `json:"type" bson:"type"` // Type of resource (e.g. Document, Image, Script). URL string `json:"url" bson:"url"` // URL of the resource. SHA256 string `json:"sha256" bson:"sha256"` // SHA256 hash of the resource (if downloaded). Content string `json:"content" bson:"content"` // String content of the resource (if downloaded). }
ResourceDataEntry contains metadata information on downloaded resource files such as JavaScript snippets.
type ResourcesData ¶
type ResourcesData []ResourceDataEntry
ResourcesData is a collection of ResourceDataEntry items.
type Visit ¶
type Visit struct { VisitID string `json:"visit_id"` // ID of the visit as defined by DevTools. Requests []*network.RequestWillBeSentReply `json:"requests"` // All requests performed by this visit (list of DevTools RequestWillBeSentReply events). Response *network.ResponseReceivedReply `json:"response"` // Response to the visit. Resources []RequestResponse `json:"resources"` // All resources loaded by the visit (in the form of an aggregate of request->response). Error *network.LoadingFailedReply `json:"error"` // Any error the visit might have triggered. }
Visit contains details of a document loaded on the browser frame.