httpx

package
v1.0.4-0...-cd4b18d Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 7, 2021 License: MIT Imports: 28 Imported by: 0

Documentation

Overview

Package httpx containst the httpx common funcionality

Index

Constants

View Source
const (
	// HTTP defines the plain http scheme
	HTTP = "http"
	// HTTPS defines the secure http scheme
	HTTPS = "https"
	// HTTPorHTTPS defines the both http and https scheme
	HTTPorHTTPS = "http|https"
)

Variables

View Source
var CSPHeaders []string = []string{
	"Content-Security-Policy",
	"Content-Security-Policy-Report-Only",
	"X-Content-Security-Policy-Report-Only",
	"X-Webkit-Csp-Report-Only",
}

CSPHeaders is an incomplete list of most common CSP headers

View Source
var DefaultOptions = Options{
	Threads:  25,
	Timeout:  30 * time.Second,
	RetryMax: 5,
	Unsafe:   false,
	CdnCheck: true,

	VHostIgnoreStatusCode:    false,
	VHostIgnoreContentLength: true,
	VHostIgnoreNumberOfWords: false,
	VHostIgnoreNumberOfLines: false,
	VHostStripHTML:           false,
	VHostSimilarityRatio:     85,
	DefaultUserAgent:         "httpx - Open-source project (github.com/projectdiscovery/httpx)",
}

DefaultOptions contains the default options

Functions

func Decodebig5

func Decodebig5(s []byte) ([]byte, error)

Decodebig5 converts BIG5 to UTF-8

func Decodegbk

func Decodegbk(s []byte) ([]byte, error)

Decodegbk converts GBK to UTF-8

func Encodebig5

func Encodebig5(s []byte) ([]byte, error)

Encodebig5 converts UTF-8 to BIG5

func ExtractTitle

func ExtractTitle(r *Response) (title string)

ExtractTitle from a response

Types

type CSPData

type CSPData struct {
	Domains []string `json:"domains,omitempty"`
}

CSPData contains the Content-Security-Policy domain list

type CustomCallback

type CustomCallback func(response *Response) (bool, error)

CustomCallback used in custom filters

type Filter

type Filter interface {
	Filter(response *Response) (bool, error)
}

Filter defines a generic filter interface to apply to responses

type FilterCustom

type FilterCustom struct {
	CallBacks []CustomCallback
}

FilterCustom defines a filter with callback functions applied

func (FilterCustom) Filter

func (f FilterCustom) Filter(response *Response) (bool, error)

Filter a response with custom callbacks

type FilterRegex

type FilterRegex struct {
	Regexs []string
}

FilterRegex defines a filter of type regex

func (FilterRegex) Filter

func (f FilterRegex) Filter(response *Response) (bool, error)

Filter a response with regexes

type FilterString

type FilterString struct {
	Keywords []string
}

FilterString defines a filter of type string

func (FilterString) Filter

func (f FilterString) Filter(response *Response) (bool, error)

Filter a response with strings filtering

type HTTPX

type HTTPX struct {
	Filters []Filter
	Options *Options

	CustomHeaders   map[string]string
	RequestOverride *RequestOverride

	Dialer *fastdialer.Dialer
	// contains filtered or unexported fields
}

HTTPX represent an instance of the library client

func New

func New(options *Options) (*HTTPX, error)

New httpx instance

func (*HTTPX) AddFilter

func (h *HTTPX) AddFilter(f Filter)

AddFilter cascade

func (*HTTPX) CSPGrab

func (h *HTTPX) CSPGrab(r *http.Response) *CSPData

CSPGrab fills the CSPData

func (*HTTPX) CdnCheck

func (h *HTTPX) CdnCheck(ip string) (bool, error)

CdnCheck verifies if the given ip is part of Cdn ranges

func (*HTTPX) Do

func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error)

Do http request

func (*HTTPX) IsVirtualHost

func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error)

IsVirtualHost checks if the target endpoint is a virtual host

func (*HTTPX) NewRequest

func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error)

NewRequest from url

func (*HTTPX) SetCustomHeaders

func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]string)

SetCustomHeaders on the provided request

func (*HTTPX) SupportHTTP2

func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool

SupportHTTP2 checks if the target host supports HTTP2

func (*HTTPX) SupportPipeline

func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool

SupportPipeline checks if the target host supports HTTP1.1 pipelining by sending x probes and reading back responses expecting at least 2 with HTTP/1.1 or HTTP/1.0

func (*HTTPX) TLSGrab

func (h *HTTPX) TLSGrab(r *http.Response) *TLSData

TLSGrab fills the TLSData

func (*HTTPX) Verify

func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error)

Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true

type Options

type Options struct {
	DefaultUserAgent string
	RequestOverride  RequestOverride
	HTTPProxy        string
	SocksProxy       string
	Threads          int
	CdnCheck         bool
	// Timeout is the maximum time to wait for the request
	Timeout time.Duration
	// RetryMax is the maximum number of retries
	RetryMax      int
	CustomHeaders map[string]string
	// VHostSimilarityRatio 1 - 100
	VHostSimilarityRatio int
	FollowRedirects      bool
	FollowHostRedirects  bool
	Unsafe               bool
	// VHOSTs options
	VHostIgnoreStatusCode    bool
	VHostIgnoreContentLength bool
	VHostIgnoreNumberOfWords bool
	VHostIgnoreNumberOfLines bool
	VHostStripHTML           bool
}

Options contains configuration options for the client

type RequestOverride

type RequestOverride struct {
	URIPath string
}

RequestOverride contains the URI path to override the request

type Response

type Response struct {
	StatusCode    int
	Headers       map[string][]string
	Data          []byte
	ContentLength int
	Raw           string
	Words         int
	Lines         int
	TLSData       *TLSData
	CSPData       *CSPData
	HTTP2         bool
	Pipeline      bool
	Duration      time.Duration
}

Response contains the response to a server

func (*Response) GetHeader

func (r *Response) GetHeader(name string) string

GetHeader value

func (*Response) GetHeaderPart

func (r *Response) GetHeaderPart(name, sep string) string

GetHeaderPart with offset

type TLSData

type TLSData struct {
	DNSNames         []string `json:"dns_names,omitempty"`
	Emails           []string `json:"emails,omitempty"`
	CommonName       []string `json:"common_name,omitempty"`
	Organization     []string `json:"organization,omitempty"`
	IssuerCommonName []string `json:"issuer_common_name,omitempty"`
	IssuerOrg        []string `json:"issuer_organization,omitempty"`
}

TLSData contains the relevant Transport Layer Security information

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL