http

package

v0.0.0-...-ed82f56 Latest Latest Go to latest Published: Aug 10, 2023 License: MIT Imports: 22 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/musabgultekin/quantumscraper

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func FasthttpHTTPDialerProxy(proxy string) fasthttp.DialFunc
func FasthttpHTTPDialerProxyTimeout(proxy string, timeout time.Duration) fasthttp.DialFunc
func FetchReliableDNSRecords(url string) ([]string, error)
func Get(requestURI string) ([]byte, int, error)
func GetFast(requestURI string) ([]byte, int, error)
func GetProxyUrl() func(*http.Request) (*url.URL, error)
type DNSRecord

Constants ¶

View Source

const DefaultMaxBody int64 = 1024 * 1024 * 1024 // 1GB

Variables ¶

View Source

var DnsResolver = &net.Resolver{
	Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
		d := &net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}
		i := int(atomic.AddUint64(&currentResolverIndex, 1)) % len(dnsResolvers)
		return d.DialContext(ctx, "udp", dnsResolvers[i]+":53")
	},
}

DNSResolver resolves DNS nameservers randomly that previously loaded.

View Source

var FasthttpDialer = &fasthttp.TCPDialer{
	Concurrency:      1000,
	DNSCacheDuration: fasthttp.DefaultDNSCacheDuration * 60,
	Resolver:         DnsResolver,
}

Functions ¶

func FasthttpHTTPDialerProxy ¶

func FasthttpHTTPDialerProxy(proxy string) fasthttp.DialFunc

FasthttpHTTPDialer returns a fasthttp.DialFunc that dials using the provided HTTP proxy.

Example usage:

c := &fasthttp.Client{
	Dial: fasthttpproxy.FasthttpHTTPDialer("username:password@localhost:9050"),
}

func FasthttpHTTPDialerProxyTimeout ¶

func FasthttpHTTPDialerProxyTimeout(proxy string, timeout time.Duration) fasthttp.DialFunc

FasthttpHTTPDialerTimeout returns a fasthttp.DialFunc that dials using the provided HTTP proxy using the given timeout.

Example usage:

c := &fasthttp.Client{
	Dial: fasthttpproxy.FasthttpHTTPDialerTimeout("username:password@localhost:9050", time.Second * 2),

func FetchReliableDNSRecords ¶

func FetchReliableDNSRecords(url string) ([]string, error)

func Get ¶

func Get(requestURI string) ([]byte, int, error)

func GetFast ¶

func GetFast(requestURI string) ([]byte, int, error)

func GetProxyUrl ¶

func GetProxyUrl() func(*http.Request) (*url.URL, error)

Types ¶

type DNSRecord ¶

type DNSRecord struct {
	IP          string  `json:"ip"`
	Name        string  `json:"name"`
	ASNumber    int     `json:"as_number"`
	ASOrg       string  `json:"as_org"`
	CountryID   string  `json:"country_id"`
	City        string  `json:"city"`
	Version     string  `json:"version"`
	Error       string  `json:"error"`
	DNSSEC      bool    `json:"dnssec"`
	Reliability float64 `json:"reliability"`
	CheckedAt   string  `json:"checked_at"`
	CreatedAt   string  `json:"created_at"`
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL