http_utils

package

v0.0.0-...-2a45534 Latest Latest Go to latest Published: Apr 27, 2024 License: GPL-3.0 Imports: 21 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/pyneda/sukyan

Documentation ¶

Index ¶

Constants
func BuildRequestFromHistoryItem(historyItem *db.History) (*http.Request, error)
func ClassifyHTTPResponseHeader(headerKey string) string
func CreateHistoryFromHttpResponse(response *http.Response, responseData FullResponseData, ...) (*db.History, error)
func CreateHttp2Client() *http.Client
func CreateHttp2Transport() *http2.Transport
func CreateHttp3Client() *http.Client
func CreateHttp3Transport() *http3.RoundTripper
func CreateHttpClient() *http.Client
func CreateHttpTransport() *http.Transport
func HeadersToString(headersMap map[string][]string) string
func JoinCookies(cookies []*http.Cookie) string
func ParseCookies(cookieStr string) []*http.Cookie
func ReadHttpResponseAndCreateHistory(response *http.Response, options HistoryCreationOptions) (*db.History, error)
func ReadResponseBodyData(response *http.Response) (body []byte, size int, err error)
func SendRequest(client *http.Client, req *http.Request) (*http.Response, error)
func SetRequestHeadersFromHistoryItem(request *http.Request, historyItem *db.History) error
type FullResponseData
- func ReadFullResponse(response *http.Response, createNewBodyStream bool) (FullResponseData, io.ReadCloser, error)
type HistoryCreationOptions
type HostRateLimiter
- func NewHostRateLimiter(hostName string, rate float64, maxTokens float64) *HostRateLimiter
- func (h *HostRateLimiter) AddRequest(request *http.Request) <-chan *ResponseWrapper
- func (h *HostRateLimiter) GetNextQueuedRequest() *QueuedRequest
- func (h *HostRateLimiter) ProcessQueue(client RequestExecutor)
- func (h *HostRateLimiter) RecordResponseTime(responseTime float64)
type MimeType
- func GetApplicationMimeTypes() (mimeTypes []MimeType)
- func GetAudioMimeTypes() (mimeTypes []MimeType)
- func GetFontMimeTypes() (mimeTypes []MimeType)
- func GetImageMimeTypes() (mimeTypes []MimeType)
- func GetMimeTypes() (mimeTypes []MimeType)
- func GetTextMimeTypes() (mimeTypes []MimeType)
- func GetVideoMimeTypes() (mimeTypes []MimeType)
type MimeTypeChecker
- func NewMimeTypeChecker() MimeTypeChecker
- func (c *MimeTypeChecker) GetMimeTypeFromContentTypeString(contentType string) MimeType
- func (c *MimeTypeChecker) GetMimeTypeFromURL(url string) MimeType
type QueuedRequest
type RequestExecutor
type RequestHeaders
type ResponseBodyData
type ResponseWrapper
type TokenBucket
- func NewTokenBucket(rate float64, maxTokens float64, minRate float64) *TokenBucket
- func (tb *TokenBucket) AdjustRate(newRate float64)
- func (tb *TokenBucket) HasToken() bool

Constants ¶

View Source

const (
	MimeTypeImgGroup         = "image"
	MimeTypeTextGroup        = "text"
	MimeTypeApplicationGroup = "application"
	MimeTypeModelGroup       = "model"
	MimeTypeVideoGroup       = "video"
	MimeTypeFontGroup        = "font"
	MimeTypeAudioGroup       = "audio"
)

var MimeTypes sync.Map

View Source

const (
	UPPER_THRESHOLD = 2   // 2 seconds
	LOWER_THRESHOLD = 0.3 // 300 milliseconds
	MIN_RATE        = 3   // 3 requests per second
)

Variables ¶

This section is empty.

Functions ¶

func BuildRequestFromHistoryItem ¶

func BuildRequestFromHistoryItem(historyItem *db.History) (*http.Request, error)

BuildRequestFromHistoryItem gets a history item and returns an http.Request with the same data

func ClassifyHTTPResponseHeader ¶

func ClassifyHTTPResponseHeader(headerKey string) string

ClassifyHTTPResponseHeader classifies a given HTTP response header key by its purpose.

func CreateHistoryFromHttpResponse ¶

func CreateHistoryFromHttpResponse(response *http.Response, responseData FullResponseData, options HistoryCreationOptions) (*db.History, error)

func CreateHttp2Client ¶

func CreateHttp2Client() *http.Client

CreateHttp2Client creates an HTTP/2 client.

func CreateHttp2Transport ¶

func CreateHttp2Transport() *http2.Transport

CreateHttp2Transport creates an HTTP/2 transport.

func CreateHttp3Client ¶

func CreateHttp3Client() *http.Client

CreateHttp3Client creates an HTTP/3 client.

func CreateHttp3Transport ¶

func CreateHttp3Transport() *http3.RoundTripper

CreateHttp3Transport creates an HTTP/3 transport.

func CreateHttpClient ¶

func CreateHttpClient() *http.Client

CreateHttpClient creates a regular HTTP client.

func CreateHttpTransport ¶

func CreateHttpTransport() *http.Transport

CreateHttpTransport creates an HTTP transport with no pre-defined http version.

func HeadersToString ¶

func HeadersToString(headersMap map[string][]string) string

func JoinCookies ¶

func JoinCookies(cookies []*http.Cookie) string

JoinCookies is a helper function to join cookies into a string

func ParseCookies ¶

func ParseCookies(cookieStr string) []*http.Cookie

ParseCookies is a helper function to parse multiple cookies from a string

func ReadHttpResponseAndCreateHistory ¶

func ReadHttpResponseAndCreateHistory(response *http.Response, options HistoryCreationOptions) (*db.History, error)

func ReadResponseBodyData ¶

func ReadResponseBodyData(response *http.Response) (body []byte, size int, err error)

ReadResponseBodyData reads an http response body and returns it as string + its length as bytes

func SendRequest ¶

func SendRequest(client *http.Client, req *http.Request) (*http.Response, error)

SendRequest sends an http request and returns the response ensuring that the Request body is still readable so we can dump it

func SetRequestHeadersFromHistoryItem ¶

func SetRequestHeadersFromHistoryItem(request *http.Request, historyItem *db.History) error

Types ¶

type FullResponseData ¶

type FullResponseData struct {
	Body      []byte
	BodySize  int
	Raw       []byte
	RawString string
	RawSize   int
	// contains filtered or unexported fields
}

func ReadFullResponse ¶

func ReadFullResponse(response *http.Response, createNewBodyStream bool) (FullResponseData, io.ReadCloser, error)

ReadResponseBodyData should be replaced by this

type HistoryCreationOptions ¶

type HistoryCreationOptions struct {
	Source              string
	WorkspaceID         uint
	TaskID              uint
	CreateNewBodyStream bool
	PlaygroundSessionID uint
}

type HostRateLimiter ¶

type HostRateLimiter struct {
	// contains filtered or unexported fields
}

func NewHostRateLimiter ¶

func NewHostRateLimiter(hostName string, rate float64, maxTokens float64) *HostRateLimiter

func (*HostRateLimiter) AddRequest ¶

func (h *HostRateLimiter) AddRequest(request *http.Request) <-chan *ResponseWrapper

func (*HostRateLimiter) GetNextQueuedRequest ¶

func (h *HostRateLimiter) GetNextQueuedRequest() *QueuedRequest

func (*HostRateLimiter) ProcessQueue ¶

func (h *HostRateLimiter) ProcessQueue(client RequestExecutor)

func (*HostRateLimiter) RecordResponseTime ¶

func (h *HostRateLimiter) RecordResponseTime(responseTime float64)

type MimeType ¶

type MimeType struct {
	Extensions             []string
	MimeTypes              []string
	Group                  string
	Name                   string
	Code                   string
	CreateIssueOnDetection bool
	PassiveScanCallbacks   []interface{}
}

MimeType holds information about a specific mime type

func GetApplicationMimeTypes ¶

func GetApplicationMimeTypes() (mimeTypes []MimeType)

func GetAudioMimeTypes ¶

func GetAudioMimeTypes() (mimeTypes []MimeType)

func GetFontMimeTypes ¶

func GetFontMimeTypes() (mimeTypes []MimeType)

func GetImageMimeTypes ¶

func GetImageMimeTypes() (mimeTypes []MimeType)

func GetMimeTypes ¶

func GetMimeTypes() (mimeTypes []MimeType)

func GetTextMimeTypes ¶

func GetTextMimeTypes() (mimeTypes []MimeType)

func GetVideoMimeTypes ¶

func GetVideoMimeTypes() (mimeTypes []MimeType)

type MimeTypeChecker ¶

type MimeTypeChecker struct {
	Available []MimeType
}

MimeTypeChecker exposes few functions to detect mime type

func NewMimeTypeChecker ¶

func NewMimeTypeChecker() MimeTypeChecker

func (*MimeTypeChecker) GetMimeTypeFromContentTypeString ¶

func (c *MimeTypeChecker) GetMimeTypeFromContentTypeString(contentType string) MimeType

func (*MimeTypeChecker) GetMimeTypeFromURL ¶

func (c *MimeTypeChecker) GetMimeTypeFromURL(url string) MimeType

type QueuedRequest ¶

type QueuedRequest struct {
	Request   *http.Request
	Response  chan *ResponseWrapper
	QueueTime time.Time
}

type RequestExecutor ¶

type RequestExecutor interface {
	Do(req *http.Request) (*http.Response, error)
}

type RequestHeaders ¶

type RequestHeaders map[string][]string

type ResponseBodyData ¶

type ResponseBodyData struct {
	Content string
	Size    int
	// contains filtered or unexported fields
}

type ResponseWrapper ¶

type ResponseWrapper struct {
	Response  *http.Response
	Request   *http.Request
	QueueTime time.Time
	SentTime  time.Time
}

type TokenBucket ¶

type TokenBucket struct {
	// contains filtered or unexported fields
}

func NewTokenBucket ¶

func NewTokenBucket(rate float64, maxTokens float64, minRate float64) *TokenBucket

func (*TokenBucket) AdjustRate ¶

func (tb *TokenBucket) AdjustRate(newRate float64)

func (*TokenBucket) HasToken ¶

func (tb *TokenBucket) HasToken() bool

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL