Documentation
¶
Index ¶
- Constants
- Variables
- func AddCookies(reqUrl string, cookies []*http.Cookie, req *http.Request)
- func AddHeaders(headers map[string]string, defaultUserAgent string, req *http.Request)
- func AddParams(params map[string]string, req *http.Request)
- func CallRequest(reqArgs *RequestArgs) (*http.Response, error)
- func CallRequestWithData(reqArgs *RequestArgs, data map[string]string) (*http.Response, error)
- func CheckInternetConnection()
- func CheckVer(url string, ver string, showProg bool, progBar progress.Progress) (bool, error)
- func DlToFile(res *http.Response, url, filePath string) error
- func DownloadUrl(filePath string, queue chan struct{}, reqArgs *RequestArgs, ...) error
- func DownloadUrls(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config) error
- func DownloadUrlsWithHandler(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config, ...) error
- func GetDefaultRandomDelay() time.Duration
- func GetHttpClient(reqArgs *RequestArgs) *http.Client
- func GetLastPartOfUrl(url string) string
- func GetRandomDelay(delayInfo *RetryDelay) time.Duration
- func GetRandomTime(min, max float32) time.Duration
- func IsHttp3Supported(site string, isApi bool) bool
- func LoadJsonFromBytes(body []byte, format any) error
- func LoadJsonFromResponse(res *http.Response, format any) error
- func ParamsToString(params map[string]string) string
- func ReadResBody(res *http.Response) ([]byte, error)
- type DlOptions
- type GithubApiRes
- type RequestArgs
- type RequestHandler
- type RetryDelay
- type ToDownload
Constants ¶
const CLI_REPO_URL = "https://api.github.com/repos/KJHJason/Cultured-Downloader-CLI/releases/latest"
Variables ¶
var ( // Since the URLs below will be redirected to Fantia's AWS S3 URL, // we need to use HTTP/2 as it is not supported by HTTP/3 yet. FANTIA_ALBUM_URL = regexp.MustCompile( `^https://fantia.jp/posts/[\d]+/album_image`, ) FANTIA_DOWNLOAD_URL = regexp.MustCompile( `^https://fantia.jp/posts/[\d]+/download/[\d]+`, ) HTTP3_SUPPORT_ARR = [...]string{ "https://www.pixiv.net", "https://app-api.pixiv.net", "https://www.google.com", "https://drive.google.com", } )
Functions ¶
func AddCookies ¶
add cookies to the request
func AddHeaders ¶
add headers to the request
func CallRequest ¶
func CallRequest(reqArgs *RequestArgs) (*http.Response, error)
CallRequest is used to make a request to a URL and return the response
If the request fails, it will retry the request again up to the defined max retries in the constants.go in utils package
func CallRequestWithData ¶
Sends a request with the given data
func CheckInternetConnection ¶
func CheckInternetConnection()
Check for active internet connection (To be used at the start of the program)
func DownloadUrl ¶
func DownloadUrl(filePath string, queue chan struct{}, reqArgs *RequestArgs, overwriteExistingFile bool) error
DownloadUrl is used to download a file from a URL
Note: If the file already exists, the download process will be skipped
func DownloadUrls ¶
func DownloadUrls(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config) error
Same as DownloadUrlsWithHandler but uses the default request handler (CallRequest)
func DownloadUrlsWithHandler ¶
func DownloadUrlsWithHandler(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config, reqHandler RequestHandler) error
DownloadUrls is used to download multiple files from URLs concurrently
Note: If the file already exists, the download process will be skipped
func GetDefaultRandomDelay ¶
Returns a random time.Duration between the defined min and max delay values in the contants.go file
func GetHttpClient ¶
func GetHttpClient(reqArgs *RequestArgs) *http.Client
Get a new HTTP/2 or HTTP/3 client based on the request arguments
func GetLastPartOfUrl ¶
Returns the last part of the given URL string
func GetRandomDelay ¶
func GetRandomDelay(delayInfo *RetryDelay) time.Duration
Returns a random time.Duration between the given min and max arguments in the RetryDelay struct
func GetRandomTime ¶
Returns a random time.Duration between the given min and max arguments
func IsHttp3Supported ¶
Returns a boolean value indicating whether the specified site supports HTTP/3
Usually, the API endpoints of a site do not support HTTP/3, so the isApi parameter must be provided.
func LoadJsonFromBytes ¶
func LoadJsonFromResponse ¶
Read the response body and unmarshal it into a interface and returns it
func ParamsToString ¶
Converts a map of string back to a string
Types ¶
type DlOptions ¶
type DlOptions struct {
// Parent context for the download process
Context context.Context
// MaxConcurrency is the maximum number of concurrent downloads
MaxConcurrency int
// Cookies is a list of cookies to be used in the download process
Cookies []*http.Cookie
// Headers is a map of headers to be used in the download process
Headers map[string]string
// UseHttp3 is a flag to enable HTTP/3
// Otherwise, HTTP/2 will be used by default
UseHttp3 bool
// RetryDelay is the delay between retries
RetryDelay *RetryDelay
// Prog bar
DownloadProgressBar progress.Progress
}
type GithubApiRes ¶
type RequestArgs ¶
type RequestArgs struct {
// Main Request Options
Method string
Url string
Timeout int
// Additional Request Options
Headers map[string]string
Params map[string]string
Cookies []*http.Cookie
UserAgent string
DisableCompression bool
// HTTP/2 and HTTP/3 Options
Http2 bool
Http3 bool
// Check status will check the status code of the response for 200 OK.
// If the status code is not 200 OK, it will retry several times and
// if the status code is still not 200 OK, it will return an error.
// Otherwise, it will return the response regardless of the status code.
CheckStatus bool
RetryDelay *RetryDelay
// Context is used to cancel the request if needed.
// E.g. if the user presses Ctrl+C, we can use context.WithCancel(context.Background())
Context context.Context
// RequestHandler is the main function that will be called to make the request.
RequestHandler RequestHandler
}
func (*RequestArgs) ValidateArgs ¶
func (args *RequestArgs) ValidateArgs()
ValidateArgs validates the arguments of the request
Will panic if the arguments are invalid as this is a developer error
type RequestHandler ¶
type RequestHandler func(reqArgs *RequestArgs) (*http.Response, error)
Source Files