source

package
v1.0.15 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 17, 2023 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source
const (
	LastModifiedLayout = http.TimeFormat
	ExpireLayout       = http.TimeFormat
)
View Source
const (
	// Range is different with HTTP Range, it's without "bytes=" prefix
	Range = "X-Dragonfly-Range" // startIndex-endIndex
)
View Source
const (
	TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT"
)
View Source
const (
	UnknownSourceFileLen = -2
)

Variables

View Source
var (
	ListMetadataScheme       = "d7ylist"
	ListMetadataOriginScheme = "X-Dragonfly-List-Origin-Scheme"
	ListMetadataExpire       = "X-Dragonfly-List-Expire"
)
View Source
var (
	// ErrResourceNotReachable represents the url resource is a not reachable.
	ErrResourceNotReachable = errors.New("resource is not reachable")

	// ErrNoClientFound represents no source client to resolve url
	ErrNoClientFound = errors.New("no source client found")

	// ErrClientNotSupportList represents the source client not support list action
	ErrClientNotSupportList = errors.New("source client not support list")

	// ErrClientNotSupportContentLength represents the source client not support get content length action
	ErrClientNotSupportContentLength = errors.New("source client not support get content length")

	// ErrClientNotSupportGetMetadata represents the source client not support get metadata
	ErrClientNotSupportGetMetadata = errors.New("source client not support get metadata")
)
View Source
var ProxyEnv = "D7Y_SOURCE_PROXY"

Functions

func CanonicalHeaderKey

func CanonicalHeaderKey(s string) string

func CheckResponseCode

func CheckResponseCode(respCode int, allowed []int) error

CheckResponseCode returns UnexpectedStatusError if the given response code is not one of the allowed status codes; otherwise nil.

func DefaultTransport

func DefaultTransport() *http.Transport

func GetContentLength

func GetContentLength(request *Request) (int64, error)

func GetLastModified

func GetLastModified(request *Request) (int64, error)

func InitSourceClients

func InitSourceClients(opts map[string]interface{}) error

InitSourceClients will initialize all resource clients which registered by RegisterBuilder.

func IsExpired

func IsExpired(request *Request, info *ExpireInfo) (bool, error)

func IsNoClientFoundError

func IsNoClientFoundError(err error) bool

func IsResourceNotReachableError

func IsResourceNotReachableError(err error) bool

func IsSupportRange

func IsSupportRange(request *Request) (bool, error)

func ListClients

func ListClients() []string

func ParseToHTTPClient

func ParseToHTTPClient(optionYaml []byte) (*http.Client, error)

func Register

func Register(scheme string, resourceClient ResourceClient, adaptor RequestAdapter, hooks ...Hook) error

func RegisterBuilder

func RegisterBuilder(scheme string, builder ResourceClientBuilder, opts ...RegisterOption)

RegisterBuilder register ResourceClientBuilder into global resourceClientBuilder, the InitSourceClients will use it.

func RequestEq

func RequestEq(url string) gomock.Matcher

RequestEq for gomock

func UnRegister

func UnRegister(scheme string)

func UnRegisterBuilder

func UnRegisterBuilder(scheme string)

func UpdatePluginDir

func UpdatePluginDir(pluginDir string)

func UpdateTransportOption

func UpdateTransportOption(transport *http.Transport, optionYaml []byte) error

func WithContentLength

func WithContentLength(length int64) func(*Response)

func WithExpireInfo

func WithExpireInfo(info ExpireInfo) func(*Response)

WithExpireInfo is used for no-http resource client to set expire info

func WithHeader

func WithHeader(header map[string]string) func(*Response)

func WithStatus

func WithStatus(code int, status string) func(*Response)

func WithTemporary

func WithTemporary(temporary bool) func(*Response)

func WithValidate

func WithValidate(validate func() error) func(*Response)

Types

type ClientManager

type ClientManager interface {
	// Register registers a source client with scheme
	Register(scheme string, resourceClient ResourceClient, adapter RequestAdapter, hook ...Hook) error

	// UnRegister revoke a source client from manager
	UnRegister(scheme string)

	// GetClient gets a source client by scheme
	GetClient(scheme string, options ...Option) (ResourceClient, bool)

	// ListClients lists all supported client scheme
	ListClients() []string
}

func NewManager

func NewManager() ClientManager

type Director

type Director interface {
	Direct(rawURL *url.URL, urlMeta *commonv1.UrlMeta) error
}

Director will handle request with some actions, like: 1. inject auth information for target url and metadata, eg: fetch docker config for different users 2. rewrite a common request into an unique request, eg: oras://harbor/user:latest to oras://harbor/user:lastest?digest=sha256:12345

func HasDirector

func HasDirector(scheme string) (Director, bool)

func NewPlainDirector

func NewPlainDirector(
	direct func(url *url.URL, urlMeta *commonv1.UrlMeta) error) Director

type ExpireInfo

type ExpireInfo struct {
	Expire       string // Mon, 02 Jan 2006 15:04:05 GMT
	LastModified string // Mon, 02 Jan 2006 15:04:05 GMT
	ETag         string
}
type Header map[string][]string

A Header represents the key-value pairs in a Dragonfly source header.

The keys should be in canonical form, as returned by CanonicalHeaderKey.

func (Header) Add

func (h Header) Add(key, value string)

Add adds the key, value pair to the header. It appends to any existing values associated with key. The key is case insensitive; it is canonicalized by CanonicalHeaderKey.

func (Header) Clone

func (h Header) Clone() Header

Clone returns a copy of h or nil if h is nil.

func (Header) Del

func (h Header) Del(key string)

Del deletes the values associated with key. The key is case insensitive; it is canonicalized by CanonicalHeaderKey.

func (Header) Get

func (h Header) Get(key string) string

Get gets the first value associated with the given key. If there are no values associated with the key, Get returns "". It is case insensitive; textproto.CanonicalMIMEHeaderKey is used to canonicalize the provided key. To use non-canonical keys, access the map directly.

func (Header) Set

func (h Header) Set(key, value string)

Set sets the header entries associated with key to the single element value. It replaces any existing values associated with key. The key is case insensitive; it is canonicalized by textproto.CanonicalMIMEHeaderKey. To use non-canonical keys, assign to the map directly.

func (Header) Values

func (h Header) Values(key string) []string

Values returns all values associated with the given key. It is case insensitive; textproto.CanonicalMIMEHeaderKey is used to canonicalize the provided key. To use non-canonical keys, access the map directly. The returned slice is not a copy.

type Hook

type Hook interface {
	BeforeRequest(request *Request) error
	AfterResponse(response *Response) error
}

Hook TODO hook

type ListMetadata

type ListMetadata struct {
	URLEntries []URLEntry
}

type ListMetadataClient

type ListMetadataClient struct {
}

func (*ListMetadataClient) Download

func (lm *ListMetadataClient) Download(request *Request) (*Response, error)

func (*ListMetadataClient) GetContentLength

func (lm *ListMetadataClient) GetContentLength(request *Request) (int64, error)

func (*ListMetadataClient) GetLastModified

func (lm *ListMetadataClient) GetLastModified(request *Request) (int64, error)

func (*ListMetadataClient) IsExpired

func (lm *ListMetadataClient) IsExpired(request *Request, info *ExpireInfo) (bool, error)

func (*ListMetadataClient) IsSupportRange

func (lm *ListMetadataClient) IsSupportRange(request *Request) (bool, error)

type Metadata

type Metadata struct {
	Status     string
	StatusCode int
	Header     Header
	// SupportRange indicates source supports partial download, like Range in http request
	SupportRange bool
	// ContentLength indicates the current content length for the target request
	// ContentLength int64
	// TotalContentLength indicates the total content length for the target request
	// eg, for http response header:
	//      Content-Range: bytes 0-9/2443
	// 2443 is the TotalContentLength, 10 is the ContentLength
	TotalContentLength int64

	Validate  func() error
	Temporary bool
}

func GetMetadata

func GetMetadata(request *Request) (*Metadata, error)

type Option

type Option func(c *clientManager)

type RegisterOption

type RegisterOption func(scheme string)

RegisterOption is used for extra options when registering, like mark target scheme protocol should inject auth information

func WithDirector

func WithDirector(director Director) RegisterOption

type Request

type Request struct {
	URL    *url.URL
	Header Header
	// contains filtered or unexported fields
}

func NewListMetadataRequest

func NewListMetadataRequest(request *Request) *Request

func NewRequest

func NewRequest(rawURL string) (*Request, error)

func NewRequestWithContext

func NewRequestWithContext(ctx context.Context, rawURL string, header map[string]string) (*Request, error)

func NewRequestWithHeader

func NewRequestWithHeader(rawURL string, header map[string]string) (*Request, error)

func (*Request) Clone

func (r *Request) Clone(ctx context.Context) *Request

Clone returns a deep copy of r with its context changed to ctx. The provided ctx must be non-nil.

For an outgoing client request, the context controls the entire lifetime of a request and its response: obtaining a connection, sending the request, and reading the response headers and body.

func (*Request) Context

func (r *Request) Context() context.Context

Context returns the request's context. To change the context, use WithContext.

The returned context is always non-nil; it defaults to the background context.

For outgoing client requests, the context controls cancellation.

For incoming server requests, the context is canceled when the client's connection closes, the request is canceled (with HTTP/2), or when the ServeHTTP method returns.

func (*Request) WithContext

func (r *Request) WithContext(ctx context.Context) *Request

WithContext returns a shallow copy of r with its context changed to ctx. The provided ctx must be non-nil.

For outgoing client request, the context controls the entire lifetime of a request and its response: obtaining a connection, sending the request, and reading the response headers and body.

To create a new request with a context, use NewRequestWithContext. To change the context of a request, such as an incoming request you want to modify before sending back out, use Request.Clone. Between those two uses, it's rare to need WithContext.

type RequestAdapter

type RequestAdapter func(request *Request) *Request

type ResourceClient

type ResourceClient interface {
	// GetContentLength get length of resource content
	// return source.UnknownSourceFileLen if response status is not StatusOK and StatusPartialContent
	GetContentLength(request *Request) (int64, error)

	// IsSupportRange checks if resource supports breakpoint continuation
	// return false if response status is not StatusPartialContent
	IsSupportRange(request *Request) (bool, error)

	// IsExpired checks if a resource received or stored is the same.
	// return false and non-nil err to prevent the source from exploding if
	// fails to get the result, it is considered that the source has not expired
	IsExpired(request *Request, info *ExpireInfo) (bool, error)

	// Download downloads from source
	Download(request *Request) (*Response, error)

	// GetLastModified gets last modified timestamp milliseconds of resource
	GetLastModified(request *Request) (int64, error)
}

ResourceClient defines the API interface to interact with source.

func LoadPlugin

func LoadPlugin(dir, scheme string) (ResourceClient, error)

type ResourceClientBuilder

type ResourceClientBuilder interface {
	// Build return the target resource with custom option
	Build(optionYaml []byte) (resourceClient ResourceClient, adaptor RequestAdapter, hooks []Hook, err error)
}

ResourceClientBuilder is used to build resource client with custom option

func NewPlainResourceClientBuilder

func NewPlainResourceClientBuilder(
	build func(optionYaml []byte) (resourceClient ResourceClient, adaptor RequestAdapter, hooks []Hook, err error)) ResourceClientBuilder

type ResourceLister

type ResourceLister interface {
	// List only list current level resources in request url
	// the request may represent a single file or a directory
	// if the request represent a directory, the result should return all file entries and subdirectory entries below the request directory
	// if the request represent a single file, the result should return a single file entry or empty slice
	List(request *Request) (urls []URLEntry, err error)
}

ResourceLister defines the interface to list all downloadable resources in request url

type ResourceMetadataGetter

type ResourceMetadataGetter interface {
	GetMetadata(request *Request) (*Metadata, error)
}

ResourceMetadataGetter defines the API interface to get metadata for special resource The metadata will be used for concurrent multiple pieces downloading

type Response

type Response struct {
	Status        string
	StatusCode    int
	Header        Header
	Body          io.ReadCloser
	ContentLength int64
	// Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this,
	// otherwise return status code to original client
	Validate func() error
	// Temporary indecates the error whether the error is temporary, if is true, we can retry it later
	Temporary bool
}

func Download

func Download(request *Request) (*Response, error)

func NewResponse

func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response

func (*Response) ExpireInfo

func (resp *Response) ExpireInfo() ExpireInfo

type URLEntry

type URLEntry struct {
	//URL download url
	URL *url.URL

	// Name returns the name of the file (or subdirectory) described by the entry.
	// Name will be used in recursive downloading as file name or subdirectory name
	// This name is only the final element of the path (the commonv1 name), not the entire path.
	// For example, Name would return "hello.go" not "home/gopher/hello.go".
	Name string
	// IsDir reports whether the entry describes a directory.
	IsDir bool
	// Entry attribute, like oss md5 and expire time
	Attribute map[string]string
}

URLEntry is an entry which read from url with specific protocol It is used in recursive downloading

func List

func List(request *Request) ([]URLEntry, error)

type UnexpectedStatusCodeError

type UnexpectedStatusCodeError struct {
	// contains filtered or unexported fields
}

UnexpectedStatusCodeError is returned when a source responds with neither an error nor with a status code indicating success.

func (UnexpectedStatusCodeError) Error

Error implements interface error

func (UnexpectedStatusCodeError) Got

Got is the actual status code returned by source.

Directories

Path Synopsis
clients
Package mocks is a generated GoMock package.
Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL