downloader

package
v0.0.1-dev.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 3, 2023 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

Functions

This section is empty.

Types

type FileProxyFactory

type FileProxyFactory struct {
	// contains filtered or unexported fields
}

read proxy file and produce Proxy line format: {address} {Port}

func NewFileProxyFactory

func NewFileProxyFactory(fileUrl string) (*FileProxyFactory, error)

func (*FileProxyFactory) GetProxy

func (this *FileProxyFactory) GetProxy() (IProxy, error)

func (*FileProxyFactory) ReturnProxy

func (this *FileProxyFactory) ReturnProxy(proxy IProxy)

type IDownloader

type IDownloader interface {
	Download(request ent.IRequest) (ent.IPage, error)
	SetDownloadTimeout(dt time.Duration)
	OnSuccess(request ent.IRequest)
	OnError(request ent.IRequest, err error)
	Validate(page ent.IPage) (bool, string)
}

type IProxy

type IProxy interface {
	GetId() int
	GetHost() string
	GetPort() int
	GetUsername() string
	GetPassword() string
}

type IProxyFactory

type IProxyFactory interface {
	GetProxy() (IProxy, error)
	ReturnProxy(proxy IProxy)
}

type Proxy

type Proxy struct {
	Id       int    `json:"id"`
	Host     string `json:"host"`
	Port     int    `json:"port"`
	Username string `json:"username"`
	Password string `json:"password"`
}

func NewProxy

func NewProxy(id int, host string, port int, username, password string) Proxy

func (Proxy) GetHost

func (this Proxy) GetHost() string

func (Proxy) GetId

func (this Proxy) GetId() int

func (Proxy) GetPassword

func (this Proxy) GetPassword() string

func (Proxy) GetPort

func (this Proxy) GetPort() int

func (Proxy) GetUsername

func (this Proxy) GetUsername() string

type SimpleDownloader

type SimpleDownloader struct {
	// contains filtered or unexported fields
}

func NewSimpleDownloader

func NewSimpleDownloader(downloadTimeout time.Duration, provider IProxyFactory) *SimpleDownloader

func (*SimpleDownloader) Download

func (rd *SimpleDownloader) Download(request ent.IRequest) (ent.IPage, error)

func (*SimpleDownloader) OnError

func (rd *SimpleDownloader) OnError(request ent.IRequest, err error)

func (*SimpleDownloader) OnSuccess

func (rd *SimpleDownloader) OnSuccess(request ent.IRequest)

func (*SimpleDownloader) SetDownloadTimeout

func (rd *SimpleDownloader) SetDownloadTimeout(dt time.Duration)

func (*SimpleDownloader) Validate

func (rd *SimpleDownloader) Validate(ent.IPage) (bool, string)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL