utils

package

v0.3.1 Latest Latest Go to latest Published: May 21, 2022 License: MIT Imports: 9 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/DiheChen/go-asmr-spider

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func PathExists(path string) bool
func SingleThreadDownload(url, path string, headers map[string]string) error
type BlockMetaData
type MultiThreadDownloader
- func NewDownloader(url string, path string, name string, threadCount int, ...) *MultiThreadDownloader
- func (m *MultiThreadDownloader) Download() error
type TaskQueue
type Worker
- func NewWorker() *Worker
- func (w *Worker) Run(wq chan TaskQueue, owner *WorkerPool)
type WorkerPool
- func NewWorkerPool(WorkerCount int) *WorkerPool
- func (wp *WorkerPool) Start()

Constants ¶

This section is empty.

Variables ¶

View Source

var Client = sync.Pool{
	New: func() interface{} {
		return &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
				TLSClientConfig: &tls.Config{
					MaxVersion: tls.VersionTLS12,
				},
			},
		}
	},
}

View Source

var (
	ErrUnsupportedMultiThreading = errors.New("unsupported multi-threading")
)

Functions ¶

func PathExists ¶ added in v0.3.0

func PathExists(path string) bool

func SingleThreadDownload ¶ added in v0.3.0

func SingleThreadDownload(url, path string, headers map[string]string) error

Types ¶

type BlockMetaData ¶ added in v0.3.0

type BlockMetaData struct {
	BeginOffset    int64
	EndOffset      int64
	DownloadedSize int64
}

type MultiThreadDownloader ¶ added in v0.3.0

type MultiThreadDownloader struct {
	Url         string
	SavePath    string
	FileName    string
	FullPath    string
	Client      *http.Client
	Headers     map[string]string
	Blocks      []*BlockMetaData
	ThreadCount int
}

func NewDownloader ¶ added in v0.3.0

func NewDownloader(url string, path string, name string, threadCount int, headers map[string]string) *MultiThreadDownloader

func (*MultiThreadDownloader) Download ¶ added in v0.3.0

func (m *MultiThreadDownloader) Download() error

type TaskQueue ¶ added in v0.3.0

type TaskQueue chan *MultiThreadDownloader

type Worker ¶ added in v0.3.0

type Worker struct {
	TaskChan TaskQueue
}

func NewWorker ¶ added in v0.3.0

func NewWorker() *Worker

func (*Worker) Run ¶ added in v0.3.0

func (w *Worker) Run(wq chan TaskQueue, owner *WorkerPool)

type WorkerPool ¶ added in v0.3.0

type WorkerPool struct {
	sync.WaitGroup

	WorkerCount int
	TaskQueue   TaskQueue
	WorkerQueue chan TaskQueue
}

func NewWorkerPool ¶ added in v0.3.0

func NewWorkerPool(WorkerCount int) *WorkerPool

func (*WorkerPool) Start ¶ added in v0.3.0

func (wp *WorkerPool) Start()

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL