Documentation
¶
Overview ¶
Package downloader provides the sync and async file download functionality.
Index ¶
- Variables
- func IsValid(f *slack.File) bool
- type Client
- func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <-chan *slack.File) (chan struct{}, error)
- func (c *Client) DownloadFile(dir string, f slack.File) (string, error)
- func (c *Client) SaveFile(ctx context.Context, dir string, f *slack.File) (int64, error)
- func (c *Client) Start(ctx context.Context)
- func (c *Client) Stop()
- type Downloader
- type FilenameFunc
- type Option
Examples ¶
Constants ¶
This section is empty.
Variables ¶
var ErrNoFS = errors.New("fs adapter not initialised")
var ErrNotStarted = errors.New("downloader not started")
Functions ¶
Types ¶
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client is the instance of the downloader.
func New ¶
func New(client Downloader, fs fsadapter.FS, opts ...Option) *Client
New initialises new file downloader.
Example (Advanced) ¶
package main
import (
"context"
"fmt"
"github.com/rusq/slackdump/v2/downloader"
"github.com/rusq/slackdump/v2/fsadapter"
"github.com/slack-go/slack"
"golang.org/x/time/rate"
)
func main() {
client := slack.New("token")
// initialise the filesystem (files.zip archive)
fs, err := fsadapter.NewZipFile("files.zip")
if err != nil {
fmt.Println("failed to initialise the file system")
return
}
defer fs.Close()
dl := downloader.New(
client,
fs,
downloader.Retries(100), // 100 retries when rate limited
downloader.Limiter(rate.NewLimiter(20, 1)), // rate limit
downloader.Workers(8), // number of download workers
)
f := &slack.File{}
if n, err := dl.SaveFile(context.Background(), "some_dir", f); err != nil {
fmt.Printf("failed to save the file: %s", err)
} else {
fmt.Printf("downloaded: %d bytes", n)
}
}
Output:
Example (Basic) ¶
package main
import (
"context"
"fmt"
"github.com/rusq/slackdump/v2/downloader"
"github.com/rusq/slackdump/v2/fsadapter"
"github.com/slack-go/slack"
)
func main() {
client := slack.New("token")
fs := fsadapter.NewDirectory("files/")
dl := downloader.New(
client,
fs,
)
f := &slack.File{}
if n, err := dl.SaveFile(context.Background(), "some_dir", f); err != nil {
fmt.Printf("failed to save the file: %s", err)
} else {
fmt.Printf("downloaded: %d bytes", n)
}
}
Output:
func (*Client) AsyncDownloader ¶
func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <-chan *slack.File) (chan struct{}, error)
AsyncDownloader starts Client.worker goroutines to download files concurrently. It will download any file that is received on fileDlQueue channel. It returns the "done" channel and an error. "done" channel will be closed once all downloads are complete.
func (*Client) DownloadFile ¶
DownloadFile requires a started downloader, otherwise it will return ErrNotStarted. Will place the file to the download queue, and save the file to the directory that was specified when Start was called. If the file buffer is full, will block until it becomes empty. It returns the filepath within the filesystem.
type Downloader ¶
type Downloader interface {
// GetFile retrieves a given file from its private download URL
GetFile(downloadURL string, writer io.Writer) error
}
Downloader is the file downloader interface. It exists primarily for mocking in tests.
type FilenameFunc ¶ added in v2.2.0
FilenameFunc is the file naming function that should return the output filename for slack.File.
var Filename FilenameFunc = stdFilenameFn
Filename returns name of the file generated from the slack.File.
type Option ¶
type Option func(*Client)
Option is the function signature for the option functions.
func Logger ¶ added in v2.0.1
Logger allows to use an external log library, that satisfies the logger.Interface.
func WithNameFunc ¶ added in v2.2.0
func WithNameFunc(fn FilenameFunc) Option
Source Files