notionapi: github.com/kjk/notionapi/caching_downloader Index | Files

package caching_downloader

import "github.com/kjk/notionapi/caching_downloader"

Index

Package Files

cache.go caching_downloader.go serialize_http_cache.go

func GetCacheFileNameFromURL Uses

func GetCacheFileNameFromURL(uri string) string

GetCacheFileNameFromURL returns name of the file in cache for this URL. The name is files/${sha1OfURL}.${ext} It's a consistent, one-way transform

func Sha1OfURL Uses

func Sha1OfURL(uri string) string

Sha1OfURL returns sha1 of url

type Cache Uses

type Cache interface {
    // ReadFile reads a file with a given name from cache
    ReadFile(string) ([]byte, error)
    // WriteFile writes a file with a given name to cache
    WriteFile(string, []byte) error
    // GetPageIDs returns ids of pages in the cache
    GetPageIDs() ([]string, error)
    // Remove removes a file with a given name from cache
    Remove(string)
}

Cache describes a caching interface

type DirectoryCache Uses

type DirectoryCache struct {
    Dir string
    // contains filtered or unexported fields
}

DirectoryCache implements disk-based Cache interface

func NewDirectoryCache Uses

func NewDirectoryCache(dir string) (*DirectoryCache, error)

NewDirectoryCache returns a new DirectoryCache which caches files in a directory

func (*DirectoryCache) GetPageIDs Uses

func (c *DirectoryCache) GetPageIDs() ([]string, error)

GetPageIDs returns ids of pages in the cache

func (*DirectoryCache) ReadFile Uses

func (c *DirectoryCache) ReadFile(name string) ([]byte, error)

ReadFile reads a file with a given name from cache

func (*DirectoryCache) Remove Uses

func (c *DirectoryCache) Remove(name string)

Remove removes a file with a given name from cache

func (*DirectoryCache) WriteFile Uses

func (c *DirectoryCache) WriteFile(name string, data []byte) error

WriteFile writes a file with a given name to cache

type Downloader Uses

type Downloader struct {
    Client *notionapi.Client
    // cached pages are stored in Cache as ${pageID}.txt files
    Cache Cache
    // NoReadCache disables reading from cache i.e. downloaded pages
    // will be written to cache but not read from it
    NoReadCache bool
    // if true, we'll re-download a page if a newer version is
    // on the server
    RedownloadNewerVersions bool
    // maps id of the page (in the no-dash format) to a cached Page
    IdToPage map[string]*notionapi.Page
    // maps id of the page (in the no-dash format) to latest version
    // of the page available on the server.
    // if doesn't exist, we haven't yet queried the server for the
    // version
    IdToPageLatestVersion map[string]int64

    // for diagnostics, number of downloaded pages
    DownloadedCount int
    // number of pages we got from cache
    FromCacheCount int

    // for diagnostics, number of downloaded files
    DownloadedFilesCount int
    // number of files we got from cache
    FilesFromCacheCount int

    EventObserver func(interface{})
    // contains filtered or unexported fields
}

Downloader implements optimized (cached) downloading of pages from the server. Cache of pages is stored in CacheDir. We return pages from cache. If RedownloadNewerVersions is true, we'll re-download latest version of the page (as opposed to returning possibly outdated version from cache). We do it more efficiently than just blindly re-downloading.

func New Uses

func New(cache Cache, client *notionapi.Client) *Downloader

New returns a new Downloader which caches page loads on disk and can return pages from that cache

func (*Downloader) DownloadFile Uses

func (d *Downloader) DownloadFile(uri string) (*notionapi.DownloadFileResponse, error)

DownloadFile downloads a file, caching in the cache

func (*Downloader) DownloadPage Uses

func (d *Downloader) DownloadPage(pageID string) (*notionapi.Page, error)

func (*Downloader) DownloadPagesRecursively Uses

func (d *Downloader) DownloadPagesRecursively(startPageID string) ([]*notionapi.Page, error)

func (*Downloader) GetClientCopy Uses

func (d *Downloader) GetClientCopy() *notionapi.Client

GetClientCopy returns a copy of client

func (*Downloader) NameForPageID Uses

func (d *Downloader) NameForPageID(pageID string) string

NameForPageID returns name of the file used for storing HTTP cache for a given page

func (*Downloader) ReadPageFromCache Uses

func (d *Downloader) ReadPageFromCache(pageID string) (*notionapi.Page, error)

ReadPageFromCache returns a page read from the cache

type EventDidDownload Uses

type EventDidDownload struct {
    // if page, PageID is set
    PageID string
    // if file, URL is set
    FileURL string
    // how long did it take to download
    Duration time.Duration
}

EventDidDownload is for logging. Emitted when page or file is downloaded

type EventDidReadFromCache Uses

type EventDidReadFromCache struct {
    // if page, PageID is set
    PageID string
    // if file, URL is set
    FileURL string
    // how long did it take to download
    Duration time.Duration
}

EventDidReadFromCache is for logging. Emitted when page or file is read from cache.

type EventError Uses

type EventError struct {
    Error string
}

EventError is for logging. Emitted when there's error to log

type EventGotVersions Uses

type EventGotVersions struct {
    Count    int
    Duration time.Duration
}

EventGotVersions is for logging. Emitted

Package caching_downloader imports 15 packages (graph) and is imported by 1 packages. Updated 2019-09-19. Refresh now. Tools for package owners.