felix

type Attempter ¶

type Attempter interface {
	// Next returns if and when the next attempt is scheduled
	Next(key string) (bool, time.Duration, error)
	// Inc increments the number of attempts by 1
	Inc(key string) error
}

Attempter is used by Fetcher to determine if and when the next fetch attempt should be made.

func NewAttempter(ds Datastore, next NextAttemptFunc) Attempter

type Config ¶

type Config struct {
	FetchInterval    time.Duration  `yaml:"fetchInterval"`
	UserAgent        string         `yaml:"userAgent"`
	Port             int            `yaml:"port"`
	FeedOutputMaxAge time.Duration  `yaml:"feedOutputMaxAge"`
	CleanupInterval  time.Duration  `yaml:"cleanupInterval"`
	CleanupMaxAge    time.Duration  `yaml:"cleanupMaxAge"`
	Feeds            []FeedConfig   `yaml:"feeds"`
	ItemFilters      []FilterConfig `yaml:"itemFilters"`
	LinkFilters      []FilterConfig `yaml:"linkFilters"`
}

Config contains the configuration

func ConfigFromFile(filename string) (Config, error)

func NewConfig() Config

type Datastore ¶

type Datastore interface {
	LastAttempt(key string) (time.Time, int, error)
	IncAttempt(key string) error
	StoreItem(item Item) (bool, error)
	StoreLink(link Link) (bool, error)
	GetItems(maxAge time.Duration) ([]Item, error)
	GetLinks(maxAge time.Duration) ([]Link, error)
	Cleanup(maxAge time.Duration) error
	Close() error
}

Datastore is used to store and retrieve items, links, etc.

type DefaultLogger ¶

type DefaultLogger struct {
	// contains filtered or unexported fields
}

DefaultLogger is the default logger implementation and logs to stdout.

func NewLogger() *DefaultLogger

func (l *DefaultLogger) Debug(msg string, keyvals ...interface{})

func (l *DefaultLogger) Error(msg string, keyvals ...interface{})

func (l *DefaultLogger) Fatal(msg string, keyvals ...interface{})

func (l *DefaultLogger) Info(msg string, keyvals ...interface{})

func (l *DefaultLogger) SetOutput(w io.Writer)

func (l *DefaultLogger) Warn(msg string, keyvals ...interface{})

type Emitter ¶

type Emitter interface {
	EmitItem(item Item)
	EmitLink(link Link)
	EmitFollow(follow string)
}

Emitter is used by a Scanner to emit Items, Links and Follow URLs that should be processed.

type FeedConfig ¶

type FeedConfig struct {
	Type          string
	URL           string
	FetchInterval time.Duration
}

FeedConfig contains the configuration of a single feed.

type Fetcher ¶

type Fetcher struct {
	// contains filtered or unexported fields
}

Fetcher is the default fetcher.

func NewFetcher(url string, source Source, scanner Scanner, attempt Attempter, items chan<- Item, links chan<- Link) *Fetcher

func (f *Fetcher) SetLogger(log Logger)

func (f *Fetcher) Start(quit <-chan struct{})

type FilterConfig ¶

type FilterConfig struct {
	Type string
	// contains filtered or unexported fields
}

FilterConfig is the common configuration of all filter types. See FilterConfig.Unmarshal for unmarshaling of the raw config value for more specific types.

func (fc *FilterConfig) Unmarshal(v interface{}) error

func (fc *FilterConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

type Item ¶

type Item struct {
	Title   string
	URL     string
	PubDate time.Time
}

Item is a feed item that should be scraped for links.

type ItemFilter ¶

type ItemFilter interface {
	Filter(item Item, next func(Item))
}

ItemFilter wraps the Filter method for items.

Filter evaluates the given item, optionally modifies it, and passes it to the next filter in the filter chain, if it matches the filter criteria.

func ItemTitleFilter(titles ...string) ItemFilter

type ItemFilterFunc ¶

type ItemFilterFunc func(Item, func(Item))

ItemFilterFunc is an adapter to allow the use of ordinary functions as filters. If f is a function with the appropriate signature, ItemFilterFunc(f) is a ItemFilter that calls f.

func (f ItemFilterFunc) Filter(item Item, next func(Item))

type ItemTitleFilterConfig ¶

type ItemTitleFilterConfig struct {
	Type   string
	Titles []string
}

ItemTitleFilterConfig contains the configuration of a ItemTitleFilter.

type Link ¶

type Link struct {
	Title string
	URL   string
}

Link is a link that was found in a feed or scraped from a page (Item)

type LinkDomainFilterConfig ¶

type LinkDomainFilterConfig struct {
	Domains []string
}

LinkDomainFilterConfig contains the configuration of a LinkDomainFilter.

type LinkDuplicatesFilterConfig ¶ added in v0.5.0

type LinkDuplicatesFilterConfig struct {
	Size int
}

type LinkFilenameAsTitleFilterConfig ¶ added in v0.2.0

type LinkFilenameAsTitleFilterConfig struct {
	TrimExt bool `yaml:"trimExt"`
}

LinkFilenameAsTitleFilterConfig contains the configuration of a LinkFilenameAsTitleFilter.

type LinkFilter ¶

type LinkFilter interface {
	Filter(link Link, next func(Link))
}

LinkFilter wraps the Filter method for links.

Filter evaluates the given link, optionally modifies it, and passes it to the next filter in the filter chain, if it matches the filter criteria.

func LinkDomainFilter(domains ...string) LinkFilter

func LinkDuplicatesFilter(size int) LinkFilter

func LinkFilenameAsTitleFilter(trimExt bool) LinkFilter

func LinkURLRegexFilter(exprs ...string) (LinkFilter, error)

func LinkUploadedExpandFilenameFilter(source Source) LinkFilter

type LinkFilterFunc ¶

type LinkFilterFunc func(Link, func(Link))

LinkFilterFunc is an adapter to allow the use of ordinary functions as filters. If f is a function with the appropriate signature, LinkFilterFunc(f) is a LinkFilter that calls f.

func (f LinkFilterFunc) Filter(link Link, next func(Link))

type LinkURLRegexFilterConfig ¶

type LinkURLRegexFilterConfig struct {
	Exprs []string
}

LinkURLRegexFilterConfig contains the configuration of a LinkURLRegexFilter.

type Logger ¶

type Logger interface {
	Debug(msg string, keyvals ...interface{})
	Info(msg string, keyvals ...interface{})
	Warn(msg string, keyvals ...interface{})
	Error(msg string, keyvals ...interface{})
	Fatal(msg string, keyvals ...interface{})
}

Logger is the standardized interface for all Loggers used in this project.

type NextAttemptFunc ¶ added in v0.3.0

type NextAttemptFunc func(last time.Time, attempts int) (bool, time.Duration)

A NextAttemptFunc returns if and when the next attempt is scheduled for the given key.

func FibNextAttemptFunc(baseInterval time.Duration, maxAttempts int) NextAttemptFunc

func PeriodicNextAttemptFunc(fetchInterval time.Duration) NextAttemptFunc

type NopLogger ¶

type NopLogger struct{}

NopLogger is a no-op implementation of the Logger interface.

func (NopLogger) Debug(msg string, keyvals ...interface{})

func (NopLogger) Error(msg string, keyvals ...interface{})

func (NopLogger) Fatal(msg string, keyvals ...interface{})

func (NopLogger) Info(msg string, keyvals ...interface{})

func (NopLogger) Warn(msg string, keyvals ...interface{})

type ScanFunc ¶

type ScanFunc func(context.Context, io.Reader, Emitter) error

ScanFunc is an adapter to allow the use of ordinary functions as scanners. If f is a function with the appropriate signature, ScanFunc(f) is a Scanner that calls f.

func (f ScanFunc) Scan(ctx context.Context, r io.Reader, e Emitter) error

type Scanner ¶

type Scanner interface {
	Scan(context.Context, io.Reader, Emitter) error
}

Scanner scans the contents of the reader and emits items, links or follow URLs.

type Source ¶

type Source interface {
	Get(ctx context.Context, url string) (io.Reader, error)
}

Source retrieves the resource from the given URL and returns a reader for the content. If applicable, the reader shall only return UTF-8 encoded text. TODO: Find a better name?

func NewHTTPSource(client *http.Client) Source

type Stringer ¶

type Stringer interface {
	String() string
}

Stringer is an optional interface for all filters to provide a 'native' textual representation.