services

package
v0.0.0-...-33037e1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 20, 2020 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

View Source
const MaxAttempts = 3
View Source
const MaxFetchBytes = 4096

Variables

This section is empty.

Functions

func CanRetry

func CanRetry(statusCode int, attempts uint, err error) bool

func NewHTTPClient

func NewHTTPClient() *http.Client

Types

type Bytes

type Bytes []byte

type FetchHelper

type FetchHelper struct {
}

func (FetchHelper) Try

func (hlp FetchHelper) Try(fetcher *Fetcher, url string, attempts uint) (*pkg.UrlScraped, uint, error)

type Fetcher

type Fetcher struct {
	Client *http.Client
}

func (Fetcher) Fetch

func (f Fetcher) Fetch(url string) (*pkg.UrlScraped, int, error)

type FetcherIntf

type FetcherIntf interface {
	Fetch(url string) (Bytes, error)
}

type Publisher

type Publisher interface {
	Publish(queue string, payload []byte) error
}

type RedisPublisher

type RedisPublisher struct {
	// contains filtered or unexported fields
}

func NewRedisPublisher

func NewRedisPublisher() *RedisPublisher

func (*RedisPublisher) Publish

func (p *RedisPublisher) Publish(queue string, payload []byte) error

type RedisSubscriber

type RedisSubscriber struct {
	PubSub *redis.PubSub
	// contains filtered or unexported fields
}

func NewRedisSubscriber

func NewRedisSubscriber(queue string) *RedisSubscriber

func (*RedisSubscriber) Close

func (s *RedisSubscriber) Close() error

func (*RedisSubscriber) Consume

func (s *RedisSubscriber) Consume(out chan string)

Consume messages.

type Scraper

type Scraper struct {
}

func (Scraper) Scrape

func (s Scraper) Scrape(buf []byte)

type Subscriber

type Subscriber interface {
	Subscribe(queue string, payload []byte) error
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL