core

package
v0.0.0-...-a3971e5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 13, 2024 License: MIT Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultUserAgent = "verssion_bot/1.0 (https://verssion.one)"
	MaxConcurrency   = 5
)
View Source 
const (
	CacheOK = 6 * time.Hour
)
View Source 
const DBURL = "postgresql:///w"

Variables

View Source 
var ErrCuratedNotFound = errors.New("curated ID not found")

Functions

func CanonicalPage 

func CanonicalPage(doc *html.Node) (string, error)

func HtmlMarkdown 

func HtmlMarkdown(src string) template.HTML

markdown to html (properly escaped)

func StableVersion 

func StableVersion(n *html.Node) (string, string)

func TextMarkdown 

func TextMarkdown(src string) string

markdown to plain text

func Title 

func Title(page string) string

title version of a wikipage path

func Titles 

func Titles(pages []string) []string

func WikiBasePage 

func WikiBasePage(href string) string

given a full wikipedia url returns the page name also works with parts of the wikipedia url

Types

type Curated 

type Curated struct {
	CustomTitle string
	Created     time.Time
	LastUsed    time.Time
	LastUpdated time.Time
	Pages       []string
}

func (*Curated) DefaultTitle 

func (c *Curated) DefaultTitle() string

func (*Curated) Title 

func (c *Curated) Title() string

type DB 

type DB interface {
	Last(string) (*Page, error) // Last spider
	Current(limit int, order SortBy) ([]Page, error)
	CurrentIn(...string) ([]Page, error)
	History(...string) ([]Page, error) // Newest first
	Store(Page) error
	Known() ([]string, error)

	CreateCurated() (string, error)
	LoadCurated(string) (*Curated, error) // will return (nil, nil) on not found
	CuratedSetPages(string, []string) error
	CuratedSetUsed(string) error
	CuratedSetTitle(string, string) error
}

type ErrNoVersion 

type ErrNoVersion struct {
	Page string
}

func (ErrNoVersion) Error 

func (e ErrNoVersion) Error() string

type ErrNotFound 

type ErrNotFound struct {
	Page string
}

func (ErrNotFound) Error 

func (e ErrNotFound) Error() string

type ErrRedirect 

type ErrRedirect struct {
	Page, To string
}

func (ErrRedirect) Error 

func (e ErrRedirect) Error() string
type Link struct {
	Rel   string
	Href  string
	Title string
}
func FindHeadLink(doc *html.Node) ([]Link, error)

FindHeadLink returns all <link> nodes

type Memory 

type Memory struct {
	// contains filtered or unexported fields
}

func NewMemory 

func NewMemory() *Memory

func (*Memory) CreateCurated 

func (m *Memory) CreateCurated() (string, error)

func (*Memory) CuratedSetPages 

func (m *Memory) CuratedSetPages(id string, pages []string) error

func (*Memory) CuratedSetTitle 

func (m *Memory) CuratedSetTitle(id string, title string) error

func (*Memory) CuratedSetUsed 

func (m *Memory) CuratedSetUsed(id string) error

func (*Memory) Current 

func (m *Memory) Current(limit int, order SortBy) ([]Page, error)

func (*Memory) CurrentIn 

func (m *Memory) CurrentIn(pages ...string) ([]Page, error)

func (*Memory) History 

func (m *Memory) History(pages ...string) ([]Page, error)

func (*Memory) Known 

func (m *Memory) Known() ([]string, error)

func (*Memory) Last 

func (m *Memory) Last(page string) (*Page, error)

func (*Memory) LoadCurated 

func (m *Memory) LoadCurated(id string) (*Curated, error)

func (*Memory) Store 

func (m *Memory) Store(p Page) error

type Page 

type Page struct {
	Page          string
	T             time.Time
	StableVersion string
	Homepage      string
}

type Postgres 

type Postgres struct {
	// contains filtered or unexported fields
}

func NewPostgres 

func NewPostgres(url string) (*Postgres, error)

func (*Postgres) CreateCurated 

func (p *Postgres) CreateCurated() (string, error)

func (*Postgres) CuratedSetPages 

func (p *Postgres) CuratedSetPages(id string, pages []string) error

pages must be unique

func (*Postgres) CuratedSetTitle 

func (p *Postgres) CuratedSetTitle(id, title string) error

func (*Postgres) CuratedSetUsed 

func (p *Postgres) CuratedSetUsed(id string) error

func (*Postgres) Current 

func (p *Postgres) Current(limit int, order SortBy) ([]Page, error)

func (*Postgres) CurrentIn 

func (p *Postgres) CurrentIn(pages ...string) ([]Page, error)

func (*Postgres) History 

func (p *Postgres) History(pages ...string) ([]Page, error)

History of a list of page. Newest first.

func (*Postgres) Known 

func (p *Postgres) Known() ([]string, error)

func (*Postgres) Last 

func (p *Postgres) Last(page string) (*Page, error)

func (*Postgres) LoadCurated 

func (p *Postgres) LoadCurated(id string) (*Curated, error)

func (*Postgres) Store 

func (p *Postgres) Store(e Page) error

func (*Postgres) StoreCurated 

func (p *Postgres) StoreCurated(cur Curated) error

type SortBy 

type SortBy int
const (
	SpiderT SortBy = iota
	Alphabet
)

func (SortBy) OrderBy 

func (s SortBy) OrderBy() string

type Spider 

type Spider interface {
	// Get the named page
	Spider(page string) (*Page, error)
}

type SpiderCache 

type SpiderCache struct {
	// contains filtered or unexported fields
}

SpiderCache wraps a Spider implementation

func NewSpiderCache 

func NewSpiderCache(spider Spider) *SpiderCache

func (*SpiderCache) Spider 

func (u *SpiderCache) Spider(page string) (*Page, error)

type Table 

type Table struct {
	Rows [][]string
}

func FindTables 

func FindTables(doc *html.Node) ([]Table, error)

FindTables returns all top-level tables

type WikipediaSpider 

type WikipediaSpider struct {
	URL       func(string) string
	UserAgent string
	// contains filtered or unexported fields
}

func NewWikipediaSpider 

func NewWikipediaSpider(url func(string) string) *WikipediaSpider

NewWikipediaSpider makes a spider. url makes a URL from a page.

func (*WikipediaSpider) Spider 

func (spider *WikipediaSpider) Spider(page string) (*Page, error)

Spider downloads and parses given wikipage

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.