etl

package

v0.0.0-...-0d6871a Latest Latest Go to latest Published: May 13, 2022 License: AGPL-3.0 Imports: 35 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/knakk/sirkulator

Documentation ¶

Overview ¶

package etl

Index ¶

func CorporationFromAuthority(rec marc.Record) (sirkulator.Resource, error)
func PersonFromAuthority(rec marc.Record) (sirkulator.Resource, error)
type FileFetch
type HarvestNBLinksJob
- func (h *HarvestNBLinksJob) Name() string
- func (h *HarvestNBLinksJob) Run(ctx context.Context, w io.Writer) error
type HarvestSNLDescriptions
- func (h *HarvestSNLDescriptions) Name() string
- func (h *HarvestSNLDescriptions) Run(ctx context.Context, w io.Writer) error
type HarvestSNLLinksJob
- func (h *HarvestSNLLinksJob) Name() string
- func (h *HarvestSNLLinksJob) Run(ctx context.Context, w io.Writer) error
type HarvestWikipediaLinks
- func (h *HarvestWikipediaLinks) Name() string
- func (h *HarvestWikipediaLinks) Run(ctx context.Context, w io.Writer) error
type HarvestWikipediaSummaries
- func (h *HarvestWikipediaSummaries) Name() string
- func (h *HarvestWikipediaSummaries) Run(ctx context.Context, w io.Writer) error
type ImportEntry
type Ingestion
type Ingestor
- func NewIngestor(db *sqlitex.Pool, idx *search.Index) *Ingestor
- func NewPreviewIngestor(db *sqlitex.Pool) *Ingestor
- func (ig *Ingestor) Ingest(ctx context.Context, data Ingestion, persist bool) ([]sirkulator.SimpleResource, error)
- func (ig *Ingestor) IngestISBN(ctx context.Context, id string, persist bool) ImportEntry
type UpdateSNLDescriptions
- func (u *UpdateSNLDescriptions) Name() string
- func (u *UpdateSNLDescriptions) Run(ctx context.Context, w io.Writer) error

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func CorporationFromAuthority ¶

func CorporationFromAuthority(rec marc.Record) (sirkulator.Resource, error)

func PersonFromAuthority ¶

func PersonFromAuthority(rec marc.Record) (sirkulator.Resource, error)

Types ¶

type FileFetch ¶

type FileFetch struct {
	ResourceID string
	URL        string
}

type HarvestNBLinksJob ¶

type HarvestNBLinksJob struct {
	DB *sqlitex.Pool
}

func (*HarvestNBLinksJob) Name ¶

func (h *HarvestNBLinksJob) Name() string

func (*HarvestNBLinksJob) Run ¶

func (h *HarvestNBLinksJob) Run(ctx context.Context, w io.Writer) error

type HarvestSNLDescriptions ¶

type HarvestSNLDescriptions struct {
	DB *sqlitex.Pool
}

func (*HarvestSNLDescriptions) Name ¶

func (h *HarvestSNLDescriptions) Name() string

func (*HarvestSNLDescriptions) Run ¶

func (h *HarvestSNLDescriptions) Run(ctx context.Context, w io.Writer) error

type HarvestSNLLinksJob ¶

type HarvestSNLLinksJob struct {
	DB *sqlitex.Pool
}

func (*HarvestSNLLinksJob) Name ¶

func (h *HarvestSNLLinksJob) Name() string

func (*HarvestSNLLinksJob) Run ¶

func (h *HarvestSNLLinksJob) Run(ctx context.Context, w io.Writer) error

type HarvestWikipediaLinks ¶

type HarvestWikipediaLinks struct {
	DB *sqlitex.Pool
}

func (*HarvestWikipediaLinks) Name ¶

func (h *HarvestWikipediaLinks) Name() string

func (*HarvestWikipediaLinks) Run ¶

func (h *HarvestWikipediaLinks) Run(ctx context.Context, w io.Writer) error

type HarvestWikipediaSummaries ¶

type HarvestWikipediaSummaries struct {
	DB *sqlitex.Pool
}

func (*HarvestWikipediaSummaries) Name ¶

func (h *HarvestWikipediaSummaries) Name() string

func (*HarvestWikipediaSummaries) Run ¶

func (h *HarvestWikipediaSummaries) Run(ctx context.Context, w io.Writer) error

type ImportEntry ¶

type ImportEntry struct {
	Source    string // bibsys, openlibrary, discogs etc
	Exists    bool   // allready in local DB
	Error     string // not found, unable to map to local metadata, insuficcient data etc
	Resources []sirkulator.SimpleResource
}

type Ingestion ¶

type Ingestion struct {
	Resources []sirkulator.Resource
	Relations []sirkulator.Relation
	Covers    []FileFetch
}

type Ingestor ¶

type Ingestor struct {

	// Options
	UseRemote     bool // if true, use external sources in additin to local
	ImageDownload bool // if true, download images found in imported records
	ImageAsync    bool // if true, download images after IngestISBN has returned
	ImageWidth    int  // scale to this with, calculating width to preserve aspect ratio
	// contains filtered or unexported fields
}

func NewIngestor ¶

func NewIngestor(db *sqlitex.Pool, idx *search.Index) *Ingestor

func NewPreviewIngestor ¶

func NewPreviewIngestor(db *sqlitex.Pool) *Ingestor

func (*Ingestor) Ingest ¶

func (ig *Ingestor) Ingest(ctx context.Context, data Ingestion, persist bool) ([]sirkulator.SimpleResource, error)

Ingest will merge the ingestion with locally matching resources before storing the data to db and trigger indexing of documents. if persist=false, nothing is persisted, and the returnet results represents a preview of which resources would have been stored if persist=true.

func (*Ingestor) IngestISBN ¶

func (ig *Ingestor) IngestISBN(ctx context.Context, id string, persist bool) ImportEntry

IngestISBN will try to ingest a publication and related resources given an ISBN-number, optionally persisting it to DB. TODO refactor to remove duplication

type UpdateSNLDescriptions ¶

type UpdateSNLDescriptions struct {
	DB *sqlitex.Pool
}

func (*UpdateSNLDescriptions) Name ¶

func (u *UpdateSNLDescriptions) Name() string

func (*UpdateSNLDescriptions) Run ¶

func (u *UpdateSNLDescriptions) Run(ctx context.Context, w io.Writer) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL