_sedefaults

func DoGetRequest ¶

func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name) error

func DoPostRequest ¶

func DoPostRequest(urll string, requestData io.Reader, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name) error

func FieldsFromDOM ¶ added in v0.12.1

func FieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)

Fetches from DOM via dompaths and sanitizes. Returns (url, title, description).

func InitializeCollectors ¶

func InitializeCollectors(ctx context.Context, engineName engines.Name, options engines.Options, settings config.Settings, timings config.Timings, relay *bucket.Relay) (*colly.Collector, *colly.Collector)

it's okay to return pointers to collectors since colly.NewCollector() returns a pointer

func PageFromContext ¶

func PageFromContext(ctx *colly.Context, seName engines.Name) int

func Prepare ¶

func Prepare(ctx context.Context, info engines.Info, support engines.SupportedSettings, options *engines.Options, settings *config.Settings) (context.Context, error)

sending options and settings as pointers since they are modified

func RawFieldsFromDOM ¶ added in v0.12.1

func RawFieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)

Fetches from DOM via dompaths. Returns (url, title, description).

func SanitizeDescription ¶ added in v0.12.1

func SanitizeDescription(descText string) string

func SanitizeFields ¶ added in v0.12.1

func SanitizeFields(linkText string, titleText string, descText string) (string, string, string)

func SanitizeTitle ¶ added in v0.12.1

func SanitizeTitle(titleText string) string

func SanitizeURL ¶ added in v0.12.1

func SanitizeURL(urlText string) string