grab

package
v0.0.0-...-7116982 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 12, 2022 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36"

Functions

func Crawl

func Crawl(g *Grab)

func GrabMeta

func GrabMeta(g *Grab) error

func IdToUrl

func IdToUrl(id string) string

func PrintJson

func PrintJson(json map[string]interface{})

func UrlToId

func UrlToId(urlStr string) string

Reverses a url's domain. Can be better for storage id http://bar.foo.com:8983/to/index.html?a=b" becomes "com.foo.bar:http:8983/to/index.html?a=b"

Types

type Grab

type Grab struct {
	Url        string
	Data       []byte
	Html       Html
	Json       Json
	StatusCode int
	Headers    map[string]string
}

func GrabUrl

func GrabUrl(url string) (*Grab, error)

func (*Grab) Header

func (g *Grab) Header(key string) string

func (Grab) Item

func (g Grab) Item() *Item

func (Grab) Summary

func (g Grab) Summary() string

type Html

type Html struct {
	Data       string
	Meta       map[string]string
	Body       string
	Tags       []string
	Paragraphs []string
}

type InMemoryCookieJar

type InMemoryCookieJar struct {
	// contains filtered or unexported fields
}

func (InMemoryCookieJar) Cookies

func (jar InMemoryCookieJar) Cookies(u *url.URL) []*http.Cookie

func (InMemoryCookieJar) SetCookies

func (jar InMemoryCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie)

type Item

type Item struct {
	Id          string
	Title       string
	Description string
	ImageUrl    string
	Url         string
	Tags        []string
}

type Json

type Json struct {
	// contains filtered or unexported fields
}

func ParseJson

func ParseJson(bytes []byte) Json

func (Json) Float

func (json Json) Float(key interface{}) float64

func (Json) Get

func (json Json) Get(key interface{}) Json

func (Json) Int

func (json Json) Int(key interface{}) int

func (Json) Str

func (json Json) Str(key string) string

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL