parser

package

v0.0.0-...-39d03c2 Latest Latest Go to latest Published: Jan 9, 2022 License: MIT Imports: 16 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/scrapnode/scrapnode

Documentation ¶

Index ¶

Constants
func ExecFilters(data interface{}, refs map[string]interface{}, functions []string) string
func GetInt(data interface{}) int64
func GetRef(ref map[string]interface{}, value string) string
func GetText(data interface{}) string
func Md5(data string) string
func New(engine ParserEngine, options map[string]interface{}) pipeline.Pipeline
func Scope(data string, scope string) string
func Url(data string, base string) string
type DynamicParser
- func (parser *DynamicParser) Exec(ctx *context.Context) (*context.Context, error)
- func (parser *DynamicParser) SetNext(next pipeline.Pipeline)
type ExtractorPropSchema
- func ParseSchema(schema string) ExtractorPropSchema
type ExtractorSchema
type HTMLExtractor
- func (extractor *HTMLExtractor) GetBlocks(node *html.Node) []*html.Node
- func (extractor *HTMLExtractor) GetProps(node *html.Node) map[string]interface{}
type HTMLParser
- func (parser *HTMLParser) Exec(ctx *context.Context) (*context.Context, error)
- func (parser *HTMLParser) SetNext(next pipeline.Pipeline)
type HTMLParserMethods
- func (methods *HTMLParserMethods) GetAttr(node *html.Node, key string) string
- func (methods *HTMLParserMethods) GetHTML(node *html.Node) string
- func (methods *HTMLParserMethods) GetText(node *html.Node, buf *bytes.Buffer) string
type JSONExtractor
- func (extractor *JSONExtractor) GetBlocks(node map[string]interface{}) []map[string]interface{}
- func (extractor *JSONExtractor) GetProps(node map[string]interface{}) map[string]interface{}
type JSONParser
- func (parser *JSONParser) Exec(ctx *context.Context) (*context.Context, error)
- func (parser *JSONParser) SetNext(next pipeline.Pipeline)
type JSONParserMethods
- func (methods *JSONParserMethods) GetInt(node interface{}) int64
- func (methods *JSONParserMethods) GetRaw(node interface{}) interface{}
- func (methods *JSONParserMethods) GetText(node interface{}) string
type Parser
type ParserEngine
type RefsSetter
- func (setter *RefsSetter) Set(item map[string]interface{}, schema map[string]interface{})
type ValuesSetter
- func (setter *ValuesSetter) Set(item map[string]interface{}, schema map[string]interface{})

Constants ¶

View Source

const (
	MetaItemSchema   string = "__PARSER_ITEM_SCHEMA__"
	MetaGlobalSchema string = "__PARSER_GLOBAL_SCHEMA__"
)

View Source

const (
	Separator     = "::"
	BlockSelector = "block_selectors"
	PropsSelector = "props_selectors"
	RefsSetters   = "refs_setters"
	ValuesSetters = "values_setters"
	RootSelector  = "$root"
)

Variables ¶

This section is empty.

Functions ¶

func ExecFilters ¶

func ExecFilters(data interface{}, refs map[string]interface{}, functions []string) string

func GetInt ¶

func GetInt(data interface{}) int64

func GetRef ¶

func GetRef(ref map[string]interface{}, value string) string

func GetText ¶

func GetText(data interface{}) string

func Md5 ¶

func Md5(data string) string

func New ¶

func New(engine ParserEngine, options map[string]interface{}) pipeline.Pipeline

func Scope ¶

func Scope(data string, scope string) string

func Url ¶

func Url(data string, base string) string

Types ¶

type DynamicParser ¶

type DynamicParser struct {
	// contains filtered or unexported fields
}

func (*DynamicParser) Exec ¶

func (parser *DynamicParser) Exec(ctx *context.Context) (*context.Context, error)

func (*DynamicParser) SetNext ¶

func (parser *DynamicParser) SetNext(next pipeline.Pipeline)

type ExtractorPropSchema ¶

type ExtractorPropSchema struct {
	Selector string
	Method   string
	Attr     string
	Filters  []string
}

func ParseSchema ¶

func ParseSchema(schema string) ExtractorPropSchema

type ExtractorSchema ¶

type ExtractorSchema struct {
	Block string
	Props map[string]interface{}
}

type HTMLExtractor ¶

type HTMLExtractor struct {
	Refs   map[string]interface{}
	Schema ExtractorSchema
	// contains filtered or unexported fields
}

func (*HTMLExtractor) GetBlocks ¶

func (extractor *HTMLExtractor) GetBlocks(node *html.Node) []*html.Node

func (*HTMLExtractor) GetProps ¶

func (extractor *HTMLExtractor) GetProps(node *html.Node) map[string]interface{}

type HTMLParser ¶

type HTMLParser struct {
	Options map[string]interface{}
	// contains filtered or unexported fields
}

func (*HTMLParser) Exec ¶

func (parser *HTMLParser) Exec(ctx *context.Context) (*context.Context, error)

func (*HTMLParser) SetNext ¶

func (parser *HTMLParser) SetNext(next pipeline.Pipeline)

type HTMLParserMethods ¶

type HTMLParserMethods struct {
}

func (*HTMLParserMethods) GetAttr ¶

func (methods *HTMLParserMethods) GetAttr(node *html.Node, key string) string

func (*HTMLParserMethods) GetHTML ¶

func (methods *HTMLParserMethods) GetHTML(node *html.Node) string

func (*HTMLParserMethods) GetText ¶

func (methods *HTMLParserMethods) GetText(node *html.Node, buf *bytes.Buffer) string

type JSONExtractor ¶

type JSONExtractor struct {
	Refs   map[string]interface{}
	Schema ExtractorSchema
	// contains filtered or unexported fields
}

func (*JSONExtractor) GetBlocks ¶

func (extractor *JSONExtractor) GetBlocks(node map[string]interface{}) []map[string]interface{}

func (*JSONExtractor) GetProps ¶

func (extractor *JSONExtractor) GetProps(node map[string]interface{}) map[string]interface{}

type JSONParser ¶

type JSONParser struct {
	Options map[string]interface{}
	// contains filtered or unexported fields
}

func (*JSONParser) Exec ¶

func (parser *JSONParser) Exec(ctx *context.Context) (*context.Context, error)

func (*JSONParser) SetNext ¶

func (parser *JSONParser) SetNext(next pipeline.Pipeline)

type JSONParserMethods ¶

type JSONParserMethods struct {
}

func (*JSONParserMethods) GetInt ¶

func (methods *JSONParserMethods) GetInt(node interface{}) int64

func (*JSONParserMethods) GetRaw ¶

func (methods *JSONParserMethods) GetRaw(node interface{}) interface{}

func (*JSONParserMethods) GetText ¶

func (methods *JSONParserMethods) GetText(node interface{}) string

type Parser ¶

type Parser interface {
	Exec(mctx *context.Context) (*context.Context, error)
}

type ParserEngine ¶

type ParserEngine string

const (
	EngineHTML    ParserEngine = "html"
	EngineJSON    ParserEngine = "json"
	EngineDynamic ParserEngine = "dynamic"
)

type RefsSetter ¶

type RefsSetter struct {
	Refs map[string]interface{}
}

func (*RefsSetter) Set ¶

func (setter *RefsSetter) Set(item map[string]interface{}, schema map[string]interface{})

type ValuesSetter ¶

type ValuesSetter struct {
}

func (*ValuesSetter) Set ¶

func (setter *ValuesSetter) Set(item map[string]interface{}, schema map[string]interface{})

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL