internal

type Collector struct {
	*colly.Collector // extend colly.Collector
	// contains filtered or unexported fields
}

type Exporter struct {
	// contains filtered or unexported fields
}

type PipelineItem struct {
	Selector  string `toml:"selector"`
	Action    string `toml:"action"`
	Attribute string `toml:"attr"`
	Column    string `toml:"column,omitempty"`
	VisitOnce bool   `toml:"visit_once,omitempty"`
}

type Script struct {
	Main struct {
		URL            string   `toml:"url"`
		File           string   `toml:"file"`
		Format         string   `toml:"format"`
		UserAgent      string   `toml:"user_agent,omitempty"`
		Delay          int      `toml:"delay,omitempty"`
		AllowedDomains []string `toml:"allowed_domains,omitempty"`
	} `toml:"main"`
	Pipeline []*PipelineItem `toml:"pipeline"`
}

type ScriptParser struct{}