gotana

type DAO interface {
	SaveItem(name string, data []byte) error
	GetItems(name string) []string
	CountItems(name string) int64
	ProcessItem(items string) genericStruct
	ProcessItems(items []string) []genericStruct
}

type DisplayExtension struct {
}

type Engine struct {
	Meta   *EngineMeta
	Config *ScraperConfig
	// contains filtered or unexported fields
}

type EngineMeta struct {
	ScraperStats  map[string]*ScraperMeta
	Started       time.Time
	RequestsTotal int
	LastRequest   *http.Request
	LastResponse  *http.Response
	// contains filtered or unexported fields
}

type Extension interface {
	ScraperStarted(scraper *Scraper)
	ScraperStopped(scraper *Scraper)
	ItemScraped(scraper *Scraper, item SaveableItem)
}

type Extractable interface {
	Extract(io.ReadCloser, func(string))
}

type HealthCheckResource struct {
	// contains filtered or unexported fields
}

type LinkExtractor struct {
	Extractable
}

type ListByScraperResource struct {
	// contains filtered or unexported fields
}

type RedisDAO struct {
	// contains filtered or unexported fields
}

type RequestMiddlewareFunc func(request *http.Request) *http.Request

type SaveInRedisExtension struct {
}

type SaveableItem interface {
	Scraper() *Scraper
	Validate() bool
	RecordData() ([]byte, error)
}

type ScrapedItem struct {
	Url      string
	FinalUrl string `json:"-"`

	BodyBytes []byte `json:"-"`
	// contains filtered or unexported fields
}

type Scraper struct {
	Name       string
	Domain     string
	Scheme     string
	BaseUrl    string
	CurrentUrl string
	// contains filtered or unexported fields
}

type ScraperConfig struct {
	Project      string `required:"true"`
	HttpAddress  string
	TcpAddress   string
	RedisAddress string
	Scrapers     []struct {
		RequestLimit int `required:"true"`
		Extractor    string
		Name         string `required:"true"`
		Url          string `required:"true"`
		Patterns     []struct {
			Type    string `required:"true"`
			Pattern string `required:"true"`
		}
	}
}

type ScraperMeta struct {
	// contains filtered or unexported fields
}

type ScraperMixin struct {
	Proxy ScrapedItem
}

type ScraperParams struct {
	Name         string
	Url          string
	RequestLimit int
	Extractor    Extractable
}

type ScrapingHandlerFunc func(ScrapedItem, chan<- SaveableItem)

type StatsResource struct {
	// contains filtered or unexported fields
}

type TCPCommand func(message string, conn net.Conn, server *TCPServer)

type TCPMessage struct {
	// contains filtered or unexported fields
}

type TCPServer struct {
	// contains filtered or unexported fields
}

type URLPattern struct {
	Type    string `required:"true"`
	Pattern string `required:"true"`
}