spynet

spynet v0.1.0

Changelog

• Init

Libraries

• HTML parser

Examples

package main

import (
	"fmt"
	"net/http"
	
	"github.com/CURVoid/spynet/rule"
	"github.com/CURVoid/spynet"
)

func handler(r *http.Response) string {
	return r.Request.URL.String()
}

func main() {
	start := "https://books.toscrape.com/index.html"
	hosts := []string{"books.toscrape.com"}
	rules := []rule.Rule{F
		rule.New([]string{"category"}, []string{}),
		rule.New([]string{"catalogue"}, []string{"category"}),
	}

	crawler := spynet.New[string](start, hosts, rules, handler)

	outputs, err := crawler.Crawl()
	if err != nil {
		panic(err)
	}

	fmt.Printf("passed - %d\n", len(outputs))
	for _, output := range outputs {
		println(output)
	}
}

Roadmap

• add proxies support