spynet v0.1.0
Changelog
Libraries
Examples
package main
import (
"fmt"
"net/http"
"github.com/CURVoid/netspy"
)
func handler(r *http.Response) string {
return r.Request.URL.String()
}
func main() {
start := "https://books.toscrape.com/index.html"
hosts := []string{"books.toscrape.com"}
rules := []netspy.Rule{
netspy.NewRule([]string{"category"}, []string{}),
netspy.NewRule([]string{"catalogue"}, []string{"category"}),
}
crawler := spynet.NewCrawler[string](start, hosts, rules, handler)
outputs, err := crawler.Crawl()
if err != nil {
panic(err)
}
fmt.Printf("passed - %d\n", len(outputs))
for _, output := range outputs {
println(output)
}
}
Roadmap