api

type Fetcher interface {
	Fetch(ctx context.Context, req *Request) (*Response, error)
	Close() error
}

type FilterRule struct {
	Hostname string
	Allow    []*regexp.Regexp
	Disallow []*regexp.Regexp
}

type MetricsMonitor interface {
	IncTotalRequests()
	IncSuccessfulRequests()
	IncFailedRequests()

	IncTotalLink()
	IncCrawledLink()
	IncSkippedLink()
	IncDuplicatedLink()

	Metrics() map[string]int64
}

type Param struct {
	Proxy       string
	UserAgent   string
	Referer     string
	MaxBodySize int64
	Timeout     time.Duration
}

type ParsedURL struct {
	Hash string
	Root string
	URL  *url.URL
}

type Queue interface {
	Push(ctx context.Context, req *Request) error
	Pop(ctx context.Context) (*Request, error)
	Len() int32
	Close() error
}

type RateLimit struct {
	Hostname string
	Rate     string
}

type Request struct {
	Target *ParsedURL
	Param  *Param
	Depth  int32
}

type Response struct {
	URL         *ParsedURL
	Status      int
	Body        []byte
	NextURLs    []*ParsedURL
	Depth       int32
	ElapsedTime time.Duration
	Err         error
}

type Store interface {
	HasVisited(ctx context.Context, u *ParsedURL) (bool, error)
	Close() error
}