Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
var DefaultBasicAuthConfig = BasicAuthConfig{}
DefaultBasicAuthConfig is the default BasicAuth middleware config.
var DefaultDecoderConfig struct{}
DefaultDecoderConfig is the default Decoder middleware config.
var DefaultDownLimiterConfig = DownLimiterConfig{ ConcurrentRequestsPerDomain: 1024, DownloadDelayPerDomain: 0, Timeout: 3 * time.Second, }
DefaultDownLimiterConfig is the default DownLimiter middleware config.
var DefaultLoggingConfig struct{}
DefaultLoggingConfig is the default Logging middleware config.
var DefaultRetryConfig = RetryConfig{
RetryTimes: 3,
}
DefaultRetryConfig is the default Retry middleware config.
var DefaultStatsCollectorConfig = StatsCollectorConfig{}
DefaultStatsCollectorConfig is the default StatsCollector middleware config.
Functions ¶
This section is empty.
Types ¶
type BasicAuth ¶
type BasicAuth struct {
// contains filtered or unexported fields
}
BasicAuth for decode response body to utf8
func NewBasicAuthWithConfig ¶
func NewBasicAuthWithConfig(config BasicAuthConfig) *BasicAuth
NewBasicAuthWithConfig returns a BasicAuth middleware with config. See: `NewBasicAuth()`.
func (*BasicAuth) Next ¶
func (p *BasicAuth) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.Middleware.Next
type BasicAuthConfig ¶
BasicAuthConfig defines the config for BasicAuth middleware.
type Decoder ¶
type Decoder struct {
// contains filtered or unexported fields
}
Decoder for decode response body to utf8
func NewDecoderWithConfig ¶
func NewDecoderWithConfig(config DecoderConfig) *Decoder
NewDecoderWithConfig returns a Decoder middleware with config. See: `NewDecoder()`.
func (*Decoder) Next ¶
func (p *Decoder) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.Middleware.Next
type DecoderConfig ¶
type DecoderConfig struct{}
DecoderConfig defines the config for Decoder middleware.
type DownLimiter ¶
type DownLimiter struct {
// contains filtered or unexported fields
}
DownLimiter for limit downloader
func NewDownLimiter ¶
func NewDownLimiter() *DownLimiter
NewDownLimiter returns a DownLimiter instance
func NewDownLimiterWithConfig ¶
func NewDownLimiterWithConfig(config DownLimiterConfig) *DownLimiter
NewDownLimiterWithConfig returns a DownLimiter middleware with config. See: `DownLimiter()`.
func (*DownLimiter) Next ¶
func (m *DownLimiter) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.Middleware.Next
type DownLimiterConfig ¶
type DownLimiterConfig struct {
// ConcurrentRequestsPerDomain 每个域名下请求并行限制
ConcurrentRequestsPerDomain uint64
// DownloadDelayPerDomain 每个域名下请求的延时
DownloadDelayPerDomain time.Duration
// Timeout 请求超时
Timeout time.Duration
}
DownLimiterConfig defines the config for DownLimiter middleware.
type Logging ¶
type Logging struct {
// contains filtered or unexported fields
}
Logging log access
func NewLoggingWithConfig ¶
func NewLoggingWithConfig(config LoggingConfig) *Logging
NewLoggingWithConfig returns a Logging middleware with config. See: `NewLogging()`.
func (*Logging) Next ¶
func (m *Logging) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.Next
type LoggingConfig ¶
type LoggingConfig struct{}
LoggingConfig defines the config for Logging middleware.
type Retry ¶
type Retry struct {
RetryConfig
}
Retry defines a middleware which Retrys from panics anywhere in the chain
func NewRetryWithConfig ¶
func NewRetryWithConfig(config RetryConfig) *Retry
NewRetryWithConfig returns a Retry middleware with config. See: `NewRetry()`.
func (*Retry) Next ¶
func (m *Retry) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.Middleware.Next
type RetryConfig ¶
type RetryConfig struct {
RetryTimes int
}
RetryConfig defines the config for Retry middleware.
type StatsCollector ¶
type StatsCollector struct {
// contains filtered or unexported fields
}
StatsCollector for collecting scraping stats
func NewStatsCollector ¶
func NewStatsCollector() *StatsCollector
NewStatsCollector returns a StatsCollector instance
func NewStatsCollectorWithConfig ¶
func NewStatsCollectorWithConfig(config StatsCollectorConfig) *StatsCollector
NewStatsCollectorWithConfig returns a StatsCollector middleware with config. See: `NewStatsCollector()`.
func (*StatsCollector) Close ¶
func (s *StatsCollector) Close(c *mellivora.Engine)
Close implement mellivora.Closer
func (*StatsCollector) Next ¶
func (s *StatsCollector) Next(handleFunc mellivora.MiddlewareFunc) mellivora.MiddlewareFunc
Next implement mellivora.StatsCollector.Next
type StatsCollectorConfig ¶
type StatsCollectorConfig struct{}
StatsCollectorConfig defines the config for StatsCollector middleware.
Source Files