Documentation ¶
Overview ¶
Package arachne is a generated GoMock package.
Package arachne is a generated GoMock package.
Package arachne is a generated GoMock package.
Index ¶
- type HTTPClient
- type LogLevel
- type Logger
- type MockHTTPClient
- type MockHTTPClientMockRecorder
- type MockLogger
- func (m *MockLogger) Criticalf(format string, v ...interface{})
- func (m *MockLogger) Debugf(format string, v ...interface{})
- func (m *MockLogger) EXPECT() *MockLoggerMockRecorder
- func (m *MockLogger) Errorf(format string, v ...interface{})
- func (m *MockLogger) Infof(format string, v ...interface{})
- func (m *MockLogger) Warnf(format string, v ...interface{})
- type MockLoggerMockRecorder
- func (mr *MockLoggerMockRecorder) Criticalf(format interface{}, v ...interface{}) *gomock.Call
- func (mr *MockLoggerMockRecorder) Debugf(format interface{}, v ...interface{}) *gomock.Call
- func (mr *MockLoggerMockRecorder) Errorf(format interface{}, v ...interface{}) *gomock.Call
- func (mr *MockLoggerMockRecorder) Infof(format interface{}, v ...interface{}) *gomock.Call
- func (mr *MockLoggerMockRecorder) Warnf(format interface{}, v ...interface{}) *gomock.Call
- type MockWorkerQueue
- type MockWorkerQueueMockRecorder
- type Request
- type Response
- type Worker
- type WorkerQueue
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type HTTPClient ¶
HTTPClient is like http.Client
type LogLevel ¶
type LogLevel int
LogLevel : DebugLevel, InfoLevel, WarnLevel, ErrorLevel, CriticalLevel
type Logger ¶
type Logger interface { Debugf(format string, v ...interface{}) Infof(format string, v ...interface{}) Warnf(format string, v ...interface{}) Errorf(format string, v ...interface{}) Criticalf(format string, v ...interface{}) }
Logger is interface for logging in arachne crawler.
type MockHTTPClient ¶
type MockHTTPClient struct {
// contains filtered or unexported fields
}
MockHTTPClient is a mock of HTTPClient interface
func NewMockHTTPClient ¶
func NewMockHTTPClient(ctrl *gomock.Controller) *MockHTTPClient
NewMockHTTPClient creates a new mock instance
func (*MockHTTPClient) EXPECT ¶
func (m *MockHTTPClient) EXPECT() *MockHTTPClientMockRecorder
EXPECT returns an object that allows the caller to indicate expected use
type MockHTTPClientMockRecorder ¶
type MockHTTPClientMockRecorder struct {
// contains filtered or unexported fields
}
MockHTTPClientMockRecorder is the mock recorder for MockHTTPClient
func (*MockHTTPClientMockRecorder) Do ¶
func (mr *MockHTTPClientMockRecorder) Do(req interface{}) *gomock.Call
Do indicates an expected call of Do
type MockLogger ¶
type MockLogger struct {
// contains filtered or unexported fields
}
MockLogger is a mock of Logger interface
func NewMockLogger ¶
func NewMockLogger(ctrl *gomock.Controller) *MockLogger
NewMockLogger creates a new mock instance
func (*MockLogger) Criticalf ¶
func (m *MockLogger) Criticalf(format string, v ...interface{})
Criticalf mocks base method
func (*MockLogger) Debugf ¶
func (m *MockLogger) Debugf(format string, v ...interface{})
Debugf mocks base method
func (*MockLogger) EXPECT ¶
func (m *MockLogger) EXPECT() *MockLoggerMockRecorder
EXPECT returns an object that allows the caller to indicate expected use
func (*MockLogger) Errorf ¶
func (m *MockLogger) Errorf(format string, v ...interface{})
Errorf mocks base method
func (*MockLogger) Infof ¶
func (m *MockLogger) Infof(format string, v ...interface{})
Infof mocks base method
func (*MockLogger) Warnf ¶
func (m *MockLogger) Warnf(format string, v ...interface{})
Warnf mocks base method
type MockLoggerMockRecorder ¶
type MockLoggerMockRecorder struct {
// contains filtered or unexported fields
}
MockLoggerMockRecorder is the mock recorder for MockLogger
func (*MockLoggerMockRecorder) Criticalf ¶
func (mr *MockLoggerMockRecorder) Criticalf(format interface{}, v ...interface{}) *gomock.Call
Criticalf indicates an expected call of Criticalf
func (*MockLoggerMockRecorder) Debugf ¶
func (mr *MockLoggerMockRecorder) Debugf(format interface{}, v ...interface{}) *gomock.Call
Debugf indicates an expected call of Debugf
func (*MockLoggerMockRecorder) Errorf ¶
func (mr *MockLoggerMockRecorder) Errorf(format interface{}, v ...interface{}) *gomock.Call
Errorf indicates an expected call of Errorf
func (*MockLoggerMockRecorder) Infof ¶
func (mr *MockLoggerMockRecorder) Infof(format interface{}, v ...interface{}) *gomock.Call
Infof indicates an expected call of Infof
func (*MockLoggerMockRecorder) Warnf ¶
func (mr *MockLoggerMockRecorder) Warnf(format interface{}, v ...interface{}) *gomock.Call
Warnf indicates an expected call of Warnf
type MockWorkerQueue ¶
type MockWorkerQueue struct {
// contains filtered or unexported fields
}
MockWorkerQueue is a mock of WorkerQueue interface
func NewMockWorkerQueue ¶
func NewMockWorkerQueue(ctrl *gomock.Controller) *MockWorkerQueue
NewMockWorkerQueue creates a new mock instance
func (*MockWorkerQueue) EXPECT ¶
func (m *MockWorkerQueue) EXPECT() *MockWorkerQueueMockRecorder
EXPECT returns an object that allows the caller to indicate expected use
func (*MockWorkerQueue) PublishRequest ¶
func (m *MockWorkerQueue) PublishRequest(request *Request) error
PublishRequest mocks base method
func (*MockWorkerQueue) RetryRequest ¶
func (m *MockWorkerQueue) RetryRequest(request *Request) error
RetryRequest mocks base method
func (*MockWorkerQueue) SubscribeRequests ¶
func (m *MockWorkerQueue) SubscribeRequests(ctx context.Context) (<-chan *Request, error)
SubscribeRequests mocks base method
type MockWorkerQueueMockRecorder ¶
type MockWorkerQueueMockRecorder struct {
// contains filtered or unexported fields
}
MockWorkerQueueMockRecorder is the mock recorder for MockWorkerQueue
func (*MockWorkerQueueMockRecorder) PublishRequest ¶
func (mr *MockWorkerQueueMockRecorder) PublishRequest(request interface{}) *gomock.Call
PublishRequest indicates an expected call of PublishRequest
func (*MockWorkerQueueMockRecorder) RetryRequest ¶
func (mr *MockWorkerQueueMockRecorder) RetryRequest(request interface{}) *gomock.Call
RetryRequest indicates an expected call of RetryRequest
func (*MockWorkerQueueMockRecorder) SubscribeRequests ¶
func (mr *MockWorkerQueueMockRecorder) SubscribeRequests(ctx interface{}) *gomock.Call
SubscribeRequests indicates an expected call of SubscribeRequests
type Request ¶
type Request struct { URL string Method string Header http.Header Body []byte Priority int64 QueueName string Meta map[string]interface{} // contains filtered or unexported fields }
Request is a domain model that represents http request.
func NewGetRequest ¶
NewGetRequest creates simple GET request.
func NewRequestFromHTTPRequest ¶
NewRequestFromHTTPRequest constructs Request from http.Request
func (*Request) BodyReader ¶
BodyReader returns io.Reader of Body
func (*Request) HTTPRequest ¶
HTTPRequest constructs http.Request from Request
type Response ¶
Response is a domain model that represents http response.
func NewResponseFromHTTPResponse ¶
NewResponseFromHTTPResponse constructs Response from http.Response
func (*Response) ContentType ¶
ContentType returns detected Content-Type of the response body. If it cannot determine a more specific one, it returns "application/octet-stream".
func (*Response) Follow ¶
Follow creates a url whose url schema and host is the same as those of response.
func (*Response) FollowRequest ¶
FollowRequest creates a simple GET request whose the schema and the host of the url is the same as those of response.
type Worker ¶
type Worker struct { WorkerQueue WorkerQueue HTTPClient HTTPClient Logger Logger RequestMiddlewares []func(request *Request) ResponseMiddlewares []func(response *Response) Spider func(response *Response) ([]*Request, error) }
Worker handles scheduled requests.
func (*Worker) RetryMiddleware ¶
RetryMiddleware is request middleware that remove request in worker pipeline and send request to worker queue if Request.Meta['retry'] flas is true.