commoncrawl

type BlobReader struct {
	// contains filtered or unexported fields
}

type Controller struct {
	// contains filtered or unexported fields
}

type ControllerConfiguration struct {
	AWSRegion         string             // Required - AWS region to use for AWS services
	TaskConfiguration *TaskConfiguration // Required - Input channel of filenames to process
	InputSource       InputSource        // Required - Produces work items to be processed
	EndOfProcess      chan<- bool        // Required - Output channel for end of process signal
	RecordReader      RecordReader       // Required - Reads an open file into a stream of records

	Plugins map[string]Plugin // Plugins that will provide specific mappers and reducers for processing

	// TODO: configure output writer from plugin
	OutputFileWriter OutputFileWriter   // Optional - Writer to save data to output files
	StatusWriter     StatusWriter       // Optional - Sends status notifications.
	HTTPClient       HTTPClient         // Optional - Client to make http calls to S3. Defaults to http.DefaultClient
	Clock            clockwork.Clock    // Optional - Pauses and checks the time. Defaults to an instance of RealClock
	Logger           logrus.FieldLogger // Optional - passes in a logger that will be used by all components

}

type DefaultWorkItemReader struct{}

type Downloader struct {
	// contains filtered or unexported fields
}

type FileDownload interface {
	// Open gets a handle on an open data file
	Open() (io.ReadCloser, error)
	// Filename returns the name of the data file
	Filename() string
	// WorkItemID returns the work item associated with the file
	WorkItemID() string
}

type FileDownloadFactory interface {
	// NewFileDownload returns a new FileDownload object
	//
	// * workItemID - Work item associated with the file
	// * filename - Full path to the file to download
	NewFileDownload(workItemID string, filename string) FileDownload
}

type FileMockHTTPClient struct {
	Clock   clockwork.Clock
	Latency time.Duration
}

type FileStatus struct {
	LastUpdate       time.Time
	RecordsParsed    int
	RecordsProcessed int
	OutputsMapped    int
	OutputsReduced   int
	EndOfFile        bool
}

type FilesystemInput struct {
	// contains filtered or unexported fields
}

type FlushRequest struct {
	WorkItemID string                                  // ID of work item data to flush
	Callback   chan<- map[string]map[string]OutputItem // Callback for flushed data
}

type HTTPClient interface {
	// Get issues a GET to the specified URL
	Get(url string) (*http.Response, error)
}

type HTTPFileDownload struct {
	// contains filtered or unexported fields
}

type HTTPFileDownloadFactory struct {
	// contains filtered or unexported fields
}

type InputSource interface {
	// Run launches its own goroutine.
	// Begins reading input and producing WorkItem instances for processing.
	Run()
	// WorkItemsOutput returns an output channel of WorkItem
	WorkItemsOutput() <-chan *WorkItem
	// CompletedWorkItemsInput returns an input channel of completed work item ids
	CompletedWorkItemsInput() chan<- string
}

type JSONReader struct {
	*BlobReader
	// contains filtered or unexported fields
}

type LambdaHandler struct {
	// contains filtered or unexported fields
}

type LocalFileDownload struct {
	// contains filtered or unexported fields
}

type LocalFileDownloadFactory struct {
}

type LocalOutputFileWriter struct{}

type LogStatusWriter struct {
	// contains filtered or unexported fields
}

type MockHTTPClient struct {
	mock.Mock
}

type MockInputSource struct {
	mock.Mock
}

type MockOutputFileWriter struct {
	mock.Mock
}

type MockStatusWriter struct {
	mock.Mock
}

type Monitor struct {
	// contains filtered or unexported fields
}

type OutputFileWriter interface {
	// Write out the specified file
	WriteOutputFile(filepath string, data []byte) error
}

type OutputFilter interface {
	// TestOutputItem returns true if the output item should be saved
	TestOutputItem(item OutputItem) bool
}

type OutputItem interface {
	// OutputSet indicates the output folder for the data
	OutputSet() string
	// Key is the unique identifier for this output.
	Key() string
	// Value is the data contained by the output. This can be the same object as the OutputItem.
	Value() interface{}
}

type OutputItems []OutputItem

type OutputReducer interface {
	// Reduce merges two OutputItems with the same key into a single item.
	// The details of how the merge is performed is dependent on the implementation.
	Reduce(existingItem OutputItem, newItem OutputItem) OutputItem
}

type OutputStore struct {
	// contains filtered or unexported fields
}

type OutputWriter struct {
	// contains filtered or unexported fields
}

type ParsedRecord struct {
	WorkItemID string      // Name of the work item associated with the file
	SourceFile string      // Name of the file where the record originated
	Record     interface{} // Contents of the record
}

type Parser struct {
	// contains filtered or unexported fields
}

type Plugin interface {
	// Creates a new plugin instance with the desired configuration
	NewInstance(config *PluginConfiguration) PluginInstance
}

type PluginConfiguration struct {
	PluginName   string            // Indicate which plugin to use
	ConfigData   map[string]string // Custom configuration data for the plugin
	OutputSet    string            // Name of output data set - subfolder will be named after this
	OutputShards int               // Number of output shards for this plugin configuration
}

type PluginInstance interface {
	// Mapper that will process map-phase records for this plugin
	Mapper() RecordMapper
	// Mapper that will process reduce-phase records for this plugin.
	// Reduce-phase records are json blobs of the output items produced
	// by the map-phase mapper.
	ReductionMapper() RecordMapper
	// Reducer to use for this plugin
	Reducer() OutputReducer
	// Filter for output before it is saved
	OutputFilter() OutputFilter
}

type ProcessStatus struct {
	LastUpdate       time.Time // Last time the process received any kind of update
	RecordsParsed    int       // Total # of records parsed
	RecordsProcessed int       // Total # of records fully processed
	OutputsMapped    int       // Total # of outputs mapped
	OutputsReduced   int       // Total # of outputs reduced
	FilesInProcess   int       // # of files currently being processed
	FilesParsed      int       // Total # of files parsed
	OutputFilesSaved int       // Total # of output files that have been saved
	ProcessComplete  bool      // True if process is completed
}

type RecordMapper interface {
	// Maps a record into zero to many OutputItems.
	//
	// * record - A record object containing data about a webpage
	// * output - Each OutputItem that is sent to this callback will be reduced in the output.
	MapRecord(record interface{}, output func(OutputItem))
}

type RecordOutput struct {
	WorkItemID string     // Name of the work item associated with the file
	SourceFile string     // Name of the file where the data originated
	OutputItem OutputItem // Output data
}

type RecordProcessor struct {
	// contains filtered or unexported fields
}

type RecordReader interface {
	// Reads a file and sends a stream of record objects to callback
	//
	// * file - The open file that should be read
	// * callback - Should be called for each record that is read
	Read(file io.ReadCloser, filename string, callback func(interface{}))
}

type S3FileDownload struct {
	// contains filtered or unexported fields
}

type S3FileDownloadFactory struct {
	// contains filtered or unexported fields
}

type S3OutputFileWriter struct {
	// contains filtered or unexported fields
}

type S3WorkItemReader struct {
	// contains filtered or unexported fields
}

type SQSInput struct {
	// contains filtered or unexported fields
}

type SingleInputSource struct {
	// contains filtered or unexported fields
}

type StatusWriter interface {
	// WriteStatus sends a notification about the current state of the system.
	WriteStatus(status ProcessStatus)
}

type TaskConfiguration struct {
	// How many threads are used for parsing data files
	DownloadPoolSize int
	// S3 bucket for input files. Necessary for pulling data from private buckets.
	InputBucket          string
	OutputBucket         string
	OutputFolder         string
	Phase                string
	InputFilename        string
	DatasetHost          string // Used to download non-commoncrawl data files through http
	PluginConfigurations []*PluginConfiguration
}

type WARCReader struct {
	// contains filtered or unexported fields
}

type WorkItem struct {
	WorkItemID  string   // ID of the work item
	SourceFiles []string // Source files to be processed
}

type WorkItemReader interface {
	// ReadWorkItem parses the message into a `WorkItem` instance
	ReadWorkItem(message string) (*WorkItem, error)
}

type WorkItemStatus struct {
	WorkItem       *WorkItem
	CompletedFiles []string
}