pipeline

type Job struct {
	// Name is a human-readable name of the job.
	Name string
	// Description is a human-readable description for the job.
	Description string
	// Tries is the current number of performed tries
	Tries uint

	// MaximumTries is the maximum number of tries before ignoring the job.
	MaximumTries uint
	// Status is the current status of the job.
	Status Status
	// JobCompletedCallback is called with the resulting record whenever a job is completed.
	JobCompletedCallback JobCompletedCallback
	// JobFailedCallback is called with the error whenver a job fails.
	JobFailedCallback JobFailedCallback
	// PerformJobCallback is called whenever the job is performed.
	PerformJobCallback PerformJobCallback
	// contains filtered or unexported fields
}

type JobCompletedCallback func(job *Job, records ...*warc.Record)

type JobFailedCallback func(job *Job, err error)

type JobHandler func(job *Job) ([]*warc.Record, error)

type PerformJobCallback func(job *Job)

type Pool struct {
	Parallelism uint
	Workers     []*Worker
	// contains filtered or unexported fields
}

type Status int

type Worker struct {
	// contains filtered or unexported fields
}