warc

type CustomHTTPClient struct {
	http.Client
	WARCWriter             chan *RecordBatch
	WARCWriterDoneChannels []chan bool
	WaitGroup              *WaitGroupWithCount

	ErrChan chan *Error

	TempDir               string
	FullOnDisk            bool
	MaxReadBeforeTruncate int
	DataTotal             *ratecounter.Counter
	// contains filtered or unexported fields
}

type DedupeOptions struct {
	LocalDedupe   bool
	CDXDedupe     bool
	CDXURL        string
	CDXCookie     string
	SizeThreshold int
}

type Error struct {
	Err  error
	Func string
}

type HTTPClientSettings struct {
	RotatorSettings       *RotatorSettings
	DedupeOptions         DedupeOptions
	Proxy                 string
	DecompressBody        bool
	SkipHTTPStatusCodes   []int
	VerifyCerts           bool
	TempDir               string
	FullOnDisk            bool
	MaxReadBeforeTruncate int
	FollowRedirects       bool
	TCPTimeout            time.Duration
	TLSHandshakeTimeout   time.Duration
	RandomLocalIP         bool
}

type Header map[string]string

type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	ReaderAt
	io.Closer
	FileName() string
}

type ReadWriteSeekCloser interface {
	ReadSeekCloser
	io.Writer
}

type Reader struct {
	// contains filtered or unexported fields
}

type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err error)
}

type Record struct {
	Header  Header
	Content ReadWriteSeekCloser
}

type RecordBatch struct {
	Records     []*Record
	Done        chan bool
	CaptureTime string
}

type RotatorSettings struct {
	// Content of the warcinfo record that will be written
	// to all WARC files
	WarcinfoContent Header
	// Prefix used for WARC filenames, WARC 1.1 specifications
	// recommend to name files this way:
	// Prefix-Timestamp-Serial-Crawlhost.warc.gz
	Prefix string
	// Compression algorithm to use
	Compression string
	// WarcSize is in MegaBytes
	WarcSize float64
	// Directory where the created WARC files will be stored,
	// default will be the current directory
	OutputDirectory string
	// WARCWriterPoolSize defines the number of parallel WARC writers
	WARCWriterPoolSize int
}

type WaitGroupWithCount struct {
	sync.WaitGroup
	// contains filtered or unexported fields
}

type Writer struct {
	FileName     string
	Compression  string
	GZIPWriter   *gzip.Writer
	PGZIPWriter  *pgzip.Writer
	ZSTDWriter   *zstd.Encoder
	FileWriter   *bufio.Writer
	ParallelGZIP bool
}