parquet

type BloomFilter interface {
	// Implement the io.ReaderAt interface as a mechanism to allow reading the
	// raw bits of the filter.
	io.ReaderAt

	// Returns the size of the bloom filter (in bytes).
	Size() int64

	// Tests whether the given value is present in the filter.
	//
	// A non-nil error may be returned if reading the filter failed. This may
	// happen if the filter was lazily loaded from a storage medium during the
	// call to Check for example. Applications that can guarantee that the
	// filter was in memory at the time Check was called can safely ignore the
	// error, which would always be nil in this case.
	Check(value Value) (bool, error)
}

type BloomFilterColumn interface {
	// Returns the path of the column that the filter applies to.
	Path() []string

	// Returns the hashing algorithm used when inserting values into a bloom
	// filter.
	Hash() bloom.Hash

	// NewFilter constructs a new bloom filter configured to hold the given
	// number of values and bits of filter per value.
	NewFilter(numValues int64, bitsPerValue uint) bloom.MutableFilter
}

type Buffer struct {
	// contains filtered or unexported fields
}

type BufferedPage interface {
	Page

	// Returns a copy of the page which does not share any of the buffers, but
	// contains the same values, repetition and definition levels.
	Clone() BufferedPage

	// Returns a new page which is as slice of the receiver between row indexes
	// i and j.
	Slice(i, j int64) BufferedPage

	// Expose the lists of repetition and definition levels of the page.
	//
	// The returned slices may be empty when the page has no repetition or
	// definition levels.
	RepetitionLevels() []int8
	DefinitionLevels() []int8

	// Writes the page to the given encoder.
	WriteTo(encoding.Encoder) error
}

type Column struct {
	// contains filtered or unexported fields
}

type ColumnBuffer interface {
	// Exposes a read-only view of the column buffer.
	ColumnChunk

	// Allows reading rows back from the column by calling ReadRowAt.
	RowReaderAt

	// Provides the main mechanism for writing values to the column via the
	// WriteRow method. WriteRow must be called with a row containing only
	// the value for this column; unless this is a repeated column, the row
	// must contain a single value.
	RowWriter

	// The column implements ValueWriter as a mechanism to optimize the copy
	// of values into the buffer in contexts where the row information is
	// provided by the values because the repetition and definition levels
	// are set.
	ValueWriter

	// For indexed columns, returns the underlying dictionary holding the column
	// values. If the column is not indexed, nil is returned.
	Dictionary() Dictionary

	// Returns a copy of the column. The returned copy shares no memory with
	// the original, mutations of either column will not modify the other.
	Clone() ColumnBuffer

	// Returns the column as a BufferedPage.
	Page() BufferedPage

	// Clears all rows written to the column.
	Reset()

	// Returns the current capacity of the column (rows).
	Cap() int

	// Returns the number of rows currently written to the column.
	Len() int

	// Compares rows at index i and j and reports whether i < j.
	Less(i, j int) bool

	// Swaps rows at index i and j.
	Swap(i, j int)

	// Returns the size of the column buffer in bytes.
	Size() int64
}

type ColumnChunk interface {
	// Returns the column type.
	Type() Type

	// Returns the index of this column in its parent row group.
	Column() int

	// Returns a reader exposing the pages of the column.
	Pages() Pages

	// Returns the components of the page index for this column chunk,
	// containing details about the content and location of pages within the
	// chunk.
	//
	// Note that the returned value may be the same across calls to these
	// methods, programs must treat those as read-only.
	//
	// If the column chunk does not have a page index, the methods return nil.
	ColumnIndex() ColumnIndex
	OffsetIndex() OffsetIndex
	BloomFilter() BloomFilter

	// Returns the number of values in the column chunk.
	//
	// This quantity may differ from the number of rows in the parent row group
	// because repeated columns may hold zero or more values per row.
	NumValues() int64
}

type ColumnIndex interface {
	// NumPages returns the number of paged in the column index.
	NumPages() int

	// Returns the number of null values in the page at the given index.
	NullCount(int) int64

	// Tells whether the page at the given index contains null values only.
	NullPage(int) bool

	// PageIndex return min/max bounds for the page at the given index in the
	// column.
	MinValue(int) Value
	MaxValue(int) Value

	// IsAscending returns true if the column index min/max values are sorted
	// in ascending order (based on the ordering rules of the column's logical
	// type).
	IsAscending() bool

	// IsDescending returns true if the column index min/max values are sorted
	// in descending order (based on the ordering rules of the column's logical
	// type).
	IsDescending() bool
}

type ColumnIndexer interface {
	// Resets the column indexer state.
	Reset()

	// Add a page to the column indexer.
	IndexPage(numValues, numNulls int64, min, max Value)

	// Generates a format.ColumnIndex value from the current state of the
	// column indexer.
	//
	// The returned value may reference internal buffers, in which case the
	// values remain valid until the next call to IndexPage or Reset on the
	// column indexer.
	ColumnIndex() format.ColumnIndex
}

type ColumnReader interface {
	ValueReader

	// Returns the type of values read.
	Type() Type

	// Returns the column number of values read.
	Column() int

	// Resets the reader state to read values from the given decoder.
	//
	// Column readers created from parquet types are initialized to an empty
	// state and will return io.EOF on every read until a decoder is installed
	// via a call to Reset.
	Reset(decoder encoding.Decoder)
}

type CompressedPage interface {
	Page

	// Returns a representation of the page header.
	PageHeader() PageHeader

	// Returns a reader exposing the content of the compressed page.
	PageData() io.Reader

	// Returns the size of the page data.
	PageSize() int64

	// CRC returns the IEEE CRC32 checksum of the page.
	CRC() uint32
}

type Conversion interface {
	// Applies the conversion logic on the src row, returning the result
	// appended to dst.
	Convert(dst, src Row) (Row, error)
	// Converts the given column index in the target schema to the original
	// column index in the source schema of the conversion.
	Column(int) int
	// Returns the target schema of the conversion.
	Schema() *Schema
}

type ConvertError struct {
	Reason string
	Path   []string
	From   Node
	To     Node
}

type DataPageHeader interface {
	PageHeader

	// Returns whether the page is compressed, according to the codec given as
	// argument and details stored in the page header.
	IsCompressed(format.CompressionCodec) bool

	// Returns the encoding of the repetition level section.
	RepetitionLevelEncoding() format.Encoding

	// Returns the encoding of the definition level section.
	DefinitionLevelEncoding() format.Encoding

	// Returns the number of null values in the page.
	NullCount() int64

	// Returns the minimum value in the page based on the ordering rules of the
	// column's logical type.
	//
	// As an optimization, the method may return the same slice across multiple
	// calls. Programs must treat the returned value as immutable to prevent
	// unpredictable behaviors.
	//
	// If the page only contains only null values, an empty slice is returned.
	MinValue() []byte

	// Returns the maximum value in the page based on the ordering rules of the
	// column's logical type.
	//
	// As an optimization, the method may return the same slice across multiple
	// calls. Programs must treat the returned value as immutable to prevent
	// unpredictable behaviors.
	//
	// If the page only contains only null values, an empty slice is returned.
	MaxValue() []byte
}

type DataPageHeaderV1 struct {
	// contains filtered or unexported fields
}

type DataPageHeaderV2 struct {
	// contains filtered or unexported fields
}

type Dictionary interface {
	// Returns the type that the dictionary was created from.
	Type() Type

	// Returns the number of value indexed in the dictionary.
	Len() int

	// Returns the dictionary value at the given index.
	Index(index int32) Value

	// Inserts values from the second slice to the dictionary and writes the
	// indexes at which each value was inserted to the first slice.
	//
	// The method panics if the length of the indexes slice is smaller than the
	// length of the values slice.
	Insert(indexes []int32, values []Value)

	// Given an array of dictionary indexes, lookup the values into the array
	// of values passed as second argument.
	//
	// The method panics if len(indexes) > len(values), or one of the indexes
	// is negative or greater than the highest index in the dictionary.
	Lookup(indexes []int32, values []Value)

	// Returns the min and max values found in the given indexes.
	Bounds(indexed []int32) (min, max Value)

	// Resets the dictionary to its initial state, removing all values.
	Reset()

	// Returns a BufferedPage representing the content of the dictionary.
	//
	// The returned page shares the underlying memory of the buffer, it remains
	// valid to use until the dictionary's Reset method is called.
	Page() BufferedPage
}

type DictionaryPageHeader struct {
	// contains filtered or unexported fields
}

type File struct {
	// contains filtered or unexported fields
}

type FileConfig struct {
	SkipPageIndex    bool
	SkipBloomFilters bool
}

type FileOption interface {
	ConfigureFile(*FileConfig)
}

type Group map[string]Node

type IndexedNode interface {
	Node

	// ChildByIndex returns the child node at the given index.
	ChildByIndex(index int) Node

	// ValueByIndex returns the sub-value of base at the given index.
	ValueByIndex(base reflect.Value, index int) reflect.Value
}

type Kind int8

type Node interface {
	// Returns a human-readable representation of the parquet node.
	String() string

	// For leaf nodes, returns the type of values of the parquet column.
	//
	// Calling this method on non-leaf nodes will panic.
	Type() Type

	// Returns whether the parquet column is optional.
	Optional() bool

	// Returns whether the parquet column is repeated.
	Repeated() bool

	// Returns whether the parquet column is required.
	Required() bool

	// Returns the number of child nodes.
	//
	// The method returns zero on leaf nodes.
	NumChildren() int

	// Returns the sorted list of child node names.
	//
	// The method returns an empty slice on leaf nodes.
	//
	// As an optimization, the returned slice may be the same across calls to
	// this method. Applications should treat the return value as immutable.
	ChildNames() []string

	// Returns the child node associated with the given name, or nil if the
	// name did not exist.
	//
	// The method panics if it is called on a leaf node.
	ChildByName(name string) Node

	// ValueByName is returns the sub-value with the given name in base.
	ValueByName(base reflect.Value, name string) reflect.Value

	// Returns the list of encodings used by the node and its children.
	//
	// The method may return an empty slice to indicate that only the plain
	// encoding is used.
	//
	// As an optimization, the returned slice may be the same across calls to
	// this method. Applications should treat the return value as immutable.
	Encoding() []encoding.Encoding

	// Returns the list of compression codecs used by the node and its children.
	//
	// The method may return an empty slice to indicate that no compression was
	// configured on the node.
	//
	// As an optimization, the returned slice may be the same across calls to
	// this method. Applications should treat the return value as immutable.
	Compression() []compress.Codec

	// Returns the Go type that best represents the parquet node.
	//
	// For leaf nodes, this will be one of bool, int32, int64, deprecated.Int96,
	// float32, float64, string, []byte, or [N]byte.
	//
	// For groups, the method returns a struct type.
	//
	// If the method is called on a repeated node, the method returns a slice of
	// the underlying type.
	//
	// For optional nodes, the method returns a pointer of the underlying type.
	//
	// For nodes that were constructed from Go values (e.g. using SchemaOf), the
	// method returns the original Go type.
	GoType() reflect.Type
}

type OffsetIndex interface {
	// NumPages returns the number of pages in the offset index.
	NumPages() int

	// Offset returns the offset starting from the beginning of the file for the
	// page at the given index.
	Offset(int) int64

	// CompressedPageSize returns the size of the page at the given index
	// (in bytes).
	CompressedPageSize(int) int64

	// FirstRowIndex returns the the first row in the page at the given index.
	//
	// The returned row index is based on the row group that the page belongs
	// to, the first row has index zero.
	FirstRowIndex(int) int64
}

type Page interface {
	// Returns the column index that this page belongs to.
	Column() int

	// If the page contains indexed values, calling this method returns the
	// dictionary in which the values are looked up. Otherwise, the method
	// returns nil.
	Dictionary() Dictionary

	// Returns the number of rows, values, and nulls in the page. The number of
	// rows may be less than the number of values in the page if the page is
	// part of a repeated column.
	NumRows() int64
	NumValues() int64
	NumNulls() int64

	// Returns the min and max values currently buffered in the writer.
	Bounds() (min, max Value)

	// Returns the size of the page in bytes (uncompressed).
	Size() int64

	// Returns a reader exposing the values contained in the page.
	//
	// Depending on the underlying implementation, the returned reader may
	// support reading an array of typed Go values by implementing interfaces
	// like parquet.Int32Reader. Applications should use type assertions on
	// the returned reader to determine whether those optimizations are
	// available.
	Values() ValueReader

	// Buffer returns the page as a BufferedPage, which may be the page itself
	// if it was already buffered.
	//
	// Compressed pages will be consumed to create the returned buffered page,
	// their content will no be readable anymore after the call.
	Buffer() BufferedPage
}

type PageBufferPool interface {
	GetPageBuffer() io.ReadWriter
	PutPageBuffer(io.ReadWriter)
}

type PageHeader interface {
	// Returns the number of values in the page (including nulls).
	NumValues() int64

	// Returns the page encoding.
	Encoding() format.Encoding

	// Returns the parquet format page type.
	PageType() format.PageType
}

type PageReader interface {
	ReadPage() (Page, error)
}

type PageWriter interface {
	WritePage(Page) (int64, error)
}

type Pages interface {
	PageReader
	RowSeeker
}

type Reader struct {
	// contains filtered or unexported fields
}

type ReaderConfig struct {
	Schema *Schema
}

type ReaderOption interface {
	ConfigureReader(*ReaderConfig)
}

type RequiredReader[T plain.Type] interface {
	// Read values into the data slice, returning the number of values read, or
	// an error if less than len(data) values could be read, or io.EOF if the
	// end of the sequence was reached.
	//
	// For columns of type BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY, T is byte and
	// the data is PLAIN encoded.
	//
	// If the column is of type FIXED_LEN_BYTE_ARRAY, the data slice length must
	// be a multiple of the column size.
	ReadRequired(data []T) (int, error)
}

type RequiredWriter[T plain.Type] interface {
	// Write values from the data slice, returning the number of values written,
	// or an error if less than len(data) values were written.
	//
	// For columns of type BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY, T is byte and
	// the data is PLAIN encoded.
	//
	// If the column is of type FIXED_LEN_BYTE_ARRAY, the data slice length must
	// be a multiple of the column size.
	WriteRequired(data []T) (int, error)
}

type Row []Value

type RowGroup interface {
	// Returns the number of rows in the group.
	NumRows() int64

	// Returns the number of leaf columns in the group.
	NumColumns() int

	// Returns the leaf column at the given index in the group.
	//
	// If the underlying implementation is not read-only, the returned
	// parquet.ColumnChunk may implement other interfaces: for example,
	// parquet.ColumnBuffer if the chunk is backed by an in-memory buffer,
	// or typed writer interfaces like parquet.Int32Writer depending on the
	// underlying type of values that can be written to the chunk.
	Column(int) ColumnChunk

	// Returns the schema of rows in the group.
	Schema() *Schema

	// Returns the list of sorting columns describing how rows are sorted in the
	// group.
	//
	// The method will return an empty slice if the rows are not sorted.
	SortingColumns() []SortingColumn

	// Returns a reader exposing the rows of the row group.
	//
	// As an optimization, the returned parquet.Rows object may implement
	// parquet.RowWriterTo, and test the RowWriter it receives for an
	// implementation of the parquet.RowGroupWriter interface.
	//
	// This optimization mechanism is leveraged by the parquet.CopyRows function
	// to skip the generic row-by-row copy algorithm and delegate the copy logic
	// to the parquet.Rows object.
	Rows() Rows
}

type RowGroupConfig struct {
	ColumnBufferSize int
	SortingColumns   []SortingColumn
	Schema           *Schema
}

type RowGroupOption interface {
	ConfigureRowGroup(*RowGroupConfig)
}

type RowGroupReader interface {
	ReadRowGroup() (RowGroup, error)
}

type RowGroupWriter interface {
	WriteRowGroup(RowGroup) (int64, error)
}

type RowReadSeeker interface {
	RowReader
	RowSeeker
}

type RowReader interface {
	ReadRow(Row) (Row, error)
}

type RowReaderAt interface {
	ReadRowAt(Row, int64) (Row, error)
}

type RowReaderFrom interface {
	ReadRowsFrom(RowReader) (int64, error)
}

type RowReaderWithSchema interface {
	RowReader
	Schema() *Schema
}

type RowSeeker interface {
	SeekToRow(int64) error
}

type RowWriter interface {
	WriteRow(Row) error
}

type RowWriterAt interface {
	WriteRowAt(Row, int64) error
}

type RowWriterTo interface {
	WriteRowsTo(RowWriter) (int64, error)
}

type RowWriterWithSchema interface {
	RowWriter
	Schema() *Schema
}

type Rows interface {
	RowReaderWithSchema
	RowSeeker
}

type Schema struct {
	// contains filtered or unexported fields
}

type SortConfig struct {
	MaxRepetitionLevel int
	MaxDefinitionLevel int
	Descending         bool
	NullsFirst         bool
}

type SortFunc func(a, b []Value) int

type SortOption interface {
	ConfigureSort(*SortConfig)
}

type SortingColumn interface {
	// Returns the path of the column in the row group schema, omitting the name
	// of the root node.
	Path() []string

	// Returns true if the column will sort values in descending order.
	Descending() bool

	// Returns true if the column will put null values at the beginning.
	NullsFirst() bool
}

type TimeUnit interface {
	// Returns the precision of the time unit as a time.Duration value.
	Duration() time.Duration
	// Converts the TimeUnit value to its representation in the parquet thrift
	// format.
	TimeUnit() format.TimeUnit
}

type Type interface {
	// Returns a human-readable representation of the parquet type.
	String() string

	// Returns the Kind value representing the underlying physical type.
	//
	// The method panics if it is called on a group type.
	Kind() Kind

	// For integer and floating point physical types, the method returns the
	// size of values in bits.
	//
	// For fixed-length byte arrays, the method returns the size of elements
	// in bytes.
	//
	// For other types, the value is zero.
	Length() int

	// Compares two values and returns a negative integer if a < b, positive if
	// a > b, or zero if a == b.
	//
	// The values' Kind must match the type, otherwise the result is undefined.
	//
	// The method panics if it is called on a group type.
	Compare(a, b Value) int

	// ColumnOrder returns the type's column order. For group types, this method
	// returns nil.
	//
	// The order describes the comparison logic implemented by the Less method.
	//
	// As an optimization, the method may return the same pointer across
	// multiple calls. Applications must treat the returned value as immutable,
	// mutating the value will result in undefined behavior.
	ColumnOrder() *format.ColumnOrder

	// Returns the physical type as a *format.Type value. For group types, this
	// method returns nil.
	//
	// As an optimization, the method may return the same pointer across
	// multiple calls. Applications must treat the returned value as immutable,
	// mutating the value will result in undefined behavior.
	PhysicalType() *format.Type

	// Returns the logical type as a *format.LogicalType value. When the logical
	// type is unknown, the method returns nil.
	//
	// As an optimization, the method may return the same pointer across
	// multiple calls. Applications must treat the returned value as immutable,
	// mutating the value will result in undefined behavior.
	LogicalType() *format.LogicalType

	// Returns the logical type's equivalent converted type. When there are
	// no equivalent converted type, the method returns nil.
	//
	// As an optimization, the method may return the same pointer across
	// multiple calls. Applications must treat the returned value as immutable,
	// mutating the value will result in undefined behavior.
	ConvertedType() *deprecated.ConvertedType

	// Creates a column indexer for values of this type.
	//
	// The size limit is a hint to the column indexer that it is allowed to
	// truncate the page boundaries to the given size. Only BYTE_ARRAY and
	// FIXED_LEN_BYTE_ARRAY types currently take this value into account.
	//
	// A value of zero or less means no limits.
	//
	// The method panics if it is called on a group type.
	NewColumnIndexer(sizeLimit int) ColumnIndexer

	// Creates a dictionary holding values of this type.
	//
	// The method panics if it is called on a group type.
	NewDictionary(columnIndex, bufferSize int) Dictionary

	// Creates a row group buffer column for values of this type.
	//
	// Column buffers are created using the index of the column they are
	// accumulating values in memory for (relative to the parent schema),
	// and the size of their memory buffer.
	//
	// The buffer size is given in bytes, because we want to control memory
	// consumption of the application, which is simpler to achieve with buffer
	// size expressed in bytes rather than number of elements.
	//
	// Note that the buffer size is not a hard limit, it defines the initial
	// capacity of the column buffer, but may grow as needed. Programs can use
	// the Size method of the column buffer (or the parent row group, when
	// relevant) to determine how many bytes are being used, and perform a flush
	// of the buffers to a storage layer.
	//
	// The method panics if it is called on a group type.
	NewColumnBuffer(columnIndex, bufferSize int) ColumnBuffer

	// Creates a reader for columns of this type.
	//
	// Column readers are created using the index of the column they are reading
	// values from (relative to the parent schema). The column index will be set
	// on values read from the reader.
	//
	// The buffer size is given in bytes, because we want to control memory
	// consumption of the application, which is simpler to achieve with buffer
	// size expressed in bytes rather than number of elements.
	//
	// The returned reader may implement extensions that can be tested via type
	// assertions. For example, on a INT32 type, the reader could implement the
	// parquet.Int32Reader interface to allow programs to more efficiently read
	// columns of INT32 values.
	NewColumnReader(columnIndex, bufferSize int) ColumnReader

	// Reads a dictionary with values of this type from the decoder passed as
	// argument.
	//
	// The number of values is a hint to optimize the allocation of memory
	// buffers for the dictionary. Callers that don't know how many values will
	// be decoded should pass zero for numValues.
	ReadDictionary(columnIndex, numValues int, decoder encoding.Decoder) (Dictionary, error)
}

type Value struct {
	// contains filtered or unexported fields
}

type ValueReader interface {
	// Read values into the buffer passed as argument and return the number of
	// values read. When all values have been read, the error will be io.EOF.
	ReadValues([]Value) (int, error)
}

type ValueReaderFrom interface {
	ReadValuesFrom(ValueReader) (int64, error)
}

type ValueWriter interface {
	// Write values from the buffer passed as argument and returns the number
	// of values written.
	WriteValues([]Value) (int, error)
}

type ValueWriterTo interface {
	WriteValuesTo(ValueWriter) (int64, error)
}

type WrappedNode interface {
	Node
	// Unwrap returns the underlying base node.
	//
	// Note that Unwrap is not intended to recursively unwrap multiple layers of
	// wrappers, it returns the immediate next layer.
	Unwrap() Node
}

type Writer struct {
	// contains filtered or unexported fields
}

type WriterConfig struct {
	CreatedBy            string
	ColumnPageBuffers    PageBufferPool
	ColumnIndexSizeLimit int
	PageBufferPool       PageBufferPool
	PageBufferSize       int
	DataPageVersion      int
	DataPageStatistics   bool
	KeyValueMetadata     map[string]string
	Schema               *Schema
	BloomFilters         []BloomFilterColumn
}

type WriterOption interface {
	ConfigureWriter(*WriterConfig)
}