format

type AesGcmCtrV1 struct {
	// AAD prefix.
	AadPrefix []byte `thrift:"1,optional"`

	// Unique file identifier part of AAD suffix.
	AadFileUnique []byte `thrift:"2,optional"`

	// In files encrypted with AAD prefix without storing it,
	// readers must supply the prefix.
	SupplyAadPrefix bool `thrift:"3,optional"`
}

type AesGcmV1 struct {
	// AAD prefix.
	AadPrefix []byte `thrift:"1,optional"`

	// Unique file identifier part of AAD suffix.
	AadFileUnique []byte `thrift:"2,optional"`

	// In files encrypted with AAD prefix without storing it,
	// readers must supply the prefix.
	SupplyAadPrefix bool `thrift:"3,optional"`
}

type BloomFilterAlgorithm struct {
	Block *SplitBlockAlgorithm `thrift:"1"`
}

type BloomFilterCompression struct {
	Uncompressed *BloomFilterUncompressed `thrift:"1"`
}

type BloomFilterHash struct {
	XxHash *XxHash `thrift:"1"`
}

type BloomFilterHeader struct {
	// The size of bitset in bytes.
	NumBytes int32 `thrift:"1,required"`
	// The algorithm for setting bits.
	Algorithm BloomFilterAlgorithm `thrift:"2,required"`
	// The hash function used for Bloom filter.
	Hash BloomFilterHash `thrift:"3,required"`
	// The compression used in the Bloom filter.
	Compression BloomFilterCompression `thrift:"4,required"`
}

type BloomFilterUncompressed struct{}

type BoundaryOrder int32

type BsonType struct{}

type ColumnChunk struct {
	// File where column data is stored.  If not set, assumed to be same file as
	// metadata.  This path is relative to the current file.
	FilePath string `thrift:"1,optional"`

	// Byte offset in file_path to the ColumnMetaData.
	FileOffset int64 `thrift:"2,required"`

	// Column metadata for this chunk. This is the same content as what is at
	// file_path/file_offset. Having it here has it replicated in the file
	// metadata.
	MetaData ColumnMetaData `thrift:"3,optional"`

	// File offset of ColumnChunk's OffsetIndex.
	OffsetIndexOffset int64 `thrift:"4,optional"`

	// Size of ColumnChunk's OffsetIndex, in bytes.
	OffsetIndexLength int32 `thrift:"5,optional"`

	// File offset of ColumnChunk's ColumnIndex.
	ColumnIndexOffset int64 `thrift:"6,optional"`

	// Size of ColumnChunk's ColumnIndex, in bytes.
	ColumnIndexLength int32 `thrift:"7,optional"`

	// Crypto metadata of encrypted columns.
	CryptoMetadata ColumnCryptoMetaData `thrift:"8,optional"`

	// Encrypted column metadata for this chunk.
	EncryptedColumnMetadata []byte `thrift:"9,optional"`
}

type ColumnCryptoMetaData struct {
	EncryptionWithFooterKey *EncryptionWithFooterKey `thrift:"1"`
	EncryptionWithColumnKey *EncryptionWithColumnKey `thrift:"2"`
}

type ColumnIndex struct {
	// A list of Boolean values to determine the validity of the corresponding
	// min and max values. If true, a page contains only null values, and writers
	// have to set the corresponding entries in min_values and max_values to
	// byte[0], so that all lists have the same length. If false, the
	// corresponding entries in min_values and max_values must be valid.
	NullPages []bool `thrift:"1,required"`

	// Two lists containing lower and upper bounds for the values of each page
	// determined by the ColumnOrder of the column. These may be the actual
	// minimum and maximum values found on a page, but can also be (more compact)
	// values that do not exist on a page. For example, instead of storing ""Blart
	// Versenwald III", a writer may set min_values[i]="B", max_values[i]="C".
	// Such more compact values must still be valid values within the column's
	// logical type. Readers must make sure that list entries are populated before
	// using them by inspecting null_pages.
	MinValues [][]byte `thrift:"2,required"`
	MaxValues [][]byte `thrift:"3,required"`

	// Stores whether both min_values and max_values are ordered and if so, in
	// which direction. This allows readers to perform binary searches in both
	// lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
	// if the lists are ordered.
	BoundaryOrder BoundaryOrder `thrift:"4,required"`

	// A list containing the number of null values for each page.
	NullCounts []int64 `thrift:"5,optional"`
}

type ColumnMetaData struct {
	// Type of this column.
	Type Type `thrift:"1,required"`

	// Set of all encodings used for this column. The purpose is to validate
	// whether we can decode those pages.
	Encoding []Encoding `thrift:"2,required"`

	// Path in schema.
	PathInSchema []string `thrift:"3,required"`

	// Compression codec.
	Codec CompressionCodec `thrift:"4,required"`

	// Number of values in this column.
	NumValues int64 `thrift:"5,required"`

	// Total byte size of all uncompressed pages in this column chunk (including the headers).
	TotalUncompressedSize int64 `thrift:"6,required"`

	// Total byte size of all compressed, and potentially encrypted, pages
	// in this column chunk (including the headers).
	TotalCompressedSize int64 `thrift:"7,required"`

	// Optional key/value metadata.
	KeyValueMetadata []KeyValue `thrift:"8,optional"`

	// Byte offset from beginning of file to first data page.
	DataPageOffset int64 `thrift:"9,required"`

	// Byte offset from beginning of file to root index page.
	IndexPageOffset int64 `thrift:"10,optional"`

	// Byte offset from the beginning of file to first (only) dictionary page.
	DictionaryPageOffset int64 `thrift:"11,optional"`

	// optional statistics for this column chunk.
	Statistics Statistics `thrift:"12,optional"`

	// Set of all encodings used for pages in this column chunk.
	// This information can be used to determine if all data pages are
	// dictionary encoded for example.
	EncodingStats []PageEncodingStats `thrift:"13,optional"`

	// Byte offset from beginning of file to Bloom filter data.
	BloomFilterOffset int64 `thrift:"14,optional"`
}

type ColumnOrder struct {
	// The sort orders for logical types are:
	//   UTF8 - unsigned byte-wise comparison
	//   INT8 - signed comparison
	//   INT16 - signed comparison
	//   INT32 - signed comparison
	//   INT64 - signed comparison
	//   UINT8 - unsigned comparison
	//   UINT16 - unsigned comparison
	//   UINT32 - unsigned comparison
	//   UINT64 - unsigned comparison
	//   DECIMAL - signed comparison of the represented value
	//   DATE - signed comparison
	//   TIME_MILLIS - signed comparison
	//   TIME_MICROS - signed comparison
	//   TIMESTAMP_MILLIS - signed comparison
	//   TIMESTAMP_MICROS - signed comparison
	//   INTERVAL - unsigned comparison
	//   JSON - unsigned byte-wise comparison
	//   BSON - unsigned byte-wise comparison
	//   ENUM - unsigned byte-wise comparison
	//   LIST - undefined
	//   MAP - undefined
	//
	// In the absence of logical types, the sort order is determined by the physical type:
	//   BOOLEAN - false, true
	//   INT32 - signed comparison
	//   INT64 - signed comparison
	//   INT96 (only used for legacy timestamps) - undefined
	//   FLOAT - signed comparison of the represented value (*)
	//   DOUBLE - signed comparison of the represented value (*)
	//   BYTE_ARRAY - unsigned byte-wise comparison
	//   FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
	//
	// (*) Because the sorting order is not specified properly for floating
	//     point values (relations vs. total ordering) the following
	//     compatibility rules should be applied when reading statistics:
	//     - If the min is a NaN, it should be ignored.
	//     - If the max is a NaN, it should be ignored.
	//     - If the min is +0, the row group may contain -0 values as well.
	//     - If the max is -0, the row group may contain +0 values as well.
	//     - When looking for NaN values, min and max should be ignored.
	TypeOrder *TypeDefinedOrder `thrift:"1"`
}

type CompressionCodec int32

type DataPageHeader struct {
	// Number of values, including NULLs, in this data page.
	NumValues int32 `thrift:"1,required"`

	// Encoding used for this data page.
	Encoding Encoding `thrift:"2,required"`

	// Encoding used for definition levels.
	DefinitionLevelEncoding Encoding `thrift:"3,required"`

	// Encoding used for repetition levels.
	RepetitionLevelEncoding Encoding `thrift:"4,required"`

	// Optional statistics for the data in this page.
	Statistics Statistics `thrift:"5,optional"`
}

type DataPageHeaderV2 struct {
	// Number of values, including NULLs, in this data page.
	NumValues int32 `thrift:"1,required"`
	// Number of NULL values, in this data page.
	// Number of non-null = num_values - num_nulls which is also the number of
	// values in the data section.
	NumNulls int32 `thrift:"2,required"`
	// Number of rows in this data page. which means pages change on record boundaries (r = 0).
	NumRows int32 `thrift:"3,required"`
	// Encoding used for data in this page.
	Encoding Encoding `thrift:"4,required"`

	// Length of the definition levels.
	DefinitionLevelsByteLength int32 `thrift:"5,required"`
	// Length of the repetition levels.
	RepetitionLevelsByteLength int32 `thrift:"6,required"`

	// Whether the values are compressed.
	// Which means the section of the page between
	// definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included)
	// is compressed with the compression_codec.
	// If missing it is considered compressed.
	IsCompressed *bool `thrift:"7,optional"`

	// Optional statistics for the data in this page.
	Statistics Statistics `thrift:"8,optional"`
}

type DateType struct{} // allowed for INT32

type DecimalType struct {
	Scale     int32 `thrift:"1,required"`
	Precision int32 `thrift:"2,required"`
}

type DictionaryPageHeader struct {
	// Number of values in the dictionary.
	NumValues int32 `thrift:"1,required"`

	// Encoding using this dictionary page.
	Encoding Encoding `thrift:"2,required"`

	// If true, the entries in the dictionary are sorted in ascending order.
	IsSorted bool `thrift:"3,optional"`
}

type Encoding int32

type EncryptionAlgorithm struct {
	AesGcmV1    *AesGcmV1    `thrift:"1"`
	AesGcmCtrV1 *AesGcmCtrV1 `thrift:"2"`
}

type EncryptionWithColumnKey struct {
	// Column path in schema.
	PathInSchema []string `thrift:"1,required"`

	// Retrieval metadata of column encryption key.
	KeyMetadata []byte `thrift:"2,optional"`
}

type EncryptionWithFooterKey struct{}

type EnumType struct{} // allowed for BINARY, must be encoded with UTF-8

type FieldRepetitionType int32

type FileCryptoMetaData struct {
	// Encryption algorithm. This field is only used for files
	// with encrypted footer. Files with plaintext footer store algorithm id
	// inside footer (FileMetaData structure).
	EncryptionAlgorithm EncryptionAlgorithm `thrift:"1,required"`

	// Retrieval metadata of key used for encryption of footer,
	// and (possibly) columns.
	KeyMetadata []byte `thrift:"2,optional"`
}

type FileMetaData struct {
	// Version of this file.
	Version int32 `thrift:"1,required"`

	// Parquet schema for this file.  This schema contains metadata for all the columns.
	// The schema is represented as a tree with a single root.  The nodes of the tree
	// are flattened to a list by doing a depth-first traversal.
	// The column metadata contains the path in the schema for that column which can be
	// used to map columns to nodes in the schema.
	// The first element is the root.
	Schema []SchemaElement `thrift:"2,required"`

	// Number of rows in this file.
	NumRows int64 `thrift:"3,required"`

	// Row groups in this file.
	RowGroups []RowGroup `thrift:"4,required"`

	// Optional key/value metadata.
	KeyValueMetadata []KeyValue `thrift:"5,optional"`

	// String for application that wrote this file.  This should be in the format
	// <Application> version <App Version> (build <App Build Hash>).
	// e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
	CreatedBy string `thrift:"6,optional"`

	// Sort order used for the min_value and max_value fields in the Statistics
	// objects and the min_values and max_values fields in the ColumnIndex
	// objects of each column in this file. Sort orders are listed in the order
	// matching the columns in the schema. The indexes are not necessary the same
	// though, because only leaf nodes of the schema are represented in the list
	// of sort orders.
	//
	// Without column_orders, the meaning of the min_value and max_value fields
	// in the Statistics object and the ColumnIndex object is undefined. To ensure
	// well-defined behavior, if these fields are written to a Parquet file,
	// column_orders must be written as well.
	//
	// The obsolete min and max fields in the Statistics object are always sorted
	// by signed comparison regardless of column_orders.
	ColumnOrders []ColumnOrder `thrift:"7,optional"`

	// Encryption algorithm. This field is set only in encrypted files
	// with plaintext footer. Files with encrypted footer store algorithm id
	// in FileCryptoMetaData structure.
	EncryptionAlgorithm EncryptionAlgorithm `thrift:"8,optional"`

	// Retrieval metadata of key used for signing the footer.
	// Used only in encrypted files with plaintext footer.
	FooterSigningKeyMetadata []byte `thrift:"9,optional"`
}

type IndexPageHeader struct {
}

type IntType struct {
	BitWidth int8 `thrift:"1,required"`
	IsSigned bool `thrift:"2,required"`
}

type JsonType struct{}

type KeyValue struct {
	Key   string `thrift:"1,required"`
	Value string `thrift:"2,required"`
}

type ListType struct{} // see LogicalTypes.md

type LogicalType struct {
	UTF8    *StringType  `thrift:"1"` // use ConvertedType UTF8
	Map     *MapType     `thrift:"2"` // use ConvertedType Map
	List    *ListType    `thrift:"3"` // use ConvertedType List
	Enum    *EnumType    `thrift:"4"` // use ConvertedType Enum
	Decimal *DecimalType `thrift:"5"` // use ConvertedType Decimal + SchemaElement.{Scale, Precision}
	Date    *DateType    `thrift:"6"` // use ConvertedType Date

	// use ConvertedType TimeMicros for Time{IsAdjustedToUTC: *, Unit: Micros}
	// use ConvertedType TimeMillis for Time{IsAdjustedToUTC: *, Unit: Millis}
	Time *TimeType `thrift:"7"`

	// use ConvertedType TimestampMicros for Timestamp{IsAdjustedToUTC: *, Unit: Micros}
	// use ConvertedType TimestampMillis for Timestamp{IsAdjustedToUTC: *, Unit: Millis}
	Timestamp *TimestampType `thrift:"8"`

	// 9: reserved for Interval
	Integer *IntType  `thrift:"10"` // use ConvertedType Int* or Uint*
	Unknown *NullType `thrift:"11"` // no compatible ConvertedType
	Json    *JsonType `thrift:"12"` // use ConvertedType JSON
	Bson    *BsonType `thrift:"13"` // use ConvertedType BSON
	UUID    *UUIDType `thrift:"14"` // no compatible ConvertedType
}

type MapType struct{} // see see LogicalTypes.md

type MicroSeconds struct{}

type MilliSeconds struct{}

type NanoSeconds struct{}

type NullType struct{}

type OffsetIndex struct {
	// PageLocations, ordered by increasing PageLocation.offset. It is required
	// that page_locations[i].first_row_index < page_locations[i+1].first_row_index.
	PageLocations []PageLocation `thrift:"1,required"`
}

type PageEncodingStats struct {
	// The page type (data/dic/...).
	PageType PageType `thrift:"1,required"`

	// Encoding of the page.
	Encoding Encoding `thrift:"2,required"`

	// Number of pages of this type with this encoding.
	Count int32 `thrift:"3,required"`
}

type PageHeader struct {
	// The type of the page indicates which of the *Header fields below is set.
	Type PageType `thrift:"1,required"`

	// Uncompressed page size in bytes (not including this header).
	UncompressedPageSize int32 `thrift:"2,required"`

	// Compressed (and potentially encrypted) page size in bytes, not including
	// this header.
	CompressedPageSize int32 `thrift:"3,required"`

	// The 32bit CRC for the page, to be be calculated as follows:
	// - Using the standard CRC32 algorithm
	// - On the data only, i.e. this header should not be included. 'Data'
	//   hereby refers to the concatenation of the repetition levels, the
	//   definition levels and the column value, in this exact order.
	// - On the encoded versions of the repetition levels, definition levels and
	//   column values.
	// - On the compressed versions of the repetition levels, definition levels
	//   and column values where possible;
	//   - For v1 data pages, the repetition levels, definition levels and column
	//     values are always compressed together. If a compression scheme is
	//     specified, the CRC shall be calculated on the compressed version of
	//     this concatenation. If no compression scheme is specified, the CRC
	//     shall be calculated on the uncompressed version of this concatenation.
	//   - For v2 data pages, the repetition levels and definition levels are
	//     handled separately from the data and are never compressed (only
	//     encoded). If a compression scheme is specified, the CRC shall be
	//     calculated on the concatenation of the uncompressed repetition levels,
	//     uncompressed definition levels and the compressed column values.
	//     If no compression scheme is specified, the CRC shall be calculated on
	//     the uncompressed concatenation.
	// - In encrypted columns, CRC is calculated after page encryption; the
	//   encryption itself is performed after page compression (if compressed)
	// If enabled, this allows for disabling checksumming in HDFS if only a few
	// pages need to be read.
	CRC int32 `thrift:"4,optional"`

	// Headers for page specific data. One only will be set.
	DataPageHeader       *DataPageHeader       `thrift:"5,optional"`
	IndexPageHeader      *IndexPageHeader      `thrift:"6,optional"`
	DictionaryPageHeader *DictionaryPageHeader `thrift:"7,optional"`
	DataPageHeaderV2     *DataPageHeaderV2     `thrift:"8,optional"`
}

type PageLocation struct {
	// Offset of the page in the file.
	Offset int64 `thrift:"1,required"`

	// Size of the page, including header. Sum of compressed_page_size and
	// header length.
	CompressedPageSize int32 `thrift:"2,required"`

	// Index within the RowGroup of the first row of the page; this means
	// pages change on record boundaries (r = 0).
	FirstRowIndex int64 `thrift:"3,required"`
}

type PageType int32

type RowGroup struct {
	// Metadata for each column chunk in this row group.
	// This list must have the same order as the SchemaElement list in FileMetaData.
	Columns []ColumnChunk `thrift:"1,required"`

	// Total byte size of all the uncompressed column data in this row group.
	TotalByteSize int64 `thrift:"2,required"`

	// Number of rows in this row group.
	NumRows int64 `thrift:"3,required"`

	// If set, specifies a sort ordering of the rows in this RowGroup.
	// The sorting columns can be a subset of all the columns.
	SortingColumns []SortingColumn `thrift:"4,optional"`

	// Byte offset from beginning of file to first page (data or dictionary)
	// in this row group
	FileOffset int64 `thrift:"5,optional"`

	// Total byte size of all compressed (and potentially encrypted) column data
	// in this row group.
	TotalCompressedSize int64 `thrift:"6,optional"`

	// Row group ordinal in the file.
	Ordinal int16 `thrift:"7,optional"`
}

type SchemaElement struct {
	// Data type for this field. Not set if the current element is a non-leaf node.
	Type *Type `thrift:"1,optional"`

	// If type is FixedLenByteArray, this is the byte length of the values.
	// Otherwise, if specified, this is the maximum bit length to store any of the values.
	// (e.g. a low cardinality INT col could have this set to 3).  Note that this is
	// in the schema, and therefore fixed for the entire file.
	TypeLength *int32 `thrift:"2,optional"`

	// repetition of the field. The root of the schema does not have a repetition_type.
	// All other nodes must have one.
	RepetitionType *FieldRepetitionType `thrift:"3,optional"`

	// Name of the field in the schema.
	Name string `thrift:"4,required"`

	// Nested fields.  Since thrift does not support nested fields,
	// the nesting is flattened to a single list by a depth-first traversal.
	// The children count is used to construct the nested relationship.
	// This field is not set when the element is a primitive type
	NumChildren int32 `thrift:"5,optional"`

	// DEPRECATED: When the schema is the result of a conversion from another model.
	// Used to record the original type to help with cross conversion.
	//
	// This is superseded by logicalType.
	ConvertedType *deprecated.ConvertedType `thrift:"6,optional"`

	// DEPRECATED: Used when this column contains decimal data.
	// See the DECIMAL converted type for more details.
	//
	// This is superseded by using the DecimalType annotation in logicalType.
	Scale     *int32 `thrift:"7,optional"`
	Precision *int32 `thrift:"8,optional"`

	// When the original schema supports field ids, this will save the
	// original field id in the parquet schema.
	FieldID int32 `thrift:"9,optional"`

	// The logical type of this SchemaElement
	//
	// LogicalType replaces ConvertedType, but ConvertedType is still required
	// for some logical types to ensure forward-compatibility in format v1.
	LogicalType *LogicalType `thrift:"10,optional"`
}

type SortingColumn struct {
	// The column index (in this row group)
	ColumnIdx int32 `thrift:"1,required"`

	// If true, indicates this column is sorted in descending order.
	Descending bool `thrift:"2,required"`

	// If true, nulls will come before non-null values, otherwise,
	// nulls go at the end.
	NullsFirst bool `thrift:"3,required"`
}

type SplitBlockAlgorithm struct{}

type Statistics struct {
	// DEPRECATED: min and max value of the column. Use min_value and max_value.
	//
	// Values are encoded using PLAIN encoding, except that variable-length byte
	// arrays do not include a length prefix.
	//
	// These fields encode min and max values determined by signed comparison
	// only. New files should use the correct order for a column's logical type
	// and store the values in the min_value and max_value fields.
	//
	// To support older readers, these may be set when the column order is
	// signed.
	Max []byte `thrift:"1"`
	Min []byte `thrift:"2"`
	// Count of null value in the column.
	NullCount int64 `thrift:"3"`
	// Count of distinct values occurring.
	DistinctCount int64 `thrift:"4"`
	// Min and max values for the column, determined by its ColumnOrder.
	//
	// Values are encoded using PLAIN encoding, except that variable-length byte
	// arrays do not include a length prefix.
	MaxValue []byte `thrift:"5"`
	MinValue []byte `thrift:"6"`
}

type StringType struct{} // allowed for BINARY, must be encoded with UTF-8

type TimeType struct {
	IsAdjustedToUTC bool     `thrift:"1,required"`
	Unit            TimeUnit `thrift:"2,required"`
}

type TimeUnit struct {
	Millis *MilliSeconds `thrift:"1"`
	Micros *MicroSeconds `thrift:"2"`
	Nanos  *NanoSeconds  `thrift:"3"`
}

type TimestampType struct {
	IsAdjustedToUTC bool     `thrift:"1,required"`
	Unit            TimeUnit `thrift:"2,required"`
}

type Type int32

type TypeDefinedOrder struct{}

type UUIDType struct{} // allowed for FIXED[16], must encode raw UUID bytes

type XxHash struct{}