parquet-go: github.com/xitongsys/parquet-go/layout Index | Files

package layout

import "github.com/xitongsys/parquet-go/layout"

Index

Package Files

chunk.go dictpage.go page.go rowgroup.go table.go

func DecodeDictChunk Uses

func DecodeDictChunk(chunk *Chunk)

Decode a dict chunk

func ReadDataPageValues Uses

func ReadDataPageValues(bytesReader *bytes.Reader, encodingMethod parquet.Encoding, dataType parquet.Type, convertedType parquet.ConvertedType, cnt uint64, bitWidth uint64) ([]interface{}, error)

Read data page values

func ReadPageHeader Uses

func ReadPageHeader(thriftReader *thrift.TBufferedTransport) (*parquet.PageHeader, error)

Read page header

type Chunk Uses

type Chunk struct {
    Pages       []*Page
    ChunkHeader *parquet.ColumnChunk
}

Chunk stores the ColumnChunk in parquet file

func PagesToChunk Uses

func PagesToChunk(pages []*Page) *Chunk

Convert several pages to one chunk

func PagesToDictChunk Uses

func PagesToDictChunk(pages []*Page) *Chunk

Convert several pages to one chunk with dict page first

func ReadChunk Uses

func ReadChunk(thriftReader *thrift.TBufferedTransport, schemaHandler *schema.SchemaHandler, chunkHeader *parquet.ColumnChunk) (*Chunk, error)

Read one chunk from parquet file (Deprecated)

type DictRecType Uses

type DictRecType struct {
    DictMap   map[interface{}]int32
    DictSlice []interface{}
    Type      parquet.Type
}

func NewDictRec Uses

func NewDictRec(pT parquet.Type) *DictRecType

type Page Uses

type Page struct {
    //Header of a page
    Header *parquet.PageHeader
    //Table to store values
    DataTable *Table
    //Compressed data of the page, which is written in parquet file
    RawData []byte
    //Compress type: gzip/snappy/zstd/none
    CompressType parquet.CompressionCodec
    //Parquet type of the values in the page
    DataType parquet.Type
    //Path in schema(include the root)
    Path []string
    //Maximum of the values
    MaxVal interface{}
    //Minimum of the values
    MinVal interface{}
    //Tag info
    Info *common.Tag

    PageSize int32
}

Page is used to store the page data

func DictRecToDictPage Uses

func DictRecToDictPage(dictRec *DictRecType, pageSize int32, compressType parquet.CompressionCodec) (*Page, int64)

func NewDataPage Uses

func NewDataPage() *Page

Create a new data page

func NewDictPage Uses

func NewDictPage() *Page

Create a new dict page

func NewPage Uses

func NewPage() *Page

Create a new page

func ReadPage Uses

func ReadPage(thriftReader *thrift.TBufferedTransport, schemaHandler *schema.SchemaHandler, colMetaData *parquet.ColumnMetaData) (*Page, int64, int64, error)

Read page from parquet file

func ReadPage2 Uses

func ReadPage2(thriftReader *thrift.TBufferedTransport, schemaHandler *schema.SchemaHandler, colMetaData *parquet.ColumnMetaData) (*Page, int64, int64, error)

This is a test function

func ReadPageRawData Uses

func ReadPageRawData(thriftReader *thrift.TBufferedTransport, schemaHandler *schema.SchemaHandler, colMetaData *parquet.ColumnMetaData) (*Page, error)

Read page RawData

func TableToDataPages Uses

func TableToDataPages(table *Table, pageSize int32, compressType parquet.CompressionCodec) ([]*Page, int64)

Convert a table to data pages

func TableToDictDataPages Uses

func TableToDictDataPages(dictRec *DictRecType, table *Table, pageSize int32, bitWidth int32, compressType parquet.CompressionCodec) ([]*Page, int64)

Convert a table to dict data pages

func TableToDictPage Uses

func TableToDictPage(table *Table, pageSize int32, compressType parquet.CompressionCodec) (*Page, int64)

Convert a table to dict page

func (*Page) DataPageCompress Uses

func (page *Page) DataPageCompress(compressType parquet.CompressionCodec) []byte

Compress the data page to parquet file

func (*Page) DataPageV2Compress Uses

func (page *Page) DataPageV2Compress(compressType parquet.CompressionCodec) []byte

Compress data page v2 to parquet file

func (*Page) Decode Uses

func (page *Page) Decode(dictPage *Page)

Decode dict page

func (*Page) DictDataPageCompress Uses

func (page *Page) DictDataPageCompress(compressType parquet.CompressionCodec, bitWidth int32) []byte

Compress the data page to parquet file

func (*Page) DictPageCompress Uses

func (page *Page) DictPageCompress(compressType parquet.CompressionCodec, pT parquet.Type) []byte

Compress the dict page to parquet file

func (*Page) EncodingValues Uses

func (page *Page) EncodingValues(valuesBuf []interface{}) []byte

Encoding values

func (*Page) GetRLDLFromRawData Uses

func (self *Page) GetRLDLFromRawData(schemaHandler *schema.SchemaHandler) (int64, int64, error)

Get RepetitionLevels and Definitions from RawData

func (*Page) GetValueFromRawData Uses

func (self *Page) GetValueFromRawData(schemaHandler *schema.SchemaHandler) error

Get values from raw data

type RowGroup Uses

type RowGroup struct {
    Chunks         []*Chunk
    RowGroupHeader *parquet.RowGroup
}

RowGroup stores the RowGroup in parquet file

func NewRowGroup Uses

func NewRowGroup() *RowGroup

Create a RowGroup

func ReadRowGroup Uses

func ReadRowGroup(rowGroupHeader *parquet.RowGroup, PFile source.ParquetFile, schemaHandler *schema.SchemaHandler, NP int64) (*RowGroup, error)

Read one RowGroup from parquet file (Deprecated)

func (*RowGroup) RowGroupToTableMap Uses

func (rowGroup *RowGroup) RowGroupToTableMap() *map[string]*Table

Convert a RowGroup to table map

type Table Uses

type Table struct {
    //Repetition type of the values: REQUIRED/OPTIONAL/REPEATED
    RepetitionType parquet.FieldRepetitionType
    //Parquet type
    Type parquet.Type
    //Path of this column
    Path []string
    //Maximum of definition levels
    MaxDefinitionLevel int32
    //Maximum of repetition levels
    MaxRepetitionLevel int32

    //Parquet values
    Values []interface{}
    //Definition Levels slice
    DefinitionLevels []int32
    //Repetition Levels slice
    RepetitionLevels []int32

    //Tag info
    Info *common.Tag
}

Table is the core data structure used to store the values

func NewEmptyTable Uses

func NewEmptyTable() *Table

func NewTableFromTable Uses

func NewTableFromTable(src *Table) *Table

func (*Table) Merge Uses

func (self *Table) Merge(tables ...*Table)

Merge several tables to one table(the first table)

func (*Table) Pop Uses

func (self *Table) Pop(numRows int64) *Table

Package layout imports 14 packages (graph) and is imported by 3 packages. Updated 2019-09-22. Refresh now. Tools for package owners.