parquet-go: Index | Files

package reader

import ""


Package Files

columnbuffer.go columnreader.go reader.go

type ColumnBufferType Uses

type ColumnBufferType struct {
    PFile        source.ParquetFile
    ThriftReader *thrift.TBufferedTransport

    Footer        *parquet.FileMetaData
    SchemaHandler *schema.SchemaHandler

    PathStr       string
    RowGroupIndex int64
    ChunkHeader   *parquet.ColumnChunk

    ChunkReadValues int64

    DictPage *layout.Page

    DataTable        *layout.Table
    DataTableNumRows int64

func NewColumnBuffer Uses

func NewColumnBuffer(pFile source.ParquetFile, footer *parquet.FileMetaData, schemaHandler *schema.SchemaHandler, pathStr string) (*ColumnBufferType, error)

func (*ColumnBufferType) NextRowGroup Uses

func (self *ColumnBufferType) NextRowGroup() error

func (*ColumnBufferType) ReadPage Uses

func (self *ColumnBufferType) ReadPage() error

func (*ColumnBufferType) ReadPageForSkip Uses

func (self *ColumnBufferType) ReadPageForSkip() (*layout.Page, error)

func (*ColumnBufferType) ReadRows Uses

func (self *ColumnBufferType) ReadRows(num int64) (*layout.Table, int64)

func (*ColumnBufferType) SkipRows Uses

func (self *ColumnBufferType) SkipRows(num int64) int64

type ParquetReader Uses

type ParquetReader struct {
    SchemaHandler *schema.SchemaHandler
    NP            int64 //parallel number
    Footer        *parquet.FileMetaData
    PFile         source.ParquetFile

    ColumnBuffers map[string]*ColumnBufferType

    //One reader can only read one type objects
    ObjType        reflect.Type
    ObjPartialType reflect.Type

func NewParquetColumnReader Uses

func NewParquetColumnReader(pFile source.ParquetFile, np int64) (*ParquetReader, error)

NewParquetColumnReader creates a parquet column reader

func NewParquetReader Uses

func NewParquetReader(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetReader, error)

Create a parquet reader: obj is a object with schema tags or a JSON schema string

func (*ParquetReader) GetFooterSize Uses

func (self *ParquetReader) GetFooterSize() (uint32, error)

Get the footer size

func (*ParquetReader) GetNumRows Uses

func (self *ParquetReader) GetNumRows() int64

func (*ParquetReader) Read Uses

func (self *ParquetReader) Read(dstInterface interface{}) error

Read rows of parquet file and unmarshal all to dst

func (*ParquetReader) ReadByNumber Uses

func (self *ParquetReader) ReadByNumber(maxReadNumber int) ([]interface{}, error)

Read maxReadNumber objects

func (*ParquetReader) ReadColumnByIndex Uses

func (self *ParquetReader) ReadColumnByIndex(index int64, num int64) (values []interface{}, rls []int32, dls []int32, err error)

ReadColumnByIndex reads column by index. The index of first column is 0.

func (*ParquetReader) ReadColumnByPath Uses

func (self *ParquetReader) ReadColumnByPath(pathStr string, num int64) (values []interface{}, rls []int32, dls []int32, err error)

ReadColumnByPath reads column by path in schema.

func (*ParquetReader) ReadFooter Uses

func (self *ParquetReader) ReadFooter() error

Read footer from parquet file

func (*ParquetReader) ReadPartial Uses

func (self *ParquetReader) ReadPartial(dstInterface interface{}, prefixPath string) error

Read rows of parquet file and unmarshal all to dst

func (*ParquetReader) ReadPartialByNumber Uses

func (self *ParquetReader) ReadPartialByNumber(maxReadNumber int, prefixPath string) ([]interface{}, error)

Read maxReadNumber partial objects

func (*ParquetReader) ReadStop Uses

func (self *ParquetReader) ReadStop()

Stop Read

func (*ParquetReader) RenameSchema Uses

func (self *ParquetReader) RenameSchema()

Rename schema name to inname

func (*ParquetReader) SetSchemaHandlerFromJSON Uses

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error

func (*ParquetReader) SkipRows Uses

func (self *ParquetReader) SkipRows(num int64) error

Skip rows of parquet file

func (*ParquetReader) SkipRowsByIndex Uses

func (self *ParquetReader) SkipRowsByIndex(index int64, num int64)

func (*ParquetReader) SkipRowsByPath Uses

func (self *ParquetReader) SkipRowsByPath(pathStr string, num int64) error

Package reader imports 13 packages (graph) and is imported by 4 packages. Updated 2019-12-17. Refresh now. Tools for package owners.