doc

package
v1.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 6, 2023 License: MIT Imports: 8 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func BoolType

func BoolType(se *sch.SchemaElement)

func Float32Type

func Float32Type(se *sch.SchemaElement)

func Float64Type

func Float64Type(se *sch.SchemaElement)

func Gzip

func Gzip(p *ParquetWriter) error

func Int32Type

func Int32Type(se *sch.SchemaElement)

func Int64Type

func Int64Type(se *sch.SchemaElement)

func MaxPageSize

func MaxPageSize(m int) func(*ParquetWriter) error

MaxPageSize is the maximum number of rows in each row groups' page.

func Snappy

func Snappy(p *ParquetWriter) error

func StringType

func StringType(se *sch.SchemaElement)

func Uint32Type

func Uint32Type(se *sch.SchemaElement)

func Uint64Type

func Uint64Type(se *sch.SchemaElement)

func Uncompressed

func Uncompressed(p *ParquetWriter) error

Types

type Document

type Document struct {
	DocID int64  `parquet:"docid"`
	Links *Link  `parquet:"link"`
	Names []Name `parquet:"names"`
}

type Field

type Field interface {
	Add(r Document)
	Write(w io.Writer, meta *parquet.Metadata) error
	Schema() parquet.Field
	Scan(r *Document)
	Read(r io.ReadSeeker, pg parquet.Page) error
	Name() string
	Levels() ([]uint8, []uint8)
}

func Fields

func Fields(compression compression) []Field

type Int64Field

type Int64Field struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewInt64Field

func NewInt64Field(read func(r Document) int64, write func(r *Document, vals []int64), path []string, opts ...func(*parquet.RequiredField)) *Int64Field

func (*Int64Field) Add

func (f *Int64Field) Add(r Document)

func (*Int64Field) Levels

func (f *Int64Field) Levels() ([]uint8, []uint8)

func (*Int64Field) Read

func (f *Int64Field) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Int64Field) Scan

func (f *Int64Field) Scan(r *Document)

func (*Int64Field) Schema

func (f *Int64Field) Schema() parquet.Field

func (*Int64Field) Write

func (f *Int64Field) Write(w io.Writer, meta *parquet.Metadata) error

type Int64OptionalField

type Int64OptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewInt64OptionalField

func NewInt64OptionalField(read func(r Document, vals []int64, defs, reps []uint8) ([]int64, []uint8, []uint8), write func(r *Document, vals []int64, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *Int64OptionalField

func (*Int64OptionalField) Add

func (f *Int64OptionalField) Add(r Document)

func (*Int64OptionalField) Levels

func (f *Int64OptionalField) Levels() ([]uint8, []uint8)

func (*Int64OptionalField) Read

func (*Int64OptionalField) Scan

func (f *Int64OptionalField) Scan(r *Document)

func (*Int64OptionalField) Schema

func (f *Int64OptionalField) Schema() parquet.Field

func (*Int64OptionalField) Write

func (f *Int64OptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Language

type Language struct {
	Code    string  `parquet:"code"`
	Country *string `parquet:"country"`
}

type Levels

type Levels struct {
	Name string
	Defs []uint8
	Reps []uint8
}
type Link struct {
	Backward []int64 `parquet:"backward"`
	Forward  []int64 `parquet:"forward"`
}

type Name

type Name struct {
	Languages []Language `parquet:"languages"`
	URL       *string    `parquet:"url"`
}

type ParquetReader

type ParquetReader struct {
	// contains filtered or unexported fields
}

ParquetReader reads one page from a row group.

func NewParquetReader

func NewParquetReader(r io.ReadSeeker, opts ...func(*ParquetReader)) (*ParquetReader, error)

func (*ParquetReader) Error

func (p *ParquetReader) Error() error

func (*ParquetReader) Levels

func (p *ParquetReader) Levels() []Levels

func (*ParquetReader) Next

func (p *ParquetReader) Next() bool

func (*ParquetReader) Rows

func (p *ParquetReader) Rows() int64

func (*ParquetReader) Scan

func (p *ParquetReader) Scan(x *Document)

type ParquetWriter

type ParquetWriter struct {
	// contains filtered or unexported fields
}

ParquetWriter reprents a row group

func NewParquetWriter

func NewParquetWriter(w io.Writer, opts ...func(*ParquetWriter) error) (*ParquetWriter, error)

func (*ParquetWriter) Add

func (p *ParquetWriter) Add(rec Document)

func (*ParquetWriter) Close

func (p *ParquetWriter) Close() error

func (*ParquetWriter) Write

func (p *ParquetWriter) Write() error

type StringOptionalField

type StringOptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewStringOptionalField

func NewStringOptionalField(read func(r Document, vals []string, def, rep []uint8) ([]string, []uint8, []uint8), write func(r *Document, vals []string, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *StringOptionalField

func (*StringOptionalField) Add

func (f *StringOptionalField) Add(r Document)

func (*StringOptionalField) Levels

func (f *StringOptionalField) Levels() ([]uint8, []uint8)

func (*StringOptionalField) Read

func (*StringOptionalField) Scan

func (f *StringOptionalField) Scan(r *Document)

func (*StringOptionalField) Schema

func (f *StringOptionalField) Schema() parquet.Field

func (*StringOptionalField) Write

func (f *StringOptionalField) Write(w io.Writer, meta *parquet.Metadata) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL