Documentation
¶
Index ¶
- func ArrowScalarToParquetValue(sc scalar.Scalar) (parquet.Value, error)
- func ColToWriter(col int, writers []MultiColumnWriter) writer.ValueWriter
- func ParquetRowGroupToArrowSchema(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema, ...) (*arrow.Schema, error)
- func ParquetSchemaToArrowSchema(ctx context.Context, schema *parquet.Schema, s *dynparquet.Schema, ...) (*arrow.Schema, error)
- func Project(r arrow.Record, projections []logicalplan.Expr) arrow.Record
- func RecordDynamicCols(record arrow.Record) (columns map[string][]string)
- func RecordToDynamicRow(pqSchema *parquet.Schema, record arrow.Record, dyncols map[string][]string, ...) (*dynparquet.DynamicRow, error)
- func RecordToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, r arrow.Record) error
- func RecordToRow(final *parquet.Schema, record arrow.Record, index int) (parquet.Row, error)
- func RecordsToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, recs []arrow.Record) error
- func SerializeRecord(r arrow.Record, schema *dynparquet.Schema) (*dynparquet.SerializedBuffer, error)
- func SingleMatchingColumn(distinctColumns []logicalplan.Expr, fields []parquet.Field) bool
- type MultiColumnWriter
- type ParquetConverter
- func (c *ParquetConverter) Close()
- func (c *ParquetConverter) Convert(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema) error
- func (c *ParquetConverter) Fields() []builder.ColumnBuilder
- func (c *ParquetConverter) NewRecord() arrow.Record
- func (c *ParquetConverter) NumRows() int
- func (c *ParquetConverter) Reset()
- type PreExprVisitorFunc
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ColToWriter ¶
func ColToWriter(col int, writers []MultiColumnWriter) writer.ValueWriter
func ParquetRowGroupToArrowSchema ¶
func ParquetRowGroupToArrowSchema(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema, options logicalplan.IterOptions) (*arrow.Schema, error)
ParquetRowGroupToArrowSchema converts a parquet row group to an arrow schema.
func ParquetSchemaToArrowSchema ¶
func ParquetSchemaToArrowSchema(ctx context.Context, schema *parquet.Schema, s *dynparquet.Schema, options logicalplan.IterOptions) (*arrow.Schema, error)
func RecordToDynamicRow ¶
func RecordToDynamicRow(pqSchema *parquet.Schema, record arrow.Record, dyncols map[string][]string, index int) (*dynparquet.DynamicRow, error)
func RecordToFile ¶
func RecordToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, r arrow.Record) error
func RecordToRow ¶
RecordToRow converts an arrow record with dynamic columns into a row using a dynamic parquet schema.
func RecordsToFile ¶
func RecordsToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, recs []arrow.Record) error
func SerializeRecord ¶
func SerializeRecord(r arrow.Record, schema *dynparquet.Schema) (*dynparquet.SerializedBuffer, error)
func SingleMatchingColumn ¶
func SingleMatchingColumn(distinctColumns []logicalplan.Expr, fields []parquet.Field) bool
SingleMatchingColumn returns true if there is only a single matching column for the given column matchers.
Types ¶
type MultiColumnWriter ¶
type MultiColumnWriter struct {
// contains filtered or unexported fields
}
type ParquetConverter ¶
type ParquetConverter struct {
// contains filtered or unexported fields
}
ParquetConverter converts parquet.RowGroups into arrow.Records. The converted results are accumulated in the converter and can be retrieved by calling NewRecord, at which point the converter is reset.
func NewParquetConverter ¶
func NewParquetConverter( pool memory.Allocator, iterOpts logicalplan.IterOptions, ) *ParquetConverter
func (*ParquetConverter) Close ¶
func (c *ParquetConverter) Close()
func (*ParquetConverter) Convert ¶
func (c *ParquetConverter) Convert(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema) error
func (*ParquetConverter) Fields ¶
func (c *ParquetConverter) Fields() []builder.ColumnBuilder
func (*ParquetConverter) NewRecord ¶
func (c *ParquetConverter) NewRecord() arrow.Record
func (*ParquetConverter) NumRows ¶
func (c *ParquetConverter) NumRows() int
func (*ParquetConverter) Reset ¶
func (c *ParquetConverter) Reset()
type PreExprVisitorFunc ¶
type PreExprVisitorFunc func(expr logicalplan.Expr) bool
func (PreExprVisitorFunc) PostVisit ¶
func (f PreExprVisitorFunc) PostVisit(_ logicalplan.Expr) bool
func (PreExprVisitorFunc) PreVisit ¶
func (f PreExprVisitorFunc) PreVisit(expr logicalplan.Expr) bool
func (PreExprVisitorFunc) Visit ¶
func (f PreExprVisitorFunc) Visit(_ logicalplan.Expr) bool
Source Files
Directories
¶
| Path | Synopsis |
|---|---|
|
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.
|
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. |
Click to show internal directories.
Click to hide internal directories.