writer

package
v1.7.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 26, 2021 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CSVWriter

type CSVWriter struct {
	ParquetWriter
}

func NewCSVWriter

func NewCSVWriter(md []string, pfile source.ParquetFile, np int64) (*CSVWriter, error)

Create CSV writer

func NewCSVWriterFromWriter added in v1.7.0

func NewCSVWriterFromWriter(md []string, w io.Writer, np int64) (*CSVWriter, error)

func (*CSVWriter) WriteString

func (w *CSVWriter) WriteString(recsi interface{}) error

Write string values to parquet file

type JSONWriter

type JSONWriter struct {
	ParquetWriter
}

func NewJSONWriter

func NewJSONWriter(jsonSchema string, pfile source.ParquetFile, np int64) (*JSONWriter, error)

Create JSON writer

func NewJSONWriterFromWriter added in v1.7.0

func NewJSONWriterFromWriter(jsonSchema string, w io.Writer, np int64) (*JSONWriter, error)

type ParquetWriter

type ParquetWriter struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	Offset          int64

	Objs              []interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64

	PagesMapBuf map[string][]*layout.Page
	Size        int64
	NumRows     int64

	DictRecs map[string]*layout.DictRecType

	ColumnIndexes []*parquet.ColumnIndex
	OffsetIndexes []*parquet.OffsetIndex

	MarshalFunc func(src []interface{}, sh *schema.SchemaHandler) (*map[string]*layout.Table, error)
}

ParquetWriter is a writer parquet file

func NewParquetWriter

func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error)

Create a parquet handler. Obj is a object with tags or JSON schema string.

func NewParquetWriterFromWriter added in v1.7.0

func NewParquetWriterFromWriter(w io.Writer, obj interface{}, np int64) (*ParquetWriter, error)

func (*ParquetWriter) Flush

func (pw *ParquetWriter) Flush(flag bool) error

Flush the write buffer to parquet file

func (*ParquetWriter) RenameSchema

func (pw *ParquetWriter) RenameSchema()

Rename schema name to exname in tags

func (*ParquetWriter) SetSchemaHandlerFromJSON

func (pw *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error

func (*ParquetWriter) Write

func (pw *ParquetWriter) Write(src interface{}) error

Write one object to parquet file

func (*ParquetWriter) WriteStop

func (pw *ParquetWriter) WriteStop() error

Write the footer and stop writing

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL