writer

package
v1.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 17, 2019 License: Apache-2.0 Imports: 13 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CSVWriter

type CSVWriter struct {
	ParquetWriter
}

func NewCSVWriter

func NewCSVWriter(md []string, pfile source.ParquetFile, np int64) (*CSVWriter, error)

Create CSV writer

func (*CSVWriter) WriteString

func (self *CSVWriter) WriteString(recsi interface{}) error

Write string values to parquet file

type JSONWriter

type JSONWriter struct {
	ParquetWriter
}

func NewJSONWriter

func NewJSONWriter(jsonSchema string, pfile source.ParquetFile, np int64) (*JSONWriter, error)

Create JSON writer

type ParquetWriter

type ParquetWriter struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	Offset          int64

	Objs              []interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64

	PagesMapBuf map[string][]*layout.Page
	Size        int64
	NumRows     int64

	DictRecs map[string]*layout.DictRecType

	MarshalFunc func(src []interface{}, bgn int, end int, sh *schema.SchemaHandler) (*map[string]*layout.Table, error)
}

ParquetWriter is a writer parquet file

func NewParquetWriter

func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error)

Create a parquet handler

func (*ParquetWriter) Flush

func (self *ParquetWriter) Flush(flag bool) error

Flush the write buffer to parquet file

func (*ParquetWriter) RenameSchema

func (self *ParquetWriter) RenameSchema()

Rename schema name to exname in tags

func (*ParquetWriter) SetSchemaHandlerFromJSON

func (self *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error

func (*ParquetWriter) Write

func (self *ParquetWriter) Write(src interface{}) error

Write one object to parquet file

func (*ParquetWriter) WriteStop

func (self *ParquetWriter) WriteStop() error

Write the footer and stop writing

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL