import "github.com/apache/arrow/go/arrow/ipc"
dict.go file_reader.go file_writer.go flight_data_reader.go flight_data_writer.go ipc.go message.go metadata.go reader.go writer.go
const (
MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0
MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0
MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1
MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0
)const (
MessageNone = MessageType(flatbuf.MessageHeaderNONE)
MessageSchema = MessageType(flatbuf.MessageHeaderSchema)
MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch)
MessageRecordBatch = MessageType(flatbuf.MessageHeaderRecordBatch)
MessageTensor = MessageType(flatbuf.MessageHeaderTensor)
MessageSparseTensor = MessageType(flatbuf.MessageHeaderSparseTensor)
)Magic string identifying an Apache Arrow file.
type FileReader struct {
// contains filtered or unexported fields
}FileReader is an Arrow file reader.
func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error)
NewFileReader opens an Arrow file using the provided reader r.
func (f *FileReader) Close() error
Close cleans up resources used by the File. Close does not close the underlying reader.
func (f *FileReader) NumDictionaries() int
func (f *FileReader) NumRecords() int
func (f *FileReader) Read() (rec array.Record, err error)
Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
The returned record value is valid until the next call to Read. Users need to call Retain on that Record to keep it valid for longer.
ReadAt reads the i-th record from the underlying stream and an error, if any.
Record returns the i-th record from the file. The returned value is valid until the next call to Record. Users need to call Retain on that Record to keep it valid for longer.
func (f *FileReader) Schema() *arrow.Schema
func (f *FileReader) Version() MetadataVersion
type FileWriter struct {
// contains filtered or unexported fields
}FileWriter is an Arrow file writer.
func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error)
NewFileWriter opens an Arrow file using the provided writer w.
func (f *FileWriter) Close() error
func (f *FileWriter) Write(rec array.Record) error
type FlightDataReader struct {
// contains filtered or unexported fields
}FlightDataReader reads records from a stream of messages
func NewFlightDataReader(r FlightDataStreamReader, opts ...Option) (*FlightDataReader, error)
NewFlightDataReader returns a reader that will produce records from a flight data stream
implementation is generally based on the ipc.Reader, expecting the first message to be the schema with the subsequent messages being the record batches.
func (f *FlightDataReader) Err() error
Err returns the last error encounted during the iteration of the stream.
func (f *FlightDataReader) Next() bool
Next returns whether a record was able to be extracted from the stream or not.
func (f *FlightDataReader) Read() (array.Record, error)
Read reads the current record from the flight stream and an error, if any. When we reach the end of the flight stream it will return (nil, io.EOF). Also calls release on the previous existing record if any.
func (f *FlightDataReader) Record() array.Record
Record returns the current record that has been extracted from the stream. It is valid until the next call to Next or Read
func (f *FlightDataReader) Release()
Release decreases the refcount by 1. When the refcount is 0 the memory is freed. Release may be called simultaneously from multiple goroutines.
func (f *FlightDataReader) Retain()
Retain increases the refcount by 1. Retain can be called by multiple goroutines simultaneously.
func (f *FlightDataReader) Schema() *arrow.Schema
Schema returns the schema of the underlying records as described by the first message received.
type FlightDataStreamReader interface {
Recv() (*flight.FlightData, error)
}FlightDataStreamReader wraps a grpc stream for receiving FlightData objects
type FlightDataStreamWriter interface {
Send(*flight.FlightData) error
}FlightDataStreamWriter wraps a grpc stream for sending FlightData
type FlightDataWriter struct {
// contains filtered or unexported fields
}FlightDataWriter is a stream writer for writing with Flight RPC
func NewFlightDataWriter(w FlightDataStreamWriter, opts ...Option) *FlightDataWriter
NewFlightDataWriter returns a writer for writing array Records to a flight data stream.
func (w *FlightDataWriter) Close() (err error)
func (w *FlightDataWriter) Write(rec array.Record) error
Write the provided record to the underlying stream
type Message struct {
// contains filtered or unexported fields
}Message is an IPC message, including metadata and body.
NewMessage creates a new message from the metadata and body buffers. NewMessage panics if any of these buffers is nil.
Release decreases the reference count by 1. Release may be called simultaneously from multiple goroutines. When the reference count goes to zero, the memory is freed.
Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.
func (msg *Message) Type() MessageType
func (msg *Message) Version() MetadataVersion
type MessageReader struct {
// contains filtered or unexported fields
}MessageReader reads messages from an io.Reader.
func NewMessageReader(r io.Reader) *MessageReader
NewMessageReader returns a reader that reads messages from an input stream.
func (r *MessageReader) Message() (*Message, error)
Message returns the current message that has been extracted from the underlying stream. It is valid until the next call to Message.
func (r *MessageReader) Release()
Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.
func (r *MessageReader) Retain()
Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.
type MessageType flatbuf.MessageHeader
MessageType represents the type of Message in an Arrow format.
func (m MessageType) String() string
type MetadataVersion flatbuf.MetadataVersion
MetadataVersion represents the Arrow metadata version.
func (m MetadataVersion) String() string
type Option func(*config)
Option is a functional option to configure opening or creating Arrow files and streams.
WithAllocator specifies the Arrow memory allocator used while building records.
WithFooterOffset specifies the Arrow footer position in bytes.
WithSchema specifies the Arrow schema to be used for reading or writing.
type Reader struct {
// contains filtered or unexported fields
}Reader reads records from an io.Reader. Reader expects a schema (plus any dictionaries) as the first messages in the stream, followed by records.
NewReader returns a reader that reads records from an input stream.
Err returns the last error encountered during the iteration over the underlying stream.
Next returns whether a Record could be extracted from the underlying stream.
Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
Record returns the current record that has been extracted from the underlying stream. It is valid until the next call to Next.
Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.
Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.
type Writer struct {
// contains filtered or unexported fields
}Writer is an Arrow stream writer.
NewWriter returns a writer that writes records to the provided output stream.
Package ipc imports 17 packages (graph) and is imported by 5 packages. Updated 2020-09-29. Refresh now. Tools for package owners.