arrow: github.com/apache/arrow/go/arrow/ipc Index | Files

package ipc

import "github.com/apache/arrow/go/arrow/ipc"

Index

Package Files

dict.go file_reader.go file_writer.go flight_data_reader.go flight_data_writer.go ipc.go message.go metadata.go reader.go writer.go

Constants

const (
    MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0
    MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0
    MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1
    MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0
)
const (
    MessageNone            = MessageType(flatbuf.MessageHeaderNONE)
    MessageSchema          = MessageType(flatbuf.MessageHeaderSchema)
    MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch)
    MessageRecordBatch     = MessageType(flatbuf.MessageHeaderRecordBatch)
    MessageTensor          = MessageType(flatbuf.MessageHeaderTensor)
    MessageSparseTensor    = MessageType(flatbuf.MessageHeaderSparseTensor)
)

Variables

var Magic = []byte("ARROW1")

Magic string identifying an Apache Arrow file.

func FlightInfoSchemaBytes Uses

func FlightInfoSchemaBytes(schema *arrow.Schema, mem memory.Allocator) []byte

func SchemaFromFlightInfo Uses

func SchemaFromFlightInfo(b []byte) (*arrow.Schema, error)

type FileReader Uses

type FileReader struct {
    // contains filtered or unexported fields
}

FileReader is an Arrow file reader.

func NewFileReader Uses

func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error)

NewFileReader opens an Arrow file using the provided reader r.

func (*FileReader) Close Uses

func (f *FileReader) Close() error

Close cleans up resources used by the File. Close does not close the underlying reader.

func (*FileReader) NumDictionaries Uses

func (f *FileReader) NumDictionaries() int

func (*FileReader) NumRecords Uses

func (f *FileReader) NumRecords() int

func (*FileReader) Read Uses

func (f *FileReader) Read() (rec array.Record, err error)

Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).

The returned record value is valid until the next call to Read. Users need to call Retain on that Record to keep it valid for longer.

func (*FileReader) ReadAt Uses

func (f *FileReader) ReadAt(i int64) (array.Record, error)

ReadAt reads the i-th record from the underlying stream and an error, if any.

func (*FileReader) Record Uses

func (f *FileReader) Record(i int) (array.Record, error)

Record returns the i-th record from the file. The returned value is valid until the next call to Record. Users need to call Retain on that Record to keep it valid for longer.

func (*FileReader) Schema Uses

func (f *FileReader) Schema() *arrow.Schema

func (*FileReader) Version Uses

func (f *FileReader) Version() MetadataVersion

type FileWriter Uses

type FileWriter struct {
    // contains filtered or unexported fields
}

FileWriter is an Arrow file writer.

func NewFileWriter Uses

func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error)

NewFileWriter opens an Arrow file using the provided writer w.

func (*FileWriter) Close Uses

func (f *FileWriter) Close() error

func (*FileWriter) Write Uses

func (f *FileWriter) Write(rec array.Record) error

type FlightDataReader Uses

type FlightDataReader struct {
    // contains filtered or unexported fields
}

FlightDataReader reads records from a stream of messages

func NewFlightDataReader Uses

func NewFlightDataReader(r FlightDataStreamReader, opts ...Option) (*FlightDataReader, error)

NewFlightDataReader returns a reader that will produce records from a flight data stream

implementation is generally based on the ipc.Reader, expecting the first message to be the schema with the subsequent messages being the record batches.

func (*FlightDataReader) Err Uses

func (f *FlightDataReader) Err() error

Err returns the last error encounted during the iteration of the stream.

func (*FlightDataReader) Next Uses

func (f *FlightDataReader) Next() bool

Next returns whether a record was able to be extracted from the stream or not.

func (*FlightDataReader) Read Uses

func (f *FlightDataReader) Read() (array.Record, error)

Read reads the current record from the flight stream and an error, if any. When we reach the end of the flight stream it will return (nil, io.EOF). Also calls release on the previous existing record if any.

func (*FlightDataReader) Record Uses

func (f *FlightDataReader) Record() array.Record

Record returns the current record that has been extracted from the stream. It is valid until the next call to Next or Read

func (*FlightDataReader) Release Uses

func (f *FlightDataReader) Release()

Release decreases the refcount by 1. When the refcount is 0 the memory is freed. Release may be called simultaneously from multiple goroutines.

func (*FlightDataReader) Retain Uses

func (f *FlightDataReader) Retain()

Retain increases the refcount by 1. Retain can be called by multiple goroutines simultaneously.

func (*FlightDataReader) Schema Uses

func (f *FlightDataReader) Schema() *arrow.Schema

Schema returns the schema of the underlying records as described by the first message received.

type FlightDataStreamReader Uses

type FlightDataStreamReader interface {
    Recv() (*flight.FlightData, error)
}

FlightDataStreamReader wraps a grpc stream for receiving FlightData objects

type FlightDataStreamWriter Uses

type FlightDataStreamWriter interface {
    Send(*flight.FlightData) error
}

FlightDataStreamWriter wraps a grpc stream for sending FlightData

type FlightDataWriter Uses

type FlightDataWriter struct {
    // contains filtered or unexported fields
}

FlightDataWriter is a stream writer for writing with Flight RPC

func NewFlightDataWriter Uses

func NewFlightDataWriter(w FlightDataStreamWriter, opts ...Option) *FlightDataWriter

NewFlightDataWriter returns a writer for writing array Records to a flight data stream.

func (*FlightDataWriter) Close Uses

func (w *FlightDataWriter) Close() (err error)

func (*FlightDataWriter) Write Uses

func (w *FlightDataWriter) Write(rec array.Record) error

Write the provided record to the underlying stream

type Message Uses

type Message struct {
    // contains filtered or unexported fields
}

Message is an IPC message, including metadata and body.

func NewMessage Uses

func NewMessage(meta, body *memory.Buffer) *Message

NewMessage creates a new message from the metadata and body buffers. NewMessage panics if any of these buffers is nil.

func (*Message) BodyLen Uses

func (msg *Message) BodyLen() int64

func (*Message) Release Uses

func (msg *Message) Release()

Release decreases the reference count by 1. Release may be called simultaneously from multiple goroutines. When the reference count goes to zero, the memory is freed.

func (*Message) Retain Uses

func (msg *Message) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

func (*Message) Type Uses

func (msg *Message) Type() MessageType

func (*Message) Version Uses

func (msg *Message) Version() MetadataVersion

type MessageReader Uses

type MessageReader struct {
    // contains filtered or unexported fields
}

MessageReader reads messages from an io.Reader.

func NewMessageReader Uses

func NewMessageReader(r io.Reader) *MessageReader

NewMessageReader returns a reader that reads messages from an input stream.

func (*MessageReader) Message Uses

func (r *MessageReader) Message() (*Message, error)

Message returns the current message that has been extracted from the underlying stream. It is valid until the next call to Message.

func (*MessageReader) Release Uses

func (r *MessageReader) Release()

Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.

func (*MessageReader) Retain Uses

func (r *MessageReader) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

type MessageType Uses

type MessageType flatbuf.MessageHeader

MessageType represents the type of Message in an Arrow format.

func (MessageType) String Uses

func (m MessageType) String() string

type MetadataVersion Uses

type MetadataVersion flatbuf.MetadataVersion

MetadataVersion represents the Arrow metadata version.

func (MetadataVersion) String Uses

func (m MetadataVersion) String() string

type Option Uses

type Option func(*config)

Option is a functional option to configure opening or creating Arrow files and streams.

func WithAllocator Uses

func WithAllocator(mem memory.Allocator) Option

WithAllocator specifies the Arrow memory allocator used while building records.

func WithFooterOffset Uses

func WithFooterOffset(offset int64) Option

WithFooterOffset specifies the Arrow footer position in bytes.

func WithSchema Uses

func WithSchema(schema *arrow.Schema) Option

WithSchema specifies the Arrow schema to be used for reading or writing.

type ReadAtSeeker Uses

type ReadAtSeeker interface {
    io.Reader
    io.Seeker
    io.ReaderAt
}

type Reader Uses

type Reader struct {
    // contains filtered or unexported fields
}

Reader reads records from an io.Reader. Reader expects a schema (plus any dictionaries) as the first messages in the stream, followed by records.

func NewReader Uses

func NewReader(r io.Reader, opts ...Option) (*Reader, error)

NewReader returns a reader that reads records from an input stream.

func (*Reader) Err Uses

func (r *Reader) Err() error

Err returns the last error encountered during the iteration over the underlying stream.

func (*Reader) Next Uses

func (r *Reader) Next() bool

Next returns whether a Record could be extracted from the underlying stream.

func (*Reader) Read Uses

func (r *Reader) Read() (array.Record, error)

Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).

func (*Reader) Record Uses

func (r *Reader) Record() array.Record

Record returns the current record that has been extracted from the underlying stream. It is valid until the next call to Next.

func (*Reader) Release Uses

func (r *Reader) Release()

Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.

func (*Reader) Retain Uses

func (r *Reader) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

func (*Reader) Schema Uses

func (r *Reader) Schema() *arrow.Schema

type Writer Uses

type Writer struct {
    // contains filtered or unexported fields
}

Writer is an Arrow stream writer.

func NewWriter Uses

func NewWriter(w io.Writer, opts ...Option) *Writer

NewWriter returns a writer that writes records to the provided output stream.

func (*Writer) Close Uses

func (w *Writer) Close() error

func (*Writer) Write Uses

func (w *Writer) Write(rec array.Record) error

Package ipc imports 17 packages (graph) and is imported by 5 packages. Updated 2020-09-29. Refresh now. Tools for package owners.