arrow: github.com/apache/arrow/go/arrow Index | Examples | Files | Directories

package arrow

import "github.com/apache/arrow/go/arrow"

Package arrow provides an implementation of Apache Arrow.

Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic operations on modern hardware. It also provides computational libraries and zero-copy streaming messaging and inter-process communication.

Basics

The fundamental data structure in Arrow is an Array, which holds a sequence of values of the same type. An array consists of memory holding the data and an additional validity bitmap that indicates if the corresponding entry in the array is valid (not null). If the array has no null entries, it is possible to omit this bitmap.

This example shows how one can slice an array. The initial (float64) array is:

[1, 2, 3, (null), 4, 5]

and the sub-slice is:

[3, (null), 4]

Code:

pool := memory.NewGoAllocator()

b := array.NewFloat64Builder(pool)
defer b.Release()

b.AppendValues(
    []float64{1, 2, 3, -1, 4, 5},
    []bool{true, true, true, false, true, true},
)

arr := b.NewFloat64Array()
defer arr.Release()

fmt.Printf("array = %v\n", arr)

sli := array.NewSlice(arr, 2, 5).(*array.Float64)
defer sli.Release()

fmt.Printf("slice = %v\n", sli)

Output:

array = [1 2 3 (null) 4 5]
slice = [3 (null) 4]

Code:

pool := memory.NewGoAllocator()

b := array.NewFloat64Builder(pool)
defer b.Release()

raw := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
b.AppendValues(raw, nil)

arr := b.NewFloat64Array()
defer arr.Release()

f64 := tensor.NewFloat64(arr.Data(), []int64{2, 5}, nil, []string{"x", "y"})
defer f64.Release()

for _, i := range [][]int64{
    []int64{0, 0},
    []int64{0, 1},
    []int64{0, 2},
    []int64{0, 3},
    []int64{0, 4},
    []int64{1, 0},
    []int64{1, 1},
    []int64{1, 2},
    []int64{1, 3},
    []int64{1, 4},
} {
    fmt.Printf("arr%v = %v\n", i, f64.Value(i))
}

Output:

arr[0 0] = 1
arr[0 1] = 2
arr[0 2] = 3
arr[0 3] = 4
arr[0 4] = 5
arr[1 0] = 6
arr[1 1] = 7
arr[1 2] = 8
arr[1 3] = 9
arr[1 4] = 10

Code:

pool := memory.NewGoAllocator()

b := array.NewFloat64Builder(pool)
defer b.Release()

raw := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
b.AppendValues(raw, nil)

arr := b.NewFloat64Array()
defer arr.Release()

f64 := tensor.NewFloat64(arr.Data(), []int64{2, 5}, []int64{8, 16}, []string{"x", "y"})
defer f64.Release()

for _, i := range [][]int64{
    []int64{0, 0},
    []int64{0, 1},
    []int64{0, 2},
    []int64{0, 3},
    []int64{0, 4},
    []int64{1, 0},
    []int64{1, 1},
    []int64{1, 2},
    []int64{1, 3},
    []int64{1, 4},
} {
    fmt.Printf("arr%v = %v\n", i, f64.Value(i))
}

Output:

arr[0 0] = 1
arr[0 1] = 3
arr[0 2] = 5
arr[0 3] = 7
arr[0 4] = 9
arr[1 0] = 2
arr[1 1] = 4
arr[1 2] = 6
arr[1 3] = 8
arr[1 4] = 10

This example demonstrates creating an array, sourcing the values and null bitmaps directly from byte slices. The null count is set to UnknownNullCount, instructing the array to calculate the null count from the bitmap when NullN is called.

Code:

// create LSB packed bits with the following pattern:
// 01010011 11000101
data := memory.NewBufferBytes([]byte{0xca, 0xa3})

// create LSB packed validity (null) bitmap, where every 4th element is null:
// 11101110 11101110
nullBitmap := memory.NewBufferBytes([]byte{0x77, 0x77})

// Create a boolean array and lazily determine NullN using UnknownNullCount
bools := array.NewBoolean(16, data, nullBitmap, array.UnknownNullCount)
defer bools.Release()

// Show the null count
fmt.Printf("NullN()  = %d\n", bools.NullN())

// Enumerate the values.
n := bools.Len()
for i := 0; i < n; i++ {
    fmt.Printf("bools[%d] = ", i)
    if bools.IsNull(i) {
        fmt.Println("(null)")
    } else {
        fmt.Printf("%t\n", bools.Value(i))
    }
}

Output:

NullN()  = 4
bools[0] = false
bools[1] = true
bools[2] = false
bools[3] = (null)
bools[4] = false
bools[5] = false
bools[6] = true
bools[7] = (null)
bools[8] = true
bools[9] = true
bools[10] = false
bools[11] = (null)
bools[12] = false
bools[13] = true
bools[14] = false
bools[15] = (null)

This example shows how to create a List array. The resulting array should be:

[[0, 1, 2], [], [3], [4, 5], [6, 7, 8], [], [9]]

Code:

pool := memory.NewGoAllocator()

lb := array.NewListBuilder(pool, arrow.PrimitiveTypes.Int64)
defer lb.Release()

vb := lb.ValueBuilder().(*array.Int64Builder)
defer vb.Release()

vb.Reserve(10)

lb.Append(true)
vb.Append(0)
vb.Append(1)
vb.Append(2)

lb.AppendNull()

lb.Append(true)
vb.Append(3)

lb.Append(true)
vb.Append(4)
vb.Append(5)

lb.Append(true)
vb.Append(6)
vb.Append(7)
vb.Append(8)

lb.AppendNull()

lb.Append(true)
vb.Append(9)

arr := lb.NewArray().(*array.List)
defer arr.Release()

fmt.Printf("NullN()   = %d\n", arr.NullN())
fmt.Printf("Len()     = %d\n", arr.Len())
fmt.Printf("Offsets() = %v\n", arr.Offsets())

offsets := arr.Offsets()[1:]

varr := arr.ListValues().(*array.Int64)

pos := 0
for i := 0; i < arr.Len(); i++ {
    if !arr.IsValid(i) {
        fmt.Printf("List[%d]   = (null)\n", i)
        continue
    }
    fmt.Printf("List[%d]   = [", i)
    for j := pos; j < int(offsets[i]); j++ {
        if j != pos {
            fmt.Printf(", ")
        }
        fmt.Printf("%v", varr.Value(j))
    }
    pos = int(offsets[i])
    fmt.Printf("]\n")
}

Output:

NullN()   = 2
Len()     = 7
Offsets() = [0 3 3 4 6 9 9 10]
List[0]   = [0, 1, 2]
List[1]   = (null)
List[2]   = [3]
List[3]   = [4, 5]
List[4]   = [6, 7, 8]
List[5]   = (null)
List[6]   = [9]

This example demonstrates how to build an array of int64 values using a builder and Append. Whilst convenient for small arrays,

Code:

// Create an allocator.
pool := memory.NewGoAllocator()

// Create an int64 array builder.
builder := array.NewInt64Builder(pool)
defer builder.Release()

builder.Append(1)
builder.Append(2)
builder.Append(3)
builder.AppendNull()
builder.Append(5)
builder.Append(6)
builder.Append(7)
builder.Append(8)

// Finish building the int64 array and reset the builder.
ints := builder.NewInt64Array()
defer ints.Release()

// Enumerate the values.
for i, v := range ints.Int64Values() {
    fmt.Printf("ints[%d] = ", i)
    if ints.IsNull(i) {
        fmt.Println("(null)")
    } else {
        fmt.Println(v)
    }
}
fmt.Printf("ints = %v\n", ints)

Output:

ints[0] = 1
ints[1] = 2
ints[2] = 3
ints[3] = (null)
ints[4] = 5
ints[5] = 6
ints[6] = 7
ints[7] = 8
ints = [1 2 3 (null) 5 6 7 8]

Code:

pool := memory.NewGoAllocator()

schema := arrow.NewSchema(
    []arrow.Field{
        arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
        arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
    },
    nil,
)

b := array.NewRecordBuilder(pool, schema)
defer b.Release()

b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4, 5, 6}, nil)
b.Field(0).(*array.Int32Builder).AppendValues([]int32{7, 8, 9, 10}, []bool{true, true, false, true})
b.Field(1).(*array.Float64Builder).AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)

rec := b.NewRecord()
defer rec.Release()

for i, col := range rec.Columns() {
    fmt.Printf("column[%d] %q: %v\n", i, rec.ColumnName(i), col)
}

Output:

column[0] "f1-i32": [1 2 3 4 5 6 7 8 (null) 10]
column[1] "f2-f64": [1 2 3 4 5 6 7 8 9 10]

Code:

pool := memory.NewGoAllocator()

schema := arrow.NewSchema(
    []arrow.Field{
        arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
        arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
    },
    nil,
)

b := array.NewRecordBuilder(pool, schema)
defer b.Release()

b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4, 5, 6}, nil)
b.Field(0).(*array.Int32Builder).AppendValues([]int32{7, 8, 9, 10}, []bool{true, true, false, true})
b.Field(1).(*array.Float64Builder).AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)

rec1 := b.NewRecord()
defer rec1.Release()

b.Field(0).(*array.Int32Builder).AppendValues([]int32{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
b.Field(1).(*array.Float64Builder).AppendValues([]float64{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)

rec2 := b.NewRecord()
defer rec2.Release()

itr, err := array.NewRecordReader(schema, []array.Record{rec1, rec2})
if err != nil {
    log.Fatal(err)
}
defer itr.Release()

n := 0
for itr.Next() {
    rec := itr.Record()
    for i, col := range rec.Columns() {
        fmt.Printf("rec[%d][%q]: %v\n", n, rec.ColumnName(i), col)
    }
    n++
}

Output:

rec[0]["f1-i32"]: [1 2 3 4 5 6 7 8 (null) 10]
rec[0]["f2-f64"]: [1 2 3 4 5 6 7 8 9 10]
rec[1]["f1-i32"]: [11 12 13 14 15 16 17 18 19 20]
rec[1]["f2-f64"]: [11 12 13 14 15 16 17 18 19 20]

This example shows how to create a Struct array. The resulting array should be:

[{‘joe’, 1}, {null, 2}, null, {‘mark’, 4}]

Code:

pool := memory.NewGoAllocator()

dtype := arrow.StructOf([]arrow.Field{
    {Name: "f1", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint8)},
    {Name: "f2", Type: arrow.PrimitiveTypes.Int32},
}...)

sb := array.NewStructBuilder(pool, dtype)
defer sb.Release()

f1b := sb.FieldBuilder(0).(*array.ListBuilder)
defer f1b.Release()
f1vb := f1b.ValueBuilder().(*array.Uint8Builder)
defer f1vb.Release()

f2b := sb.FieldBuilder(1).(*array.Int32Builder)
defer f2b.Release()

sb.Reserve(4)
f1vb.Reserve(7)
f2b.Reserve(3)

sb.Append(true)
f1b.Append(true)
f1vb.AppendValues([]byte("joe"), nil)
f2b.Append(1)

sb.Append(true)
f1b.AppendNull()
f2b.Append(2)

sb.AppendNull()

sb.Append(true)
f1b.Append(true)
f1vb.AppendValues([]byte("mark"), nil)
f2b.Append(4)

arr := sb.NewArray().(*array.Struct)
defer arr.Release()

fmt.Printf("NullN() = %d\n", arr.NullN())
fmt.Printf("Len()   = %d\n", arr.Len())

list := arr.Field(0).(*array.List)
defer list.Release()

offsets := list.Offsets()

varr := list.ListValues().(*array.Uint8)
defer varr.Release()

ints := arr.Field(1).(*array.Int32)
defer ints.Release()

for i := 0; i < arr.Len(); i++ {
    if !arr.IsValid(i) {
        fmt.Printf("Struct[%d] = (null)\n", i)
        continue
    }
    fmt.Printf("Struct[%d] = [", i)
    pos := int(offsets[i])
    switch {
    case list.IsValid(pos):
        fmt.Printf("[")
        for j := offsets[i]; j < offsets[i+1]; j++ {
            if j != offsets[i] {
                fmt.Printf(", ")
            }
            fmt.Printf("%v", string(varr.Value(int(j))))
        }
        fmt.Printf("], ")
    default:
        fmt.Printf("(null), ")
    }
    fmt.Printf("%d]\n", ints.Value(i))
}

Output:

NullN() = 1
Len()   = 4
Struct[0] = [[j, o, e], 1]
Struct[1] = [[], 2]
Struct[2] = (null)
Struct[3] = [[m, a, r, k], 4]

Code:

pool := memory.NewGoAllocator()

schema := arrow.NewSchema(
    []arrow.Field{
        arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
        arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
    },
    nil,
)

b := array.NewRecordBuilder(pool, schema)
defer b.Release()

b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4, 5, 6}, nil)
b.Field(0).(*array.Int32Builder).AppendValues([]int32{7, 8, 9, 10}, []bool{true, true, false, true})
b.Field(1).(*array.Float64Builder).AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)

rec1 := b.NewRecord()
defer rec1.Release()

b.Field(0).(*array.Int32Builder).AppendValues([]int32{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
b.Field(1).(*array.Float64Builder).AppendValues([]float64{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)

rec2 := b.NewRecord()
defer rec2.Release()

tbl := array.NewTableFromRecords(schema, []array.Record{rec1, rec2})
defer tbl.Release()

tr := array.NewTableReader(tbl, 5)
defer tr.Release()

n := 0
for tr.Next() {
    rec := tr.Record()
    for i, col := range rec.Columns() {
        fmt.Printf("rec[%d][%q]: %v\n", n, rec.ColumnName(i), col)
    }
    n++
}

Output:

rec[0]["f1-i32"]: [1 2 3 4 5]
rec[0]["f2-f64"]: [1 2 3 4 5]
rec[1]["f1-i32"]: [6 7 8 (null) 10]
rec[1]["f2-f64"]: [6 7 8 9 10]
rec[2]["f1-i32"]: [11 12 13 14 15]
rec[2]["f2-f64"]: [11 12 13 14 15]
rec[3]["f1-i32"]: [16 17 18 19 20]
rec[3]["f2-f64"]: [16 17 18 19 20]

Index

Examples

Package Files

datatype.go datatype_binary.go datatype_fixedwidth.go datatype_nested.go datatype_null.go datatype_numeric.gen.go doc.go schema.go type_string.go type_traits_boolean.go type_traits_numeric.gen.go

Constants

const (
    // Date32SizeBytes specifies the number of bytes required to store a single Date32 in memory
    Date32SizeBytes = int(unsafe.Sizeof(Date32(0)))
)
const (
    // Date64SizeBytes specifies the number of bytes required to store a single Date64 in memory
    Date64SizeBytes = int(unsafe.Sizeof(Date64(0)))
)
const (
    // Float32SizeBytes specifies the number of bytes required to store a single float32 in memory
    Float32SizeBytes = int(unsafe.Sizeof(float32(0)))
)
const (
    // Float64SizeBytes specifies the number of bytes required to store a single float64 in memory
    Float64SizeBytes = int(unsafe.Sizeof(float64(0)))
)
const (
    // Int16SizeBytes specifies the number of bytes required to store a single int16 in memory
    Int16SizeBytes = int(unsafe.Sizeof(int16(0)))
)
const (
    // Int32SizeBytes specifies the number of bytes required to store a single int32 in memory
    Int32SizeBytes = int(unsafe.Sizeof(int32(0)))
)
const (
    // Int64SizeBytes specifies the number of bytes required to store a single int64 in memory
    Int64SizeBytes = int(unsafe.Sizeof(int64(0)))
)
const (
    // Int8SizeBytes specifies the number of bytes required to store a single int8 in memory
    Int8SizeBytes = int(unsafe.Sizeof(int8(0)))
)
const (
    // Time32SizeBytes specifies the number of bytes required to store a single Time32 in memory
    Time32SizeBytes = int(unsafe.Sizeof(Time32(0)))
)
const (
    // Time64SizeBytes specifies the number of bytes required to store a single Time64 in memory
    Time64SizeBytes = int(unsafe.Sizeof(Time64(0)))
)
const (
    // TimestampSizeBytes specifies the number of bytes required to store a single Timestamp in memory
    TimestampSizeBytes = int(unsafe.Sizeof(Timestamp(0)))
)
const (
    // Uint16SizeBytes specifies the number of bytes required to store a single uint16 in memory
    Uint16SizeBytes = int(unsafe.Sizeof(uint16(0)))
)
const (
    // Uint32SizeBytes specifies the number of bytes required to store a single uint32 in memory
    Uint32SizeBytes = int(unsafe.Sizeof(uint32(0)))
)
const (
    // Uint64SizeBytes specifies the number of bytes required to store a single uint64 in memory
    Uint64SizeBytes = int(unsafe.Sizeof(uint64(0)))
)
const (
    // Uint8SizeBytes specifies the number of bytes required to store a single uint8 in memory
    Uint8SizeBytes = int(unsafe.Sizeof(uint8(0)))
)

Variables

var (
    Int64Traits     int64Traits
    Uint64Traits    uint64Traits
    Float64Traits   float64Traits
    Int32Traits     int32Traits
    Uint32Traits    uint32Traits
    Float32Traits   float32Traits
    Int16Traits     int16Traits
    Uint16Traits    uint16Traits
    Int8Traits      int8Traits
    Uint8Traits     uint8Traits
    TimestampTraits timestampTraits
    Time32Traits    time32Traits
    Time64Traits    time64Traits
    Date32Traits    date32Traits
    Date64Traits    date64Traits
)
var (
    BinaryTypes = struct {
        Binary BinaryDataType
        String BinaryDataType
    }{
        Binary: &BinaryType{},
        String: &StringType{},
    }
)
var BooleanTraits booleanTraits
var (
    FixedWidthTypes = struct {
        Boolean  FixedWidthDataType
        Time32s  FixedWidthDataType
        Time32ms FixedWidthDataType
        Time64us FixedWidthDataType
        Time64ns FixedWidthDataType
    }{
        Boolean:  &BooleanType{},
        Time32s:  &Time32Type{Unit: Second},
        Time32ms: &Time32Type{Unit: Millisecond},
        Time64us: &Time64Type{Unit: Microsecond},
        Time64ns: &Time64Type{Unit: Nanosecond},
    }
)
var (
    PrimitiveTypes = struct {
        Int8    DataType
        Int16   DataType
        Int32   DataType
        Int64   DataType
        Uint8   DataType
        Uint16  DataType
        Uint32  DataType
        Uint64  DataType
        Float32 DataType
        Float64 DataType
        Date32  DataType
        Date64  DataType
    }{

        Int8:    &Int8Type{},
        Int16:   &Int16Type{},
        Int32:   &Int32Type{},
        Int64:   &Int64Type{},
        Uint8:   &Uint8Type{},
        Uint16:  &Uint16Type{},
        Uint32:  &Uint32Type{},
        Uint64:  &Uint64Type{},
        Float32: &Float32Type{},
        Float64: &Float64Type{},
        Date32:  &Date32Type{},
        Date64:  &Date64Type{},
    }
)

type BinaryDataType Uses

type BinaryDataType interface {
    DataType
    // contains filtered or unexported methods
}

type BinaryType Uses

type BinaryType struct{}

func (*BinaryType) ID Uses

func (t *BinaryType) ID() Type

func (*BinaryType) Name Uses

func (t *BinaryType) Name() string

type BooleanType Uses

type BooleanType struct{}

func (*BooleanType) BitWidth Uses

func (t *BooleanType) BitWidth() int

BitWidth returns the number of bits required to store a single element of this data type in memory.

func (*BooleanType) ID Uses

func (t *BooleanType) ID() Type

func (*BooleanType) Name Uses

func (t *BooleanType) Name() string

type DataType Uses

type DataType interface {
    ID() Type
    // Name is name of the data type.
    Name() string
}

DataType is the representation of an Arrow type.

type Date32 Uses

type Date32 int32

type Date32Type Uses

type Date32Type struct{}

func (*Date32Type) BitWidth Uses

func (t *Date32Type) BitWidth() int

func (*Date32Type) ID Uses

func (t *Date32Type) ID() Type

func (*Date32Type) Name Uses

func (t *Date32Type) Name() string

type Date64 Uses

type Date64 int64

type Date64Type Uses

type Date64Type struct{}

func (*Date64Type) BitWidth Uses

func (t *Date64Type) BitWidth() int

func (*Date64Type) ID Uses

func (t *Date64Type) ID() Type

func (*Date64Type) Name Uses

func (t *Date64Type) Name() string

type Field Uses

type Field struct {
    Name     string   // Field name
    Type     DataType // The field's data type
    Nullable bool     // Fields can be nullable
    Metadata Metadata // The field's metadata, if any
}

func (Field) Equal Uses

func (f Field) Equal(o Field) bool

func (Field) HasMetadata Uses

func (f Field) HasMetadata() bool

type FixedSizeBinaryType Uses

type FixedSizeBinaryType struct {
    ByteWidth int
}

func (*FixedSizeBinaryType) BitWidth Uses

func (t *FixedSizeBinaryType) BitWidth() int

func (*FixedSizeBinaryType) ID Uses

func (*FixedSizeBinaryType) ID() Type

func (*FixedSizeBinaryType) Name Uses

func (*FixedSizeBinaryType) Name() string

type FixedWidthDataType Uses

type FixedWidthDataType interface {
    DataType
    // BitWidth returns the number of bits required to store a single element of this data type in memory.
    BitWidth() int
}

FixedWidthDataType is the representation of an Arrow type that requires a fixed number of bits in memory for each element.

type Float32Type Uses

type Float32Type struct{}

func (*Float32Type) BitWidth Uses

func (t *Float32Type) BitWidth() int

func (*Float32Type) ID Uses

func (t *Float32Type) ID() Type

func (*Float32Type) Name Uses

func (t *Float32Type) Name() string

type Float64Type Uses

type Float64Type struct{}

func (*Float64Type) BitWidth Uses

func (t *Float64Type) BitWidth() int

func (*Float64Type) ID Uses

func (t *Float64Type) ID() Type

func (*Float64Type) Name Uses

func (t *Float64Type) Name() string

type Int16Type Uses

type Int16Type struct{}

func (*Int16Type) BitWidth Uses

func (t *Int16Type) BitWidth() int

func (*Int16Type) ID Uses

func (t *Int16Type) ID() Type

func (*Int16Type) Name Uses

func (t *Int16Type) Name() string

type Int32Type Uses

type Int32Type struct{}

func (*Int32Type) BitWidth Uses

func (t *Int32Type) BitWidth() int

func (*Int32Type) ID Uses

func (t *Int32Type) ID() Type

func (*Int32Type) Name Uses

func (t *Int32Type) Name() string

type Int64Type Uses

type Int64Type struct{}

func (*Int64Type) BitWidth Uses

func (t *Int64Type) BitWidth() int

func (*Int64Type) ID Uses

func (t *Int64Type) ID() Type

func (*Int64Type) Name Uses

func (t *Int64Type) Name() string

type Int8Type Uses

type Int8Type struct{}

func (*Int8Type) BitWidth Uses

func (t *Int8Type) BitWidth() int

func (*Int8Type) ID Uses

func (t *Int8Type) ID() Type

func (*Int8Type) Name Uses

func (t *Int8Type) Name() string

type ListType Uses

type ListType struct {
    // contains filtered or unexported fields
}

ListType describes a nested type in which each array slot contains a variable-size sequence of values, all having the same relative type.

func ListOf Uses

func ListOf(t DataType) *ListType

ListOf returns the list type with element type t. For example, if t represents int32, ListOf(t) represents []int32.

ListOf panics if t is nil or invalid.

func (*ListType) Elem Uses

func (t *ListType) Elem() DataType

Elem returns the ListType's element type.

func (*ListType) ID Uses

func (*ListType) ID() Type

func (*ListType) Name Uses

func (*ListType) Name() string

type Metadata Uses

type Metadata struct {
    // contains filtered or unexported fields
}

func MetadataFrom Uses

func MetadataFrom(kv map[string]string) Metadata

func NewMetadata Uses

func NewMetadata(keys, values []string) Metadata

func (Metadata) Keys Uses

func (md Metadata) Keys() []string

func (Metadata) Len Uses

func (md Metadata) Len() int

func (Metadata) Values Uses

func (md Metadata) Values() []string

type NullType Uses

type NullType struct{}

NullType describes a degenerate array, with zero physical storage.

var (
    Null *NullType
)

func (*NullType) ID Uses

func (*NullType) ID() Type

func (*NullType) Name Uses

func (*NullType) Name() string

type Schema Uses

type Schema struct {
    // contains filtered or unexported fields
}

Schema is a sequence of Field values, describing the columns of a table or a record batch.

func NewSchema Uses

func NewSchema(fields []Field, metadata *Metadata) *Schema

NewSchema returns a new Schema value from the slice of fields and metadata.

NewSchema panics if there are duplicated fields. NewSchema panics if there is a field with an invalid DataType.

func (*Schema) Equal Uses

func (sc *Schema) Equal(o *Schema) bool

Equal returns whether two schema are equal. Equal does not compare the metadata.

func (*Schema) Field Uses

func (sc *Schema) Field(i int) Field

func (*Schema) FieldByName Uses

func (sc *Schema) FieldByName(n string) (Field, bool)

func (*Schema) FieldIndex Uses

func (sc *Schema) FieldIndex(n string) int

FieldIndex returns the index of the named field or -1.

func (*Schema) Fields Uses

func (sc *Schema) Fields() []Field

func (*Schema) HasField Uses

func (sc *Schema) HasField(n string) bool

func (*Schema) HasMetadata Uses

func (sc *Schema) HasMetadata() bool

func (*Schema) Metadata Uses

func (sc *Schema) Metadata() Metadata

type StringType Uses

type StringType struct{}

func (*StringType) ID Uses

func (t *StringType) ID() Type

func (*StringType) Name Uses

func (t *StringType) Name() string

type StructType Uses

type StructType struct {
    // contains filtered or unexported fields
}

StructType describes a nested type parameterized by an ordered sequence of relative types, called its fields.

func StructOf Uses

func StructOf(fs ...Field) *StructType

StructOf returns the struct type with fields fs.

StructOf panics if there are duplicated fields. StructOf panics if there is a field with an invalid DataType.

func (*StructType) Field Uses

func (t *StructType) Field(i int) Field

func (*StructType) FieldByName Uses

func (t *StructType) FieldByName(name string) (Field, bool)

func (*StructType) Fields Uses

func (t *StructType) Fields() []Field

func (*StructType) ID Uses

func (*StructType) ID() Type

func (*StructType) Name Uses

func (*StructType) Name() string

type Time32 Uses

type Time32 int32

type Time32Type Uses

type Time32Type struct {
    Unit TimeUnit
}

Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight.

func (*Time32Type) BitWidth Uses

func (*Time32Type) BitWidth() int

func (*Time32Type) ID Uses

func (*Time32Type) ID() Type

func (*Time32Type) Name Uses

func (*Time32Type) Name() string

type Time64 Uses

type Time64 int64

type Time64Type Uses

type Time64Type struct {
    Unit TimeUnit
}

Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight.

func (*Time64Type) BitWidth Uses

func (*Time64Type) BitWidth() int

func (*Time64Type) ID Uses

func (*Time64Type) ID() Type

func (*Time64Type) Name Uses

func (*Time64Type) Name() string

type TimeUnit Uses

type TimeUnit int
const (
    Nanosecond TimeUnit = iota
    Microsecond
    Millisecond
    Second
)

func (TimeUnit) String Uses

func (u TimeUnit) String() string

type Timestamp Uses

type Timestamp int64

type TimestampType Uses

type TimestampType struct {
    Unit     TimeUnit
    TimeZone string
}

TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z). The zero-value is a nanosecond and time zone neutral. Time zone neutral can be considered UTC without having "UTC" as a time zone.

func (*TimestampType) BitWidth Uses

func (*TimestampType) BitWidth() int

BitWidth returns the number of bits required to store a single element of this data type in memory.

func (*TimestampType) ID Uses

func (*TimestampType) ID() Type

func (*TimestampType) Name Uses

func (*TimestampType) Name() string

type Type Uses

type Type int

Type is a logical type. They can be expressed as either a primitive physical type (bytes or bits of some fixed size), a nested type consisting of other data types, or another data type (e.g. a timestamp encoded as an int64)

const (
    // NULL type having no physical storage
    NULL Type = iota

    // BOOL is a 1 bit, LSB bit-packed ordering
    BOOL

    // UINT8 is an Unsigned 8-bit little-endian integer
    UINT8

    // INT8 is a Signed 8-bit little-endian integer
    INT8

    // UINT16 is an Unsigned 16-bit little-endian integer
    UINT16

    // INT16 is a Signed 16-bit little-endian integer
    INT16

    // UINT32 is an Unsigned 32-bit little-endian integer
    UINT32

    // INT32 is a Signed 32-bit little-endian integer
    INT32

    // UINT64 is an Unsigned 64-bit little-endian integer
    UINT64

    // INT64 is a Signed 64-bit little-endian integer
    INT64

    // HALF_FLOAT is a 2-byte floating point value
    HALF_FLOAT

    // FLOAT32 is a 4-byte floating point value
    FLOAT32

    // FLOAT64 is an 8-byte floating point value
    FLOAT64

    // STRING is a UTF8 variable-length string
    STRING

    // BINARY is a Variable-length byte type (no guarantee of UTF8-ness)
    BINARY

    // FIXED_SIZE_BINARY is a binary where each value occupies the same number of bytes
    FIXED_SIZE_BINARY

    // DATE32 is int32 days since the UNIX epoch
    DATE32

    // DATE64 is int64 milliseconds since the UNIX epoch
    DATE64

    // TIMESTAMP is an exact timestamp encoded with int64 since UNIX epoch
    // Default unit millisecond
    TIMESTAMP

    // TIME32 is a signed 32-bit integer, representing either seconds or
    // milliseconds since midnight
    TIME32

    // TIME64 is a signed 64-bit integer, representing either microseconds or
    // nanoseconds since midnight
    TIME64

    // INTERVAL is YEAR_MONTH or DAY_TIME interval in SQL style
    INTERVAL

    // DECIMAL is a precision- and scale-based decimal type. Storage type depends on the
    // parameters.
    DECIMAL

    // LIST is a list of some logical data type
    LIST

    // STRUCT of logical types
    STRUCT

    // UNION of logical types
    UNION

    // DICTIONARY aka Category type
    DICTIONARY

    // MAP is a repeated struct logical type
    MAP
)

func (Type) String Uses

func (i Type) String() string

type Uint16Type Uses

type Uint16Type struct{}

func (*Uint16Type) BitWidth Uses

func (t *Uint16Type) BitWidth() int

func (*Uint16Type) ID Uses

func (t *Uint16Type) ID() Type

func (*Uint16Type) Name Uses

func (t *Uint16Type) Name() string

type Uint32Type Uses

type Uint32Type struct{}

func (*Uint32Type) BitWidth Uses

func (t *Uint32Type) BitWidth() int

func (*Uint32Type) ID Uses

func (t *Uint32Type) ID() Type

func (*Uint32Type) Name Uses

func (t *Uint32Type) Name() string

type Uint64Type Uses

type Uint64Type struct{}

func (*Uint64Type) BitWidth Uses

func (t *Uint64Type) BitWidth() int

func (*Uint64Type) ID Uses

func (t *Uint64Type) ID() Type

func (*Uint64Type) Name Uses

func (t *Uint64Type) Name() string

type Uint8Type Uses

type Uint8Type struct{}

func (*Uint8Type) BitWidth Uses

func (t *Uint8Type) BitWidth() int

func (*Uint8Type) ID Uses

func (t *Uint8Type) ID() Type

func (*Uint8Type) Name Uses

func (t *Uint8Type) Name() string

Directories

PathSynopsis
arrayPackage array provides implementations of various Arrow array types.
csvPackage csv reads CSV files and presents the extracted data as records, also writes data as record into CSV files
internal/bitutil
internal/cpuPackage cpu implements processor feature detection used by the Go standard library.
internal/debugPackage debug provides APIs for conditional runtime assertions and debug logging.
internal/testing/tools
mathPackage math provides optimized mathematical functions for processing Arrow arrays.
memoryPackage memory provides support for allocating and manipulating memory at a low level.
tensorPackage tensor provides types that implement n-dimensional arrays.

Package arrow imports 7 packages (graph) and is imported by 5 packages. Updated 2019-03-18. Refresh now. Tools for package owners.