schema

package
v0.0.17 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 14, 2024 License: Apache-2.0 Imports: 9 Imported by: 1

Documentation

Overview

Package schema defines schema of YT tables.

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func CompareRows

func CompareRows(a, b []any, s *Schema) int

CompareRows orders two set of row keys according to schema.

Keys may contain:

  • nil
  • bool
  • string
  • []byte
  • float32, float64
  • int, int32, int64, int16, int8
  • uint, uint32, uint64, uint16, uint8

Passing any other value to this function results in panic.

Currently, this function does not handle NaN-s correctly.

The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

Types

type AggregateFunction

type AggregateFunction string
const (
	AggregateSum AggregateFunction = "sum"
	AggregateMin AggregateFunction = "min"
	AggregateMax AggregateFunction = "max"
)

func (AggregateFunction) MarshalText

func (a AggregateFunction) MarshalText() (text []byte, err error)

func (*AggregateFunction) UnmarshalText

func (a *AggregateFunction) UnmarshalText(text []byte) error

type Any

type Any any

type Column

type Column struct {
	Name string `yson:"name"`

	// Type and Required represent "legacy" row schema.
	//
	// When creating Column fill either Type and Required, or ComplexType but not both.
	//
	// When reading schema from YT, both "legacy" and "new" fields are available.
	Type     Type `yson:"type,omitempty"`
	Required bool `yson:"required,omitempty"`

	// ComplexType is "new" row schema.
	//
	//   Type == TypeInt64 && Required == true is equivalent to ComplexType == TypeInt64
	//   Type == TypeInt64 && Required == false is equivalent to ComplexType == Optional{Item: TypeInt64}
	ComplexType ComplexType `yson:"type_v3,omitempty"`

	SortOrder SortOrder `yson:"sort_order,omitempty"`

	// Lock specifies lock group.
	//
	// Used by the MVCC mechanism of dynamic tables. Columns from different lock groups might be modified concurrently.
	Lock       string            `yson:"lock,omitempty"`
	Expression string            `yson:"expression,omitempty"`
	Aggregate  AggregateFunction `yson:"aggregate,omitempty"`

	// Group specifies column group. When using columnar format (optimize_for=scan), columns groups are stored together.
	Group string `yson:"group,omitempty"`
}

Column specifies schema of a single column.

See https://yt.yandex-team.ru/docs/description/storage/static_schema

func (Column) MarshalYSON

func (c Column) MarshalYSON(w *yson.Writer) error

func (Column) NormalizeType

func (c Column) NormalizeType() Column

type ComplexType

type ComplexType interface {
	// contains filtered or unexported methods
}

type Date

type Date uint64

Date is YT type representing number of days since beginning of the unix epoch.

func NewDate

func NewDate(t time.Time) (Date, error)

func (Date) Time

func (t Date) Time() time.Time

type Datetime

type Datetime uint64

Datetime is YT type representing number of seconds since beginning of the unix epoch.

func NewDatetime

func NewDatetime(t time.Time) (Datetime, error)

func (Datetime) Time

func (t Datetime) Time() time.Time

type Decimal

type Decimal struct {
	Precision int `yson:"precision"`
	Scale     int `yson:"scale"`
}

func (Decimal) MarshalYSON

func (d Decimal) MarshalYSON(w *yson.Writer) error

type Dict

type Dict struct {
	Key   ComplexType `yson:"key"`
	Value ComplexType `yson:"value"`
}

func (Dict) MarshalYSON

func (d Dict) MarshalYSON(w *yson.Writer) error

type Interval

type Interval int64

Interval is YT type representing distance between two Timestamps-s in microseconds.

func NewInterval

func NewInterval(d time.Duration) (Interval, error)

func (Interval) Duration

func (i Interval) Duration() time.Duration

type List

type List struct {
	Item ComplexType `yson:"item"`
}

func (List) MarshalYSON

func (l List) MarshalYSON(w *yson.Writer) error

type Optional

type Optional struct {
	Item ComplexType `yson:"item"`
}

func (Optional) MarshalYSON

func (o Optional) MarshalYSON(w *yson.Writer) error

type RangeError

type RangeError struct {
	MinValue any
	MaxValue any
}

func (RangeError) Error

func (r RangeError) Error() string

type Schema

type Schema struct {
	// Schema is strict by default. Change this option only if you know that you are doing.
	Strict     *bool `yson:"strict,attr,omitempty"`
	UniqueKeys bool  `yson:"unique_keys,attr"`

	Columns []Column `yson:",value"`
}

Schema describe schema of YT table.

Schema is strict by default.

See https://yt.yandex-team.ru/docs/description/storage/static_schema

func Infer

func Infer(value any) (s Schema, err error)

Infer infers Schema from go struct.

By default Infer creates column for each struct field.

Column name is inferred from the value of yson tag, by the usual rules of the yson.Marshal.

`yson:",key"` tag corresponds to sort_order=ascending scheme attribute.

`ytgroup:""` tag is used to fill "group" scheme attribute.

Type of the column corresponds to go type of the field.

Mapping between go and YT types is defined as follows:

Go types int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64 and bool are mapped to equivalent YT types. NOTE: int8 and uint8 types are not supported.

Go type []byte is mapped to YT type string.

Go types implementing encoding.BinaryMarshaler interface are mapper to YT type string.

Go type string is mapped to YT type utf8.

Go types implementing encoding.TextMarshaler interface are mapped to YT type utf8.

Example
package main

import (
	"fmt"

	"go.ytsaurus.tech/yt/go/schema"
)

type MyStruct struct {
	IntColumn    int
	AnyValue     any
	Optional     *int
	StringColumn string `yson:"custom_column_name"`
	SkipMe       int    `yson:"-"`
}

var MySchema = schema.MustInfer(&MyStruct{})

func main() {
	for _, column := range MySchema.Columns {
		var kind string
		if column.Required {
			kind = "required"
		} else {
			kind = "optional"
		}

		fmt.Printf("%q is %s column of type %s\n", column.Name, kind, column.Type)
	}

}
Output:

"IntColumn" is required column of type int64
"AnyValue" is optional column of type any
"Optional" is optional column of type int64
"custom_column_name" is required column of type utf8

func InferMap

func InferMap(value any) (s Schema, err error)

InferMap infers Schema from go map[string]any.

InferMap creates column for each key value pair. Column name inferred from key itself, and column type inferred from the type of value.

To avoid ambiguity key type should always be string, while value type doesn't matter.

func MergeSchemas

func MergeSchemas(lhs, rhs Schema) Schema

MergeSchemas merges two schemas inferred from rows

If column have different types, then result column will be of type Any. Result schema does not have key columns, and columns have following order: columns from `lhs` schema, then columns from `rhs` schema that does not exist in `lhs` schema.

func MustInfer

func MustInfer(value any) Schema

MustInfer infers Schema from go struct.

MustInfer panics on errors.

func MustInferMap

func MustInferMap(value any) (s Schema)

MustInferMap infers Schema from go map[string]any.

MustInferMap panics on errors.

func (Schema) Append

func (s Schema) Append(column ...Column) Schema

func (Schema) Copy

func (s Schema) Copy() Schema

func (Schema) Equal

func (s Schema) Equal(other Schema) bool

func (*Schema) IsStrict

func (s *Schema) IsStrict() bool

func (Schema) KeyColumns

func (s Schema) KeyColumns() []string

KeyColumns returns list of columns names marked as sorted.

func (Schema) Normalize

func (s Schema) Normalize() Schema

Normalize converts in-memory Schema to canonical representation.

  • Strict flag is explicitly set to %true.
  • Old types are converted to new types.

func (Schema) Prepend

func (s Schema) Prepend(column ...Column) Schema

func (Schema) SortedBy

func (s Schema) SortedBy(keyColumns ...string) Schema

SortedBy returns copy of the schema, with keyColumns (and only them) marked as sorted.

func (Schema) WithUniqueKeys

func (s Schema) WithUniqueKeys() Schema

WithUniqueKeys returns copy of schema with UniqueKeys attribute set.

type SortOrder

type SortOrder string
const (
	SortNone       SortOrder = ""
	SortAscending  SortOrder = "ascending"
	SortDescending SortOrder = "descending"
)

func (SortOrder) MarshalText

func (o SortOrder) MarshalText() (text []byte, err error)

func (*SortOrder) UnmarshalText

func (o *SortOrder) UnmarshalText(text []byte) error

type Struct

type Struct struct {
	Members []StructMember `yson:"members"`
}

func (Struct) MarshalYSON

func (s Struct) MarshalYSON(w *yson.Writer) error

type StructMember

type StructMember struct {
	Name string      `yson:"name"`
	Type ComplexType `yson:"type"`
}

func (StructMember) MarshalYSON

func (m StructMember) MarshalYSON(w *yson.Writer) error

type Tagged

type Tagged struct {
	Tag  string      `yson:"tag"`
	Item ComplexType `yson:"item"`
}

func (Tagged) MarshalYSON

func (t Tagged) MarshalYSON(w *yson.Writer) error

type Timestamp

type Timestamp uint64

Timestamp is YT type representing number of microseconds since beginning of the unix epoch.

func NewTimestamp

func NewTimestamp(t time.Time) (Timestamp, error)

func (Timestamp) Time

func (t Timestamp) Time() time.Time

type Tuple

type Tuple struct {
	Elements []TupleElement `yson:"elements"`
}

func (Tuple) MarshalYSON

func (t Tuple) MarshalYSON(w *yson.Writer) error

type TupleElement

type TupleElement struct {
	Type ComplexType `yson:"type"`
}

func (TupleElement) MarshalYSON

func (e TupleElement) MarshalYSON(w *yson.Writer) error

type Type

type Type string
const (
	TypeInt64   Type = "int64"
	TypeInt32   Type = "int32"
	TypeInt16   Type = "int16"
	TypeInt8    Type = "int8"
	TypeUint64  Type = "uint64"
	TypeUint32  Type = "uint32"
	TypeUint16  Type = "uint16"
	TypeUint8   Type = "uint8"
	TypeFloat32 Type = "float"
	TypeFloat64 Type = "double"

	TypeBytes  Type = "string"
	TypeString Type = "utf8"

	// NOTE: "boolean" was renamed into "bool" in type_v3, and "any" was renamed into "yson"
	//
	// We keep old in-memory representation for this types for compatibility reasons.
	// Code that only operates with constants provided by this package should not care.
	TypeBoolean Type = "boolean"

	TypeAny Type = "any"

	TypeDate      Type = "date"
	TypeDatetime  Type = "datetime"
	TypeTimestamp Type = "timestamp"
	TypeInterval  Type = "interval"
)

func (Type) MarshalText

func (t Type) MarshalText() (text []byte, err error)

func (Type) String

func (t Type) String() string

func (*Type) UnmarshalText

func (t *Type) UnmarshalText(text []byte) error

type TypeName

type TypeName string
const (
	TypeNameDecimal  TypeName = "decimal"
	TypeNameOptional TypeName = "optional"
	TypeNameList     TypeName = "list"
	TypeNameStruct   TypeName = "struct"
	TypeNameTuple    TypeName = "tuple"
	TypeNameVariant  TypeName = "variant"
	TypeNameDict     TypeName = "dict"
	TypeNameTagged   TypeName = "tagged"
)

type Variant

type Variant struct {
	Members  []StructMember `yson:"members"`
	Elements []TupleElement `yson:"elements"`
}

func (Variant) MarshalYSON

func (v Variant) MarshalYSON(w *yson.Writer) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL