tablecodec

package
v0.0.0-...-503c688 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 8, 2023 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const (

	// RecordRowKeyLen is public for calculating average row size.
	RecordRowKeyLen = prefixLen + idLen /*handle*/

	// MaxOldEncodeValueLen is the maximum len of the old encoding of index value.
	MaxOldEncodeValueLen = 9

	// CommonHandleFlag is the flag used to decode the common handle in an unique index value.
	CommonHandleFlag byte = 127
	// PartitionIDFlag is the flag used to decode the partition ID in global index value.
	PartitionIDFlag byte = 126
	// IndexVersionFlag is the flag used to decode the index's version info.
	IndexVersionFlag byte = 125
	// RestoreDataFlag is the flag that RestoreData begin with.
	// See rowcodec.Encoder.Encode and rowcodec.row.toBytes
	RestoreDataFlag byte = rowcodec.CodecVer
)
View Source
const IndexIDMask = 0xffffffffffff

IndexIDMask used to get index id from index ID/temp index ID.

View Source
const TableSplitKeyLen = 1 + idLen

TableSplitKeyLen is the length of key 't{table_id}' which is used for table split.

View Source
const TempIndexPrefix = 0x7fff000000000000

TempIndexPrefix used to generate temporary index ID from index ID.

Variables

This section is empty.

Functions

func CutCommonHandle

func CutCommonHandle(key kv.Key, length int) (values [][]byte, b []byte, err error)

CutCommonHandle cuts encoded common handle key into colIDs to bytes slices. The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data if it is non-unique index.

func CutIndexKey

func CutIndexKey(key kv.Key, colIDs []int64) (values map[int64][]byte, b []byte, err error)

CutIndexKey cuts encoded index key into colIDs to bytes slices map. The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data if it is non-unique index.

func CutIndexKeyNew

func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err error)

CutIndexKeyNew cuts encoded index key into colIDs to bytes slices. The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data if it is non-unique index.

func CutIndexPrefix

func CutIndexPrefix(key kv.Key) []byte

CutIndexPrefix cuts the index prefix.

func CutRowKeyPrefix

func CutRowKeyPrefix(key kv.Key) []byte

CutRowKeyPrefix cuts the row key prefix.

func CutRowNew

func CutRowNew(data []byte, colIDs map[int64]int) ([][]byte, error)

CutRowNew cuts encoded row into byte slices and return columns' byte slice. Row layout: colID1, value1, colID2, value2, .....

func DecodeColumnValue

func DecodeColumnValue(data []byte, ft *types.FieldType, loc *time.Location) (types.Datum, error)

DecodeColumnValue decodes data to a Datum according to the column info.

func DecodeColumnValueWithDatum

func DecodeColumnValueWithDatum(data []byte, ft *types.FieldType, loc *time.Location, result *types.Datum) error

DecodeColumnValueWithDatum decodes data to an existing Datum according to the column info.

func DecodeHandleInUniqueIndexValue

func DecodeHandleInUniqueIndexValue(data []byte, isCommonHandle bool) (kv.Handle, error)

DecodeHandleInUniqueIndexValue decodes handle in data.

func DecodeHandleToDatumMap

func DecodeHandleToDatumMap(handle kv.Handle, handleColIDs []int64,
	cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error)

DecodeHandleToDatumMap decodes a handle into datum map.

func DecodeIndexHandle

func DecodeIndexHandle(key, value []byte, colsLen int) (kv.Handle, error)

DecodeIndexHandle uses to decode the handle from index key/value.

func DecodeIndexKV

func DecodeIndexKV(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error)

DecodeIndexKV uses to decode index key values.

`colsLen` is expected to be index columns count.
`columns` is expected to be index columns + handle columns(if hdStatus is not HandleNotNeeded).

func DecodeIndexKey

func DecodeIndexKey(key kv.Key) (tableID int64, indexID int64, indexValues []string, err error)

DecodeIndexKey decodes the key and gets the tableID, indexID, indexValues.

func DecodeKeyHead

func DecodeKeyHead(key kv.Key) (tableID int64, indexID int64, isRecordKey bool, err error)

DecodeKeyHead decodes the key's head and gets the tableID, indexID. isRecordKey is true when is a record key.

func DecodeMetaKey

func DecodeMetaKey(ek kv.Key) (key []byte, field []byte, err error)

DecodeMetaKey decodes the key and get the meta key and meta field.

func DecodeRecordKey

func DecodeRecordKey(key kv.Key) (tableID int64, handle kv.Handle, err error)

DecodeRecordKey decodes the key and gets the tableID, handle.

func DecodeRowKey

func DecodeRowKey(key kv.Key) (kv.Handle, error)

DecodeRowKey decodes the key and gets the handle.

func DecodeRowToDatumMap

func DecodeRowToDatumMap(b []byte, cols map[int64]*types.FieldType, loc *time.Location) (map[int64]types.Datum, error)

DecodeRowToDatumMap decodes a byte slice into datums. Row layout: colID1, value1, colID2, value2, ..... Default value columns, generated columns and handle columns are unprocessed.

func DecodeRowWithMap

func DecodeRowWithMap(b []byte, cols map[int64]*types.FieldType, loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error)

DecodeRowWithMap decodes a byte slice into datums with an existing row map. Row layout: colID1, value1, colID2, value2, .....

func DecodeRowWithMapNew

func DecodeRowWithMapNew(b []byte, cols map[int64]*types.FieldType,
	loc *time.Location, row map[int64]types.Datum) (map[int64]types.Datum, error)

DecodeRowWithMapNew decode a row to datum map.

func DecodeTableID

func DecodeTableID(key kv.Key) int64

DecodeTableID decodes the table ID of the key, if the key is not table key, returns 0.

func DecodeValuesBytesToStrings

func DecodeValuesBytesToStrings(b []byte) ([]string, error)

DecodeValuesBytesToStrings decode the raw bytes to strings for each columns. FIXME: Without the schema information, we can only decode the raw kind of the column. For instance, MysqlTime is internally saved as uint64.

func EncodeHandleInUniqueIndexValue

func EncodeHandleInUniqueIndexValue(h kv.Handle, isUntouched bool) []byte

EncodeHandleInUniqueIndexValue encodes handle in data.

func EncodeIndexSeekKey

func EncodeIndexSeekKey(tableID int64, idxID int64, encodedValue []byte) kv.Key

EncodeIndexSeekKey encodes an index value to kv.Key.

func EncodeMetaKey

func EncodeMetaKey(key []byte, field []byte) kv.Key

EncodeMetaKey encodes the key and field into meta key.

func EncodeMetaKeyPrefix

func EncodeMetaKeyPrefix(key []byte) kv.Key

EncodeMetaKeyPrefix encodes the key prefix into meta key

func EncodeOldRow

func EncodeOldRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum) ([]byte, error)

EncodeOldRow encode row data and column ids into a slice of byte. Row layout: colID1, value1, colID2, value2, ..... valBuf and values pass by caller, for reducing EncodeOldRow allocates temporary bufs. If you pass valBuf and values as nil, EncodeOldRow will allocate it.

func EncodeRecordKey

func EncodeRecordKey(recordPrefix kv.Key, h kv.Handle) kv.Key

EncodeRecordKey encodes the recordPrefix, row handle into a kv.Key.

func EncodeRow

func EncodeRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, valBuf []byte, values []types.Datum, e *rowcodec.Encoder, checksums ...uint32) ([]byte, error)

EncodeRow encode row data and column ids into a slice of byte. valBuf and values pass by caller, for reducing EncodeRow allocates temporary bufs. If you pass valBuf and values as nil, EncodeRow will allocate it.

func EncodeRowKey

func EncodeRowKey(tableID int64, encodedHandle []byte) kv.Key

EncodeRowKey encodes the table id and record handle into a kv.Key

func EncodeRowKeyWithHandle

func EncodeRowKeyWithHandle(tableID int64, handle kv.Handle) kv.Key

EncodeRowKeyWithHandle encodes the table id, row handle into a kv.Key

func EncodeTableIndexPrefix

func EncodeTableIndexPrefix(tableID, idxID int64) kv.Key

EncodeTableIndexPrefix encodes index prefix with tableID and idxID.

func EncodeTablePrefix

func EncodeTablePrefix(tableID int64) kv.Key

EncodeTablePrefix encodes the table prefix to generate a key

func EncodeTablePrefixSeekKey

func EncodeTablePrefixSeekKey(tableID int64, encodecValue []byte) kv.Key

EncodeTablePrefixSeekKey encodes the table prefix and encodecValue into a kv.Key. It used for seek justly.

func EncodeValue

func EncodeValue(sc *stmtctx.StatementContext, b []byte, raw types.Datum) ([]byte, error)

EncodeValue encodes a go value to bytes.

func GenIndexKey

func GenIndexKey(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
	phyTblID int64, indexedValues []types.Datum, h kv.Handle, buf []byte) (key []byte, distinct bool, err error)

GenIndexKey generates index key using input physical table id

func GenIndexValueForClusteredIndexVersion1

func GenIndexValueForClusteredIndexVersion1(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo, idxValNeedRestoredData bool, distinct bool, untouched bool, indexedValues []types.Datum, h kv.Handle, partitionID int64, handleRestoredData []types.Datum) ([]byte, error)

GenIndexValueForClusteredIndexVersion1 generates the index value for the clustered index with version 1(New in v5.0.0).

func GenIndexValuePortal

func GenIndexValuePortal(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo, needRestoredData bool, distinct bool, untouched bool, indexedValues []types.Datum, h kv.Handle, partitionID int64, restoredData []types.Datum) ([]byte, error)

GenIndexValuePortal is the portal for generating index value. Value layout:

+-- IndexValueVersion0  (with restore data, or common handle, or index is global)
|
|  Layout: TailLen | Options      | Padding      | [IntHandle] | [UntouchedFlag]
|  Length:   1     | len(options) | len(padding) |    8        |     1
|
|  TailLen:       len(padding) + len(IntHandle) + len(UntouchedFlag)
|  Options:       Encode some value for new features, such as common handle, new collations or global index.
|                 See below for more information.
|  Padding:       Ensure length of value always >= 10. (or >= 11 if UntouchedFlag exists.)
|  IntHandle:     Only exists when table use int handles and index is unique.
|  UntouchedFlag: Only exists when index is untouched.
|
+-- Old Encoding (without restore data, integer handle, local)
|
|  Layout: [Handle] | [UntouchedFlag]
|  Length:   8      |     1
|
|  Handle:        Only exists in unique index.
|  UntouchedFlag: Only exists when index is untouched.
|
|  If neither Handle nor UntouchedFlag exists, value will be one single byte '0' (i.e. []byte{'0'}).
|  Length of value <= 9, use to distinguish from the new encoding.
|
+-- IndexValueForClusteredIndexVersion1
|
|  Layout: TailLen |    VersionFlag  |    Version     | Options      |   [UntouchedFlag]
|  Length:   1     |        1        |      1         |  len(options) |         1
|
|  TailLen:       len(UntouchedFlag)
|  Options:       Encode some value for new features, such as common handle, new collations or global index.
|                 See below for more information.
|  UntouchedFlag: Only exists when index is untouched.
|
|  Layout of Options:
|
|     Segment:             Common Handle                 |     Global Index      |   New Collation
|     Layout:  CHandle flag | CHandle Len | CHandle      | PidFlag | PartitionID |    restoreData
|     Length:     1         | 2           | len(CHandle) |    1    |    8        |   len(restoreData)
|
|     Common Handle Segment: Exists when unique index used common handles.
|     Global Index Segment:  Exists when index is global.
|     New Collation Segment: Exists when new collation is used and index or handle contains non-binary string.
|     In v4.0, restored data contains all the index values. For example, (a int, b char(10)) and index (a, b).
|     The restored data contains both the values of a and b.
|     In v5.0, restored data contains only non-binary data(except for char and _bin). In the above example, the restored data contains only the value of b.
|     Besides, if the collation of b is _bin, then restored data is an integer indicate the spaces are truncated. Then we use sortKey
|     and the restored data together to restore original data.

func GenTableIndexPrefix

func GenTableIndexPrefix(tableID int64) kv.Key

GenTableIndexPrefix composes index prefix with tableID: "t[tableID]_i".

func GenTablePrefix

func GenTablePrefix(tableID int64) kv.Key

GenTablePrefix composes table record and index prefix: "t[tableID]".

func GenTableRecordPrefix

func GenTableRecordPrefix(tableID int64) kv.Key

GenTableRecordPrefix composes record prefix with tableID: "t[tableID]_r".

func GetIndexKeyBuf

func GetIndexKeyBuf(buf []byte, defaultCap int) []byte

GetIndexKeyBuf reuse or allocate buffer

func GetTableHandleKeyRange

func GetTableHandleKeyRange(tableID int64) (startKey, endKey []byte)

GetTableHandleKeyRange returns table handle's key range with tableID.

func GetTableIndexKeyRange

func GetTableIndexKeyRange(tableID, indexID int64) (startKey, endKey []byte)

GetTableIndexKeyRange returns table index's key range with tableID and indexID.

func IndexKVIsUnique

func IndexKVIsUnique(value []byte) bool

IndexKVIsUnique uses to judge if an index is unique, it can handle the KV committed by txn already, it doesn't consider the untouched flag.

func IndexKey2TempIndexKey

func IndexKey2TempIndexKey(key []byte)

IndexKey2TempIndexKey generates a temporary index key.

func IsIndexKey

func IsIndexKey(k []byte) bool

IsIndexKey is used to check whether the key is an index key.

func IsRecordKey

func IsRecordKey(k []byte) bool

IsRecordKey is used to check whether the key is an record key.

func IsTableKey

func IsTableKey(k []byte) bool

IsTableKey is used to check whether the key is a table key.

func IsTempIndexKey

func IsTempIndexKey(indexKey []byte) (isTemp bool)

IsTempIndexKey checks whether the input key is for a temp index.

func IsUntouchedIndexKValue

func IsUntouchedIndexKValue(k, v []byte) bool

IsUntouchedIndexKValue uses to check whether the key is index key, and the value is untouched, since the untouched index key/value is no need to commit.

func MetaPrefix

func MetaPrefix() []byte

MetaPrefix returns meta prefix 'm'.

func TablePrefix

func TablePrefix() []byte

TablePrefix returns table's prefix 't'.

func TempIndexKey2IndexKey

func TempIndexKey2IndexKey(tempIdxKey []byte)

TempIndexKey2IndexKey generates an index key from temporary index key.

func TempIndexValueIsUntouched

func TempIndexValueIsUntouched(b []byte) bool

TempIndexValueIsUntouched returns true if the value is untouched. All the temp index value has the suffix of temp key version. All the temp key versions differ from the uncommitted KV flag.

func TruncateIndexValue

func TruncateIndexValue(v *types.Datum, idxCol *model.IndexColumn, tblCol *model.ColumnInfo)

TruncateIndexValue truncate one value in the index.

func TruncateIndexValues

func TruncateIndexValues(tblInfo *model.TableInfo, idxInfo *model.IndexInfo, indexedValues []types.Datum)

TruncateIndexValues truncates the index values created using only the leading part of column values.

func TruncateToRowKeyLen

func TruncateToRowKeyLen(key kv.Key) kv.Key

TruncateToRowKeyLen truncates the key to row key length if the key is longer than row key.

func TryGetCommonPkColumnRestoredIds

func TryGetCommonPkColumnRestoredIds(tbl *model.TableInfo) []int64

TryGetCommonPkColumnRestoredIds get the IDs of primary key columns which need restored data if the table has common handle. Caller need to make sure the table has common handle.

func Unflatten

func Unflatten(datum types.Datum, ft *types.FieldType, loc *time.Location) (types.Datum, error)

Unflatten converts a raw datum to a column datum.

func UnflattenDatums

func UnflattenDatums(datums []types.Datum, fts []*types.FieldType, loc *time.Location) ([]types.Datum, error)

UnflattenDatums converts raw datums to column datums.

func VerifyTableIDForRanges

func VerifyTableIDForRanges(keyRanges *kv.KeyRanges) ([]int64, error)

VerifyTableIDForRanges verifies that all given ranges are valid to decode the table id.

Types

type HandleStatus

type HandleStatus int

HandleStatus is the handle status in index.

const (
	// HandleDefault means decode handle value as int64 or bytes when DecodeIndexKV.
	HandleDefault HandleStatus = iota
	// HandleIsUnsigned means decode handle value as uint64 when DecodeIndexKV.
	HandleIsUnsigned
	// HandleNotNeeded means no need to decode handle value when DecodeIndexKV.
	HandleNotNeeded
)

type IndexValueSegments

type IndexValueSegments struct {
	CommonHandle   []byte
	PartitionID    []byte
	RestoredValues []byte
	IntHandle      []byte
}

IndexValueSegments use to store result of SplitIndexValue.

func SplitIndexValue

func SplitIndexValue(value []byte) (segs IndexValueSegments)

SplitIndexValue splits index value into segments.

func SplitIndexValueForClusteredIndexVersion1

func SplitIndexValueForClusteredIndexVersion1(value []byte) (segs IndexValueSegments)

SplitIndexValueForClusteredIndexVersion1 splits index value into segments.

type TempIndexValue

type TempIndexValue []*TempIndexValueElem

TempIndexValue is the value of temporary index. It contains one or more element, each element represents a history index operations on the original index. A temp index value element is encoded as one of:

  • [flag 1 byte][value_length 2 bytes ] [value value_len bytes] [key_version 1 byte] {distinct normal}
  • [flag 1 byte][value value_len bytes] [key_version 1 byte] {non-distinct normal}
  • [flag 1 byte][handle_length 2 bytes] [handle handle_len bytes] [key_version 1 byte] {distinct deleted}
  • [flag 1 byte] [key_version 1 byte] {non-distinct deleted}

The temp index value is encoded as:

  • [element 1][element 2]...[element n] {for distinct values}
  • [element 1] {for non-distinct values}

func DecodeTempIndexValue

func DecodeTempIndexValue(value []byte) (TempIndexValue, error)

DecodeTempIndexValue decodes the temp index value.

func (TempIndexValue) Current

func (v TempIndexValue) Current() *TempIndexValueElem

Current returns the current latest temp index value.

func (TempIndexValue) FilterOverwritten

func (v TempIndexValue) FilterOverwritten() TempIndexValue

FilterOverwritten is used by the temp index merge process to remove the overwritten index operations. For example, the value {temp_idx_key -> [h2, h2d, h3, h1d]} recorded four operations on the original index. Since 'h2d' overwrites 'h2', we can remove 'h2' from the value.

func (TempIndexValue) IsEmpty

func (v TempIndexValue) IsEmpty() bool

IsEmpty checks whether the value is empty.

type TempIndexValueElem

type TempIndexValueElem struct {
	Value    []byte
	Handle   kv.Handle
	KeyVer   byte
	Delete   bool
	Distinct bool
}

TempIndexValueElem represents a history index operations on the original index. A temp index value element is encoded as one of:

  • [flag 1 byte][value_length 2 bytes ] [value value_len bytes] [key_version 1 byte] {distinct normal}
  • [flag 1 byte][value value_len bytes] [key_version 1 byte] {non-distinct normal}
  • [flag 1 byte][handle_length 2 bytes] [handle handle_len bytes] [key_version 1 byte] {distinct deleted}
  • [flag 1 byte] [key_version 1 byte] {non-distinct deleted}

func (*TempIndexValueElem) DecodeOne

func (v *TempIndexValueElem) DecodeOne(b []byte) (remain []byte, err error)

DecodeOne decodes one temp index value element.

func (*TempIndexValueElem) Encode

func (v *TempIndexValueElem) Encode(buf []byte) []byte

Encode encodes the temp index value.

type TempIndexValueFlag

type TempIndexValueFlag byte

TempIndexValueFlag is the flag of temporary index value.

const (
	// TempIndexValueFlagNormal means the following value is a distinct the normal index value.
	TempIndexValueFlagNormal TempIndexValueFlag = iota
	// TempIndexValueFlagNonDistinctNormal means the following value is the non-distinct normal index value.
	TempIndexValueFlagNonDistinctNormal
	// TempIndexValueFlagDeleted means the following value is the distinct and deleted index value.
	TempIndexValueFlagDeleted
	// TempIndexValueFlagNonDistinctDeleted means the following value is the non-distinct deleted index value.
	TempIndexValueFlagNonDistinctDeleted
)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL