typeutil

package
v0.0.0-...-1593278 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 7, 2023 License: Apache-2.0 Imports: 21 Imported by: 4

Documentation

Index

Constants

View Source 
const (
	// EmbeddedRole is for embedded Milvus.
	EmbeddedRole = "embedded"
	// StandaloneRole is a constant represent Standalone
	StandaloneRole = "standalone"
	// RootCoordRole is a constant represent RootCoord
	RootCoordRole = "rootcoord"
	// ProxyRole is a constant represent Proxy
	ProxyRole = "proxy"
	// QueryCoordRole is a constant represent QueryCoord
	QueryCoordRole = "querycoord"
	// QueryNodeRole is a constant represent QueryNode
	QueryNodeRole = "querynode"
	// IndexCoordRole is a constant represent IndexCoord
	IndexCoordRole = "indexcoord"
	// IndexNodeRole is a constant represent IndexNode
	DataCoordRole = "datacoord"
	// DataNodeRole is a constant represent DataNode
	DataNodeRole = "datanode"
	// IndexNodeRole is a constant represent IndexNode
	IndexNodeRole = "indexnode"
)
View Source 
const DynamicFieldMaxLength = 512
View Source 
const MaxTimestamp = Timestamp(math.MaxUint64)

MaxTimestamp is the max timestamp.

View Source 
const Unlimited int64 = -1

Variables

View Source 
var ZeroTime = time.Time{}

ZeroTime is a zero time.

View Source 
var ZeroTimestamp = Timestamp(0)

ZeroTimestamp is a zero timestamp

Functions

func AddOne 

func AddOne(data string) string

AddOne add one to last byte in string, on empty string return empty it helps with key iteration upper bound

func After 

func After(str string, sub string) string

After get substring after sub string.

func AfterN 

func AfterN(str string, sub string, sep string) []string

AfterN Split slices After(str) into all substrings separated by sep

func AppendFieldData 

func AppendFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData, idx int64)

AppendFieldData appends fields data of specified index from src to dst

func AppendIDs 

func AppendIDs(dst *schemapb.IDs, src *schemapb.IDs, idx int)

func AppendPKs 

func AppendPKs(pks *schemapb.IDs, pk interface{})

func BigEndianBytesToUint64 

func BigEndianBytesToUint64(b []byte) (uint64, error)

BigEndianBytesToUint64 converts a byte slice (big endian) to uint64.

func BytesToFloat32 

func BytesToFloat32(bytes []byte) float32

BytesToFloat32 converts a byte slice to float32.

func BytesToInt64 

func BytesToInt64(b []byte) (int64, error)

BytesToInt64 converts a byte slice to uint64.

func BytesToUint64 

func BytesToUint64(b []byte) (uint64, error)

BytesToUint64 converts a byte slice to uint64.

func CalcColumnSize 

func CalcColumnSize(column *schemapb.FieldData) int

func Clone 

func Clone[T proto.Message](msg T) T

Generic Clone for proto message

func CompareIndexParams 

func CompareIndexParams(indexParam1, indexParam2 []*commonpb.KeyValuePair) bool

CompareIndexParams compares indexParam1 with indexParam2. When all keys of indexParam1 exist in indexParam2, and the corresponding value are the same as in indexParam2, return true Otherwise return false

func ComparePK 

func ComparePK(pkA, pkB interface{}) bool

ComparePK returns if i-th PK of dataA > j-th PK of dataB

func ComparePKInSlice 

func ComparePKInSlice(data *schemapb.IDs, i, j int) bool

ComparePKInSlice returns if i-th PK < j-th PK

func DeleteFieldData 

func DeleteFieldData(dst []*schemapb.FieldData)

DeleteFieldData delete fields data appended last time

func EstimateEntitySize 

func EstimateEntitySize(fieldsData []*schemapb.FieldData, rowOffset int) (int, error)

func EstimateSizePerRecord 

func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error)

EstimateSizePerRecord returns the estimate size of a record in a collection

func Float32ToBytes 

func Float32ToBytes(float float32) []byte

Float32ToBytes converts a float to byte slice.

func GenEmptyFieldData 

func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error)

func GetAvgLengthOfVarLengthField 

func GetAvgLengthOfVarLengthField(fieldSchema *schemapb.FieldSchema) (int, error)

func GetData 

func GetData(field *schemapb.FieldData, idx int) interface{}

func GetDataSize 

func GetDataSize(fieldData *schemapb.FieldData) int

func GetDim 

func GetDim(field *schemapb.FieldSchema) (int64, error)

GetDim get dimension of field. Maybe also helpful outside.

func GetMapKeys 

func GetMapKeys(src map[string]string) []string

GetMapKeys return keys of a map

func GetPK 

func GetPK(data *schemapb.IDs, idx int64) interface{}

func GetPartitionKeyFieldSchema 

func GetPartitionKeyFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSchema, error)

GetPartitionKeyFieldSchema get partition field schema from collection schema

func GetPrimaryFieldData 

func GetPrimaryFieldData(datas []*schemapb.FieldData, primaryFieldSchema *schemapb.FieldSchema) (*schemapb.FieldData, error)

GetPrimaryFieldData get primary field data from all field data inserted from sdk

func GetPrimaryFieldSchema 

func GetPrimaryFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSchema, error)

GetPrimaryFieldSchema get primary field schema from collection schema

func GetSizeOfIDs 

func GetSizeOfIDs(data *schemapb.IDs) int

func GetVectorFieldSchema 

func GetVectorFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSchema, error)

GetVectorFieldSchema get vector field schema from collection schema.

func Hash32Bytes 

func Hash32Bytes(b []byte) (uint32, error)

Hash32Bytes hashing a byte array to uint32

func Hash32Int64 

func Hash32Int64(v int64) (uint32, error)

Hash32Int64 hashing an int64 number to uint32

func Hash32String 

func Hash32String(s string) (int64, error)

Hash32String hashing a string to int64

func Hash32Uint64 

func Hash32Uint64(v uint64) (uint32, error)

Hash32Uint64 hashing an uint64 nubmer to uint32

func HashKey2Partitions 

func HashKey2Partitions(keys *schemapb.FieldData, partitionNames []string) ([]uint32, error)

HashKey2Partitions hash partition keys to partitions

func HashPK2Channels 

func HashPK2Channels(primaryKeys *schemapb.IDs, shardNames []string) []uint32

HashPK2Channels hash primary keys to channels

func HashString2Uint32 

func HashString2Uint32(v string) uint32

HashString2Uint32 hashing a string to uint32

func Int64ToBytes 

func Int64ToBytes(v int64) []byte

Int64ToBytes converts uint64 to a byte slice.

func IsArithmetic 

func IsArithmetic(dataType schemapb.DataType) bool

IsArithmetic returns true if input is of arithmetic type, otherwise false.

func IsBoolType 

func IsBoolType(dataType schemapb.DataType) bool

IsBoolType returns true if input is a bool type, otherwise false

func IsFloatingType 

func IsFloatingType(dataType schemapb.DataType) bool

IsFloatingType returns true if input is a floating type, otherwise false

func IsIntegerType 

func IsIntegerType(dataType schemapb.DataType) bool

IsIntegerType returns true if input is an integer type, otherwise false

func IsJSONType 

func IsJSONType(dataType schemapb.DataType) bool

func IsPrimaryFieldDataExist 

func IsPrimaryFieldDataExist(datas []*schemapb.FieldData, primaryFieldSchema *schemapb.FieldSchema) bool

func IsPrimaryFieldType 

func IsPrimaryFieldType(dataType schemapb.DataType) bool

func IsStringType 

func IsStringType(dataType schemapb.DataType) bool

IsStringType returns true if input is a varChar type, otherwise false

func IsVectorType 

func IsVectorType(dataType schemapb.DataType) bool

IsVectorType returns true if input is a vector type, otherwise false

func MapEqual 

func MapEqual(left, right map[int64]int64) bool

MapEqual returns true if the two map contain the same keys and values

func MergeFieldData 

func MergeFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData) error

MergeFieldData appends fields data to dst

func MergeMap 

func MergeMap(src map[string]string, dst map[string]string) map[string]string

MergeMap merge one map to another

func NewKvPairs 

func NewKvPairs(pairs []*commonpb.KeyValuePair) *kvPairsHelper[string, string]

func ParseTimestamp 

func ParseTimestamp(data []byte) (time.Time, error)

ParseTimestamp returns a timestamp for a given byte slice.

func SelectMinPK 

func SelectMinPK[T ResultWithID](results []T, cursors []int64) int

SelectMinPK select the index of the minPK in results T of the cursors.

func ServerTypeList 

func ServerTypeList() []string

func SliceRemoveDuplicate 

func SliceRemoveDuplicate(a interface{}) (ret []interface{})

SliceRemoveDuplicate is used to dedup a Slice

func SubTimeByWallClock 

func SubTimeByWallClock(after, before time.Time) time.Duration

SubTimeByWallClock returns the duration between two different timestamps.

func SwapPK 

func SwapPK(data *schemapb.IDs, i, j int)

SwapPK swaps i-th PK with j-th PK

func Uint64ToBytes 

func Uint64ToBytes(v uint64) []byte

Uint64ToBytes converts uint64 to a byte slice.

func Uint64ToBytesBigEndian 

func Uint64ToBytesBigEndian(v uint64) []byte

Uint64ToBytesBigEndian converts uint64 to a byte slice(big endian).

func UnsafeBytes2str 

func UnsafeBytes2str(b []byte) string

#nosec G103

func UnsafeStr2bytes 

func UnsafeStr2bytes(s string) []byte

#nosec G103

func VerifyFloat 

func VerifyFloat(value float64) error

func VerifyFloats32 

func VerifyFloats32(values []float32) error

func VerifyFloats64 

func VerifyFloats64(values []float64) error

Types

type CacheOp 

type CacheOp struct {
	OpType CacheOpType
	OpKey  string
}

type CacheOpType 

type CacheOpType int32
const (
	CacheAddUserToRole CacheOpType = iota + 1
	CacheRemoveUserFromRole
	CacheGrantPrivilege
	CacheRevokePrivilege
)

type ConcurrentMap 

type ConcurrentMap[K comparable, V any] struct {
	// contains filtered or unexported fields
}

func NewConcurrentMap 

func NewConcurrentMap[K comparable, V any]() *ConcurrentMap[K, V]

func (*ConcurrentMap[K, V]) Contain 

func (m *ConcurrentMap[K, V]) Contain(key K) bool

func (*ConcurrentMap[K, V]) Get 

func (m *ConcurrentMap[K, V]) Get(key K) (V, bool)

func (*ConcurrentMap[K, V]) GetAndRemove 

func (m *ConcurrentMap[K, V]) GetAndRemove(key K) (V, bool)

func (*ConcurrentMap[K, V]) GetOrInsert 

func (m *ConcurrentMap[K, V]) GetOrInsert(key K, value V) (V, bool)

GetOrInsert returns the `value` and `loaded` on the given `key`, `value` set. If the key already exists, return the value and set `loaded` to true. If the key does not exist, insert the given `key` and `value` to map, return the value and set `loaded` to false.

func (*ConcurrentMap[K, V]) Insert 

func (m *ConcurrentMap[K, V]) Insert(key K, value V)

Insert inserts the key-value pair to the concurrent map

func (*ConcurrentMap[K, V]) Len 

func (m *ConcurrentMap[K, V]) Len() int

func (*ConcurrentMap[K, V]) Range 

func (m *ConcurrentMap[K, V]) Range(f func(key K, value V) bool)

type ConcurrentSet 

type ConcurrentSet[T comparable] struct {
	// contains filtered or unexported fields
}

func NewConcurrentSet 

func NewConcurrentSet[T comparable]() *ConcurrentSet[T]

func (*ConcurrentSet[T]) Collect 

func (set *ConcurrentSet[T]) Collect() []T

Get all elements in the set

func (*ConcurrentSet[T]) Contain 

func (set *ConcurrentSet[T]) Contain(elements ...T) bool

Check whether the elements exist

func (*ConcurrentSet[T]) Insert 

func (set *ConcurrentSet[T]) Insert(element T) bool

func (*ConcurrentSet[T]) Remove 

func (set *ConcurrentSet[T]) Remove(elements ...T)

Remove elements from the set, do nothing if set is nil or id not exists

func (*ConcurrentSet[T]) Upsert 

func (set *ConcurrentSet[T]) Upsert(elements ...T)

Insert elements into the set, do nothing if the id existed

type IntPrimaryKey 

type IntPrimaryKey = int64

IntPrimaryKey is an alias of int64

type OrderedMap 

type OrderedMap[K comparable, V any] struct {
	// contains filtered or unexported fields
}

func NewOrderedMap 

func NewOrderedMap[K comparable, V any]() *OrderedMap[K, V]

func (*OrderedMap[K, V]) Delete 

func (o *OrderedMap[K, V]) Delete(key K)

func (*OrderedMap[K, V]) Get 

func (o *OrderedMap[K, V]) Get(key K) (V, bool)

func (*OrderedMap[K, V]) Keys 

func (o *OrderedMap[K, V]) Keys() []K

func (*OrderedMap[K, V]) Set 

func (o *OrderedMap[K, V]) Set(key K, value V)

func (*OrderedMap[K, V]) SortKeys 

func (o *OrderedMap[K, V]) SortKeys(sortFunc func(keys []K))

SortKeys Sort the map keys using your sort func

type ResultWithID 

type ResultWithID interface {
	GetIds() *schemapb.IDs
}

type SchemaHelper 

type SchemaHelper struct {
	// contains filtered or unexported fields
}

SchemaHelper provides methods to get the schema of fields

func CreateSchemaHelper 

func CreateSchemaHelper(schema *schemapb.CollectionSchema) (*SchemaHelper, error)

CreateSchemaHelper returns a new SchemaHelper object

func (*SchemaHelper) GetFieldFromID 

func (helper *SchemaHelper) GetFieldFromID(fieldID int64) (*schemapb.FieldSchema, error)

GetFieldFromID returns the schema of specified field

func (*SchemaHelper) GetFieldFromName 

func (helper *SchemaHelper) GetFieldFromName(fieldName string) (*schemapb.FieldSchema, error)

GetFieldFromName is used to find the schema by field name

func (*SchemaHelper) GetFieldFromNameDefaultJSON 

func (helper *SchemaHelper) GetFieldFromNameDefaultJSON(fieldName string) (*schemapb.FieldSchema, error)

GetFieldFromNameDefaultJSON is used to find the schema by field name, if not exist, use json field

func (*SchemaHelper) GetPartitionKeyField 

func (helper *SchemaHelper) GetPartitionKeyField() (*schemapb.FieldSchema, error)

GetPartitionKeyField returns the schema of the partition key

func (*SchemaHelper) GetPrimaryKeyField 

func (helper *SchemaHelper) GetPrimaryKeyField() (*schemapb.FieldSchema, error)

GetPrimaryKeyField returns the schema of the primary key

func (*SchemaHelper) GetVectorDimFromID 

func (helper *SchemaHelper) GetVectorDimFromID(fieldID int64) (int, error)

GetVectorDimFromID returns the dimension of specified field

type Set 

type Set[T comparable] map[T]struct{}

func NewSet 

func NewSet[T comparable](elements ...T) Set[T]

func ServerTypeSet 

func ServerTypeSet() Set[string]

func (Set[T]) Clear 

func (set Set[T]) Clear()

func (Set[T]) Collect 

func (set Set[T]) Collect() []T

Get all elements in the set

func (Set[T]) Complement 

func (set Set[T]) Complement(other Set[T]) Set[T]

Complement returns the complement with the given set

func (Set[T]) Contain 

func (set Set[T]) Contain(elements ...T) bool

Check whether the elements exist

func (Set[T]) Insert 

func (set Set[T]) Insert(elements ...T)

Insert elements into the set, do nothing if the id existed

func (Set[T]) Intersection 

func (set Set[T]) Intersection(other Set[T]) Set[T]

Intersection returns the intersection with the given set

func (Set[T]) Len 

func (set Set[T]) Len() int

Len returns the number of elements in the set

func (Set[T]) Remove 

func (set Set[T]) Remove(elements ...T)

Remove elements from the set, do nothing if set is nil or id not exists

func (Set[T]) Union 

func (set Set[T]) Union(other Set[T]) Set[T]

Union returns the union with the given set

type SkipList 

type SkipList[K constraints.Ordered, V any] struct {
	// contains filtered or unexported fields
}

SkipList generic skip list. with extra truncate and traverse-after feature.

func NewSkipList 

func NewSkipList[K constraints.Ordered, V any](opts ...SkipListOption) (*SkipList[K, V], error)

NewSkipList creates a new SkipList with provided option.

func (*SkipList[K, V]) Delete 

func (sl *SkipList[K, V]) Delete(key K) (value V, ok bool)

Delete removes node with provided key. returns value if found.

func (*SkipList[K, V]) Get 

func (sl *SkipList[K, V]) Get(key K) (value V, ok bool)

Get searches skip list with provided key.

func (*SkipList[K, V]) ListAfter 

func (sl *SkipList[K, V]) ListAfter(key K, include bool) []V

ListAfter returns values after provided key. include indicates value related to key is returned or not.

func (*SkipList[K, V]) TruncateBefore 

func (sl *SkipList[K, V]) TruncateBefore(key K)

TruncateBefore truncates elements before key.

func (*SkipList[K, V]) Upsert 

func (sl *SkipList[K, V]) Upsert(key K, value V)

Insert inserts an element into skip list.

type SkipListOption 

type SkipListOption func(opt *skipListOpt)

SkipListOption setup function alias.

func WithMaxLevel 

func WithMaxLevel(maxLevel int) SkipListOption

WithMaxLevel returns SkipListOption to setup max level.

func WithSkip 

func WithSkip(skip int) SkipListOption

WithSkip returns SkipListOption to setup skip.

type Timestamp 

type Timestamp = uint64

Timestamp is an alias of uint64

type UniqueID 

type UniqueID = int64

UniqueID is an alias of int64

type UniqueSet 

type UniqueSet = Set[UniqueID]

UniqueSet is set type, which contains only UniqueIDs, the underlying type is map[UniqueID]struct{}. Create a UniqueSet instance with make(UniqueSet) like creating a map instance.

func NewUniqueSet 

func NewUniqueSet(ids ...UniqueID) UniqueSet

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.