be

package module
v0.0.0-...-42a1afe Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 24, 2022 License: Apache-2.0 Imports: 19 Imported by: 0

README

Quickstart

func ExampleNewClient() {
	protoc := be.GrpcProtocol
	if *Http {
		protoc = be.HttpProtocol
	}
	client, err := be.NewProximaSearchClient(protoc, &address)
	if err != nil {
		log.Fatal("Can't create ProximaClient instance.", err)
	}
}

func ExampleListCollections(client *be.ProximaSearchClient) {
	// List all collections
	collections, err := client.ListCollections()
	if err != nil {
		log.Fatal("Can't retrieve collections from Proxima Server.", err)
	}
	log.Printf("Collections (%d): \n", len(collections))
	for _, collection := range collections {
		log.Printf("%+v\n", collection)
	}

	// List all collections by Repo
	collections, err = client.ListCollections(be.ByRepo("repo"))
	if err != nil {
		log.Fatal("Can't retrieve collections from Proxima Server.", err)
	}
	log.Printf("Collections (%d): \n", len(collections))
	for _, collection := range collections {
		log.Printf("%+v\n", collection)
	}
}

Howto

Please go through sdk/references to get an idea how to use this package.

Look and feel

Release Rules:

Dev management: Using lightweight tag to release package, using annotation tag for develop.

Documentation

Overview

package be is client of Proxima BE project

package be is client of Proxima BE project

Index

Constants

View Source
const (
	VERSION = "0.2.0"
)

Variables

View Source
var (
	// InvalidIndex used for GenericValueList
	InvalidIndex int = -1
)

Functions

This section is empty.

Types

type Address

type Address struct {
	// Host IP address or host name
	Host string `json:"host:omitempty"`
	// Port
	Port uint32 `json:"port"`
}

Address reference to entrypoint of ProximaSE

func DefaultAddress

func DefaultAddress() *Address

DefaultAddress create one address for Proxima Search Engine

func (*Address) String

func (address *Address) String() string

String return json formatted string of Address object

type CollectionConfig

type CollectionConfig struct {
	// Name of collection
	CollectionName string `json:"collection_name,omitempty"`
	// Max documents per segment, Optional field, default is max value of system(^uint64(0) = 18446744073709551615)
	MaxDocsPerSegment uint64 `json:"max_docs_per_segment"`
	// Entity of document, optional field, default is empty
	ForwardColumns []string `json:"forward_columns,omitempty"`
	// Columns of document, empty array is illegal argument
	Columns []ColumnIndex `json:"columns"`
	// Repository of collection, Optional field, nil for collection which import from client only
	Repository *DatabaseRepository `json:"repository,omitempty"`
}

CollectionConfig configuration of collection

func (*CollectionConfig) String

func (config *CollectionConfig) String() string

String method return json formatted string of CollectionConfig

type CollectionInfo

type CollectionInfo struct {
	// Collection config
	CollectionConfig
	// Status of collection
	Status CollectionStatus `json:"status"`
	// Unique id of collection, allocated by ProximaSE
	UUID string `json:"uuid,omitempty"`
	// Latest LSN Context
	Context *LsnContext `json:"context"`
	// Magic number of collection
	MagicNumber uint64 `json:"magic_number"`
}

CollectionInfo Details of collection

func (*CollectionInfo) String

func (info *CollectionInfo) String() string

String method return json formatted string of CollectionInfo

type CollectionStat

type CollectionStat struct {
	// Name of collection
	CollectionName string
	// Storage path of collection
	CollectionPath string
	// Total documents inside collection
	TotalDocCount uint64
	// The number of segments in collection
	TotalSegmentCount uint64
	// The number of files in collection
	TotalIndexFileCount uint64
	// The total size of storage, count by bytes
	TotalIndexFileSize uint64
	// Segment stats
	SegmentStats []SegmentStat
}

CollectionStat stat of Collection

func (*CollectionStat) String

func (stat *CollectionStat) String() string

String method return json formatted string of CollectionStat

type CollectionStatus

type CollectionStatus uint32

CollectionStatus collection Status type

const (
	// Collection has been initialized, is ready for serving
	Initialized CollectionStatus = iota
	// Collection is serving
	Serving
	// Collection has been dropped permanently
	Dropped
)

Available status of collection

type ColumnIndex

type ColumnIndex struct {
	// Name of column, empty Name is illegal argument
	Name string `json:"name,omitempty"`
	// Type of column, Default is IT_PROXIMA_HNSW_INDEX
	IndexType `json:"index_type"`
	// Datatype of column
	DataType `json:"data_type"`
	// Dimension of column, Default is 0
	Dimension uint32 `json:"dimension"`
	// Customized parameters for column, Optional
	ExtraParams map[string]string `json:"extra_params,omitempty"`
}

ColumnIndex Description of column

func (*ColumnIndex) String

func (column *ColumnIndex) String() string

String return json format string of ColumnIndex

type ConnectionProtocol

type ConnectionProtocol uint32

ConnectionProtocol types

const (
	// Using GRPC to communicate with ProximaSE, which is default
	GrpcProtocol ConnectionProtocol = iota
	// Using HTTP to communicate with ProximaSE
	HttpProtocol
)

Available types of connection

type DataType

type DataType uint32

DataType is types of data

const (
	// Undefined type
	DataTypeUndefined DataType = iota
	// Binary type
	Binary DataType = 1
	// String type
	String DataType = 2
	// Boolean type
	Bool DataType = 3
	// Signed integer with 4 bytes
	Int32 DataType = 4
	// Signed integer with 8 bytes
	Int64 DataType = 5
	// Unsigned integer with 4 bytes
	Uint32 DataType = 6
	// Unsigned integer with 8 bytes
	Uint64 DataType = 7
	// Single precision float
	Float DataType = 8
	// Double precision float
	Double DataType = 9
	// Binary with 4 bytes array
	// Memory layout:  [item1 item2 item3]
	// Offset          0 ... 4 ... 8 .. 12
	VectorBinary32 DataType = 20
	// Binary with 8 bytes array
	// Memory layout:  [item1 item2 item3]
	// Offset          0 ... 8 .. 16 .. 24
	VectorBinary64 DataType = 21
	// Float with 2 bytes array,
	// Tips: Create index with fp16, the query should be float32 if no float16 type in language built-in types
	VectorFP16 DataType = 22
	// Single precision float with 4 bytes array
	VectorFP32 DataType = 23
	// Double precision float with 8 bytes array
	VectorFP64 DataType = 24
	// Signed integer with 4 bits array
	VectorInt4 DataType = 25
	// Signed integer with 8 bits array
	VectorInt8 DataType = 26
	// Signed integer with 16 bits array
	VectorInt16 DataType = 27
)

Available data types

type DatabaseRepository

type DatabaseRepository struct {
	// Derived from Repository class
	Repository
	// URI of database, invalid uri will be deferred by Proxima BE,
	// example: mysql://localhost:3033/example_database
	// Details:
	//   Schema: indicate drivers, mysql is only supported
	//   Path: indicate databases
	//
	Connection string `json:"connection,omitempty"`
	// Table name
	TableName string `json:"table_name,omitempty"`
	// User name used to login database
	User string `json:"user,omitempty"`
	// User password
	Password string `json:"password,omitempty"`
}

DatabaseRepository database repository Indicate where the data of collection coming from, Proxima BE synchronize Collection with Database automatically if create collection with Repository.

func (*DatabaseRepository) String

func (repo *DatabaseRepository) String() string

String return json format string of Repository

type Document

type Document struct {
	// Primary key of document
	PrimaryKey uint64 `json:"primary_key,omitempty"`
	// Similarity of document to query vector, bigger is more similar
	Score float32 `json:"score"`
	// Properties of document
	ForwardColumns map[string]interface{} `json:"forwards,omitempty"`
}

Document for search result

func (*Document) GetForward

func (doc *Document) GetForward(name string) (reflect.Value, error)

func (*Document) String

func (doc *Document) String() string

String return json format of document

type ErrorCode

type ErrorCode int32

ErrorCode type of Code

const (
	Success ErrorCode = 0
	// ProximaSE has been overload, please retry later
	RetryLater ErrorCode = -4009
	// Unknown error, which equals to MaxProximaSEErrorCode(refers to ProximaSE manual book)
	Unknown ErrorCode = -1000000
	// Incompatible error, current client can't work with ProximaSE
	// (please check web side of ProximaSE for release notes)
	Incompatible ErrorCode = -1000001
)

Code list of client only

type IndexType

type IndexType uint32

IndexType is types of index

const (
	// Undefined, which is not illegal argument for column configuration
	IndexTypeUndefined IndexType = iota
	// Proxima HNSW Index, Default value
	ProximaGraphIndex
)

Available index types

type ListCollectionFilter

type ListCollectionFilter interface {
	// contains filtered or unexported methods
}

ListCollectionFilter for ListCollections method

func ByRepo

func ByRepo(repo string) ListCollectionFilter

ByRepo list collections by repo

type LsnContext

type LsnContext struct {
	// Sequence number of collection
	LSN uint64 `json:"lsn"`
	// Context of collection
	Context string `json:"context,omitempty"`
}

LsnContext Log Sequence Number Context indicate specific point of collection

func (*LsnContext) String

func (context *LsnContext) String() string

String return json format of LsnContext

type MetricType

type MetricType uint32

MetricType indicates the method of distance between vector

const (
	// Undefined method, this is not illegal argument for column configuration
	MetricTypeUndefined MetricType = iota
	// Squared Euclidean
	SquaredEuclidean
	// Euclidean
	Euclidean
	// Manhattan
	Manhattan
	// Inner Product
	InnerProduct
	// Hamming
	Hamming
)

Available metric types

type OperationType

type OperationType uint32

OperationType operation type

const (
	// Insert Operation
	Insert OperationType = iota
	// Update Operation
	Update
	// Delete Operation
	Delete
)

Available operations

type ProximaSearchClient

type ProximaSearchClient interface {
	// CreateCollection method to create one collection
	//
	// Return error not nil means error, otherwise succeed
	// Example:
	//
	CreateCollection(config *CollectionConfig) error

	// DropCollection for delete collection
	//
	// return error is nil for success, otherwise failed
	DropCollection(name string) error

	// DescribeCollection method used to retrieve details of collection
	// Argument @name indicate interested collection
	// Return pointer of CollectionInfo, error not nil for failed
	DescribeCollection(name string) (*CollectionInfo, error)

	// ListCollections method, used to retrieve collections from ProximaSE
	// Available filters listed as following:
	//   ByRepo(repo string): only list collection which name of repository equals repo.
	//                        last one task effect, if multiple instance passed
	// Return pointer of CollectionInfo array, error not nil for failed
	ListCollections(filters ...ListCollectionFilter) ([]*CollectionInfo, error)

	// StatCollection used for retrieve stat of collection
	// Return pointer of CollectionStat, error not nil for failed
	StatCollection(name string) (*CollectionStat, error)

	// Write Collection
	// Return Status object, error not nil for failed
	Write(in *WriteRequest) error

	// Perform Query Request to Proxima BE
	// Return pointer of QueryResponse, error not nil for failed
	// Note: acceptable features listed below
	//     // slice/array/matrix of [int8, uint32, uint64, float32]
	//     features := []int8{1,2,3,4}
	//     features := [5]int8{1,2,3,4}
	//     features := [][]int8{{1,2,3,4},{1,2,3,4}}
	//     features := [5][5]int8{{1,2,3,4},{1,2,3,4}}
	//     features := []uint32{1,2,3,4}
	//     features := [5]uint32{1,2,3,4}
	//     features := [][]uint32{{1,2,3,4},{1,2,3,4}}
	//     features := [5][5]uint32{{1,2,3,4},{1,2,3,4}}
	//     features := []float32{1,2,3,4}
	//     features := [5]float32{1,2,3,4}
	//     features := [][]float32{{1,2,3,4},{1,2,3,4}}
	//     features := [5][5]float32{{1,2,3,4},{1,2,3,4}}
	// opts parameter is interface of QueryOptions, possible value listed below:
	//   WithTopK(int): customize topk
	//   WithRadius(float32): customize search radius
	//   WithLinearSearch(): enable linear search
	//   WithDebugMode(): enable debug mode
	//   WithParam(string, interface{}): for advance configuration
	// example:
	//     client.Query("collection", "column", []int{1,2}, WithTopK(10), WithRadius(0.6), WithLinearSearch(), WithDebugMode(), WithParam("key", 1))
	Query(collection string, column string, features interface{}, opts ...QueryOption) (*QueryResponse, error)

	// GetDocumentByKey retrieve document which primary key was indicated by param req
	// Return pointer of Document, error not nil for failed
	GetDocumentByKey(collection string, primaryKey uint64) (*Document, error)
}

ProximaSearchClient is the client API for Proxima BE service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.

func NewProximaSearchClient

func NewProximaSearchClient(conn ConnectionProtocol, address *Address) (ProximaSearchClient, error)

NewProximaSearchClient create new ProximaSearchClient object

type QueryOption

type QueryOption interface {
	// contains filtered or unexported methods
}

QueryOption for customize query params

func WithDebugMode

func WithDebugMode() QueryOption

WithDebugMode enable debug search mode, consequence bigger latench, does not recommend in product environments

func WithLinearSearch

func WithLinearSearch() QueryOption

WithLinearSearch enable linear search mode

func WithParam

func WithParam(key string, value interface{}) QueryOption

WithParam customize param, Acceptable value type listed below types of value: [[]byte, string, bool, int32, int64, uint32, uint64, float32, float64] Examples:

int32opt := WithParam("key", int32(10))
int64opt := WithParam("key", 10)

func WithRadius

func WithRadius(radius float32) QueryOption

WithRadius customize radius param, default is 0.5

func WithTopK

func WithTopK(topk uint32) QueryOption

WithTopK customize topk param, default is 100

type QueryResponse

type QueryResponse struct {
	// Debug Information, json format string
	DebugInfo string
	// Latency of request, count by ms
	Latency uint64
	// Documents of query
	Documents [][]*Document
}

QueryResponse Response of QueryRequest

func (*QueryResponse) String

func (resp *QueryResponse) String() string

String return json format of QueryResponse

type Range

type Range struct {
	// Beginning of range
	Min uint64
	// Ending of range
	Max uint64
}

Range for define limitation of properties

type Repository

type Repository struct {
	// Name of repository, empty is illegal argument
	Name string `json:"name,omitempty"`
	// Type of repository, Optional field, Default is RT_DATABASE
	Type RepositoryType `json:"type"`
}

Repository Base Description of Repository, drove class specific one type of repository

type RepositoryType

type RepositoryType uint32

RepositoryType type of repository

const (
	// Database repository
	Database RepositoryType = iota
)

Repository list

type Row

type Row struct {
	// Primary key of row record
	PrimaryKey uint64
	// Operation type
	OperationType
	// Index column value list
	IndexColumnValues []interface{}
	// Forward column value list
	ForwardColumnValues []interface{}
	// Log Sequence Number context
	*LsnContext
}

Row record

type RowMeta

type RowMeta struct {
	// Index column name list
	IndexColumnNames []string
	// Index column name list
	ForwardColumnNames []string
}

RowMeta meta for row records

type SegmentStat

type SegmentStat struct {
	// ID of segment
	SegmentID uint32
	// State of segment
	State SegmentState
	// Count of documents in segment
	DocCount uint64
	// Index files inside segment
	IndexFileCount uint64
	// Size of segment
	IndexFileSize uint64
	// Range of documents inside segment
	DocsRange Range
	// Primary key range of segment
	PrimaryKeyRange Range
	// Range of document timestamp inside segment
	TimestampRange Range
	// Range of LSN
	LSNRange Range
	// Storage path of segment
	SegmentPath string
}

SegmentStat stats of Segment

func (*SegmentStat) String

func (stat *SegmentStat) String() string

String method return json formatted string of SegmentStat

type SegmentState

type SegmentState uint32

SegmentState state of segment

const (
	// Segment has been created
	Created SegmentState = iota
	// Writing operation on Segment
	Writing
	// Segment is dumping
	Dumping
	// Compacting operation on segment
	Compacting
	// Persistent Segment
	Persist
)

Available states of segment

type Status

type Status struct {
	// Response code from ProximaSE, 0 for success, otherwise for failed
	Code int32 `json:"code"`
	// Detail message explain failed reason, Optional field
	Reason string `json:"reason,omitempty"`
}

Status object

func WrapStatus

func WrapStatus(err error) (*Status, error)

WrapStatus wrap error as Status, for further actions

func (*Status) Error

func (s *Status) Error() string

Error msg

func (*Status) IsGrpcError

func (s *Status) IsGrpcError() bool

func (*Status) OK

func (s *Status) OK() bool

OK check status

func (*Status) String

func (s *Status) String() string

String return json formatted string of Status

type Version

type Version struct {
	// Version of client
	Client string `json:"client:omitempty"`
	// Version of ProximaSE
	Server string `json:"server:omitempty"`
}

Version object

func (*Version) Compatible

func (ver *Version) Compatible() bool

Compatible check compatibility between client and server

func (*Version) String

func (version *Version) String() string

String return json formatted string of Version object

type WriteRequest

type WriteRequest struct {
	// Name of collection
	CollectionName string
	// Meta header
	Meta RowMeta
	// Row record list
	Rows []Row
	// Request ID, Optional
	RequestID string
	// Magic number, Optional
	MagicNumber uint64
}

WriteRequest object, the parameter of ProximaSEClient.Write method

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL