integration

package
v0.10.3-0...-f06509b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 25, 2024 License: Apache-2.0 Imports: 59 Imported by: 0

README

Integration test

This folder contains the integration test for Milvus components.

How to run integration test locally

Integration test still need some thirdparty components to start:

cd [milvus-folder]/deployments/docker/dev && docker-compose up -d

Run following script to start the full integration test:

cd [milvus-folder]
make milvus # milvus needs to be compiled to make cpp build ready
./scripts/run_intergration_test.sh

If you want to run single test case, you could execute command like this example

# mq, etcd, minio ready before
cd [milvus-folder]
source scripts/setenv.sh
cd tests/integration/[testcase-folder]/
go test -run "$testCaseName^" -testify.m "$subTestifyCaseName^" -race -v
Using suite

MiniClusterandMiniClusterSuite` provides lots of comment preset tool function to execute intergration test.

It is recommend to add a new test with testify/suite


import (
    // ...
    "github.com/milvus-io/milvus/tests/integration"
)

type NewSuite struct {
    integration.MiniClusterSuite
}


// Setups and teardowns, optional if no custom logic needed
// example to suite setup & teardown, same logic applies to test setup&teardown

func (s *NewSuite) SetupSuite() {
    s.MiniClusterSuite.SetupSuite()
    // customized setup
}

func (s *NewSuite) TearDownSuite() {
    s.MiniClusterSuite.TearDownSuite()
    // customized teardown
}
New folder for each new scenario

It's a known issue that integration test cases run in same process might affect due to some singleton component not fully cleaned.

As a temp solution, test cases are separated into different packages to run independently.

Documentation

Index

Constants

View Source 
const (
	IndexRaftIvfFlat         = indexparamcheck.IndexRaftIvfFlat
	IndexRaftIvfPQ           = indexparamcheck.IndexRaftIvfPQ
	IndexFaissIDMap          = indexparamcheck.IndexFaissIDMap
	IndexFaissIvfFlat        = indexparamcheck.IndexFaissIvfFlat
	IndexFaissIvfPQ          = indexparamcheck.IndexFaissIvfPQ
	IndexScaNN               = indexparamcheck.IndexScaNN
	IndexFaissIvfSQ8         = indexparamcheck.IndexFaissIvfSQ8
	IndexFaissBinIDMap       = indexparamcheck.IndexFaissBinIDMap
	IndexFaissBinIvfFlat     = indexparamcheck.IndexFaissBinIvfFlat
	IndexHNSW                = indexparamcheck.IndexHNSW
	IndexDISKANN             = indexparamcheck.IndexDISKANN
	IndexSparseInvertedIndex = indexparamcheck.IndexSparseInverted
	IndexSparseWand          = indexparamcheck.IndexSparseWand
)
View Source 
const (
	AnnsFieldKey    = "anns_field"
	TopKKey         = "topk"
	NQKey           = "nq"
	MetricTypeKey   = common.MetricTypeKey
	SearchParamsKey = common.IndexParamsKey
	RoundDecimalKey = "round_decimal"
	OffsetKey       = "offset"
	LimitKey        = "limit"
)
View Source 
const (
	BoolField           = "boolField"
	Int8Field           = "int8Field"
	Int16Field          = "int16Field"
	Int32Field          = "int32Field"
	Int64Field          = "int64Field"
	FloatField          = "floatField"
	DoubleField         = "doubleField"
	VarCharField        = "varCharField"
	JSONField           = "jsonField"
	FloatVecField       = "floatVecField"
	BinVecField         = "binVecField"
	Float16VecField     = "float16VecField"
	BFloat16VecField    = "bfloat16VecField"
	SparseFloatVecField = "sparseFloatVecField"
)

Variables

This section is empty.

Functions

func ConstructIndexParam 

func ConstructIndexParam(dim int, indexType string, metricType string) []*commonpb.KeyValuePair

func ConstructSchema 

func ConstructSchema(collection string, dim int, autoID bool, fields ...*schemapb.FieldSchema) *schemapb.CollectionSchema

func ConstructSchemaOfVecDataType 

func ConstructSchemaOfVecDataType(collection string, dim int, autoID bool, dataType schemapb.DataType) *schemapb.CollectionSchema

func ConstructSearchRequest 

func ConstructSearchRequest(
	dbName, collectionName string,
	expr string,
	vecField string,
	vectorType schemapb.DataType,
	outputFields []string,
	metricType string,
	params map[string]any,
	nq, dim int, topk, roundDecimal int,
) *milvuspb.SearchRequest

func ConstructSearchRequestWithConsistencyLevel 

func ConstructSearchRequestWithConsistencyLevel(
	dbName, collectionName string,
	expr string,
	vecField string,
	vectorType schemapb.DataType,
	outputFields []string,
	metricType string,
	params map[string]any,
	nq, dim int, topk, roundDecimal int,
	useDefaultConsistency bool,
	consistencyLevel commonpb.ConsistencyLevel,
) *milvuspb.SearchRequest

func DefaultParams 

func DefaultParams() map[string]string

func GenerateBFloat16Vectors 

func GenerateBFloat16Vectors(numRows, dim int) []byte

func GenerateBinaryVectors 

func GenerateBinaryVectors(numRows, dim int) []byte

func GenerateFloat16Vectors 

func GenerateFloat16Vectors(numRows, dim int) []byte

func GenerateFloatVectors 

func GenerateFloatVectors(numRows, dim int) []float32

func GenerateHashKeys 

func GenerateHashKeys(numRows int) []uint32

func GenerateInt64Array 

func GenerateInt64Array(numRows int, start int64) []int64

func GenerateSameInt64Array 

func GenerateSameInt64Array(numRows int, value int64) []int64

func GenerateSameStringArray 

func GenerateSameStringArray(numRows int, value string) []string

func GenerateSparseFloatArray 

func GenerateSparseFloatArray(numRows int) *schemapb.SparseFloatArray

func GenerateStringArray 

func GenerateStringArray(numRows int) []string

func GetSearchParams 

func GetSearchParams(indexType string, metricType string) map[string]any

func NewBFloat16VectorFieldData 

func NewBFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData

func NewBinaryVectorFieldData 

func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData

func NewFloat16VectorFieldData 

func NewFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData

func NewFloatVectorFieldData 

func NewFloatVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData

func NewInt64FieldData 

func NewInt64FieldData(fieldName string, numRows int) *schemapb.FieldData

func NewInt64FieldDataWithStart 

func NewInt64FieldDataWithStart(fieldName string, numRows int, start int64) *schemapb.FieldData

func NewInt64SameFieldData 

func NewInt64SameFieldData(fieldName string, numRows int, value int64) *schemapb.FieldData

func NewSparseFloatVectorFieldData 

func NewSparseFloatVectorFieldData(fieldName string, numRows int) *schemapb.FieldData

func NewStringFieldData 

func NewStringFieldData(fieldName string, numRows int) *schemapb.FieldData

func NewVarCharSameFieldData 

func NewVarCharSameFieldData(fieldName string, numRows int, value string) *schemapb.FieldData

func PrettyReplica 

func PrettyReplica(replica *querypb.Replica) string

Types

type ClusterConfig 

type ClusterConfig struct {
	// ProxyNum int
	// todo coord num can be more than 1 if enable Active-Standby
	// RootCoordNum int
	// DataCoordNum int
	// IndexCoordNum int
	// QueryCoordNum int
	QueryNodeNum int
	DataNodeNum  int
	IndexNodeNum int
}

func DefaultClusterConfig 

func DefaultClusterConfig() ClusterConfig

type CreateCollectionConfig 

type CreateCollectionConfig struct {
	DBName           string
	CollectionName   string
	ChannelNum       int
	SegmentNum       int
	RowNumPerSegment int
	Dim              int
}

type EmbedEtcdSuite 

type EmbedEtcdSuite struct {
	EtcdServer *embed.Etcd
	EtcdDir    string
}

EmbedEtcdSuite contains embed setup & teardown related logic

func (*EmbedEtcdSuite) SetupEmbedEtcd 

func (s *EmbedEtcdSuite) SetupEmbedEtcd() error

func (*EmbedEtcdSuite) TearDownEmbedEtcd 

func (s *EmbedEtcdSuite) TearDownEmbedEtcd()

type EtcdMetaWatcher 

type EtcdMetaWatcher struct {
	MetaWatcher
	// contains filtered or unexported fields
}

func (*EtcdMetaWatcher) ShowReplicas 

func (watcher *EtcdMetaWatcher) ShowReplicas() ([]*querypb.Replica, error)

func (*EtcdMetaWatcher) ShowSegments 

func (watcher *EtcdMetaWatcher) ShowSegments() ([]*datapb.SegmentInfo, error)

func (*EtcdMetaWatcher) ShowSessions 

func (watcher *EtcdMetaWatcher) ShowSessions() ([]*sessionutil.SessionRaw, error)

type MetaWatcher 

type MetaWatcher interface {
	ShowSessions() ([]*sessionutil.SessionRaw, error)
	ShowSegments() ([]*datapb.SegmentInfo, error)
	ShowReplicas() ([]*querypb.Replica, error)
}

MetaWatcher to observe meta data of milvus cluster

type MiniClusterSuite 

type MiniClusterSuite struct {
	suite.Suite
	EmbedEtcdSuite

	Cluster *MiniClusterV2
	// contains filtered or unexported fields
}

func (*MiniClusterSuite) CreateCollectionWithConfiguration 

func (s *MiniClusterSuite) CreateCollectionWithConfiguration(ctx context.Context, cfg *CreateCollectionConfig)

func (*MiniClusterSuite) SetupSuite 

func (s *MiniClusterSuite) SetupSuite()

func (*MiniClusterSuite) SetupTest 

func (s *MiniClusterSuite) SetupTest()

func (*MiniClusterSuite) TearDownSuite 

func (s *MiniClusterSuite) TearDownSuite()

func (*MiniClusterSuite) TearDownTest 

func (s *MiniClusterSuite) TearDownTest()

func (*MiniClusterSuite) WaitForFlush 

func (s *MiniClusterSuite) WaitForFlush(ctx context.Context, segIDs []int64, flushTs uint64, dbName, collectionName string)

func (*MiniClusterSuite) WaitForIndexBuilt 

func (s *MiniClusterSuite) WaitForIndexBuilt(ctx context.Context, collection, field string)

func (*MiniClusterSuite) WaitForIndexBuiltWithDB 

func (s *MiniClusterSuite) WaitForIndexBuiltWithDB(ctx context.Context, dbName, collection, field string)

func (*MiniClusterSuite) WaitForIndexBuiltWithIndexName 

func (s *MiniClusterSuite) WaitForIndexBuiltWithIndexName(ctx context.Context, collection, field, indexName string)

func (*MiniClusterSuite) WaitForLoad 

func (s *MiniClusterSuite) WaitForLoad(ctx context.Context, collection string)

func (*MiniClusterSuite) WaitForLoadRefresh 

func (s *MiniClusterSuite) WaitForLoadRefresh(ctx context.Context, dbName, collection string)

func (*MiniClusterSuite) WaitForLoadWithDB 

func (s *MiniClusterSuite) WaitForLoadWithDB(ctx context.Context, dbName, collection string)

type MiniClusterV2 

type MiniClusterV2 struct {
	ChunkManager storage.ChunkManager

	EtcdCli *clientv3.Client

	Proxy      *grpcproxy.Server
	DataCoord  *grpcdatacoord.Server
	RootCoord  *grpcrootcoord.Server
	QueryCoord *grpcquerycoord.Server

	DataCoordClient  types.DataCoordClient
	RootCoordClient  types.RootCoordClient
	QueryCoordClient types.QueryCoordClient

	ProxyClient     types.ProxyClient
	DataNodeClient  types.DataNodeClient
	QueryNodeClient types.QueryNodeClient
	IndexNodeClient types.IndexNodeClient

	DataNode  *grpcdatanode.Server
	QueryNode *grpcquerynode.Server
	IndexNode *grpcindexnode.Server

	MetaWatcher MetaWatcher

	Extension *ReportChanExtension
	// contains filtered or unexported fields
}

func StartMiniClusterV2 

func StartMiniClusterV2(ctx context.Context, opts ...OptionV2) (*MiniClusterV2, error)

func (*MiniClusterV2) AddDataNode 

func (cluster *MiniClusterV2) AddDataNode() *grpcdatanode.Server

func (*MiniClusterV2) AddQueryNode 

func (cluster *MiniClusterV2) AddQueryNode() *grpcquerynode.Server

func (*MiniClusterV2) AddQueryNodes 

func (cluster *MiniClusterV2) AddQueryNodes(k int) []*grpcquerynode.Server

func (*MiniClusterV2) GetAllQueryNodes 

func (cluster *MiniClusterV2) GetAllQueryNodes() []*grpcquerynode.Server

func (*MiniClusterV2) GetAvailablePort 

func (cluster *MiniClusterV2) GetAvailablePort() (int, error)

func (*MiniClusterV2) GetAvailablePorts 

func (cluster *MiniClusterV2) GetAvailablePorts(n int) ([]int, error)

func (*MiniClusterV2) GetContext 

func (cluster *MiniClusterV2) GetContext() context.Context

func (*MiniClusterV2) GetFactory 

func (cluster *MiniClusterV2) GetFactory() dependency.Factory

func (*MiniClusterV2) Start 

func (cluster *MiniClusterV2) Start() error

func (*MiniClusterV2) Stop 

func (cluster *MiniClusterV2) Stop() error

func (*MiniClusterV2) StopAllDataNodes 

func (cluster *MiniClusterV2) StopAllDataNodes()

func (*MiniClusterV2) StopAllQueryNodes 

func (cluster *MiniClusterV2) StopAllQueryNodes()

type OptionV2 

type OptionV2 func(cluster *MiniClusterV2)

type ReportChanExtension 

type ReportChanExtension struct {
	// contains filtered or unexported fields
}

func InitReportExtension 

func InitReportExtension() *ReportChanExtension

func NewReportChanExtension 

func NewReportChanExtension() *ReportChanExtension

func (*ReportChanExtension) GetReportChan 

func (r *ReportChanExtension) GetReportChan() <-chan any

func (*ReportChanExtension) Report 

func (r *ReportChanExtension) Report(info any) int

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.