vectorstore

type CompressedDocument ¶

type CompressedDocument struct {
	*gollum.Document
	Encoded   []byte
	Unencoded []byte
}

type CompressedVectorStore ¶

type CompressedVectorStore struct {
	Data       []CompressedDocument
	Compressor Compressor
}

func NewDummyVectorStore() *CompressedVectorStore

func NewGzipVectorStore() *CompressedVectorStore

func NewStdGzipVectorStore() *CompressedVectorStore

func NewZstdVectorStore() *CompressedVectorStore

func (ts *CompressedVectorStore) Insert(ctx context.Context, d gollum.Document) error

func (cvs *CompressedVectorStore) Query(ctx context.Context, qb QueryRequest) ([]*gollum.Document, error)

func (cvs *CompressedVectorStore) RetrieveAll(ctx context.Context) ([]gollum.Document, error)

type Compressor ¶

type Compressor interface {
	Compress(src []byte) []byte
}

Compressor is a single method interface that returns a compressed representation of an object.

type DummyCompressor ¶

type DummyCompressor struct {
}

func (g *DummyCompressor) Compress(src []byte) []byte

type GzipCompressor ¶

type GzipCompressor struct {
	// contains filtered or unexported fields
}

GzipCompressor uses the klauspost/compress gzip compressor. We generally suggest using this optimized implementation over the stdlib.

func (g *GzipCompressor) Compress(src []byte) []byte

type Heap ¶

type Heap []NodeSimilarity

Heap is a custom heap implementation, to avoid interface{} conversion. I _think_ theoretically that a memory arena would be useful here, but that feels a bit beyond the pale, even for me. In benchmarking, we see that allocations are limited by scale according to k -- since K is known, we should be able to allocate a fixed-size arena and use that. That being said... let's revisit in the future :)

func (h *Heap) Init(k int)

func (h *Heap) Len() int

func (h Heap) Less(i, j int) bool

func (h *Heap) Pop() NodeSimilarity

func (h *Heap) Push(e NodeSimilarity)

func (h Heap) Swap(i, j int)

type MemoryVectorStore ¶

type MemoryVectorStore struct {
	Documents []gollum.Document
	LLM       gollum.Embedder
}

MemoryVectorStore embeds documents on insert and stores them in memory

func NewMemoryVectorStore(llm gollum.Embedder) *MemoryVectorStore

func NewMemoryVectorStoreFromDisk(ctx context.Context, bucket *blob.Bucket, path string, llm gollum.Embedder) (*MemoryVectorStore, error)

func (m *MemoryVectorStore) Insert(ctx context.Context, d gollum.Document) error

func (m *MemoryVectorStore) Persist(ctx context.Context, bucket *blob.Bucket, path string) error

func (m *MemoryVectorStore) Query(ctx context.Context, qb QueryRequest) ([]*gollum.Document, error)

func (m *MemoryVectorStore) RetrieveAll(ctx context.Context) ([]gollum.Document, error)

type NodeSimilarity ¶

type NodeSimilarity struct {
	Document   *gollum.Document
	Similarity float32
}

type QueryRequest ¶

type QueryRequest struct {
	// Query is the text to query
	Query string
	// EmbeddingStrings is a list of strings to concatenate and embed instead of Query
	EmbeddingStrings []string
	// EmbeddingFloats is a query vector to use instead of Query
	EmbeddingFloats []float32
	// K is the number of results to return
	K int
}

QueryRequest is a struct that contains the query and optional query strings or embeddings

type StdGzipCompressor ¶

type StdGzipCompressor struct {
	// contains filtered or unexported fields
}

StdGzipCompressor uses the std gzip compressor.

func (g *StdGzipCompressor) Compress(src []byte) []byte

type VectorStore ¶

type VectorStore interface {
	Insert(context.Context, gollum.Document) error
	Query(ctx context.Context, qb QueryRequest) ([]*gollum.Document, error)
	RetrieveAll(ctx context.Context) ([]gollum.Document, error)
}

type ZstdCompressor ¶

type ZstdCompressor struct {
	// contains filtered or unexported fields
}

ZstdCompressor uses the klauspost/compress zstd compressor.

func (g *ZstdCompressor) Compress(src []byte) []byte