kvstore

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 27, 2021 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MaxTableSize        = 768 << 20
	MaxValueLogFileSize = 2048 << 10
	MaxValueLogEntries  = 100000000
)

Variables

This section is empty.

Functions

func CreateHashValue 

func CreateHashValue(ids [][]byte, unique bool) ([]byte, []byte)

func NewAATable 

func NewAATable() (map[[2]rune]uint32, map[uint32][2]rune)

func NewKVStore 

func NewKVStore(kv *KVStore, options badger.Options, flushSize int, nbOfThreads int)

func RemoveDuplicatesFromSlice 

func RemoveDuplicatesFromSlice(s [][]byte) [][]byte

Utility functions

func SplitHashValue 

func SplitHashValue(hashValue []byte) ([][]byte, error)

Types

type KC_ 

type KC_ struct {
	*KVStore
}

Hash store for values combination used in other stores

func KC_New 

func KC_New(opts badger.Options, flushSize int, nbOfThreads int) *KC_

func (*KC_) CreateKCKeyValue 

func (kc *KC_) CreateKCKeyValue(keys [][]byte) ([]byte, []byte)

type KComb 

type KComb struct {
	ProteinKeys          []uint32 `protobuf:"varint,1,rep,packed,name=ProteinKeys,proto3" json:"ProteinKeys,omitempty"`
	Count                uint32   `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*KComb) Descriptor 

func (*KComb) Descriptor() ([]byte, []int)

func (*KComb) GetCount 

func (m *KComb) GetCount() uint32

func (*KComb) GetProteinKeys 

func (m *KComb) GetProteinKeys() []uint32

func (*KComb) ProtoMessage 

func (*KComb) ProtoMessage()

func (*KComb) Reset 

func (m *KComb) Reset()

func (*KComb) String 

func (m *KComb) String() string

func (*KComb) XXX_DiscardUnknown 

func (m *KComb) XXX_DiscardUnknown()

func (*KComb) XXX_Marshal 

func (m *KComb) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*KComb) XXX_Merge 

func (m *KComb) XXX_Merge(src proto.Message)

func (*KComb) XXX_Size 

func (m *KComb) XXX_Size() int

func (*KComb) XXX_Unmarshal 

func (m *KComb) XXX_Unmarshal(b []byte) error

type KSettings 

type KSettings struct {
	Name                 string   `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
	Port                 int32    `protobuf:"varint,2,opt,name=Port,proto3" json:"Port,omitempty"`
	CreationDate         string   `protobuf:"bytes,3,opt,name=CreationDate,proto3" json:"CreationDate,omitempty"`
	OriginalFile         string   `protobuf:"bytes,4,opt,name=OriginalFile,proto3" json:"OriginalFile,omitempty"`
	DatabaseIndexed      bool     `protobuf:"varint,5,opt,name=DatabaseIndexed,proto3" json:"DatabaseIndexed,omitempty"`
	IDsIndexed           bool     `protobuf:"varint,6,opt,name=IDsIndexed,proto3" json:"IDsIndexed,omitempty"`
	NamesIndexed         bool     `protobuf:"varint,7,opt,name=NamesIndexed,proto3" json:"NamesIndexed,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*KSettings) Descriptor 

func (*KSettings) Descriptor() ([]byte, []int)

func (*KSettings) GetCreationDate 

func (m *KSettings) GetCreationDate() string

func (*KSettings) GetDatabaseIndexed 

func (m *KSettings) GetDatabaseIndexed() bool

func (*KSettings) GetIDsIndexed 

func (m *KSettings) GetIDsIndexed() bool

func (*KSettings) GetName 

func (m *KSettings) GetName() string

func (*KSettings) GetNamesIndexed 

func (m *KSettings) GetNamesIndexed() bool

func (*KSettings) GetOriginalFile 

func (m *KSettings) GetOriginalFile() string

func (*KSettings) GetPort 

func (m *KSettings) GetPort() int32

func (*KSettings) ProtoMessage 

func (*KSettings) ProtoMessage()

func (*KSettings) Reset 

func (m *KSettings) Reset()

func (*KSettings) String 

func (m *KSettings) String() string

func (*KSettings) XXX_DiscardUnknown 

func (m *KSettings) XXX_DiscardUnknown()

func (*KSettings) XXX_Marshal 

func (m *KSettings) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*KSettings) XXX_Merge 

func (m *KSettings) XXX_Merge(src proto.Message)

func (*KSettings) XXX_Size 

func (m *KSettings) XXX_Size() int

func (*KSettings) XXX_Unmarshal 

func (m *KSettings) XXX_Unmarshal(b []byte) error

type KStats 

type KStats struct {
	NumberOfProteins     uint64   `protobuf:"varint,1,opt,name=NumberOfProteins,proto3" json:"NumberOfProteins,omitempty"`
	NumberOfAA           uint64   `protobuf:"varint,2,opt,name=NumberOfAA,proto3" json:"NumberOfAA,omitempty"`
	NumberOfKmers        uint64   `protobuf:"varint,4,opt,name=NumberOfKmers,proto3" json:"NumberOfKmers,omitempty"`
	NumberOfKCombSets    uint64   `protobuf:"varint,5,opt,name=NumberOfKCombSets,proto3" json:"NumberOfKCombSets,omitempty"`
	Features             []string `protobuf:"bytes,6,rep,name=Features,proto3" json:"Features,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*KStats) Descriptor 

func (*KStats) Descriptor() ([]byte, []int)

func (*KStats) GetFeatures 

func (m *KStats) GetFeatures() []string

func (*KStats) GetNumberOfAA 

func (m *KStats) GetNumberOfAA() uint64

func (*KStats) GetNumberOfKCombSets 

func (m *KStats) GetNumberOfKCombSets() uint64

func (*KStats) GetNumberOfKmers 

func (m *KStats) GetNumberOfKmers() uint64

func (*KStats) GetNumberOfProteins 

func (m *KStats) GetNumberOfProteins() uint64

func (*KStats) ProtoMessage 

func (*KStats) ProtoMessage()

func (*KStats) Reset 

func (m *KStats) Reset()

func (*KStats) String 

func (m *KStats) String() string

func (*KStats) XXX_DiscardUnknown 

func (m *KStats) XXX_DiscardUnknown()

func (*KStats) XXX_Marshal 

func (m *KStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*KStats) XXX_Merge 

func (m *KStats) XXX_Merge(src proto.Message)

func (*KStats) XXX_Size 

func (m *KStats) XXX_Size() int

func (*KStats) XXX_Unmarshal 

func (m *KStats) XXX_Unmarshal(b []byte) error

type KV 

type KV struct {
	Key    []byte
	Val    []byte
	Unique bool
}

type KVStore 

type KVStore struct {
	DB *badger.DB

	TxBatchChannel     []TxBatch
	TxBatchChannelWG   *sync.WaitGroup
	TxBatchChannelJobs chan KV

	BatchCounter uint64
	NbOfThreads  int
	FlushSize    int
	NilVal       []byte
	Mu           sync.Mutex
}

Key Value Store

func (*KVStore) AddValueChanWorker 

func (kv *KVStore) AddValueChanWorker()

func (*KVStore) AddValueToChannel 

func (kv *KVStore) AddValueToChannel(key []byte, newVal []byte, unique bool)

func (*KVStore) Close 

func (kv *KVStore) Close()

func (*KVStore) CloseInsertChannel 

func (kv *KVStore) CloseInsertChannel()

func (*KVStore) Flush 

func (kv *KVStore) Flush()

func (*KVStore) GarbageCollect 

func (kv *KVStore) GarbageCollect(count int, ratio float64)

func (*KVStore) GetValue 

func (kv *KVStore) GetValue(key []byte) ([]byte, bool)

func (*KVStore) GetValueFromBadger 

func (kv *KVStore) GetValueFromBadger(key []byte) ([]byte, error)

func (*KVStore) GetValues 

func (kv *KVStore) GetValues(key []byte) ([][]byte, error)

func (*KVStore) MergeCombinationKeys 

func (kv *KVStore) MergeCombinationKeys(combKeys [][]byte, threadId int) []byte

func (*KVStore) OpenInsertChannel 

func (kv *KVStore) OpenInsertChannel()

OpenInsertChannell Open a channel to concurrently do batch insert in the store

func (*KVStore) UpdateValue 

func (kv *KVStore) UpdateValue(key []byte, val []byte)

type KVStores 

type KVStores struct {
	KmerStore    *K_
	KCombStore   *KC_
	ProteinStore *P_
}

func KVStoresNew 

func KVStoresNew(dbPath string, nbOfThreads int, maxSize bool, syncWrite bool, readOnly bool) *KVStores

func (*KVStores) Close 

func (kvStores *KVStores) Close()

func (*KVStores) CloseInsertChannel 

func (kvStores *KVStores) CloseInsertChannel()

func (*KVStores) Flush 

func (kvStores *KVStores) Flush()

func (*KVStores) OpenInsertChannel 

func (kvStores *KVStores) OpenInsertChannel()

type KVsToMerge 

type KVsToMerge struct {
	Key    []byte
	Values [][]byte
}

type K_ 

type K_ struct {
	*KVStore
	// contains filtered or unexported fields
}

Kmer Entries

func K_New 

func K_New(opts badger.Options, flushSize int, nbOfThreads int) *K_

func (*K_) CreateBytesKey 

func (k *K_) CreateBytesKey(kmer string) []byte

func (*K_) CreateBytesVal 

func (k *K_) CreateBytesVal(entry string) []byte

func (*K_) DecodeKmer 

func (k *K_) DecodeKmer(key []byte) string

expect kmers of length 7

func (*K_) EncodeEntry 

func (k *K_) EncodeEntry(kmer string) uint32

func (*K_) EncodeKmer 

func (k *K_) EncodeKmer(kmer string) uint32

expect kmers of length 7

type P_ 

type P_ struct {
	*KVStore
}

Hash store for values combination used in other stores

func P_New 

func P_New(opts badger.Options, flushSize int, nbOfThreads int) *P_

type Protein 

type Protein struct {
	EntryId              string            `protobuf:"bytes,1,opt,name=EntryId,proto3" json:"EntryId,omitempty"`
	Sequence             string            `protobuf:"bytes,2,opt,name=Sequence,proto3" json:"Sequence,omitempty"`
	Length               int32             `protobuf:"varint,3,opt,name=Length,proto3" json:"Length,omitempty"`
	Features             map[string]string `` /* 157-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Protein) Descriptor 

func (*Protein) Descriptor() ([]byte, []int)

func (*Protein) GetEntryId 

func (m *Protein) GetEntryId() string

func (*Protein) GetFeatures 

func (m *Protein) GetFeatures() map[string]string

func (*Protein) GetLength 

func (m *Protein) GetLength() int32

func (*Protein) GetSequence 

func (m *Protein) GetSequence() string

func (*Protein) ProtoMessage 

func (*Protein) ProtoMessage()

func (*Protein) Reset 

func (m *Protein) Reset()

func (*Protein) String 

func (m *Protein) String() string

func (*Protein) XXX_DiscardUnknown 

func (m *Protein) XXX_DiscardUnknown()

func (*Protein) XXX_Marshal 

func (m *Protein) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Protein) XXX_Merge 

func (m *Protein) XXX_Merge(src proto.Message)

func (*Protein) XXX_Size 

func (m *Protein) XXX_Size() int

func (*Protein) XXX_Unmarshal 

func (m *Protein) XXX_Unmarshal(b []byte) error

type TxBatch 

type TxBatch struct {
	NbOfTx       int
	TxBufferSize int
	Entries      map[string]KV
	Mu           sync.Mutex
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.