storage

package

v0.0.0-...-3a65784 Latest Latest Go to latest Published: Apr 16, 2024 License: Apache-2.0 Imports: 17 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/dsg-uwaterloo/oblishard

Links

Open Source Insights

Documentation ¶

Index ¶

func Decrypt(s string, key []byte) (string, error)
func Encrypt(s string, key []byte) (string, error)
func GetNextReverseLexicographicPath(evictionCount int, treeHeight int) (nextPath int)
func GetRandomPathAndStorageID(treeHeight int, storageCount int) (path int, storageID int)
type BlockInfo
type BlockOffsetStatus
type IntSet
type MockStorageHandler
- func NewMockStorageHandler(levelCount int, maxAccessCount int) *MockStorageHandler
type StorageHandler
- func NewStorageHandler(treeHeight int, Z int, S int, shift int, redisEndpoints []config.RedisEndpoint) *StorageHandler

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func Decrypt ¶

func Decrypt(s string, key []byte) (string, error)

func Encrypt ¶

func Encrypt(s string, key []byte) (string, error)

func GetNextReverseLexicographicPath ¶

func GetNextReverseLexicographicPath(evictionCount int, treeHeight int) (nextPath int)

evictionCount starts from zero and goes forward

func GetRandomPathAndStorageID ¶

func GetRandomPathAndStorageID(treeHeight int, storageCount int) (path int, storageID int)

It returns valid randomly chosen path and storageID.

Types ¶

type BlockInfo ¶

type BlockInfo struct {
	Value string
	Path  int
}

type BlockOffsetStatus ¶

type BlockOffsetStatus struct {
	Offset     int
	IsReal     bool
	BlockFound string
}

type IntSet ¶

type IntSet map[int]struct{}

func (IntSet) Add ¶

func (s IntSet) Add(item int)

func (IntSet) Contains ¶

func (s IntSet) Contains(item int) bool

func (IntSet) Remove ¶

func (s IntSet) Remove(item int)

type MockStorageHandler ¶

type MockStorageHandler struct {
	// contains filtered or unexported fields
}

func NewMockStorageHandler ¶

func NewMockStorageHandler(levelCount int, maxAccessCount int) *MockStorageHandler

func (*MockStorageHandler) BatchGetAccessCount ¶

func (m *MockStorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)

func (*MockStorageHandler) BatchGetBlockOffset ¶

func (m *MockStorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (offsets map[int]BlockOffsetStatus, err error)

func (*MockStorageHandler) BatchReadBlock ¶

func (m *MockStorageHandler) BatchReadBlock(offsets map[int]int, storageID int) (values map[int]string, err error)

func (*MockStorageHandler) BatchReadBucket ¶

func (m *MockStorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)

func (*MockStorageHandler) BatchWriteBucket ¶

func (m *MockStorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)

func (*MockStorageHandler) GetBucketsInPaths ¶

func (m *MockStorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)

func (*MockStorageHandler) GetMaxAccessCount ¶

func (m *MockStorageHandler) GetMaxAccessCount() int

func (*MockStorageHandler) GetMultipleReverseLexicographicPaths ¶

func (m *MockStorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)

func (*MockStorageHandler) GetRandomStorageID ¶

func (m *MockStorageHandler) GetRandomStorageID() int

func (*MockStorageHandler) LockStorage ¶

func (m *MockStorageHandler) LockStorage(storageID int)

func (*MockStorageHandler) UnlockStorage ¶

func (m *MockStorageHandler) UnlockStorage(storageID int)

func (*MockStorageHandler) WithCusomBatchGetBlockOffsetFunc ¶

func (m *MockStorageHandler) WithCusomBatchGetBlockOffsetFunc(f func(bucketIDs []int, storageID int, blocks []string) (offsets map[int]BlockOffsetStatus, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchGetAccessCountFunc ¶

func (m *MockStorageHandler) WithCustomBatchGetAccessCountFunc(f func(bucketIDs []int, storageID int) (counts map[int]int, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchReadBlockFunc ¶

func (m *MockStorageHandler) WithCustomBatchReadBlockFunc(f func(offsets map[int]int, storageID int) (values map[int]string, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchReadBucketFunc ¶

func (m *MockStorageHandler) WithCustomBatchReadBucketFunc(f func(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchWriteBucketFunc ¶

func (m *MockStorageHandler) WithCustomBatchWriteBucketFunc(f func(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)) *MockStorageHandler

type StorageHandler ¶

type StorageHandler struct {
	Z int // the maximum number of real blocks in each bucket
	S int // the number of dummy blocks in each bucket
	// contains filtered or unexported fields
}

StorageHandler is responsible for handling one or multiple storage shards.

func NewStorageHandler ¶

func NewStorageHandler(treeHeight int, Z int, S int, shift int, redisEndpoints []config.RedisEndpoint) *StorageHandler

func (*StorageHandler) BatchGetAccessCount ¶

func (s *StorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)

It returns the number of times a bucket was accessed for multiple buckets. This is helpful to know when to do an early reshuffle.

func (*StorageHandler) BatchGetAllMetaData ¶

func (s *StorageHandler) BatchGetAllMetaData(bucketIDs []int, storageID int) (map[int]map[string]int, error)

Returns a map of bucketID to a map of block to position. It returns all the valid real and dummy blocks in the bucket. The invalidated blocks are not returned.

func (*StorageHandler) BatchGetBlockOffset ¶

func (s *StorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (blockoffsetStatuses map[int]BlockOffsetStatus, err error)

func (*StorageHandler) BatchPushDataAndMetadata ¶

func (s *StorageHandler) BatchPushDataAndMetadata(bucketId int, valueData []string, valueMetadata []string, pipe redis.Pipeliner) (dataCmd *redis.BoolCmd, metadataCmd *redis.BoolCmd)

func (*StorageHandler) BatchReadBlock ¶

func (s *StorageHandler) BatchReadBlock(bucketOffsets map[int]int, storageID int) (values map[int]string, err error)

It reads multiple blocks from multiple buckets and returns the values.

func (*StorageHandler) BatchReadBucket ¶

func (s *StorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)

It reads multiple buckets from a single storage shard.

func (*StorageHandler) BatchWriteBucket ¶

func (s *StorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)

It writes blocks to multiple buckets in a single storage shard.

func (*StorageHandler) GetBucketsInPaths ¶

func (s *StorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)

GetBucketsInPaths return all the bucket ids for the passed paths.

func (*StorageHandler) GetMaxAccessCount ¶

func (s *StorageHandler) GetMaxAccessCount() int

func (*StorageHandler) GetMultipleReverseLexicographicPaths ¶

func (s *StorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)

func (*StorageHandler) GetRandomStorageID ¶

func (s *StorageHandler) GetRandomStorageID() int

func (*StorageHandler) InitDatabase ¶

func (s *StorageHandler) InitDatabase() error

func (*StorageHandler) LockStorage ¶

func (s *StorageHandler) LockStorage(storageID int)

func (*StorageHandler) UnlockStorage ¶

func (s *StorageHandler) UnlockStorage(storageID int)

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL