storage

package
v0.0.0-...-b78b3a4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 16, 2019 License: GPL-3.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ErrInit = iota
	ErrNotFound
	ErrIO
	ErrUnauthorized
	ErrInvalidValue
	ErrDataOverflow
	ErrNothingToReturn
	ErrCorruptData
	ErrInvalidSignature
	ErrNotSynced
	ErrPeriodDepth
	ErrCnt
)
View Source
const (
	DataChunk = 0
	TreeChunk = 1
)
View Source
const (
	BMTHash     = "BMT"
	SHA3Hash    = "SHA3" //
	DefaultHash = BMTHash
)
View Source
const (
	ChunkProcessors = 8
)
View Source
const KeyLength = 32
View Source
const MaxPO = 16

Variables

View Source
var (
	ErrChunkNotFound    = errors.New("chunk not found")
	ErrFetching         = errors.New("chunk still fetching")
	ErrChunkInvalid     = errors.New("invalid chunk")
	ErrChunkForward     = errors.New("cannot forward")
	ErrChunkUnavailable = errors.New("chunk unavailable")
	ErrChunkTimeout     = errors.New("timeout")
)
View Source
var ZeroAddr = Address(common.Hash{}.Bytes())

Functions

func BytesToU64

func BytesToU64(data []byte) uint64

func IsZeroAddr

func IsZeroAddr(addr Address) bool

func NewHasherStore

func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore

func Proximity

func Proximity(one, other []byte) (ret int)

func PutChunks

func PutChunks(store *LocalStore, chunks ...*Chunk)

func U64ToBytes

func U64ToBytes(val uint64) []byte

Types

type Address

type Address []byte

func PyramidAppend

func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)

func PyramidSplit

func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)

func TreeSplit

func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (k Address, wait func(context.Context) error, err error)

func (Address) Hex

func (a Address) Hex() string

func (Address) Log

func (a Address) Log() string

func (Address) MarshalJSON

func (a Address) MarshalJSON() (out []byte, err error)

func (Address) Size

func (a Address) Size() uint

func (Address) String

func (a Address) String() string

func (*Address) UnmarshalJSON

func (a *Address) UnmarshalJSON(value []byte) error

type AddressCollection

type AddressCollection []Address

func NewAddressCollection

func NewAddressCollection(l int) AddressCollection

func (AddressCollection) Len

func (c AddressCollection) Len() int

func (AddressCollection) Less

func (c AddressCollection) Less(i, j int) bool

func (AddressCollection) Swap

func (c AddressCollection) Swap(i, j int)

type Chunk

type Chunk struct {
	Addr  Address //
	SData []byte  //
	Size  int64   //
	C    chan bool //
	ReqC chan bool //
	// contains filtered or unexported fields
}

func GenerateRandomChunk

func GenerateRandomChunk(dataSize int64) *Chunk

func GenerateRandomChunks

func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk)

func NewChunk

func NewChunk(addr Address, reqC chan bool) *Chunk

func (*Chunk) GetErrored

func (c *Chunk) GetErrored() error

func (*Chunk) SetErrored

func (c *Chunk) SetErrored(err error)

func (*Chunk) String

func (c *Chunk) String() string

func (*Chunk) WaitToStore

func (c *Chunk) WaitToStore() error

type ChunkData

type ChunkData []byte

func (ChunkData) Data

func (c ChunkData) Data() []byte

func (ChunkData) Size

func (c ChunkData) Size() int64

type ChunkStore

type ChunkStore interface {
	Put(context.Context, *Chunk) //
	Get(context.Context, Address) (*Chunk, error)
	Close()
}

type ChunkValidator

type ChunkValidator interface {
	Validate(addr Address, data []byte) bool
}

type ChunkerParams

type ChunkerParams struct {
	// contains filtered or unexported fields
}

type ContentAddressValidator

type ContentAddressValidator struct {
	Hasher SwarmHasher
}

func NewContentAddressValidator

func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator

func (*ContentAddressValidator) Validate

func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool

type DBAPI

type DBAPI struct {
	// contains filtered or unexported fields
}

func NewDBAPI

func NewDBAPI(loc *LocalStore) *DBAPI

func (*DBAPI) CurrentBucketStorageIndex

func (d *DBAPI) CurrentBucketStorageIndex(po uint8) uint64

func (*DBAPI) Get

func (d *DBAPI) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*DBAPI) GetOrCreateRequest

func (d *DBAPI) GetOrCreateRequest(ctx context.Context, addr Address) (*Chunk, bool)

func (*DBAPI) Iterator

func (d *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error

func (*DBAPI) Put

func (d *DBAPI) Put(ctx context.Context, chunk *Chunk)

type FileStore

type FileStore struct {
	ChunkStore
	// contains filtered or unexported fields
}

func NewFileStore

func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore

func NewLocalFileStore

func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error)

func (*FileStore) HashSize

func (f *FileStore) HashSize() int

func (*FileStore) Retrieve

func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool)

func (*FileStore) Store

func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error)

type FileStoreParams

type FileStoreParams struct {
	Hash string
}

func NewFileStoreParams

func NewFileStoreParams() *FileStoreParams

type Getter

type Getter interface {
	Get(context.Context, Reference) (ChunkData, error)
}

type HashWithLength

type HashWithLength struct {
	hash.Hash
}

func (*HashWithLength) ResetWithLength

func (h *HashWithLength) ResetWithLength(length []byte)

type Hasher

type Hasher func() hash.Hash

type JoinerParams

type JoinerParams struct {
	ChunkerParams
	// contains filtered or unexported fields
}

type LDBDatabase

type LDBDatabase struct {
	// contains filtered or unexported fields
}

func NewLDBDatabase

func NewLDBDatabase(file string) (*LDBDatabase, error)

func (*LDBDatabase) Close

func (db *LDBDatabase) Close()

func (*LDBDatabase) Delete

func (db *LDBDatabase) Delete(key []byte) error

func (*LDBDatabase) Get

func (db *LDBDatabase) Get(key []byte) ([]byte, error)

func (*LDBDatabase) LastKnownTD

func (db *LDBDatabase) LastKnownTD() []byte

func (*LDBDatabase) NewIterator

func (db *LDBDatabase) NewIterator() iterator.Iterator

func (*LDBDatabase) Put

func (db *LDBDatabase) Put(key []byte, value []byte)

func (*LDBDatabase) Write

func (db *LDBDatabase) Write(batch *leveldb.Batch) error

type LDBStore

type LDBStore struct {
	// contains filtered or unexported fields
}

func NewLDBStore

func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error)

func NewMockDbStore

func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error)

func (*LDBStore) Cleanup

func (s *LDBStore) Cleanup()

func (*LDBStore) Close

func (s *LDBStore) Close()

func (*LDBStore) CurrentBucketStorageIndex

func (s *LDBStore) CurrentBucketStorageIndex(po uint8) uint64

func (*LDBStore) CurrentStorageIndex

func (s *LDBStore) CurrentStorageIndex() uint64

func (*LDBStore) Export

func (s *LDBStore) Export(out io.Writer) (int64, error)

func (*LDBStore) Get

func (s *LDBStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*LDBStore) Import

func (s *LDBStore) Import(in io.Reader) (int64, error)

func (*LDBStore) Put

func (s *LDBStore) Put(ctx context.Context, chunk *Chunk)

func (*LDBStore) ReIndex

func (s *LDBStore) ReIndex()

func (*LDBStore) Size

func (s *LDBStore) Size() uint64

func (*LDBStore) SyncIterator

func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error

type LDBStoreParams

type LDBStoreParams struct {
	*StoreParams
	Path string
	Po   func(Address) uint8
}

func NewLDBStoreParams

func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams

type LazyChunkReader

type LazyChunkReader struct {
	Ctx context.Context
	// contains filtered or unexported fields
}

func TreeJoin

func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader

func (*LazyChunkReader) Context

func (r *LazyChunkReader) Context() context.Context

func (*LazyChunkReader) Read

func (r *LazyChunkReader) Read(b []byte) (read int, err error)

func (*LazyChunkReader) ReadAt

func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error)

func (*LazyChunkReader) Seek

func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error)

func (*LazyChunkReader) Size

func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, err error)

type LazySectionReader

type LazySectionReader interface {
	Context() context.Context
	Size(context.Context, chan bool) (int64, error)
	io.Seeker
	io.Reader
	io.ReaderAt
}

type LazyTestSectionReader

type LazyTestSectionReader struct {
	*io.SectionReader
}

func (*LazyTestSectionReader) Context

func (r *LazyTestSectionReader) Context() context.Context

func (*LazyTestSectionReader) Size

type LocalStore

type LocalStore struct {
	Validators []ChunkValidator

	DbStore *LDBStore
	// contains filtered or unexported fields
}

func NewLocalStore

func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error)

func NewTestLocalStoreForAddr

func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error)

func (*LocalStore) Close

func (ls *LocalStore) Close()

func (*LocalStore) Get

func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*LocalStore) GetOrCreateRequest

func (ls *LocalStore) GetOrCreateRequest(ctx context.Context, addr Address) (chunk *Chunk, created bool)

func (*LocalStore) Put

func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk)

func (*LocalStore) RequestsCacheLen

func (ls *LocalStore) RequestsCacheLen() int

type LocalStoreParams

type LocalStoreParams struct {
	*StoreParams
	ChunkDbPath string
	Validators  []ChunkValidator `toml:"-"`
}

func NewDefaultLocalStoreParams

func NewDefaultLocalStoreParams() *LocalStoreParams

func (*LocalStoreParams) Init

func (p *LocalStoreParams) Init(path string)

type MapChunkStore

type MapChunkStore struct {
	// contains filtered or unexported fields
}

func NewMapChunkStore

func NewMapChunkStore() *MapChunkStore

func (*MapChunkStore) Close

func (m *MapChunkStore) Close()

func (*MapChunkStore) Get

func (m *MapChunkStore) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*MapChunkStore) Put

func (m *MapChunkStore) Put(ctx context.Context, chunk *Chunk)

type MemStore

type MemStore struct {
	// contains filtered or unexported fields
}

func NewMemStore

func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore)

func (*MemStore) Close

func (s *MemStore) Close()

func (*MemStore) Get

func (m *MemStore) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*MemStore) Put

func (m *MemStore) Put(ctx context.Context, c *Chunk)

type NetStore

type NetStore struct {
	// contains filtered or unexported fields
}

func NewNetStore

func NewNetStore(localStore *LocalStore, retrieve func(ctx context.Context, chunk *Chunk) error) *NetStore

func (*NetStore) Close

func (ns *NetStore) Close()

func (*NetStore) Get

func (ns *NetStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*NetStore) GetWithTimeout

func (ns *NetStore) GetWithTimeout(ctx context.Context, addr Address, timeout time.Duration) (chunk *Chunk, err error)

func (*NetStore) Put

func (ns *NetStore) Put(ctx context.Context, chunk *Chunk)

type Peer

type Peer interface{}

type Putter

type Putter interface {
	Put(context.Context, ChunkData) (Reference, error)
	//
	RefSize() int64
	//
	Close()
	//
	Wait(context.Context) error
}

type PyramidChunker

type PyramidChunker struct {
	// contains filtered or unexported fields
}

func NewPyramidSplitter

func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker)

func (*PyramidChunker) Append

func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error)

func (*PyramidChunker) Join

func (pc *PyramidChunker) Join(addr Address, getter Getter, depth int) LazySectionReader

func (*PyramidChunker) Split

func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error)

type PyramidSplitterParams

type PyramidSplitterParams struct {
	SplitterParams
	// contains filtered or unexported fields
}

func NewPyramidSplitterParams

func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, getter Getter, chunkSize int64) *PyramidSplitterParams

type Reference

type Reference []byte

type SplitterParams

type SplitterParams struct {
	ChunkerParams
	// contains filtered or unexported fields
}

type StoreParams

type StoreParams struct {
	Hash                       SwarmHasher `toml:"-"`
	DbCapacity                 uint64
	CacheCapacity              uint
	ChunkRequestsCacheCapacity uint
	BaseKey                    []byte
}

func NewDefaultStoreParams

func NewDefaultStoreParams() *StoreParams

func NewStoreParams

func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams

type SwarmHash

type SwarmHash interface {
	hash.Hash
	ResetWithLength([]byte)
}

type SwarmHasher

type SwarmHasher func() SwarmHash

func MakeHashFunc

func MakeHashFunc(hash string) SwarmHasher

type TreeChunker

type TreeChunker struct {
	// contains filtered or unexported fields
}

func NewTreeJoiner

func NewTreeJoiner(params *JoinerParams) *TreeChunker

func NewTreeSplitter

func NewTreeSplitter(params *TreeSplitterParams) *TreeChunker

func (*TreeChunker) Append

func (tc *TreeChunker) Append() (Address, func(), error)

func (*TreeChunker) Join

func (tc *TreeChunker) Join(ctx context.Context) *LazyChunkReader

func (*TreeChunker) Split

func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error)

type TreeEntry

type TreeEntry struct {
	// contains filtered or unexported fields
}

func NewTreeEntry

func NewTreeEntry(pyramid *PyramidChunker) *TreeEntry

type TreeSplitterParams

type TreeSplitterParams struct {
	SplitterParams
	// contains filtered or unexported fields
}

Directories

Path Synopsis
db
mem
rpc

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL