splitstore

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 15, 2021 License: Apache-2.0, MIT Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const (
	BloomFilterMinSize     = 10_000_000
	BloomFilterProbability = 0.01
)

Variables

View Source
var (
	// CompactionThreshold is the number of epochs that need to have elapsed
	// from the previously compacted epoch to trigger a new compaction.
	//
	//        |················· CompactionThreshold ··················|
	//        |                                                        |
	// =======‖≡≡≡≡≡≡≡‖-----------------------|------------------------»
	//        |       |                       |   chain -->             ↑__ current epoch
	//        |·······|                       |
	//            ↑________ CompactionCold    ↑________ CompactionBoundary
	//
	// === :: cold (already archived)
	// ≡≡≡ :: to be archived in this compaction
	// --- :: hot
	CompactionThreshold = 5 * build.Finality

	// CompactionCold is the number of epochs that will be archived to the
	// cold store on compaction. See diagram on CompactionThreshold for a
	// better sense.
	CompactionCold = build.Finality

	// CompactionBoundary is the number of epochs from the current epoch at which
	// we will walk the chain for live objects
	CompactionBoundary = 2 * build.Finality
)

Functions

This section is empty.

Types

type BloomMarkSet

type BloomMarkSet struct {
	// contains filtered or unexported fields
}

func (*BloomMarkSet) Close

func (s *BloomMarkSet) Close() error

func (*BloomMarkSet) Has

func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error)

func (*BloomMarkSet) Mark

func (s *BloomMarkSet) Mark(cid cid.Cid) error

type BloomMarkSetEnv

type BloomMarkSetEnv struct{}

func NewBloomMarkSetEnv

func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error)

func (*BloomMarkSetEnv) Close

func (e *BloomMarkSetEnv) Close() error

func (*BloomMarkSetEnv) Create

func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error)

type BoltMarkSet

type BoltMarkSet struct {
	// contains filtered or unexported fields
}

func (*BoltMarkSet) Close

func (s *BoltMarkSet) Close() error

func (*BoltMarkSet) Has

func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error)

func (*BoltMarkSet) Mark

func (s *BoltMarkSet) Mark(cid cid.Cid) error

type BoltMarkSetEnv

type BoltMarkSetEnv struct {
	// contains filtered or unexported fields
}

func NewBoltMarkSetEnv

func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error)

func (*BoltMarkSetEnv) Close

func (e *BoltMarkSetEnv) Close() error

func (*BoltMarkSetEnv) Create

func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error)

type BoltTrackingStore

type BoltTrackingStore struct {
	// contains filtered or unexported fields
}

func OpenBoltTrackingStore

func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error)

func (*BoltTrackingStore) Close

func (s *BoltTrackingStore) Close() error

func (*BoltTrackingStore) Delete

func (s *BoltTrackingStore) Delete(cid cid.Cid) error

func (*BoltTrackingStore) DeleteBatch

func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error

func (*BoltTrackingStore) ForEach

func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error

func (*BoltTrackingStore) Get

func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error)

func (*BoltTrackingStore) Put

func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error

func (*BoltTrackingStore) PutBatch

func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error

func (*BoltTrackingStore) Sync

func (s *BoltTrackingStore) Sync() error

type ChainAccessor

type ChainAccessor interface {
	GetTipsetByHeight(context.Context, abi.ChainEpoch, *types.TipSet, bool) (*types.TipSet, error)
	GetHeaviestTipSet() *types.TipSet
	SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
	WalkSnapshot(context.Context, *types.TipSet, abi.ChainEpoch, bool, bool, func(cid.Cid) error) error
}

ChainAccessor allows the Splitstore to access the chain. It will most likely be a ChainStore at runtime.

type Config

type Config struct {
	// TrackingStore is the type of tracking store to use.
	//
	// Supported values are: "bolt" (default if omitted), "mem" (for tests and readonly access).
	TrackingStoreType string

	// MarkSetType is the type of mark set to use.
	//
	// Supported values are: "bloom" (default if omitted), "bolt".
	MarkSetType string
	// DisableCompaction is the switch of splitstore compaction,
	// You should enable this option if you plan to disable move data from hot to cold.
	DisableCompaction bool
	// perform full reachability analysis (expensive) for compaction
	// You should enable this option if you plan to use the splitstore without a backing coldstore
	EnableFullCompaction bool
	// EXPERIMENTAL enable pruning of unreachable objects.
	// This has not been sufficiently tested yet; only enable if you know what you are doing.
	// Only applies if you enable full compaction.
	EnableGC bool
	// full archival nodes should enable this if EnableFullCompaction is enabled
	// do NOT enable this if you synced from a snapshot.
	// Only applies if you enabled full compaction
	Archival bool

	// drop the cold store, the node only save the hot data.
	EnableColdDrop bool
}

type MarkSet

type MarkSet interface {
	Mark(cid.Cid) error
	Has(cid.Cid) (bool, error)
	Close() error
}

MarkSet is a utility to keep track of seen CID, and later query for them.

* If the expected dataset is large, it can be backed by a datastore (e.g. bbolt). * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).

type MarkSetEnv

type MarkSetEnv interface {
	Create(name string, sizeHint int64) (MarkSet, error)
	Close() error
}

func OpenMarkSetEnv

func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error)

type MemTrackingStore

type MemTrackingStore struct {
	sync.Mutex
	// contains filtered or unexported fields
}

MemTrackingStore is a simple in-memory tracking store

func NewMemTrackingStore

func NewMemTrackingStore() *MemTrackingStore

NewMemTrackingStore creates an in-memory tracking store. This is only useful for test or situations where you don't want to open the real tracking store (eg concurrent read only access on a node's datastore)

func (*MemTrackingStore) Close

func (s *MemTrackingStore) Close() error

func (*MemTrackingStore) Delete

func (s *MemTrackingStore) Delete(cid cid.Cid) error

func (*MemTrackingStore) DeleteBatch

func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error

func (*MemTrackingStore) ForEach

func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error

func (*MemTrackingStore) Get

func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error)

func (*MemTrackingStore) Put

func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error

func (*MemTrackingStore) PutBatch

func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error

func (*MemTrackingStore) Sync

func (s *MemTrackingStore) Sync() error

type SplitStore

type SplitStore struct {
	// contains filtered or unexported fields
}

func Open

func Open(path string, us bstore.Blockstore, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Config) (*SplitStore, error)

Open opens an existing splistore, or creates a new splitstore. The splitstore is backed by the provided hot and cold stores. The returned SplitStore MUST be attached to the ChainStore with Start in order to trigger compaction.

func (*SplitStore) AllKeysChan

func (s *SplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*SplitStore) Close

func (s *SplitStore) Close() error

func (*SplitStore) CollectGarbage

func (s *SplitStore) CollectGarbage() error

func (*SplitStore) DeleteBlock

func (s *SplitStore) DeleteBlock(_ cid.Cid) error

Blockstore interface

func (*SplitStore) DeleteMany

func (s *SplitStore) DeleteMany(_ []cid.Cid) error

func (*SplitStore) Get

func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error)

func (*SplitStore) GetSize

func (s *SplitStore) GetSize(cid cid.Cid) (int, error)

func (*SplitStore) Has

func (s *SplitStore) Has(cid cid.Cid) (bool, error)

func (*SplitStore) HashOnRead

func (s *SplitStore) HashOnRead(enabled bool)

func (*SplitStore) HeadChange

func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error

func (*SplitStore) Put

func (s *SplitStore) Put(blk blocks.Block) error

func (*SplitStore) PutMany

func (s *SplitStore) PutMany(blks []blocks.Block) error

func (*SplitStore) Start

func (s *SplitStore) Start(chain ChainAccessor) error

State tracking

func (*SplitStore) View

func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error

type TrackingStore

type TrackingStore interface {
	Put(cid.Cid, abi.ChainEpoch) error
	PutBatch([]cid.Cid, abi.ChainEpoch) error
	Get(cid.Cid) (abi.ChainEpoch, error)
	Delete(cid.Cid) error
	DeleteBatch([]cid.Cid) error
	ForEach(func(cid.Cid, abi.ChainEpoch) error) error
	Sync() error
	Close() error
}

TrackingStore is a persistent store that tracks blocks that are added to the hotstore, tracking the epoch at which they are written.

func OpenTrackingStore

func OpenTrackingStore(path string, ttype string) (TrackingStore, error)

OpenTrackingStore opens a tracking store of the specified type in the specified path.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL