Documentation
¶
Overview ¶
Package indexblob manages sets of active index blobs.
Index ¶
- Constants
- type CompactOptions
- type EncryptionManager
- type IndexFormattingOptions
- type Manager
- type ManagerV0
- func (m *ManagerV0) Compact(ctx context.Context, opt CompactOptions) error
- func (m *ManagerV0) Invalidate()
- func (m *ManagerV0) ListActiveIndexBlobs(ctx context.Context) ([]Metadata, time.Time, error)
- func (m *ManagerV0) ListIndexBlobInfos(ctx context.Context) ([]Metadata, time.Time, error)
- func (m *ManagerV0) WriteIndexBlobs(ctx context.Context, dataShards []gather.Bytes, suffix blob.ID) ([]blob.Metadata, error)
- type ManagerV1
- func (m *ManagerV1) Compact(ctx context.Context, opt CompactOptions) error
- func (m *ManagerV1) CompactEpoch(ctx context.Context, blobIDs []blob.ID, outputPrefix blob.ID) error
- func (m *ManagerV1) EpochManager() *epoch.Manager
- func (m *ManagerV1) Invalidate()
- func (m *ManagerV1) ListActiveIndexBlobs(ctx context.Context) ([]Metadata, time.Time, error)
- func (m *ManagerV1) ListIndexBlobInfos(ctx context.Context) ([]Metadata, time.Time, error)
- func (m *ManagerV1) PrepareUpgradeToIndexBlobManagerV1(ctx context.Context, v0 *ManagerV0) error
- func (m *ManagerV1) WriteIndexBlobs(ctx context.Context, dataShards []gather.Bytes, suffix blob.ID) ([]blob.Metadata, error)
- type Metadata
Constants ¶
const DefaultIndexShardSize = 16e6
DefaultIndexShardSize is the maximum number of items in an index shard. It is less than 2^24, which lets V1 index use 24-bit/3-byte indexes.
const V0CleanupBlobPrefix = "l"
V0CleanupBlobPrefix is the prefix for all legacy (v0) index cleanup blobs.
const V0CompactionLogBlobPrefix = "m"
V0CompactionLogBlobPrefix is the prefix for all legacy (v0) index compactions blobs.
const V0IndexBlobPrefix = "n"
V0IndexBlobPrefix is the prefix for all legacy (v0) index blobs.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CompactOptions ¶
type CompactOptions struct {
MaxSmallBlobs int
AllIndexes bool
DropDeletedBefore time.Time
DropContents []index.ID
DisableEventualConsistencySafety bool
}
CompactOptions provides options for compaction.
type EncryptionManager ¶
type EncryptionManager struct {
// contains filtered or unexported fields
}
EncryptionManager manages encryption and caching of index blobs.
func NewEncryptionManager ¶
func NewEncryptionManager( st blob.Storage, crypter blobcrypto.Crypter, indexBlobCache *cache.PersistentCache, log logging.Logger, ) *EncryptionManager
NewEncryptionManager creates new encryption manager.
func (*EncryptionManager) EncryptAndWriteBlob ¶
func (m *EncryptionManager) EncryptAndWriteBlob(ctx context.Context, data gather.Bytes, prefix, suffix blob.ID) (blob.Metadata, error)
EncryptAndWriteBlob encrypts and writes the provided data into a blob, with name {prefix}{hash}[-{suffix}].
func (*EncryptionManager) GetEncryptedBlob ¶
func (m *EncryptionManager) GetEncryptedBlob(ctx context.Context, blobID blob.ID, output *gather.WriteBuffer) error
GetEncryptedBlob fetches and decrypts the contents of a given encrypted blob using cache first and falling back to the underlying storage.
type IndexFormattingOptions ¶
type IndexFormattingOptions interface {
GetMutableParameters(ctx context.Context) (format.MutableParameters, error)
}
IndexFormattingOptions provides options for formatting index blobs.
type Manager ¶
type Manager interface {
WriteIndexBlobs(ctx context.Context, data []gather.Bytes, suffix blob.ID) ([]blob.Metadata, error)
ListActiveIndexBlobs(ctx context.Context) ([]Metadata, time.Time, error)
Compact(ctx context.Context, opts CompactOptions) error
Invalidate()
}
Manager is the API of index blob manager as used by content manager.
type ManagerV0 ¶
type ManagerV0 struct {
// contains filtered or unexported fields
}
ManagerV0 is a V0 (legacy) implementation of index blob manager.
func NewManagerV0 ¶
func NewManagerV0( st blob.Storage, enc *EncryptionManager, timeNow func() time.Time, formattingOptions IndexFormattingOptions, log logging.Logger, ) *ManagerV0
NewManagerV0 creates new instance of ManagerV0 with all required parameters set.
func (*ManagerV0) Compact ¶
func (m *ManagerV0) Compact(ctx context.Context, opt CompactOptions) error
Compact performs compaction of index blobs by merging smaller ones into larger and registering compaction and cleanup blobs in the repository.
func (*ManagerV0) ListActiveIndexBlobs ¶
ListActiveIndexBlobs lists the metadata for active index blobs and returns the cut-off time before which all deleted index entries should be treated as non-existent.
func (*ManagerV0) ListIndexBlobInfos ¶
ListIndexBlobInfos list active blob info structs. Also returns time of latest content deletion commit.
type ManagerV1 ¶
type ManagerV1 struct {
// contains filtered or unexported fields
}
ManagerV1 is the append-only implementation of indexblob.Manager based on epoch manager.
func NewManagerV1 ¶
func NewManagerV1( st blob.Storage, enc *EncryptionManager, epochMgr *epoch.Manager, timeNow func() time.Time, formattingOptions IndexFormattingOptions, log logging.Logger, ) *ManagerV1
NewManagerV1 creates new instance of ManagerV1 with all required parameters set.
func (*ManagerV1) Compact ¶
func (m *ManagerV1) Compact(ctx context.Context, opt CompactOptions) error
Compact advances the deletion watermark.
func (*ManagerV1) CompactEpoch ¶
func (m *ManagerV1) CompactEpoch(ctx context.Context, blobIDs []blob.ID, outputPrefix blob.ID) error
CompactEpoch compacts the provided index blobs and writes a new set of blobs.
func (*ManagerV1) EpochManager ¶
EpochManager returns the epoch manager.
func (*ManagerV1) ListActiveIndexBlobs ¶
ListActiveIndexBlobs lists the metadata for active index blobs and returns the cut-off time before which all deleted index entries should be treated as non-existent.
func (*ManagerV1) ListIndexBlobInfos ¶
ListIndexBlobInfos list active blob info structs. Also returns time of latest content deletion commit.
func (*ManagerV1) PrepareUpgradeToIndexBlobManagerV1 ¶
PrepareUpgradeToIndexBlobManagerV1 prepares the repository for migrating to IndexBlobManagerV1.
func (*ManagerV1) WriteIndexBlobs ¶
func (m *ManagerV1) WriteIndexBlobs(ctx context.Context, dataShards []gather.Bytes, suffix blob.ID) ([]blob.Metadata, error)
WriteIndexBlobs writes dataShards into new index blobs with an optional blob name suffix. The writes are atomic in the sense that if any of them fails, the reader will ignore all of the indexes that share the same suffix.
Source Files