metaservice

package
v0.0.0-...-0d681e9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 3, 2024 License: Apache-2.0, MIT Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	METAS_PATH          = "metas"
	MAPPINGS_PATH       = "mappings"
	MAPPING_FILE_SUFFIX = ".json"
)
View Source 
const (
	SOURCE_CHUNK_SIZE = 127
	SLAB_CHUNK_SIZE   = 128
	NODE_SIZE         = 32
	CHUNK_NODES_NUM   = 4

	CAR_32GIB_SIZE      = uint64(1 << 35)
	CAR_2MIB_CHUNK_SIZE = uint64(SOURCE_CHUNK_SIZE * CAR_2MIB_NODE_NUM / CHUNK_NODES_NUM) // source data node = 127, no padding
	CAR_512B_CHUNK_SIZE = uint64(SOURCE_CHUNK_SIZE * CAR_512B_NODE_NUM / CHUNK_NODES_NUM) // source data node = 127, no padding

	CAR_2MIB_NODE_NUM = uint64(1 << 20 * 2 / NODE_SIZE) // = 2MIB / NODE_SIZE
	CAR_512B_NODE_NUM = uint64(1 << 9 / NODE_SIZE)      // = 512B / NODE_SIZE

	DATASET_RULE_CHALLENGE_POINTS_PER_AUDITOR = 5 // 5 point per auditor

	CAR_2MIB_CACHE_LAYER_START  = 16
	CAR_512B_CACHE_LAYER_START  = 4
	CACHE_SUFFIX                = ".cache"
	CACHE_DATASET_PROOF_PATH    = "dataset.proof"
	CACHE_CHALLENGE_PROOFS_PATH = "challenges.proofs"
	PROOFS_PATH                 = "proofs"

	// MaxLayers is the current maximum height of the rust-fil-proofs proving tree.
	MaxLayers = uint(31) // result of log2( 64 GiB / 32 )
	// MaxPieceSize is the current maximum size of the rust-fil-proofs proving tree.
	MaxPieceSize = uint64(1 << (MaxLayers + 5))
)

Variables

View Source 
var (
	StackedNulPadding [MaxLayers][]byte
	SumChunkCount     uint64
	CommpHashConfig   = &mt.Config{
		HashFunc:           NewHashFunc,
		DisableLeafHashing: true,
		Mode:               mt.ModeTreeBuild,
		RunInParallel:      true,
	}

	Once sync.Once
)

Functions

func AppendProof 

func AppendProof(base *mt.Proof, sub mt.Proof) (*mt.Proof, error)

Append base and sub proof

func CarCacheLayerStart 

func CarCacheLayerStart(carSize uint64) int

CarCacheLayerStart returns the start index of the cache layer for a given CAR size. It takes the size of the CAR as input. It returns the start index of the cache layer.

func CarChallengeCount 

func CarChallengeCount(carNum uint64) uint64

Car challenge count

func CarChunkParams 

func CarChunkParams(carSize uint64) (uint64, uint64)

CarChunkParams returns the chunk size and node number for a given CAR size. It takes the size of the CAR as input. It returns the chunk size and the number of nodes.

func DataPadding 

func DataPadding(inSlab []byte) []byte

SHA256 DataPadding function for commp

func GenCarChallenge 

func GenCarChallenge(randomness uint64, carChallengeIndex uint64, carChallengesCount uint64, carNum uint64) uint64

GenCarChallenge generates a car challenge index using randomness, the car challenge index, and the total number of car challenges. It returns the generated car challenge index.

func GenCommP 

func GenCommP(buf bytes.Buffer, cachePath string, targetPaddedSize uint64) ([]byte, uint64, error)

GenCommP is the commP generate. targetPaddedSize = 0 is use default padded size

func GenDatasetProof 

func GenDatasetProof(cachePath string) ([]byte, error)

Generate commPs Merkle-Tree root to .tcache, proofs{rootHash, leafHashes[]} cachePath: store to file path

func GenLeafChallenge 

func GenLeafChallenge(randomness uint64, carIndex uint64, leafChallengeCount uint64, carSize uint64) ([]uint64, error)

GenLeafChallenge generates a leaf challenge index using randomness, the car index, the leaf challenge index, and the size of the car. It returns the generated leaf challenge index.

func GenProof 

func GenProof(blocks []mt.DataBlock, leaf mt.DataBlock) (*mt.Proof, []byte, error)

GenProof generates a Merkle tree proof for the specified leaf block. It takes a slice of data blocks and the leaf block as input. It returns the proof, the root hash of the Merkle tree, and any error encountered.

func GenProofFromCache 

func GenProofFromCache(leaf mt.DataBlock, file string) (*mt.Proof, []byte, error)

GenProofFromCache generates a Merkle tree proof for the specified leaf block using a level cache. It takes the leaf block and the cache file path as input. It returns the proof, the root hash of the Merkle tree, and any error encountered.

func GetChallengeChunk 

func GetChallengeChunk(commCid cid.Cid, offset uint64, size uint64) ([]byte, error)

Use this function to generate data fragments of the CAR file at challenge points when creating challenge proofs.

func LeafChallengeCount 

func LeafChallengeCount(carChallengesCount uint64) []uint64

Car leaf challenge count.

func LoadSortCommp 

func LoadSortCommp(cachePath string) ([][]byte, []uint64)

LoadSortCommp loads and sorts the CommP values from the cache file. It takes the cache file path as input. It returns a slice of CommP values and a slice of their corresponding indices.

func NewDataBlockFromBytes 

func NewDataBlockFromBytes(data []byte) mt.DataBlock

No padding DataBlock

func NewDataBlocksFromBytes 

func NewDataBlocksFromBytes(bt [][]byte) []mt.DataBlock

No padding DataBlock

func NewHashFunc 

func NewHashFunc(data []byte) ([]byte, error)

SHA256 hash generate function for commp

func NewPaddedDataBlocksFromBuffer 

func NewPaddedDataBlocksFromBuffer(buf bytes.Buffer, targetPaddedSize uint64) ([]mt.DataBlock, uint64, error)

Padding DataBlock, commp leaf node use targetPaddedSize = 0 use default paddedSize

func NewPaddedDataBlocksFromDataBlocks 

func NewPaddedDataBlocksFromDataBlocks(dataBlocks []mt.DataBlock, sourcePaddedSize, targetPaddedSize uint64) ([]mt.DataBlock, error)

Padding DataBlock, commp leaf node use targetPaddedSize = 0 use default paddedSize

func PadCommP 

func PadCommP(sourceCommP []byte, sourcePaddedSize, targetPaddedSize uint64) ([]byte, error)

PadCommP is experimental, do not use it.

func SaveCommP 

func SaveCommP(rawCommP []byte, carSize uint64, cachePath string) error

SaveCommP append. struct format: map{commmp:carSize}

func VerifyChallengeProof 

func VerifyChallengeProof(cachePath string) (bool, error)

Verify challenge nodes Proof

func VerifyDatasetProof 

func VerifyDatasetProof(cachePath string, randomness uint64) (bool, *mt.Proof, error)

Verify commPs Merkle-Tree proof

Types

type CarChallenge 

type CarChallenge struct {
	CarIndex uint64
	Leaves   []uint64
}

CarChallenge struct represents the challenge information of a car, including the car index and corresponding challenges.

func GenChallenges 

func GenChallenges(randomness uint64, carNum uint64, carSize []uint64) ([]CarChallenge, error)

GenChallenges function generates challenge information for cars and returns a sequentially traversable structure.

type ChallengeProofs 

type ChallengeProofs struct {
	RandomSeed uint64
	Leaves     []string
	Siblings   [][]string
	Paths      []string
}

ChallengeProofs represents the challenge proofs data structure.

func NewChallengeProofs 

func NewChallengeProofs(randomness uint64, proofs Proofs) *ChallengeProofs

NewChallengeProofs creates a new ChallengeProofs instance from the provided randomness and proof map.

func NewChallengeProofsFromFile 

func NewChallengeProofsFromFile(filePath string) (*ChallengeProofs, error)

NewChallengeProofsFromFile creates a new ChallengeProofs instance by reading from the provided file path.

type DataBlock 

type DataBlock struct {
	Data []byte
}

DataBlock is a implementation of the DataBlock interface.

func (*DataBlock) Serialize 

func (t *DataBlock) Serialize() ([]byte, error)

Serialize returns the serialized form of the DataBlock.

type DatasetMerkletree 

type DatasetMerkletree struct {
	Root   []byte
	Leaves [][]byte
}

DatasetMerkletree represents the structure of a dataset Merkle tree.

type DatasetProof 

type DatasetProof struct {
	Root       string
	LeafHashes []string
	LeafSizes  []uint64
}

DatasetProof represents the data structure of dataset proofs.

func NewDatasetProof 

func NewDatasetProof(proof DatasetMerkletree, leafSizes []uint64) *DatasetProof

NewDatasetProof creates a new DatasetProof instance based on the provided DatasetMerkletree.

func NewDatasetProofFromFile 

func NewDatasetProofFromFile(filePath string) (*DatasetProof, error)

NewDatasetProofFromFile creates a new DatasetProof instance from the provided file path.

type MappingService 

type MappingService struct {
	// contains filtered or unexported fields
}

MappingService generates the mapping relationship from the source file to the car file.

func MappingServiceInstance 

func MappingServiceInstance(opts ...Option) *MappingService

func New 

func New(opts ...Option) *MappingService

func (*MappingService) GenerateCarWriter 

func (ms *MappingService) GenerateCarWriter(w io.Writer, path string, call bool) io.Writer

Wrapping the io.Writer interface to implement mapping recording for writing CAR files.

func (*MappingService) GenerateChunksFromMappings 

func (ms *MappingService) GenerateChunksFromMappings(path string, srcParent string, mappings []*types.ChunkMapping) error

Generating a data fragment file of a CAR file based on contiguous mapping information and data obtained from the source file. For example, generating a 2MB data fragment for which a consistency proof needs to be challenged.

func (*MappingService) GenerateDagService 

func (ms *MappingService) GenerateDagService(ds ipld.DAGService) ipld.DAGService

Creating an encapsulation interface for ipld.DAGService to implement recording of mapping files during the process of writing nodes

func (*MappingService) GenerateHelper 

func (ms *MappingService) GenerateHelper(params *helpers.DagBuilderParams, spl libs.EnhancedSplitter) (helpers.Helper, error)

Implementing the Helper interface of go-unixfs.

func (*MappingService) GenerateNodeFromSource 

func (ms *MappingService) GenerateNodeFromSource(path string, srcParent string, m *types.ChunkMapping, cidBuilder cid.Builder) (ipld.Node, error)

Node construction involving the source data.

func (*MappingService) GenerateNodeWithoutData 

func (ms *MappingService) GenerateNodeWithoutData(m *types.ChunkMapping, cidBuilder cid.Builder) (ipld.Node, error)

Node construction without involving the source data.

func (*MappingService) GetAllChunkMappings 

func (ms *MappingService) GetAllChunkMappings() ([]*types.ChunkMapping, error)

Getting all mapping information from the MappingService cache.

func (*MappingService) GetChunkMappings 

func (ms *MappingService) GetChunkMappings(dstOffset uint64, dstSize uint64) ([]*types.ChunkMapping, error)

Getting mapping information for data fragments associated with a given offset and size of car.

func (*MappingService) LoadMetaMappings 

func (ms *MappingService) LoadMetaMappings(path string) error

Loading mapping information from a file into the MappingService cache.

func (*MappingService) MetaPath 

func (ms *MappingService) MetaPath() string

Access path for metadata.

func (*MappingService) SaveMetaMappings 

func (ms *MappingService) SaveMetaMappings(path string, name string) error

Saving the cached mapping information to a file.

func (*MappingService) SetCarDataRoot 

func (ms *MappingService) SetCarDataRoot(root cid.Cid)

Set the DAG data root for the current CAR file.

func (*MappingService) SourceParentPath 

func (ms *MappingService) SourceParentPath() string

Access path for source data.

type Option 

type Option func(o *Options)

func MetaPath 

func MetaPath(path string) Option

func RawLeaves 

func RawLeaves(rawLeaves bool) Option

func SourceParentPath 

func SourceParentPath(path string) Option

type Options 

type Options struct {
	// contains filtered or unexported fields
}

type Proofs 

type Proofs struct {
	Leaves []string
	Proofs []mt.Proof
}

Proofs represents the challenge proofs data structure.

func GenChallengeProof 

func GenChallengeProof(randomness uint64, cachePath string) (*Proofs, error)

Generate challenge nodes Proofs

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.