ec

package

v0.0.0-...-303e327 Latest Latest Go to latest Published: May 22, 2023 License: Apache-2.0 Imports: 11 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/cubefs/cubefs

Documentation ¶

Overview ¶

Note:

Do not use after releasing it.
You need to ensure its safety yourself before releasing it.
You should know that its pointer would have changed after Resize.
Manually manage the DataBuf in Ranged mode, do not Split it.

MinShardSize min size per shard, fill data into shards 0-N continuously, align with zero bytes if data size less than MinShardSize*N

Length of real data less than MinShardSize*N, ShardSize = MinShardSize.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | data | align bytes | partiy | local |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Length more than MinShardSize*N, ShardSize = ceil(len(data)/N)

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | data |padding| partiy | local |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | 0 | 1 | .... | N | N+1 | ... | N+M | N+M+L |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Example:

import (

"github.com/cubefs/cubefs/blobstore/common/codemode"
"github.com/cubefs/cubefs/blobstore/common/resourcepool"

)

func xxx() {
    pool := resourcepool.NewMemPool(nil)
    codeModeInfo := codemode.GetTactic(codemode.EC15p12)
    buffer, err := NewBuffer(1024, codeModeInfo, pool)
    if err != nil {
        // ...
    }
    // release
    defer buffer.Release()

    // release it in an anonymous func if you will resize
    defer func() {
        buffer.Release()
    }()

    io.Copy(buffer.DataBuf, io.Reader)
    shards := ec.Split(buffer.ECDataBuf)
    // ...

    buffer.Resize(10240)
    // ...
}

Index ¶

Variables
type Buffer
- func NewBuffer(dataSize int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error)
- func NewRangeBuffer(dataSize, from, to int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error)
- func (b *Buffer) Release() error
- func (b *Buffer) Resize(dataSize int) error
type BufferSizes
- func GetBufferSizes(dataSize int, tactic codemode.Tactic) (BufferSizes, error)
type Config
type Encoder
- func NewEncoder(cfg Config) (Encoder, error)

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	ErrShortData       = errors.New("short data")
	ErrInvalidCodeMode = errors.New("invalid code mode")
	ErrVerify          = errors.New("shards verify failed")
	ErrInvalidShards   = errors.New("invalid shards")
)

errors

Functions ¶

This section is empty.

Types ¶

type Buffer ¶

type Buffer struct {

	// DataBuf real data buffer
	DataBuf []byte
	// ECDataBuf ec data buffer to Split,
	// is nil if Buffer is Ranged mode
	ECDataBuf []byte
	// BufferSizes all sizes
	BufferSizes
	// contains filtered or unexported fields
}

Buffer one ec blob's reused buffer Manually manage the DataBuf in Ranged mode, do not Split it

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | data | align bytes | partiy | local |
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | DataBuf | |<--DataSize->|
- - - - - - - - - - - - - - - - - | ECDataBuf | |<-- ECDataSize -->|
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | ECBuf | |<--- ECSize --->|

func NewBuffer ¶

func NewBuffer(dataSize int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error)

NewBuffer new ec buffer with data size and ec mode

func NewRangeBuffer ¶

func NewRangeBuffer(dataSize, from, to int, tactic codemode.Tactic, pool *resourcepool.MemPool) (*Buffer, error)

NewRangeBuffer ranged buffer with least data size

func (*Buffer) Release ¶

func (b *Buffer) Release() error

Release recycles the buffer into pool

func (*Buffer) Resize ¶

func (b *Buffer) Resize(dataSize int) error

Resize re-calculate ec buffer, alloc a new buffer if oversize

type BufferSizes ¶

type BufferSizes struct {
	ShardSize  int // per shard size
	DataSize   int // real data size
	ECDataSize int // ec data size only with data
	ECSize     int // ec buffer size with partiy and local shards
	From       int // ranged from
	To         int // ranged to
}

BufferSizes all sizes ECSize is sum of all ec shards size, not equal the capacity of DataBuf in Ranged mode

func GetBufferSizes ¶

func GetBufferSizes(dataSize int, tactic codemode.Tactic) (BufferSizes, error)

GetBufferSizes calculate ec buffer sizes

type Config ¶

type Config struct {
	CodeMode     codemode.Tactic
	EnableVerify bool
	Concurrency  int
}

Config ec encoder config

type Encoder ¶

type Encoder interface {
	// encode source data into shards, whatever normal ec or LRC
	Encode(shards [][]byte) error
	// reconstruct all missing shards, you should assign the missing or bad idx in shards
	Reconstruct(shards [][]byte, badIdx []int) error
	// only reconstruct data shards, you should assign the missing or bad idx in shards
	ReconstructData(shards [][]byte, badIdx []int) error
	// split source data into adapted shards size
	Split(data []byte) ([][]byte, error)
	// get data shards(No-Copy)
	GetDataShards(shards [][]byte) [][]byte
	// get parity shards(No-Copy)
	GetParityShards(shards [][]byte) [][]byte
	// get local shards(LRC model, No-Copy)
	GetLocalShards(shards [][]byte) [][]byte
	// get shards in an idc
	GetShardsInIdc(shards [][]byte, idx int) [][]byte
	// output source data into dst(io.Writer)
	Join(dst io.Writer, shards [][]byte, outSize int) error
	// verify parity shards with data shards
	Verify(shards [][]byte) (bool, error)
}

Encoder normal ec encoder, implements all these functions

func NewEncoder ¶

func NewEncoder(cfg Config) (Encoder, error)

NewEncoder return an encoder which support normal EC or LRC

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL