Documentation ¶
Index ¶
- Constants
- Variables
- func ApplyUpdates(updates ...writeaheadlog.Update) error
- func CombinedChunkIndex(numChunks, chunkIndex uint64, numCombinedChunks int) int
- func ExtractSegment(pieces [][]byte, segmentIndex int, segmentSize uint64) [][]byte
- func IsI3vFileUpdate(update writeaheadlog.Update) bool
- func LoadI3vFileFromReaderWithChunks(r io.ReadSeeker, path string, wal *writeaheadlog.WAL) (*I3vFile, Chunks, error)
- func NewRSCode(nData, nParity int) (modules.ErasureCoder, error)
- func NewRSSubCode(nData, nParity int, segmentSize uint64) (modules.ErasureCoder, error)
- type BubbledMetadata
- type CachedHealthMetadata
- type Chunk
- type Chunks
- type FileChunk
- type FileData
- type HostPublicKey
- type I3vFile
- func LoadI3vFile(path string, wal *writeaheadlog.WAL) (*I3vFile, error)
- func LoadI3vFileFromReader(r io.ReadSeeker, path string, wal *writeaheadlog.WAL) (*I3vFile, error)
- func New(i3vFilePath, source string, wal *writeaheadlog.WAL, ...) (*I3vFile, error)
- func NewFromLegacyData(fd FileData, i3vFilePath string, wal *writeaheadlog.WAL) (*I3vFile, error)
- func (sf *I3vFile) AccessTime() time.Time
- func (sf *I3vFile) AddPiece(pk types.I3vPublicKey, chunkIndex, pieceIndex uint64, merkleRoot crypto.Hash) error
- func (sf *I3vFile) AddSkylink(s modules.Skylink) error
- func (sf *I3vFile) ChangeTime() time.Time
- func (sf *I3vFile) Chunk(chunkIndex uint64) (chunk, error)
- func (sf *I3vFile) ChunkHealth(index int, offlineMap map[string]bool, goodForRenewMap map[string]bool) (float64, float64, error)
- func (sf *I3vFile) ChunkSize() uint64
- func (sf *I3vFile) CreateTime() time.Time
- func (sf *I3vFile) Delete() error
- func (sf *I3vFile) Deleted() bool
- func (sf *I3vFile) ErasureCode() modules.ErasureCoder
- func (sf *I3vFile) Expiration(contracts map[string]modules.RenterContract) types.BlockHeight
- func (sf *I3vFile) GoodPieces(chunkIndex int, offlineMap map[string]bool, goodForRenewMap map[string]bool) (uint64, uint64)
- func (sf *I3vFile) GrowNumChunks(numChunks uint64) (err error)
- func (sf *I3vFile) HasPartialChunk() bool
- func (sf *I3vFile) Health(offline map[string]bool, goodForRenew map[string]bool) (h float64, sh float64, uh float64, ush float64, nsc uint64)
- func (sf *I3vFile) HostPublicKeys() (spks []types.I3vPublicKey)
- func (sf *I3vFile) I3vFilePath() string
- func (sf *I3vFile) IsIncludedPartialChunk(chunkIndex uint64) bool
- func (sf *I3vFile) IsIncompletePartialChunk(chunkIndex uint64) bool
- func (sf *I3vFile) LastHealthCheckTime() time.Time
- func (sf *I3vFile) LocalPath() string
- func (sf *I3vFile) Lock()
- func (sf *I3vFile) MasterKey() crypto.CipherKey
- func (sf *I3vFile) Merge(newFile *I3vFile) (map[uint64]uint64, error)
- func (sf *I3vFile) Metadata() Metadata
- func (sf *I3vFile) ModTime() time.Time
- func (sf *I3vFile) Mode() os.FileMode
- func (sf *I3vFile) NumChunks() uint64
- func (sf *I3vFile) NumStuckChunks() uint64
- func (sf *I3vFile) PartialChunks() []PartialChunkInfo
- func (sf *I3vFile) PieceSize() uint64
- func (sf *I3vFile) Pieces(chunkIndex uint64) ([][]Piece, error)
- func (sf *I3vFile) Redundancy(offlineMap map[string]bool, goodForRenewMap map[string]bool) (r, ur float64, err error)
- func (sf *I3vFile) RemoveLastChunk() error
- func (sf *I3vFile) Rename(newI3vFilePath string) error
- func (sf *I3vFile) SaveHeader() error
- func (sf *I3vFile) SaveMetadata() error
- func (sf *I3vFile) SaveWithChunks(chunks Chunks) error
- func (sf *I3vFile) SetAllStuck(stuck bool) (err error)
- func (sf *I3vFile) SetChunkStatusCompleted(pci uint64) error
- func (sf *I3vFile) SetFileSize(fileSize uint64) error
- func (sf *I3vFile) SetI3vFilePath(path string)
- func (sf *I3vFile) SetLastHealthCheckTime()
- func (sf *I3vFile) SetLocalPath(path string) error
- func (sf *I3vFile) SetMode(mode os.FileMode) error
- func (sf *I3vFile) SetPartialChunks(combinedChunks []modules.PartialChunk, updates []writeaheadlog.Update) error
- func (sf *I3vFile) SetPartialsI3vFile(partialsI3vFile *I3vFile)
- func (sf *I3vFile) SetStuck(index uint64, stuck bool) (err error)
- func (sf *I3vFile) Size() uint64
- func (sf *I3vFile) Snapshot(sp modules.I3vPath) (*Snapshot, error)
- func (sf *I3vFile) SnapshotReader() (*SnapshotReader, error)
- func (sf *I3vFile) StuckChunkByIndex(index uint64) (bool, error)
- func (sf *I3vFile) UID() I3vfileUID
- func (sf *I3vFile) Unlock()
- func (sf *I3vFile) UnmanagedSetDeleted(deleted bool)
- func (sf *I3vFile) UnmanagedSetI3vFilePath(newI3vFilePath string)
- func (sf *I3vFile) UpdateAccessTime() error
- func (sf *I3vFile) UpdateUniqueID()
- func (sf *I3vFile) UpdateUsedHosts(used []types.I3vPublicKey) error
- func (sf *I3vFile) UploadProgressAndBytes() (float64, uint64, error)
- type I3vfileUID
- type Metadata
- type PartialChunkInfo
- type Piece
- type RSCode
- func (rs *RSCode) Encode(data []byte) ([][]byte, error)
- func (rs *RSCode) EncodeShards(pieces [][]byte) ([][]byte, error)
- func (rs *RSCode) Identifier() modules.ErasureCoderIdentifier
- func (rs *RSCode) MinPieces() int
- func (rs *RSCode) NumPieces() int
- func (rs *RSCode) Reconstruct(pieces [][]byte) error
- func (rs *RSCode) Recover(pieces [][]byte, n uint64, w io.Writer) error
- func (rs *RSCode) SupportsPartialEncoding() bool
- func (rs *RSCode) Type() modules.ErasureCoderType
- type RSSubCode
- func (rs *RSSubCode) Encode(data []byte) ([][]byte, error)
- func (rs *RSSubCode) EncodeShards(pieces [][]byte) ([][]byte, error)
- func (rs *RSSubCode) Identifier() modules.ErasureCoderIdentifier
- func (rs *RSSubCode) Reconstruct(pieces [][]byte) error
- func (rs *RSSubCode) Recover(pieces [][]byte, n uint64, w io.Writer) error
- func (rs *RSSubCode) SupportsPartialEncoding() bool
- func (rs *RSSubCode) Type() modules.ErasureCoderType
- type Snapshot
- func (s *Snapshot) ChunkIndexByOffset(offset uint64) (chunkIndex uint64, off uint64)
- func (s *Snapshot) ChunkSize() uint64
- func (s *Snapshot) ErasureCode() modules.ErasureCoder
- func (s *Snapshot) I3vPath() modules.I3vPath
- func (s *Snapshot) IsIncludedPartialChunk(chunkIndex uint64) (PartialChunkInfo, bool)
- func (s *Snapshot) IsIncompletePartialChunk(chunkIndex uint64) bool
- func (s *Snapshot) LocalPath() string
- func (s *Snapshot) MasterKey() crypto.CipherKey
- func (s *Snapshot) Mode() os.FileMode
- func (s *Snapshot) NumChunks() uint64
- func (s *Snapshot) PartialChunks() []PartialChunkInfo
- func (s *Snapshot) PieceSize() uint64
- func (s *Snapshot) Pieces(chunkIndex uint64) [][]Piece
- func (s *Snapshot) Size() uint64
- func (s *Snapshot) UID() I3vfileUID
- type SnapshotReader
Constants ¶
const ( CombinedChunkStatusInvalid = iota // status wasn't initialized CombinedChunkStatusInComplete // partial chunk is included in an incomplete combined chunk. CombinedChunkStatusCompleted // partial chunk is included in a completed combined chunk. )
Constants to indicate which part of the partial upload the combined chunk is currently at.
Variables ¶
var ( // ECReedSolomon is the marshaled type of the reed solomon coder. ECReedSolomon = modules.ErasureCoderType{0, 0, 0, 1} // ECReedSolomonSubShards64 is the marshaled type of the reed solomon coder // for files where every 64 bytes of an encoded piece can be decoded // separately. ECReedSolomonSubShards64 = modules.ErasureCoderType{0, 0, 0, 2} )
var ( // ErrPathOverload is an error when a file already exists at that location ErrPathOverload = errors.New("a file already exists at that location") // ErrUnknownPath is an error when a file cannot be found with the given path ErrUnknownPath = errors.New("no file known with that path") // ErrUnknownThread is an error when a I3vFile is trying to be closed by a // thread that is not in the threadMap ErrUnknownThread = errors.New("thread should not be calling Close(), does not have control of the i3vfile") // ErrDeleted is returned when an operation failed due to the i3vfile being // deleted already. ErrDeleted = errors.New("files was deleted") )
Functions ¶
func ApplyUpdates ¶
func ApplyUpdates(updates ...writeaheadlog.Update) error
ApplyUpdates is a wrapper for applyUpdates that uses the production dependencies.
func CombinedChunkIndex ¶
CombinedChunkIndex is a helper method which translates a chunk's index to the corresponding combined chunk index dependng on the number of combined chunks.
func ExtractSegment ¶
ExtractSegment is a convenience method that extracts the data of the segment at segmentIndex from pieces.
func IsI3vFileUpdate ¶
func IsI3vFileUpdate(update writeaheadlog.Update) bool
IsI3vFileUpdate is a helper method that makes sure that a wal update belongs to the I3vFile package.
func LoadI3vFileFromReaderWithChunks ¶
func LoadI3vFileFromReaderWithChunks(r io.ReadSeeker, path string, wal *writeaheadlog.WAL) (*I3vFile, Chunks, error)
LoadI3vFileFromReaderWithChunks does not only read the header of the I3vfile from disk but also the chunks which it returns separately. This is useful if the file is read from a buffer in-memory and the chunks can't be read from disk later.
func NewRSCode ¶
func NewRSCode(nData, nParity int) (modules.ErasureCoder, error)
NewRSCode creates a new Reed-Solomon encoder/decoder using the supplied parameters.
func NewRSSubCode ¶
func NewRSSubCode(nData, nParity int, segmentSize uint64) (modules.ErasureCoder, error)
NewRSSubCode creates a new Reed-Solomon encoder/decoder using the supplied parameters.
Types ¶
type BubbledMetadata ¶
type BubbledMetadata struct { Health float64 LastHealthCheckTime time.Time ModTime time.Time NumStuckChunks uint64 Redundancy float64 Size uint64 StuckHealth float64 UID I3vfileUID }
BubbledMetadata is the metadata of a i3vfile that gets bubbled
type CachedHealthMetadata ¶
CachedHealthMetadata is a healper struct that contains the i3vfile health metadata fields that are cached
type Chunk ¶
type Chunk struct {
Pieces [][]Piece
}
Chunk is an exported chunk. It contains exported pieces.
type Chunks ¶
type Chunks struct {
// contains filtered or unexported fields
}
Chunks is an exported version of a chunk slice.. It exists for convenience to make sure the caller has an exported type to pass around.
type FileChunk ¶
type FileChunk struct {
Pieces [][]Piece
}
FileChunk is a helper struct that contains data about a chunk.
type FileData ¶
type FileData struct { Name string FileSize uint64 MasterKey [crypto.EntropySize]byte ErasureCode modules.ErasureCoder RepairPath string PieceSize uint64 Mode os.FileMode Deleted bool UID I3vfileUID Chunks []FileChunk }
FileData is a helper struct that contains all the relevant information of a file. It simplifies passing the necessary data between modules and keeps the interface clean.
type HostPublicKey ¶
type HostPublicKey struct { PublicKey types.I3vPublicKey // public key of host Used bool // indicates if we currently use this host }
HostPublicKey is an entry in the HostPubKey table.
func (HostPublicKey) MarshalI3v ¶
func (hpk HostPublicKey) MarshalI3v(w io.Writer) error
MarshalI3v implements the encoding.I3vMarshaler interface.
func (*HostPublicKey) UnmarshalI3v ¶
func (hpk *HostPublicKey) UnmarshalI3v(r io.Reader) error
UnmarshalI3v implements the encoding.I3vUnmarshaler interface.
type I3vFile ¶
type I3vFile struct {
// contains filtered or unexported fields
}
I3vFile is the disk format for files uploaded to the I3v network. It contains all the necessary information to recover a file from its hosts and allows for easy constant-time updates of the file without having to read or write the whole file.
func LoadI3vFile ¶
func LoadI3vFile(path string, wal *writeaheadlog.WAL) (*I3vFile, error)
LoadI3vFile is a wrapper for loadI3vFile that uses the production dependencies.
func LoadI3vFileFromReader ¶
func LoadI3vFileFromReader(r io.ReadSeeker, path string, wal *writeaheadlog.WAL) (*I3vFile, error)
LoadI3vFileFromReader allows loading a I3vFile from a different location that directly from disk as long as the source satisfies the I3vFileSource interface.
func New ¶
func New(i3vFilePath, source string, wal *writeaheadlog.WAL, erasureCode modules.ErasureCoder, masterKey crypto.CipherKey, fileSize uint64, fileMode os.FileMode, partialsI3vFile *I3vFile, disablePartialUpload bool) (*I3vFile, error)
New create a new I3vFile.
func NewFromLegacyData ¶
NewFromLegacyData creates a new I3vFile from data that was previously loaded from a legacy file.
func (*I3vFile) AccessTime ¶
AccessTime returns the AccessTime timestamp of the file.
func (*I3vFile) AddPiece ¶
func (sf *I3vFile) AddPiece(pk types.I3vPublicKey, chunkIndex, pieceIndex uint64, merkleRoot crypto.Hash) error
AddPiece adds an uploaded piece to the file. It also updates the host table if the public key of the host is not already known.
func (*I3vFile) AddSkylink ¶
AddSkylink will add a skylink to the I3vFile.
func (*I3vFile) ChangeTime ¶
ChangeTime returns the ChangeTime timestamp of the file.
func (*I3vFile) ChunkHealth ¶
func (sf *I3vFile) ChunkHealth(index int, offlineMap map[string]bool, goodForRenewMap map[string]bool) (float64, float64, error)
ChunkHealth returns the health of the chunk which is defined as the percent of parity pieces remaining.
func (*I3vFile) CreateTime ¶
CreateTime returns the CreateTime timestamp of the file.
func (*I3vFile) Delete ¶
Delete removes the file from disk and marks it as deleted. Once the file is deleted, certain methods should return an error.
func (*I3vFile) ErasureCode ¶
func (sf *I3vFile) ErasureCode() modules.ErasureCoder
ErasureCode returns the erasure coder used by the file.
func (*I3vFile) Expiration ¶
func (sf *I3vFile) Expiration(contracts map[string]modules.RenterContract) types.BlockHeight
Expiration updates CachedExpiration with the lowest height at which any of the file's contracts will expire and returns the new value.
func (*I3vFile) GoodPieces ¶
func (sf *I3vFile) GoodPieces(chunkIndex int, offlineMap map[string]bool, goodForRenewMap map[string]bool) (uint64, uint64)
GoodPieces loops over the pieces of a chunk and tracks the number of unique pieces that are good for upload, meaning the host is online, and the number of unique pieces that are good for renew, meaning the contract is set to renew.
func (*I3vFile) GrowNumChunks ¶
GrowNumChunks increases the number of chunks in the I3vFile to numChunks. If the file already contains >= numChunks chunks then GrowNumChunks is a no-op.
func (*I3vFile) HasPartialChunk ¶
HasPartialChunk returns whether this file is supposed to have a partial chunk or not.
func (*I3vFile) Health ¶
func (sf *I3vFile) Health(offline map[string]bool, goodForRenew map[string]bool) (h float64, sh float64, uh float64, ush float64, nsc uint64)
Health calculates the health of the file to be used in determining repair priority. Health of the file is the lowest health of any of the chunks and is defined as the percent of parity pieces remaining. The NumStuckChunks will be calculated for the I3vFile and returned.
NOTE: The cached values of the health and stuck health will be set but not saved to disk as Health() does not write to disk. If the cached values need to be updated on disk then a metadata save method should be called in conjunction with Health()
health = 0 is full redundancy, health <= 1 is recoverable, health > 1 needs to be repaired from disk
func (*I3vFile) HostPublicKeys ¶
func (sf *I3vFile) HostPublicKeys() (spks []types.I3vPublicKey)
HostPublicKeys returns all the public keys of hosts the file has ever been uploaded to. That means some of those hosts might no longer be in use.
func (*I3vFile) I3vFilePath ¶
I3vFilePath returns the i3vFilePath field of the I3vFile.
func (*I3vFile) IsIncludedPartialChunk ¶
IsIncludedPartialChunk returns 'true' if the provided index points to a partial chunk which has been added to the partials i3v file already.
func (*I3vFile) IsIncompletePartialChunk ¶
IsIncompletePartialChunk returns 'true' if the provided index points to a partial chunk which hasn't been added to a partials i3vfile yet.
func (*I3vFile) LastHealthCheckTime ¶
LastHealthCheckTime returns the LastHealthCheckTime timestamp of the file
func (*I3vFile) Lock ¶
func (sf *I3vFile) Lock()
Lock acquires the I3vFile's mutex for calling Unmanaged exported methods.
func (*I3vFile) Merge ¶
Merge merges two PartialsI3vfiles into one, returning a map which translates chunk indices in newFile to indices in sf.
func (*I3vFile) Metadata ¶
Metadata returns the I3vFile's metadata, resolving any fields related to partial chunks.
func (*I3vFile) NumChunks ¶
NumChunks returns the number of chunks the file consists of. This will return the number of chunks the file consists of even if the file is not fully uploaded yet.
func (*I3vFile) NumStuckChunks ¶
NumStuckChunks returns the Number of Stuck Chunks recorded in the file's metadata
func (*I3vFile) PartialChunks ¶
func (sf *I3vFile) PartialChunks() []PartialChunkInfo
PartialChunks returns the partial chunk infos of the i3vfile.
func (*I3vFile) Pieces ¶
Pieces returns all the pieces for a chunk in a slice of slices that contains all the pieces for a certain index.
func (*I3vFile) Redundancy ¶
func (sf *I3vFile) Redundancy(offlineMap map[string]bool, goodForRenewMap map[string]bool) (r, ur float64, err error)
Redundancy returns the redundancy of the least redundant chunk. A file becomes available when this redundancy is >= 1. Assumes that every piece is unique within a file contract. -1 is returned if the file has size 0. It takes two arguments, a map of offline contracts for this file and a map that indicates if a contract is goodForRenew. The first redundancy returned is the one that should be used by the repair code and is more accurate. The other one is the redundancy presented to users.
func (*I3vFile) RemoveLastChunk ¶
RemoveLastChunk removes the last chunk of the I3vFile and truncates the file accordingly.
func (*I3vFile) Rename ¶
Rename changes the name of the file to a new one. To guarantee that renaming the file is atomic across all operating systems, we create a wal transaction that moves over all the chunks one-by-one and deletes the src file.
func (*I3vFile) SaveHeader ¶
SaveHeader saves the file's header to disk.
func (*I3vFile) SaveMetadata ¶
SaveMetadata saves the file's metadata to disk.
func (*I3vFile) SaveWithChunks ¶
SaveWithChunks saves the file's header to disk and appends the raw chunks provided at the end of the file.
func (*I3vFile) SetAllStuck ¶
SetAllStuck sets the Stuck field of all chunks to stuck.
func (*I3vFile) SetChunkStatusCompleted ¶
SetChunkStatusCompleted sets the CombinedChunkStatus field of the metadata to completed.
func (*I3vFile) SetFileSize ¶
SetFileSize changes the fileSize of the I3vFile.
func (*I3vFile) SetI3vFilePath ¶
SetI3vFilePath sets the path of the i3vfile on disk.
func (*I3vFile) SetLastHealthCheckTime ¶
func (sf *I3vFile) SetLastHealthCheckTime()
SetLastHealthCheckTime sets the LastHealthCheckTime in memory to the current time but does not update and write to disk.
NOTE: This call should be used in conjunction with a method that saves the I3vFile metadata
func (*I3vFile) SetLocalPath ¶
SetLocalPath changes the local path of the file which is used to repair the file from disk.
func (*I3vFile) SetPartialChunks ¶
func (sf *I3vFile) SetPartialChunks(combinedChunks []modules.PartialChunk, updates []writeaheadlog.Update) error
SetPartialChunks informs the I3vFile about a partial chunk that has been saved by the partial chunk set. As such it should be exclusively called by the partial chunk set. It updates the metadata of the I3vFile and also adds a new chunk to the partial I3vFile if necessary. At the end it applies the updates of the partial chunk set, the I3vFile and the partial I3vFile atomically.
func (*I3vFile) SetPartialsI3vFile ¶
SetPartialsI3vFile sets the partialsI3vFile field of the I3vFile. This is usually done for non-partials I3vFiles after loading them from disk.
func (*I3vFile) SnapshotReader ¶
func (sf *I3vFile) SnapshotReader() (*SnapshotReader, error)
SnapshotReader creates a io.ReadCloser that can be used to read the raw I3vfile from disk. Note that the underlying i3vfile holds a readlock until the SnapshotReader is closed, which means that no operations can be called to the underlying i3vfile which may cause it to grab a lock, because that will cause a deadlock.
Operations which require grabbing a readlock on the underlying i3vfile are also not okay, because if some other thread has attempted to grab a writelock on the i3vfile, the readlock will block and then the Close() statement may never be reached for the SnapshotReader.
TODO: Things upstream would be a lot easier if we could drop the requirement to hold a lock for the duration of the life of the snapshot reader.
func (*I3vFile) StuckChunkByIndex ¶
StuckChunkByIndex returns if the chunk at the index is marked as Stuck or not
func (*I3vFile) UID ¶
func (sf *I3vFile) UID() I3vfileUID
UID returns a unique identifier for this file.
func (*I3vFile) UnmanagedSetDeleted ¶
UnmanagedSetDeleted sets the deleted field of the I3vFile without holding the lock.
func (*I3vFile) UnmanagedSetI3vFilePath ¶
UnmanagedSetI3vFilePath sets the i3vFilePath field of the I3vFile without holding the lock.
func (*I3vFile) UpdateAccessTime ¶
UpdateAccessTime updates the AccessTime timestamp to the current time.
func (*I3vFile) UpdateUniqueID ¶
func (sf *I3vFile) UpdateUniqueID()
UpdateUniqueID creates a new random uid for the I3vFile.
func (*I3vFile) UpdateUsedHosts ¶
func (sf *I3vFile) UpdateUsedHosts(used []types.I3vPublicKey) error
UpdateUsedHosts updates the 'Used' flag for the entries in the pubKeyTable of the I3vFile. The keys of all used hosts should be passed to the method and the I3vFile will update the flag for hosts it knows of to 'true' and set hosts which were not passed in to 'false'.
type I3vfileUID ¶
type I3vfileUID string
I3vfileUID is a unique identifier for i3vfile which is used to track i3vfiles even after renaming them.
type Metadata ¶
type Metadata struct { UniqueID I3vfileUID `json:"uniqueid"` // unique identifier for file StaticPagesPerChunk uint8 `json:"pagesperchunk"` // number of pages reserved for storing a chunk. StaticVersion [16]byte `json:"version"` // version of the i3v file format used FileSize int64 `json:"filesize"` // total size of the file StaticPieceSize uint64 `json:"piecesize"` // size of a single piece of the file LocalPath string `json:"localpath"` // file to the local copy of the file used for repairing // Fields for encryption StaticMasterKey []byte `json:"masterkey"` // masterkey used to encrypt pieces StaticMasterKeyType crypto.CipherType `json:"masterkeytype"` StaticSharingKey []byte `json:"sharingkey"` // key used to encrypt shared pieces StaticSharingKeyType crypto.CipherType `json:"sharingkeytype"` // Fields for partial uploads DisablePartialChunk bool `json:"disablepartialchunk"` // determines whether the file should be treated like legacy files PartialChunks []PartialChunkInfo `json:"partialchunks"` // information about the partial chunk. HasPartialChunk bool `json:"haspartialchunk"` // indicates whether this file is supposed to have a partial chunk or not // The following fields are the usual unix timestamps of files. ModTime time.Time `json:"modtime"` // time of last content modification ChangeTime time.Time `json:"changetime"` // time of last metadata modification AccessTime time.Time `json:"accesstime"` // time of last access CreateTime time.Time `json:"createtime"` // time of file creation // Cached fields. These fields are cached fields and are only meant to be used // to create FileInfos for file related API endpoints. There is no guarantee // that these fields are up-to-date. Neither in memory nor on disk. Updates to // these fields aren't persisted immediately. Instead they will only be // persisted whenever another method persists the metadata or when the I3vFile // is closed. // // CachedRedundancy is the redundancy of the file on the network and is // updated within the 'Redundancy' method which is periodically called by the // repair code. // // CachedUserRedundancy is the redundancy of the file on the network as // visible to the user and is updated within the 'Redundancy' method which is // periodically called by the repair code. // // CachedHealth is the health of the file on the network and is also // periodically updated by the health check loop whenever 'Health' is called. // // CachedStuckHealth is the health of the stuck chunks of the file. It is // updated by the health check loop. CachedExpiration is the lowest height at // which any of the file's contracts will expire. Also updated periodically by // the health check loop whenever 'Health' is called. // // CachedUploadedBytes is the number of bytes of the file that have been // uploaded to the network so far. Is updated every time a piece is added to // the i3vfile. // // CachedUploadProgress is the upload progress of the file and is updated // every time a piece is added to the i3vfile. // CachedRedundancy float64 `json:"cachedredundancy"` CachedUserRedundancy float64 `json:"cacheduserredundancy"` CachedHealth float64 `json:"cachedhealth"` CachedStuckHealth float64 `json:"cachedstuckhealth"` CachedExpiration types.BlockHeight `json:"cachedexpiration"` CachedUploadedBytes uint64 `json:"cacheduploadedbytes"` CachedUploadProgress float64 `json:"cacheduploadprogress"` // Repair loop fields // // Health is the worst health of the file's unstuck chunks and // represents the percent of redundancy missing // // LastHealthCheckTime is the timestamp of the last time the I3vFile's // health was checked by Health() // // NumStuckChunks is the number of all the I3vFile's chunks that have // been marked as stuck by the repair loop. This doesn't include a potential // partial chunk at the end of the file though. Use 'numStuckChunks()' for // that instead. // // Redundancy is the cached value of the last time the file's redundancy // was checked // // StuckHealth is the worst health of any of the file's stuck chunks // Health float64 `json:"health"` LastHealthCheckTime time.Time `json:"lasthealthchecktime"` NumStuckChunks uint64 `json:"numstuckchunks"` Redundancy float64 `json:"redundancy"` StuckHealth float64 `json:"stuckhealth"` // File ownership/permission fields. Mode os.FileMode `json:"mode"` // unix filemode of the i3v file - uint32 UserID int32 `json:"userid"` // id of the user who owns the file GroupID int32 `json:"groupid"` // id of the group that owns the file // The following fields are the offsets for data that is written to disk // after the pubKeyTable. We reserve a generous amount of space for the // table and extra fields, but we need to remember those offsets in case we // need to resize later on. // // chunkOffset is the offset of the first chunk, forced to be a factor of // 4096, default 4kib // // pubKeyTableOffset is the offset of the publicKeyTable within the // file. // ChunkOffset int64 `json:"chunkoffset"` PubKeyTableOffset int64 `json:"pubkeytableoffset"` // erasure code settings. // // StaticErasureCodeType specifies the algorithm used for erasure coding // chunks. Available types are: // 0 - Invalid / Missing Code // 1 - Reed Solomon Code // // erasureCodeParams specifies possible parameters for a certain // StaticErasureCodeType. Currently params will be parsed as follows: // Reed Solomon Code - 4 bytes dataPieces / 4 bytes parityPieces // StaticErasureCodeType [4]byte `json:"erasurecodetype"` StaticErasureCodeParams [8]byte `json:"erasurecodeparams"` // Skylink tracking. If this i3vfile is known to have sectors of any // skyfiles, those skyfiles will be listed here. It should be noted that // a single i3vfile can be responsible for tracking many skyfiles. Skylinks []string `json:"skylinks"` // contains filtered or unexported fields }
Metadata is the metadata of a I3vFile and is JSON encoded.
func LoadI3vFileMetadata ¶
LoadI3vFileMetadata is a wrapper for loadI3vFileMetadata that uses the production dependencies.
type PartialChunkInfo ¶
type PartialChunkInfo struct { ID modules.CombinedChunkID `json:"id"` // ID of the combined chunk Index uint64 `json:"index"` // Index of the combined chunk within partialsI3vFile Offset uint64 `json:"offset"` // Offset of partial chunk within combined chunk Length uint64 `json:"length"` // Length of partial chunk within combined chunk Status uint8 `json:"status"` // Status of combined chunk }
PartialChunkInfo contains all the essential information about a partial chunk relevant to I3vFiles. A I3vFile with a partial chunk may contain 1 or 2 PartialChunkInfos since the partial chunk might be split between 2 combined chunks.
type Piece ¶
type Piece struct { HostPubKey types.I3vPublicKey // public key of the host MerkleRoot crypto.Hash // merkle root of the piece }
Piece is an exported piece. It contains a resolved public key instead of the table offset.
type RSCode ¶
type RSCode struct {
// contains filtered or unexported fields
}
RSCode is a Reed-Solomon encoder/decoder. It implements the modules.ErasureCoder interface.
func (*RSCode) Encode ¶
Encode splits data into equal-length pieces, some containing the original data and some containing parity data.
func (*RSCode) EncodeShards ¶
EncodeShards creates the parity shards for an already sharded input.
func (*RSCode) Identifier ¶
func (rs *RSCode) Identifier() modules.ErasureCoderIdentifier
Identifier returns an identifier for an erasure coder which can be used to identify erasure coders of the same type, dataPieces and parityPieces.
func (*RSCode) MinPieces ¶
MinPieces return the minimum number of pieces that must be present to recover the original data.
func (*RSCode) Reconstruct ¶
Reconstruct recovers the full set of encoded shards from the provided pieces, of which at least MinPieces must be non-nil.
func (*RSCode) Recover ¶
Recover recovers the original data from pieces and writes it to w. pieces should be identical to the slice returned by Encode (length and order must be preserved), but with missing elements set to nil.
func (*RSCode) SupportsPartialEncoding ¶
SupportsPartialEncoding returns false for the basic reed-solomon encoder.
func (*RSCode) Type ¶
func (rs *RSCode) Type() modules.ErasureCoderType
Type returns the erasure coders type identifier.
type RSSubCode ¶
type RSSubCode struct { RSCode // contains filtered or unexported fields }
RSSubCode is a Reed-Solomon encoder/decoder. It implements the modules.ErasureCoder interface in a way that every crypto.SegmentSize bytes of encoded data can be recovered separately.
func (*RSSubCode) Encode ¶
Encode splits data into equal-length pieces, some containing the original data and some containing parity data.
func (*RSSubCode) EncodeShards ¶
EncodeShards encodes data in a way that every segmentSize bytes of the encoded data can be decoded independently.
func (*RSSubCode) Identifier ¶
func (rs *RSSubCode) Identifier() modules.ErasureCoderIdentifier
Identifier returns an identifier for an erasure coder which can be used to identify erasure coders of the same type, dataPieces and parityPieces.
func (*RSSubCode) Reconstruct ¶
Reconstruct recovers the full set of encoded shards from the provided pieces, of which at least MinPieces must be non-nil.
func (*RSSubCode) Recover ¶
Recover accepts encoded pieces and decodes the segment at segmentIndex. The size of the decoded data is segmentSize * dataPieces.
func (*RSSubCode) SupportsPartialEncoding ¶
SupportsPartialEncoding returns true for the custom reed-solomon encoder.
func (*RSSubCode) Type ¶
func (rs *RSSubCode) Type() modules.ErasureCoderType
Type returns the erasure coders type identifier.
type Snapshot ¶
type Snapshot struct {
// contains filtered or unexported fields
}
Snapshot is a snapshot of a I3vFile. A snapshot is a deep-copy and can be accessed without locking at the cost of being a frozen readonly representation of a i3vfile which only exists in memory.
func SnapshotFromReader ¶
SnapshotFromReader reads a i3vfile from the specified reader and creates a snapshot from it.
func (*Snapshot) ChunkIndexByOffset ¶
ChunkIndexByOffset will return the chunkIndex that contains the provided offset of a file and also the relative offset within the chunk. If the offset is out of bounds, chunkIndex will be equal to NumChunk().
func (*Snapshot) ErasureCode ¶
func (s *Snapshot) ErasureCode() modules.ErasureCoder
ErasureCode returns the erasure coder used by the file.
func (*Snapshot) IsIncludedPartialChunk ¶
func (s *Snapshot) IsIncludedPartialChunk(chunkIndex uint64) (PartialChunkInfo, bool)
IsIncludedPartialChunk returns 'true' if the provided index points to a partial chunk which has been added to the partials i3v file already.
func (*Snapshot) IsIncompletePartialChunk ¶
IsIncompletePartialChunk returns 'true' if the provided index points to a partial chunk which hasn't been added to a partials i3vfile yet.
func (*Snapshot) NumChunks ¶
NumChunks returns the number of chunks the file consists of. This will return the number of chunks the file consists of even if the file is not fully uploaded yet.
func (*Snapshot) PartialChunks ¶
func (s *Snapshot) PartialChunks() []PartialChunkInfo
PartialChunks returns the snapshot's PartialChunks.
type SnapshotReader ¶
type SnapshotReader struct {
// contains filtered or unexported fields
}
SnapshotReader is a helper type that allows reading a raw I3vFile from disk while keeping the file in memory locked.
func (*SnapshotReader) Close ¶
func (sfr *SnapshotReader) Close() error
Close closes the underlying file.