archivist

package
v0.0.0-...-db4037c Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 9, 2019 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source
const CheckpointFreq = uint32(64)
View Source
const NumLevels = 11

Variables

This section is empty.

Functions

func BucketPath

func BucketPath(bucket Hash) string

func Categories

func Categories() []string

func CategoryCheckpointPath

func CategoryCheckpointPath(cat string, chk uint32) string

func DumpXdrAsJson

func DumpXdrAsJson(args []string) error

func Mirror

func Mirror(src *Archive, dst *Archive, opts *CommandOptions) error

func NextCheckpoint

func NextCheckpoint(i uint32) uint32

func PrevCheckpoint

func PrevCheckpoint(i uint32) uint32

func RangePaths

func RangePaths(r Range) []string

Returns an array of path prefixes to walk to enumerate all the objects in the provided range.

func Repair

func Repair(src *Archive, dst *Archive, opts *CommandOptions) error

func SortTxsForHash

func SortTxsForHash(txset *xdr.TransactionSet) error

func WriteFramedXdr

func WriteFramedXdr(out io.Writer, in interface{}) error

Types

type Archive

type Archive struct {
	// contains filtered or unexported fields
}

func Connect

func Connect(u string, opts ConnectOptions) (*Archive, error)

func MustConnect

func MustConnect(u string, opts ConnectOptions) *Archive

func (*Archive) BucketExists

func (a *Archive) BucketExists(bucket Hash) bool

func (*Archive) CategoryCheckpointExists

func (a *Archive) CategoryCheckpointExists(cat string, chk uint32) bool

func (*Archive) CheckBucketsMissing

func (arch *Archive) CheckBucketsMissing() map[Hash]bool

func (*Archive) CheckCheckpointFilesMissing

func (arch *Archive) CheckCheckpointFilesMissing(opts *CommandOptions) map[string][]uint32

func (*Archive) ClearCachedInfo

func (arch *Archive) ClearCachedInfo()

func (*Archive) GetCheckpointHAS

func (a *Archive) GetCheckpointHAS(chk uint32) (HistoryArchiveState, error)

func (*Archive) GetPathHAS

func (a *Archive) GetPathHAS(path string) (HistoryArchiveState, error)

func (*Archive) GetRootHAS

func (a *Archive) GetRootHAS() (HistoryArchiveState, error)

func (*Archive) GetXdrStream

func (a *Archive) GetXdrStream(pth string) (*XdrStream, error)

func (*Archive) ListAllBucketHashes

func (a *Archive) ListAllBucketHashes() (chan Hash, chan error)

func (*Archive) ListAllBuckets

func (a *Archive) ListAllBuckets() (chan string, chan error)

func (*Archive) ListBucket

func (a *Archive) ListBucket(dp DirPrefix) (chan string, chan error)

func (*Archive) ListCategoryCheckpoints

func (a *Archive) ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)

func (*Archive) NoteCheckpointFile

func (arch *Archive) NoteCheckpointFile(cat string, chk uint32, present bool)

func (*Archive) NoteExistingBucket

func (arch *Archive) NoteExistingBucket(bucket Hash)

func (*Archive) NoteReferencedBucket

func (arch *Archive) NoteReferencedBucket(bucket Hash) bool

func (*Archive) PutCheckpointHAS

func (a *Archive) PutCheckpointHAS(chk uint32, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutPathHAS

func (a *Archive) PutPathHAS(path string, has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) PutRootHAS

func (a *Archive) PutRootHAS(has HistoryArchiveState, opts *CommandOptions) error

func (*Archive) ReportBucketStats

func (arch *Archive) ReportBucketStats()

func (*Archive) ReportCheckpointStats

func (arch *Archive) ReportCheckpointStats()

func (*Archive) ReportInvalid

func (arch *Archive) ReportInvalid(opts *CommandOptions) error

func (*Archive) ReportMissing

func (arch *Archive) ReportMissing(opts *CommandOptions) error

func (*Archive) Scan

func (arch *Archive) Scan(opts *CommandOptions) error

func (*Archive) ScanAllBuckets

func (arch *Archive) ScanAllBuckets() error

func (*Archive) ScanBuckets

func (arch *Archive) ScanBuckets(opts *CommandOptions) error

func (*Archive) ScanCheckpoints

func (arch *Archive) ScanCheckpoints(opts *CommandOptions) error

func (*Archive) ScanCheckpointsFast

func (arch *Archive) ScanCheckpointsFast(opts *CommandOptions) error

func (*Archive) ScanCheckpointsSlow

func (arch *Archive) ScanCheckpointsSlow(opts *CommandOptions) error

func (*Archive) VerifyBucketEntries

func (arch *Archive) VerifyBucketEntries(h Hash) error

func (*Archive) VerifyBucketHash

func (arch *Archive) VerifyBucketHash(h Hash) error

func (*Archive) VerifyCategoryCheckpoint

func (arch *Archive) VerifyCategoryCheckpoint(cat string, chk uint32) error

func (*Archive) VerifyLedgerHeaderHistoryEntry

func (arch *Archive) VerifyLedgerHeaderHistoryEntry(entry *xdr.LedgerHeaderHistoryEntry) error

func (*Archive) VerifyTransactionHistoryEntry

func (arch *Archive) VerifyTransactionHistoryEntry(entry *xdr.TransactionHistoryEntry) error

func (*Archive) VerifyTransactionHistoryResultEntry

func (arch *Archive) VerifyTransactionHistoryResultEntry(entry *xdr.TransactionHistoryResultEntry) error

type ArchiveBackend

type ArchiveBackend interface {
	Exists(path string) bool
	GetFile(path string) (io.ReadCloser, error)
	PutFile(path string, in io.ReadCloser) error
	ListFiles(path string) (chan string, chan error)
	CanListFiles() bool
}

func MakeFsBackend

func MakeFsBackend(pth string, opts ConnectOptions) ArchiveBackend

func MakeHttpBackend

func MakeHttpBackend(base *url.URL, opts ConnectOptions) ArchiveBackend

func MakeMockBackend

func MakeMockBackend(opts ConnectOptions) ArchiveBackend

func MakeS3Backend

func MakeS3Backend(bucket string, prefix string, opts ConnectOptions) (ArchiveBackend, error)

type ByHash

type ByHash struct {
	// contains filtered or unexported fields
}

func (*ByHash) Len

func (h *ByHash) Len() int

func (*ByHash) Less

func (h *ByHash) Less(i, j int) bool

func (*ByHash) Swap

func (h *ByHash) Swap(i, j int)

type ByUint32

type ByUint32 []uint32

func (ByUint32) Len

func (a ByUint32) Len() int

func (ByUint32) Less

func (a ByUint32) Less(i, j int) bool

func (ByUint32) Swap

func (a ByUint32) Swap(i, j int)

type CommandOptions

type CommandOptions struct {
	Concurrency int
	Range       Range
	DryRun      bool
	Force       bool
	Verify      bool
	Thorough    bool
}

type ConnectOptions

type ConnectOptions struct {
	S3Region   string
	S3Endpoint string
}

type DirPrefix

type DirPrefix [3]uint8

func CheckpointPrefix

func CheckpointPrefix(seq uint32) DirPrefix

func HashPrefix

func HashPrefix(h Hash) DirPrefix

func (DirPrefix) Path

func (d DirPrefix) Path() string

func (DirPrefix) PathPrefix

func (d DirPrefix) PathPrefix(n int) string

type FsArchiveBackend

type FsArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*FsArchiveBackend) CanListFiles

func (b *FsArchiveBackend) CanListFiles() bool

func (*FsArchiveBackend) Exists

func (b *FsArchiveBackend) Exists(pth string) bool

func (*FsArchiveBackend) GetFile

func (b *FsArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*FsArchiveBackend) ListFiles

func (b *FsArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*FsArchiveBackend) PutFile

func (b *FsArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type Hash

type Hash [sha256.Size]byte

func DecodeHash

func DecodeHash(s string) (Hash, error)

func EmptyXdrArrayHash

func EmptyXdrArrayHash() Hash

func HashEmptyTxSet

func HashEmptyTxSet(previousLedgerHash Hash) Hash

func HashTxSet

func HashTxSet(txset *xdr.TransactionSet) (Hash, error)

func HashXdr

func HashXdr(x interface{}) (Hash, error)

func MustDecodeHash

func MustDecodeHash(s string) Hash

func (Hash) IsZero

func (h Hash) IsZero() bool

func (Hash) String

func (h Hash) String() string

type HistoryArchiveState

type HistoryArchiveState struct {
	Version        int    `json:"version"`
	Server         string `json:"server"`
	CurrentLedger  uint32 `json:"currentLedger"`
	CurrentBuckets [NumLevels]struct {
		Curr string `json:"curr"`
		Snap string `json:"snap"`
		Next struct {
			State  uint32 `json:"state"`
			Output string `json:"output,omitempty"`
		} `json:"next"`
	} `json:"currentBuckets"`
}

func (*HistoryArchiveState) Buckets

func (h *HistoryArchiveState) Buckets() []Hash

func (*HistoryArchiveState) LevelSummary

func (h *HistoryArchiveState) LevelSummary() (string, int)

func (*HistoryArchiveState) Range

func (h *HistoryArchiveState) Range() Range

type HttpArchiveBackend

type HttpArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*HttpArchiveBackend) CanListFiles

func (b *HttpArchiveBackend) CanListFiles() bool

func (*HttpArchiveBackend) Exists

func (b *HttpArchiveBackend) Exists(pth string) bool

func (*HttpArchiveBackend) GetFile

func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*HttpArchiveBackend) ListFiles

func (b *HttpArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*HttpArchiveBackend) PutFile

func (b *HttpArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type MockArchiveBackend

type MockArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*MockArchiveBackend) CanListFiles

func (b *MockArchiveBackend) CanListFiles() bool

func (*MockArchiveBackend) Exists

func (b *MockArchiveBackend) Exists(pth string) bool

func (*MockArchiveBackend) GetFile

func (b *MockArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*MockArchiveBackend) ListFiles

func (b *MockArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*MockArchiveBackend) PutFile

func (b *MockArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type Range

type Range struct {
	Low  uint32
	High uint32
}

func MakeRange

func MakeRange(low uint32, high uint32) Range

func (Range) Checkpoints

func (r Range) Checkpoints() chan uint32

func (Range) Clamp

func (r Range) Clamp(other Range) Range

func (Range) CollapsedString

func (r Range) CollapsedString() string

func (Range) Size

func (r Range) Size() int

func (Range) String

func (r Range) String() string

type S3ArchiveBackend

type S3ArchiveBackend struct {
	// contains filtered or unexported fields
}

func (*S3ArchiveBackend) CanListFiles

func (b *S3ArchiveBackend) CanListFiles() bool

func (*S3ArchiveBackend) Exists

func (b *S3ArchiveBackend) Exists(pth string) bool

func (*S3ArchiveBackend) GetFile

func (b *S3ArchiveBackend) GetFile(pth string) (io.ReadCloser, error)

func (*S3ArchiveBackend) ListFiles

func (b *S3ArchiveBackend) ListFiles(pth string) (chan string, chan error)

func (*S3ArchiveBackend) PutFile

func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error

type XdrStream

type XdrStream struct {
	// contains filtered or unexported fields
}

func NewXdrGzStream

func NewXdrGzStream(in io.ReadCloser) (*XdrStream, error)

func NewXdrStream

func NewXdrStream(in io.ReadCloser) *XdrStream

func (*XdrStream) Close

func (x *XdrStream) Close()

func (*XdrStream) ReadOne

func (x *XdrStream) ReadOne(in interface{}) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL