pachyderm: github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index Index | Files

package index

import "github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"

Index

Package Files

index.pb.go reader.go util.go writer.go

Variables

var (
    ErrInvalidLengthIndex = fmt.Errorf("proto: negative length found during unmarshaling")
    ErrIntOverflowIndex   = fmt.Errorf("proto: integer overflow")
)
var Op_name = map[int32]string{
    0:  "APPEND",
    1:  "OVERWRITE",
    2:  "DELETE",
}
var Op_value = map[string]int32{
    "APPEND":    0,
    "OVERWRITE": 1,
    "DELETE":    2,
}

func Generate Uses

func Generate(s string) []string

Generate generates the permutations of the passed in string and returns them sorted.

func Perm Uses

func Perm(a []rune, f func([]rune))

Perm calls f with each permutation of a.

type DataOp Uses

type DataOp struct {
    DataRefs             []*chunk.DataRef `protobuf:"bytes,1,rep,name=data_refs,json=dataRefs,proto3" json:"data_refs,omitempty"`
    Op                   Op               `protobuf:"varint,2,opt,name=op,proto3,enum=index.Op" json:"op,omitempty"`
    Tags                 []*Tag           `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"`
    XXX_NoUnkeyedLiteral struct{}         `json:"-"`
    XXX_unrecognized     []byte           `json:"-"`
    XXX_sizecache        int32            `json:"-"`
}

DataOp is a sequence of data references and an operation associated with the referenced data. Tags map identifiers to data in the resulting byte stream.

func (*DataOp) Descriptor Uses

func (*DataOp) Descriptor() ([]byte, []int)

func (*DataOp) GetDataRefs Uses

func (m *DataOp) GetDataRefs() []*chunk.DataRef

func (*DataOp) GetOp Uses

func (m *DataOp) GetOp() Op

func (*DataOp) GetTags Uses

func (m *DataOp) GetTags() []*Tag

func (*DataOp) Marshal Uses

func (m *DataOp) Marshal() (dAtA []byte, err error)

func (*DataOp) MarshalTo Uses

func (m *DataOp) MarshalTo(dAtA []byte) (int, error)

func (*DataOp) MarshalToSizedBuffer Uses

func (m *DataOp) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*DataOp) ProtoMessage Uses

func (*DataOp) ProtoMessage()

func (*DataOp) Reset Uses

func (m *DataOp) Reset()

func (*DataOp) Size Uses

func (m *DataOp) Size() (n int)

func (*DataOp) String Uses

func (m *DataOp) String() string

func (*DataOp) Unmarshal Uses

func (m *DataOp) Unmarshal(dAtA []byte) error

func (*DataOp) XXX_DiscardUnknown Uses

func (m *DataOp) XXX_DiscardUnknown()

func (*DataOp) XXX_Marshal Uses

func (m *DataOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*DataOp) XXX_Merge Uses

func (m *DataOp) XXX_Merge(src proto.Message)

func (*DataOp) XXX_Size Uses

func (m *DataOp) XXX_Size() int

func (*DataOp) XXX_Unmarshal Uses

func (m *DataOp) XXX_Unmarshal(b []byte) error
type Header struct {
    Hdr *tar.Header
    Idx *Index
}

Header is a wrapper for a tar header and index.

type Index Uses

type Index struct {
    Range  *Range  `protobuf:"bytes,1,opt,name=range,proto3" json:"range,omitempty"`
    DataOp *DataOp `protobuf:"bytes,2,opt,name=data_op,json=dataOp,proto3" json:"data_op,omitempty"`
    // Size of the content being indexed (does not include headers).
    SizeBytes            int64    `protobuf:"varint,3,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

Index stores an index to and metadata about a tar stream.

func (*Index) Descriptor Uses

func (*Index) Descriptor() ([]byte, []int)

func (*Index) GetDataOp Uses

func (m *Index) GetDataOp() *DataOp

func (*Index) GetRange Uses

func (m *Index) GetRange() *Range

func (*Index) GetSizeBytes Uses

func (m *Index) GetSizeBytes() int64

func (*Index) Marshal Uses

func (m *Index) Marshal() (dAtA []byte, err error)

func (*Index) MarshalTo Uses

func (m *Index) MarshalTo(dAtA []byte) (int, error)

func (*Index) MarshalToSizedBuffer Uses

func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Index) ProtoMessage Uses

func (*Index) ProtoMessage()

func (*Index) Reset Uses

func (m *Index) Reset()

func (*Index) Size Uses

func (m *Index) Size() (n int)

func (*Index) String Uses

func (m *Index) String() string

func (*Index) Unmarshal Uses

func (m *Index) Unmarshal(dAtA []byte) error

func (*Index) XXX_DiscardUnknown Uses

func (m *Index) XXX_DiscardUnknown()

func (*Index) XXX_Marshal Uses

func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Index) XXX_Merge Uses

func (m *Index) XXX_Merge(src proto.Message)

func (*Index) XXX_Size Uses

func (m *Index) XXX_Size() int

func (*Index) XXX_Unmarshal Uses

func (m *Index) XXX_Unmarshal(b []byte) error

type Op Uses

type Op int32

Op is the set of operations that can be associated with data.

const (
    Op_APPEND    Op  = 0
    Op_OVERWRITE Op  = 1
    Op_DELETE    Op  = 2
)

func (Op) EnumDescriptor Uses

func (Op) EnumDescriptor() ([]byte, []int)

func (Op) String Uses

func (x Op) String() string

type Range Uses

type Range struct {
    LastPath             string   `protobuf:"bytes,1,opt,name=last_path,json=lastPath,proto3" json:"last_path,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Range) Descriptor Uses

func (*Range) Descriptor() ([]byte, []int)

func (*Range) GetLastPath Uses

func (m *Range) GetLastPath() string

func (*Range) Marshal Uses

func (m *Range) Marshal() (dAtA []byte, err error)

func (*Range) MarshalTo Uses

func (m *Range) MarshalTo(dAtA []byte) (int, error)

func (*Range) MarshalToSizedBuffer Uses

func (m *Range) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Range) ProtoMessage Uses

func (*Range) ProtoMessage()

func (*Range) Reset Uses

func (m *Range) Reset()

func (*Range) Size Uses

func (m *Range) Size() (n int)

func (*Range) String Uses

func (m *Range) String() string

func (*Range) Unmarshal Uses

func (m *Range) Unmarshal(dAtA []byte) error

func (*Range) XXX_DiscardUnknown Uses

func (m *Range) XXX_DiscardUnknown()

func (*Range) XXX_Marshal Uses

func (m *Range) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Range) XXX_Merge Uses

func (m *Range) XXX_Merge(src proto.Message)

func (*Range) XXX_Size Uses

func (m *Range) XXX_Size() int

func (*Range) XXX_Unmarshal Uses

func (m *Range) XXX_Unmarshal(b []byte) error

type Reader Uses

type Reader struct {
    // contains filtered or unexported fields
}

Reader is used for reading a multi-level index.

func NewReader Uses

func NewReader(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path, prefix string) *Reader

NewReader create a new Reader.

func (*Reader) Close Uses

func (r *Reader) Close() error

Close closes the reader.

func (*Reader) Next Uses

func (r *Reader) Next() (*Header, error)

Next gets the next header in the index.

type Tag Uses

type Tag struct {
    Id                   string   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
    SizeBytes            int64    `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Tag) Descriptor Uses

func (*Tag) Descriptor() ([]byte, []int)

func (*Tag) GetId Uses

func (m *Tag) GetId() string

func (*Tag) GetSizeBytes Uses

func (m *Tag) GetSizeBytes() int64

func (*Tag) Marshal Uses

func (m *Tag) Marshal() (dAtA []byte, err error)

func (*Tag) MarshalTo Uses

func (m *Tag) MarshalTo(dAtA []byte) (int, error)

func (*Tag) MarshalToSizedBuffer Uses

func (m *Tag) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Tag) ProtoMessage Uses

func (*Tag) ProtoMessage()

func (*Tag) Reset Uses

func (m *Tag) Reset()

func (*Tag) Size Uses

func (m *Tag) Size() (n int)

func (*Tag) String Uses

func (m *Tag) String() string

func (*Tag) Unmarshal Uses

func (m *Tag) Unmarshal(dAtA []byte) error

func (*Tag) XXX_DiscardUnknown Uses

func (m *Tag) XXX_DiscardUnknown()

func (*Tag) XXX_Marshal Uses

func (m *Tag) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Tag) XXX_Merge Uses

func (m *Tag) XXX_Merge(src proto.Message)

func (*Tag) XXX_Size Uses

func (m *Tag) XXX_Size() int

func (*Tag) XXX_Unmarshal Uses

func (m *Tag) XXX_Unmarshal(b []byte) error

type Writer Uses

type Writer struct {
    // contains filtered or unexported fields
}

Writer is used for creating a multi-level index into a serialized FileSet. Each index level consists of compressed tar stream chunks. Each index tar entry has the full index in the content section.

func NewWriter Uses

func NewWriter(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path string, rangeSize ...int64) *Writer

NewWriter create a new Writer. rangeSize should not be used except for testing purposes, the defaultRangeSize will be used in a real deployment.

func (*Writer) Close Uses

func (w *Writer) Close() error

Close finishes the index, and returns the serialized top level index.

func (*Writer) WriteHeader Uses

func (w *Writer) WriteHeader(hdr *Header) error

WriteHeader writes a Header to the index.

Package index imports 14 packages (graph) and is imported by 1 packages. Updated 2019-08-13. Refresh now. Tools for package owners.