pachyderm: github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index Index | Files

package index

import "github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"

Index

Package Files

index.pb.go option.go reader.go util.go writer.go

Variables

var (
    ErrInvalidLengthIndex        = fmt.Errorf("proto: negative length found during unmarshaling")
    ErrIntOverflowIndex          = fmt.Errorf("proto: integer overflow")
    ErrUnexpectedEndOfGroupIndex = fmt.Errorf("proto: unexpected end of group")
)
var Op_name = map[int32]string{
    0:  "APPEND",
    1:  "OVERWRITE",
    2:  "DELETE",
}
var Op_value = map[string]int32{
    "APPEND":    0,
    "OVERWRITE": 1,
    "DELETE":    2,
}

func BeforeBound Uses

func BeforeBound(str string, strBound ...string) bool

BeforeBound checks if the passed in string is before the string bound (exclusive). The string bound is optional, so if no string bound is passed then it returns true.

func Generate Uses

func Generate(s string) []string

Generate generates the permutations of the passed in string and returns them sorted.

func Perm Uses

func Perm(a []rune, f func([]rune))

Perm calls f with each permutation of a.

type Copy Uses

type Copy struct {
    // contains filtered or unexported fields
}

Copy is the basic data structure to represent a copy of data from a reader to a writer.

type DataOp Uses

type DataOp struct {
    DataRefs             []*chunk.DataRef `protobuf:"bytes,1,rep,name=data_refs,json=dataRefs,proto3" json:"data_refs,omitempty"`
    Op                   Op               `protobuf:"varint,2,opt,name=op,proto3,enum=index.Op" json:"op,omitempty"`
    Tags                 []*Tag           `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"`
    XXX_NoUnkeyedLiteral struct{}         `json:"-"`
    XXX_unrecognized     []byte           `json:"-"`
    XXX_sizecache        int32            `json:"-"`
}

DataOp is a sequence of data references and an operation associated with the referenced data. Tags map identifiers to data in the resulting byte stream.

func (*DataOp) Descriptor Uses

func (*DataOp) Descriptor() ([]byte, []int)

func (*DataOp) GetDataRefs Uses

func (m *DataOp) GetDataRefs() []*chunk.DataRef

func (*DataOp) GetOp Uses

func (m *DataOp) GetOp() Op

func (*DataOp) GetTags Uses

func (m *DataOp) GetTags() []*Tag

func (*DataOp) Marshal Uses

func (m *DataOp) Marshal() (dAtA []byte, err error)

func (*DataOp) MarshalTo Uses

func (m *DataOp) MarshalTo(dAtA []byte) (int, error)

func (*DataOp) MarshalToSizedBuffer Uses

func (m *DataOp) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*DataOp) ProtoMessage Uses

func (*DataOp) ProtoMessage()

func (*DataOp) Reset Uses

func (m *DataOp) Reset()

func (*DataOp) Size Uses

func (m *DataOp) Size() (n int)

func (*DataOp) String Uses

func (m *DataOp) String() string

func (*DataOp) Unmarshal Uses

func (m *DataOp) Unmarshal(dAtA []byte) error

func (*DataOp) XXX_DiscardUnknown Uses

func (m *DataOp) XXX_DiscardUnknown()

func (*DataOp) XXX_Marshal Uses

func (m *DataOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*DataOp) XXX_Merge Uses

func (m *DataOp) XXX_Merge(src proto.Message)

func (*DataOp) XXX_Size Uses

func (m *DataOp) XXX_Size() int

func (*DataOp) XXX_Unmarshal Uses

func (m *DataOp) XXX_Unmarshal(b []byte) error
type Header struct {
    Hdr *tar.Header
    Idx *Index
}

Header is a wrapper for a tar header and index.

type Index Uses

type Index struct {
    Range  *Range  `protobuf:"bytes,1,opt,name=range,proto3" json:"range,omitempty"`
    DataOp *DataOp `protobuf:"bytes,2,opt,name=data_op,json=dataOp,proto3" json:"data_op,omitempty"`
    // Size of the content being indexed (does not include headers).
    SizeBytes            int64    `protobuf:"varint,3,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
    LastPathChunk        string   `protobuf:"bytes,4,opt,name=last_path_chunk,json=lastPathChunk,proto3" json:"last_path_chunk,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

Index stores an index to and metadata about a tar stream.

func (*Index) Descriptor Uses

func (*Index) Descriptor() ([]byte, []int)

func (*Index) GetDataOp Uses

func (m *Index) GetDataOp() *DataOp

func (*Index) GetLastPathChunk Uses

func (m *Index) GetLastPathChunk() string

func (*Index) GetRange Uses

func (m *Index) GetRange() *Range

func (*Index) GetSizeBytes Uses

func (m *Index) GetSizeBytes() int64

func (*Index) Marshal Uses

func (m *Index) Marshal() (dAtA []byte, err error)

func (*Index) MarshalTo Uses

func (m *Index) MarshalTo(dAtA []byte) (int, error)

func (*Index) MarshalToSizedBuffer Uses

func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Index) ProtoMessage Uses

func (*Index) ProtoMessage()

func (*Index) Reset Uses

func (m *Index) Reset()

func (*Index) Size Uses

func (m *Index) Size() (n int)

func (*Index) String Uses

func (m *Index) String() string

func (*Index) Unmarshal Uses

func (m *Index) Unmarshal(dAtA []byte) error

func (*Index) XXX_DiscardUnknown Uses

func (m *Index) XXX_DiscardUnknown()

func (*Index) XXX_Marshal Uses

func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Index) XXX_Merge Uses

func (m *Index) XXX_Merge(src proto.Message)

func (*Index) XXX_Size Uses

func (m *Index) XXX_Size() int

func (*Index) XXX_Unmarshal Uses

func (m *Index) XXX_Unmarshal(b []byte) error

type Op Uses

type Op int32

Op is the set of operations that can be associated with data.

const (
    Op_APPEND    Op  = 0
    Op_OVERWRITE Op  = 1
    Op_DELETE    Op  = 2
)

func (Op) EnumDescriptor Uses

func (Op) EnumDescriptor() ([]byte, []int)

func (Op) String Uses

func (x Op) String() string

type Option Uses

type Option func(r *Reader)

Option configures an index reader.

func WithPrefix Uses

func WithPrefix(prefix string) Option

WithPrefix sets a prefix filter for the read.

func WithRange Uses

func WithRange(pathRange *PathRange) Option

WithRange sets a range filter for the read.

type PathRange Uses

type PathRange struct {
    Lower, Upper string
}

PathRange is a range of paths. The range is inclusive: [Lower, Upper].

type Range Uses

type Range struct {
    Offset               int64    `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
    LastPath             string   `protobuf:"bytes,2,opt,name=last_path,json=lastPath,proto3" json:"last_path,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Range) Descriptor Uses

func (*Range) Descriptor() ([]byte, []int)

func (*Range) GetLastPath Uses

func (m *Range) GetLastPath() string

func (*Range) GetOffset Uses

func (m *Range) GetOffset() int64

func (*Range) Marshal Uses

func (m *Range) Marshal() (dAtA []byte, err error)

func (*Range) MarshalTo Uses

func (m *Range) MarshalTo(dAtA []byte) (int, error)

func (*Range) MarshalToSizedBuffer Uses

func (m *Range) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Range) ProtoMessage Uses

func (*Range) ProtoMessage()

func (*Range) Reset Uses

func (m *Range) Reset()

func (*Range) Size Uses

func (m *Range) Size() (n int)

func (*Range) String Uses

func (m *Range) String() string

func (*Range) Unmarshal Uses

func (m *Range) Unmarshal(dAtA []byte) error

func (*Range) XXX_DiscardUnknown Uses

func (m *Range) XXX_DiscardUnknown()

func (*Range) XXX_Marshal Uses

func (m *Range) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Range) XXX_Merge Uses

func (m *Range) XXX_Merge(src proto.Message)

func (*Range) XXX_Size Uses

func (m *Range) XXX_Size() int

func (*Range) XXX_Unmarshal Uses

func (m *Range) XXX_Unmarshal(b []byte) error

type Reader Uses

type Reader struct {
    // contains filtered or unexported fields
}

Reader is used for reading a multi-level index.

func NewReader Uses

func NewReader(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path string, opts ...Option) *Reader

NewReader create a new Reader.

func (*Reader) Close Uses

func (r *Reader) Close() error

Close closes the reader.

func (*Reader) Next Uses

func (r *Reader) Next() (*Header, error)

Next gets the next header in the index.

func (*Reader) Peek Uses

func (r *Reader) Peek() (*Header, error)

Peek peeks ahead in the index.

func (*Reader) ReadCopyFunc Uses

func (r *Reader) ReadCopyFunc(pathBound ...string) func() (*Copy, error)

ReadCopyFunc returns a function for copying data from the reader.

type Tag Uses

type Tag struct {
    Id                   string   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
    SizeBytes            int64    `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Tag) Descriptor Uses

func (*Tag) Descriptor() ([]byte, []int)

func (*Tag) GetId Uses

func (m *Tag) GetId() string

func (*Tag) GetSizeBytes Uses

func (m *Tag) GetSizeBytes() int64

func (*Tag) Marshal Uses

func (m *Tag) Marshal() (dAtA []byte, err error)

func (*Tag) MarshalTo Uses

func (m *Tag) MarshalTo(dAtA []byte) (int, error)

func (*Tag) MarshalToSizedBuffer Uses

func (m *Tag) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Tag) ProtoMessage Uses

func (*Tag) ProtoMessage()

func (*Tag) Reset Uses

func (m *Tag) Reset()

func (*Tag) Size Uses

func (m *Tag) Size() (n int)

func (*Tag) String Uses

func (m *Tag) String() string

func (*Tag) Unmarshal Uses

func (m *Tag) Unmarshal(dAtA []byte) error

func (*Tag) XXX_DiscardUnknown Uses

func (m *Tag) XXX_DiscardUnknown()

func (*Tag) XXX_Marshal Uses

func (m *Tag) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Tag) XXX_Merge Uses

func (m *Tag) XXX_Merge(src proto.Message)

func (*Tag) XXX_Size Uses

func (m *Tag) XXX_Size() int

func (*Tag) XXX_Unmarshal Uses

func (m *Tag) XXX_Unmarshal(b []byte) error

type Writer Uses

type Writer struct {
    // contains filtered or unexported fields
}

Writer is used for creating a multi-level index into a serialized FileSet. Each index level consists of compressed tar stream chunks. Each index tar entry has the full index in the content section.

func NewWriter Uses

func NewWriter(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path string) *Writer

NewWriter create a new Writer.

func (*Writer) Close Uses

func (w *Writer) Close() error

Close finishes the index, and returns the serialized top level index.

func (*Writer) WriteCopyFunc Uses

func (w *Writer) WriteCopyFunc(f func() (*Copy, error)) error

WriteCopyFunc executes a function for copying data to the writer.

func (*Writer) WriteHeaders Uses

func (w *Writer) WriteHeaders(hdrs []*Header) error

WriteHeaders writes a set of Header to the index.

Package index imports 15 packages (graph) and is imported by 1 packages. Updated 2019-10-19. Refresh now. Tools for package owners.