pachyderm: github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index Index | Files

package index

import "github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"

Index

Package Files

index.pb.go option.go reader.go util.go writer.go

Variables

var (
    ErrInvalidLengthIndex        = fmt.Errorf("proto: negative length found during unmarshaling")
    ErrIntOverflowIndex          = fmt.Errorf("proto: integer overflow")
    ErrUnexpectedEndOfGroupIndex = fmt.Errorf("proto: unexpected end of group")
)
var Op_name = map[int32]string{
    0:  "APPEND",
    1:  "OVERWRITE",
    2:  "DELETE",
}
var Op_value = map[string]int32{
    "APPEND":    0,
    "OVERWRITE": 1,
    "DELETE":    2,
}

func Generate Uses

func Generate(s string) []string

Generate generates the permutations of the passed in string and returns them sorted.

func Perm Uses

func Perm(a []rune, f func([]rune))

Perm calls f with each permutation of a.

type DataOp Uses

type DataOp struct {
    DataRefs             []*chunk.DataRef `protobuf:"bytes,1,rep,name=data_refs,json=dataRefs,proto3" json:"data_refs,omitempty"`
    Op                   Op               `protobuf:"varint,2,opt,name=op,proto3,enum=index.Op" json:"op,omitempty"`
    XXX_NoUnkeyedLiteral struct{}         `json:"-"`
    XXX_unrecognized     []byte           `json:"-"`
    XXX_sizecache        int32            `json:"-"`
}

DataOp is a sequence of data references and an operation associated with the referenced data.

func (*DataOp) Descriptor Uses

func (*DataOp) Descriptor() ([]byte, []int)

func (*DataOp) GetDataRefs Uses

func (m *DataOp) GetDataRefs() []*chunk.DataRef

func (*DataOp) GetOp Uses

func (m *DataOp) GetOp() Op

func (*DataOp) Marshal Uses

func (m *DataOp) Marshal() (dAtA []byte, err error)

func (*DataOp) MarshalTo Uses

func (m *DataOp) MarshalTo(dAtA []byte) (int, error)

func (*DataOp) MarshalToSizedBuffer Uses

func (m *DataOp) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*DataOp) ProtoMessage Uses

func (*DataOp) ProtoMessage()

func (*DataOp) Reset Uses

func (m *DataOp) Reset()

func (*DataOp) Size Uses

func (m *DataOp) Size() (n int)

func (*DataOp) String Uses

func (m *DataOp) String() string

func (*DataOp) Unmarshal Uses

func (m *DataOp) Unmarshal(dAtA []byte) error

func (*DataOp) XXX_DiscardUnknown Uses

func (m *DataOp) XXX_DiscardUnknown()

func (*DataOp) XXX_Marshal Uses

func (m *DataOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*DataOp) XXX_Merge Uses

func (m *DataOp) XXX_Merge(src proto.Message)

func (*DataOp) XXX_Size Uses

func (m *DataOp) XXX_Size() int

func (*DataOp) XXX_Unmarshal Uses

func (m *DataOp) XXX_Unmarshal(b []byte) error

type Index Uses

type Index struct {
    Path   string  `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
    Range  *Range  `protobuf:"bytes,2,opt,name=range,proto3" json:"range,omitempty"`
    DataOp *DataOp `protobuf:"bytes,3,opt,name=data_op,json=dataOp,proto3" json:"data_op,omitempty"`
    // Size of the content being indexed (does not include headers).
    SizeBytes            int64    `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

Index stores an index to and metadata about file sets.

func GetTopLevelIndex Uses

func GetTopLevelIndex(ctx context.Context, objC obj.Client, path string) (*Index, error)

GetTopLevelIndex gets the top level index entry for a file set, which contains metadata for the file set.

func (*Index) Descriptor Uses

func (*Index) Descriptor() ([]byte, []int)

func (*Index) GetDataOp Uses

func (m *Index) GetDataOp() *DataOp

func (*Index) GetPath Uses

func (m *Index) GetPath() string

func (*Index) GetRange Uses

func (m *Index) GetRange() *Range

func (*Index) GetSizeBytes Uses

func (m *Index) GetSizeBytes() int64

func (*Index) Marshal Uses

func (m *Index) Marshal() (dAtA []byte, err error)

func (*Index) MarshalTo Uses

func (m *Index) MarshalTo(dAtA []byte) (int, error)

func (*Index) MarshalToSizedBuffer Uses

func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Index) ProtoMessage Uses

func (*Index) ProtoMessage()

func (*Index) Reset Uses

func (m *Index) Reset()

func (*Index) Size Uses

func (m *Index) Size() (n int)

func (*Index) String Uses

func (m *Index) String() string

func (*Index) Unmarshal Uses

func (m *Index) Unmarshal(dAtA []byte) error

func (*Index) XXX_DiscardUnknown Uses

func (m *Index) XXX_DiscardUnknown()

func (*Index) XXX_Marshal Uses

func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Index) XXX_Merge Uses

func (m *Index) XXX_Merge(src proto.Message)

func (*Index) XXX_Size Uses

func (m *Index) XXX_Size() int

func (*Index) XXX_Unmarshal Uses

func (m *Index) XXX_Unmarshal(b []byte) error

type Op Uses

type Op int32

Op is the set of operations that can be associated with data.

const (
    Op_APPEND    Op  = 0
    Op_OVERWRITE Op  = 1
    Op_DELETE    Op  = 2
)

func (Op) EnumDescriptor Uses

func (Op) EnumDescriptor() ([]byte, []int)

func (Op) String Uses

func (x Op) String() string

type Option Uses

type Option func(r *Reader)

Option configures an index reader.

func WithPrefix Uses

func WithPrefix(prefix string) Option

WithPrefix sets a prefix filter for the read.

func WithRange Uses

func WithRange(pathRange *PathRange) Option

WithRange sets a range filter for the read.

type PathRange Uses

type PathRange struct {
    Lower, Upper string
}

PathRange is a range of paths. The range is inclusive: [Lower, Upper].

type Range Uses

type Range struct {
    Offset               int64    `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
    LastPath             string   `protobuf:"bytes,2,opt,name=last_path,json=lastPath,proto3" json:"last_path,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Range) Descriptor Uses

func (*Range) Descriptor() ([]byte, []int)

func (*Range) GetLastPath Uses

func (m *Range) GetLastPath() string

func (*Range) GetOffset Uses

func (m *Range) GetOffset() int64

func (*Range) Marshal Uses

func (m *Range) Marshal() (dAtA []byte, err error)

func (*Range) MarshalTo Uses

func (m *Range) MarshalTo(dAtA []byte) (int, error)

func (*Range) MarshalToSizedBuffer Uses

func (m *Range) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Range) ProtoMessage Uses

func (*Range) ProtoMessage()

func (*Range) Reset Uses

func (m *Range) Reset()

func (*Range) Size Uses

func (m *Range) Size() (n int)

func (*Range) String Uses

func (m *Range) String() string

func (*Range) Unmarshal Uses

func (m *Range) Unmarshal(dAtA []byte) error

func (*Range) XXX_DiscardUnknown Uses

func (m *Range) XXX_DiscardUnknown()

func (*Range) XXX_Marshal Uses

func (m *Range) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Range) XXX_Merge Uses

func (m *Range) XXX_Merge(src proto.Message)

func (*Range) XXX_Size Uses

func (m *Range) XXX_Size() int

func (*Range) XXX_Unmarshal Uses

func (m *Range) XXX_Unmarshal(b []byte) error

type Reader Uses

type Reader struct {
    // contains filtered or unexported fields
}

Reader is used for reading a multilevel index.

func NewReader Uses

func NewReader(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path string, opts ...Option) *Reader

NewReader create a new Reader.

func (*Reader) Iterate Uses

func (r *Reader) Iterate(f func(*Index) error, pathBound ...string) error

Iterate iterates over the indexes. pathBound is an optional parameter for specifiying the upper bound (exclusive) of the iteration.

func (*Reader) Next Uses

func (r *Reader) Next() (*Index, error)

Next returns the next index and progresses the reader.

func (*Reader) Peek Uses

func (r *Reader) Peek() (*Index, error)

Peek returns the next index without progressing the reader.

type Writer Uses

type Writer struct {
    // contains filtered or unexported fields
}

Writer is used for creating a multilevel index into a serialized file set. Each index level is a stream of byte length encoded index entries that are stored in chunk storage.

func NewWriter Uses

func NewWriter(ctx context.Context, objC obj.Client, chunks *chunk.Storage, path string) *Writer

NewWriter create a new Writer.

func (*Writer) Close Uses

func (w *Writer) Close() error

Close finishes the index, and returns the serialized top index level.

func (*Writer) WriteIndexes Uses

func (w *Writer) WriteIndexes(idxs []*Index) error

WriteIndexes writes a set of index entries.

Package index imports 12 packages (graph) and is imported by 3 packages. Updated 2020-03-30. Refresh now. Tools for package owners.