chunkenc

package

v0.1.0 Latest Latest Go to latest Published: Jun 3, 2019 License: Apache-2.0 Imports: 17 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/pracucci/loki

Links

Open Source Insights

README ¶

Chunk format

  |                 |             |
  | MagicNumber(4b) | version(1b) |
  |                 |             |
  --------------------------------------------------
  |         block-1 bytes         |  checksum (4b) |
  --------------------------------------------------
  |         block-2 bytes         |  checksum (4b) |
  --------------------------------------------------
  |         block-n bytes         |  checksum (4b) |
  --------------------------------------------------
  |         #blocks (uvarint)                      |
  --------------------------------------------------
  | #entries(uvarint) | mint, maxt (varint) | offset, len (uvarint) |
  -------------------------------------------------------------------
  | #entries(uvarint) | mint, maxt (varint) | offset, len (uvarint) |
  -------------------------------------------------------------------
  | #entries(uvarint) | mint, maxt (varint) | offset, len (uvarint) |
  -------------------------------------------------------------------
  | #entries(uvarint) | mint, maxt (varint) | offset, len (uvarint) |
  -------------------------------------------------------------------
  |                      checksum(from #blocks)                     |
  -------------------------------------------------------------------
  | metasOffset - offset to the point with #blocks |
  --------------------------------------------------

Documentation ¶

Index ¶

Constants
Variables
func NewFacade(c Chunk) encoding.Chunk
type Chunk
- func NewDumbChunk() Chunk
type CompressionReader
type CompressionWriter
type Encoding
- func (e Encoding) String() string
type Facade
type LazyChunk
- func (c LazyChunk) Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error)
type MemChunk

Constants ¶

View Source

const GzipLogChunk = encoding.Encoding(128)

GzipLogChunk is a cortex encoding type for our chunks.

Variables ¶

View Source

var (
	ErrChunkFull       = errors.New("chunk full")
	ErrOutOfOrder      = errors.New("entry out of order")
	ErrInvalidSize     = errors.New("invalid size")
	ErrInvalidFlag     = errors.New("invalid flag")
	ErrInvalidChecksum = errors.New("invalid checksum")
)

Errors returned by the chunk interface.

Functions ¶

func NewFacade ¶

func NewFacade(c Chunk) encoding.Chunk

NewFacade makes a new Facade.

Types ¶

type Chunk ¶

type Chunk interface {
	Bounds() (time.Time, time.Time)
	SpaceFor(*logproto.Entry) bool
	Append(*logproto.Entry) error
	Iterator(from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error)
	Size() int
	Bytes() ([]byte, error)
}

Chunk is the interface for the compressed logs chunk format.

func NewDumbChunk ¶

func NewDumbChunk() Chunk

NewDumbChunk returns a new chunk that isn't very good.

type CompressionReader ¶

type CompressionReader interface {
	Read(p []byte) (int, error)
	Reset(r io.Reader) error
}

CompressionReader reads the compressed data.

type CompressionWriter ¶

type CompressionWriter interface {
	Write(p []byte) (int, error)
	Close() error
	Flush() error
	Reset(w io.Writer)
}

CompressionWriter is the writer that compresses the data passed to it.

type Encoding ¶

type Encoding uint8

Encoding is the identifier for a chunk encoding.

const (
	EncNone Encoding = iota
	EncGZIP
	EncDumb
)

The different available encodings.

func (Encoding) String ¶

func (e Encoding) String() string

type Facade ¶

type Facade struct {
	encoding.Chunk
	// contains filtered or unexported fields
}

Facade for compatibility with cortex chunk type, so we can use it's chunk store.

func (Facade) Encoding ¶

func (Facade) Encoding() encoding.Encoding

Encoding implements encoding.Chunk.

func (Facade) LokiChunk ¶

func (f Facade) LokiChunk() Chunk

LokiChunk returns the chunkenc.Chunk.

func (Facade) Marshal ¶

func (f Facade) Marshal(w io.Writer) error

Marshal implements encoding.Chunk.

func (*Facade) UnmarshalFromBuf ¶

func (f *Facade) UnmarshalFromBuf(buf []byte) error

UnmarshalFromBuf implements encoding.Chunk.

type LazyChunk ¶

type LazyChunk struct {
	Chunk   chunk.Chunk
	Fetcher *chunk.Fetcher
}

LazyChunk loads the chunk when it is accessed.

func (LazyChunk) Iterator ¶

func (c LazyChunk) Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction) (iter.EntryIterator, error)

Iterator returns an entry iterator.

type MemChunk ¶

type MemChunk struct {
	// contains filtered or unexported fields
}

MemChunk implements compressed log chunks.

func NewByteChunk ¶

func NewByteChunk(b []byte) (*MemChunk, error)

NewByteChunk returns a MemChunk on the passed bytes.

func NewMemChunk ¶

func NewMemChunk(enc Encoding) *MemChunk

NewMemChunk returns a new in-mem chunk for query.

func NewMemChunkSize ¶

func NewMemChunkSize(enc Encoding, blockSize int) *MemChunk

NewMemChunkSize returns a new in-mem chunk. Mainly for config push size.

func (*MemChunk) Append ¶

func (c *MemChunk) Append(entry *logproto.Entry) error

Append implements Chunk.

func (*MemChunk) Bounds ¶

func (c *MemChunk) Bounds() (fromT, toT time.Time)

Bounds implements Chunk.

func (*MemChunk) Bytes ¶

func (c *MemChunk) Bytes() ([]byte, error)

Bytes implements Chunk.

func (*MemChunk) Close ¶

func (c *MemChunk) Close() error

Close implements Chunk. TODO: Fix this to check edge cases.

func (*MemChunk) Encoding ¶

func (c *MemChunk) Encoding() Encoding

Encoding implements Chunk.

func (*MemChunk) Iterator ¶

func (c *MemChunk) Iterator(mintT, maxtT time.Time, direction logproto.Direction) (iter.EntryIterator, error)

Iterator implements Chunk.

func (*MemChunk) Size ¶

func (c *MemChunk) Size() int

Size implements Chunk.

func (*MemChunk) SpaceFor ¶

func (c *MemChunk) SpaceFor(*logproto.Entry) bool

SpaceFor implements Chunk.

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL