cof

package module
v0.0.0-...-b729e24 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 2, 2022 License: BSD-3-Clause Imports: 10 Imported by: 0

README

COF

Column Oriented Files. This is a package i extracted from a bigger project that i stumbled upon when i was looking through ~/projects directory on an old laptop that i had. The code was written in 2017. I want to see if adding genrics to it would make it better, but i'll leave that to a later time. I'm pushing it here just so i can refer to it when i need to.

The other parts which i didn't publish are:

  • Query Execution Engine: it accepts a json object query that can do aggregate functions, filteration, group by.
  • Web UI to interface with the query execution engine.

Docs from 2017

Writing to a file

urlCol := NewStringColumn()

schema := Table{
    "url": NewStringCol(),
    "serverIP": NewStringCol(),
    "clientIP": NewStringCol(),
    "timestamp": NewStringCol(),
}

store.New(schema)

bulk := []Record{}
record1 := Record{
    "url": "https://google.com",
    "serverIP": "94.0.0.1",
    "clientIP": "127.0.0.1",
    "timestamp": "1520000",
}
bulk = append(bulk, record1)
store.Batch(bulk)

Documentation

Overview

Package cof only writes to data to files it has no idea how you fetch data reading and fetching is implemented by fetcher

Index

Constants

View Source 
const MaxSize = 1e9

Variables

View Source 
var (
	ErrBadType = errors.New("given value was of a bad type")
)
View Source 
var (
	ErrNoTimeColumn = errors.New(`missing "time" column in Table`)
)

Functions

This section is empty.

Types

type BitsetColumn 

type BitsetColumn struct {
	// contains filtered or unexported fields
}

func NewBitsetCol 

func NewBitsetCol() *BitsetColumn

func (*BitsetColumn) Add 

func (i *BitsetColumn) Add(value interface{}) error

func (*BitsetColumn) Reset 

func (i *BitsetColumn) Reset() error

func (*BitsetColumn) Rows 

func (i *BitsetColumn) Rows() int

func (*BitsetColumn) Type 

func (c *BitsetColumn) Type() string

func (*BitsetColumn) WriteTo 

func (i *BitsetColumn) WriteTo(w io.Writer) (int64, error)

type BlockHeader 

type BlockHeader struct {
	Min  int64
	Max  int64
	Rows int64
	Size int64
}

type ColOffset 

type ColOffset struct {
	Name   string
	Offset int64
}

type Column 

type Column interface {
	Type() string
	WriteTo(io.Writer) (int64, error)
	Reset() error
	Add(value interface{}) error
	Rows() int
}

type ColumnValue 

type ColumnValue struct {
	Index int
	Value interface{}
}

func (*ColumnValue) Reset 

func (c *ColumnValue) Reset()

type DeltaColumn 

type DeltaColumn struct {
	// contains filtered or unexported fields
}

func NewDeltaCol 

func NewDeltaCol() *DeltaColumn

func (*DeltaColumn) Add 

func (i *DeltaColumn) Add(value interface{}) error

func (*DeltaColumn) Max 

func (i *DeltaColumn) Max() int64

func (*DeltaColumn) Min 

func (i *DeltaColumn) Min() int64

func (*DeltaColumn) Reset 

func (i *DeltaColumn) Reset() error

func (*DeltaColumn) Rows 

func (i *DeltaColumn) Rows() int

func (*DeltaColumn) Type 

func (c *DeltaColumn) Type() string

func (*DeltaColumn) WriteTo 

func (i *DeltaColumn) WriteTo(w io.Writer) (int64, error)

type DeltaIterator 

type DeltaIterator struct {
	// contains filtered or unexported fields
}

func (*DeltaIterator) Err 

func (iter *DeltaIterator) Err() error

func (*DeltaIterator) Next 

func (iter *DeltaIterator) Next() (*[]ColumnValue, error)

func (*DeltaIterator) Pos 

func (iter *DeltaIterator) Pos() int64

func (*DeltaIterator) Reset 

func (iter *DeltaIterator) Reset()

type Field 

type Field struct {
	Name string
	Type string
}

type GroupIterator 

type GroupIterator struct {
	TotalRows int64
	// contains filtered or unexported fields
}

func (*GroupIterator) Next 

func (gi *GroupIterator) Next() ([]Record, int, error)

type IntColumn 

type IntColumn struct {
	// contains filtered or unexported fields
}

func NewIntCol 

func NewIntCol() *IntColumn

func (*IntColumn) Add 

func (i *IntColumn) Add(value interface{}) error

func (*IntColumn) Max 

func (i *IntColumn) Max() int64

func (*IntColumn) Min 

func (i *IntColumn) Min() int64

func (*IntColumn) Reset 

func (i *IntColumn) Reset() error

func (*IntColumn) Rows 

func (i *IntColumn) Rows() int

func (*IntColumn) Type 

func (c *IntColumn) Type() string

func (*IntColumn) WriteTo 

func (i *IntColumn) WriteTo(w io.Writer) (int64, error)

type Iterator 

type Iterator interface {
	Next() (*[]ColumnValue, error)
	Err() error
	Pos() int64
}

func NewBitsetIterator 

func NewBitsetIterator(vectorSize int, start, end int64, buf []byte) (Iterator, error)

func NewDeltaIterator 

func NewDeltaIterator(vectorSize int, start, end int64, buf []byte) Iterator

func NewIntIterator 

func NewIntIterator(vectorSize int, start, end int64, buf []byte) Iterator

func NewStringIterator 

func NewStringIterator(vectorSize int, start, end int64, buf []byte) Iterator

type MinMaxer 

type MinMaxer interface {
	Max() int64
	Min() int64
}

type RLEDeltaColumn 

type RLEDeltaColumn struct {
	// contains filtered or unexported fields
}

func NewRLEDeltaCol 

func NewRLEDeltaCol() *RLEDeltaColumn

func (*RLEDeltaColumn) Add 

func (i *RLEDeltaColumn) Add(value interface{}) error

func (*RLEDeltaColumn) Max 

func (i *RLEDeltaColumn) Max() int64

func (*RLEDeltaColumn) Min 

func (i *RLEDeltaColumn) Min() int64

func (*RLEDeltaColumn) Reset 

func (i *RLEDeltaColumn) Reset() error

func (*RLEDeltaColumn) Rows 

func (i *RLEDeltaColumn) Rows() int

func (*RLEDeltaColumn) Type 

func (c *RLEDeltaColumn) Type() string

func (*RLEDeltaColumn) WriteTo 

func (i *RLEDeltaColumn) WriteTo(w io.Writer) (int64, error)

WriteTo compresses and writes the encoded data to w

type Reader 

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader 

func NewReader(buf []byte) *Reader

func (*Reader) DecodeGroup 

func (r *Reader) DecodeGroup(group *types.Group) ([]byte, error)

func (*Reader) Footer 

func (r *Reader) Footer() *types.Footer

func (*Reader) ReadFooter 

func (r *Reader) ReadFooter() error

func (*Reader) ReadGroup 

func (r *Reader) ReadGroup(group *types.Group, size int, columns ...string) (*GroupIterator, error)

type Record 

type Record map[string]interface{}

type Shard 

type Shard struct {
	// contains filtered or unexported fields
}

func (*Shard) AddColumn 

func (s *Shard) AddColumn(name string, col Column)

func (*Shard) Flush 

func (s *Shard) Flush(w io.Writer) error

type Store 

type Store struct {
	BlockSize int

	BatchSize int
	Footer    *types.Footer
	// contains filtered or unexported fields
}

func New 

func New(schema Table) (*Store, error)

func (*Store) Batch 

func (s *Store) Batch(records []Record) error

func (*Store) ResetAll 

func (s *Store) ResetAll() error

func (*Store) ShouldFlush 

func (s *Store) ShouldFlush() bool

func (*Store) WriteFooter 

func (s *Store) WriteFooter(w io.Writer) (int64, error)

func (*Store) WriteGroup 

func (s *Store) WriteGroup(w io.Writer) (int64, error)

type StringCol 

type StringCol struct {
	// contains filtered or unexported fields
}

func NewStringCol 

func NewStringCol() *StringCol

func (*StringCol) Add 

func (c *StringCol) Add(ival interface{}) error

func (*StringCol) Dict 

func (c *StringCol) Dict() map[string]int64

func (*StringCol) ReadFrom 

func (c *StringCol) ReadFrom(rdr io.Reader) []string

func (*StringCol) Reset 

func (c *StringCol) Reset() error

func (*StringCol) Rows 

func (s *StringCol) Rows() int

func (*StringCol) Type 

func (c *StringCol) Type() string

func (*StringCol) WriteTo 

func (c *StringCol) WriteTo(w io.Writer) (int64, error)

type Table 

type Table map[string]Column

Source Files

View all Source files

Directories

Path Synopsis
Package types is a generated protocol buffer package.
 Package types is a generated protocol buffer package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.