pdk

type AttrMapper ¶

type AttrMapper struct {
	Mapper  Mapper
	Parsers []Parser
	Fields  []int
}

AttrMapper is a struct for mapping some set of data fields to a value for sending to Pilosa as a SetColumnAttrs query

type BinaryFloatMapper ¶

type BinaryFloatMapper struct {
	Min      float64
	Max      float64
	BitDepth int
	// contains filtered or unexported fields
}

BinaryFloatMapper is a Mapper for float types, mapping to a set of buckets representing the value in a binary sense

func (m BinaryFloatMapper) ID(fi ...interface{}) (rowIDs []int64, err error)

type BinaryIntMapper ¶

type BinaryIntMapper struct {
	Min      int64
	Max      int64
	BitDepth int
	// contains filtered or unexported fields
}

BinaryIntMapper is a Mapper for int types, mapping to a set of buckets representing the value in a binary sense

func (m BinaryIntMapper) ID(ii ...interface{}) (rowIDs []int64, err error)

type Bit ¶

type Bit struct {
	// contains filtered or unexported fields
}

type BitMapper ¶

type BitMapper struct {
	Frame   string
	Mapper  Mapper
	Parsers []Parser
	Fields  []int
}

BitMapper is a struct for mapping some set of data fields to a (frame, id) combination for sending to Pilosa as a SetBit query

type BoltTranslator ¶

type BoltTranslator struct {
	Db *bolt.DB
	// contains filtered or unexported fields
}

BoltTranslator is a Translator which stores the two way val/id mapping in boltdb. It only accepts string values to map.

func NewBoltTranslator(filename string, frames ...string) (bt *BoltTranslator, err error)

func (bt *BoltTranslator) BulkAdd(frame string, values [][]byte) error

func (bt *BoltTranslator) Close() error

func (bt *BoltTranslator) Get(frame string, id uint64) (val interface{})

func (bt *BoltTranslator) GetID(frame string, val interface{}) (id uint64, err error)

type BoolMapper ¶

type BoolMapper struct {
}

BoolMapper is a trivial Mapper for boolean types

func (m BoolMapper) ID(bi ...interface{}) (rowIDs []int64, err error)

type BucketVLock ¶

type BucketVLock struct {
	// contains filtered or unexported fields
}

func NewBucketVLock() BucketVLock

func (b BucketVLock) Lock(val []byte)

func (b BucketVLock) Unlock(val []byte)

type Bytes ¶

type Bytes uint64

func (b Bytes) String() string

T: Terabyte
G: Gigabyte
M: Megabyte
K: Kilobyte
B: Byte

type ChanBitIterator ¶

type ChanBitIterator chan pcli.Bit

func NewChanBitIterator() ChanBitIterator

func (c ChanBitIterator) NextBit() (pcli.Bit, error)

type ChanValIterator ¶

type ChanValIterator chan pcli.FieldValue

func NewChanValIterator() ChanValIterator

func (c ChanValIterator) NextValue() (pcli.FieldValue, error)

type CustomMapper ¶

type CustomMapper struct {
	Func   func(...interface{}) interface{}
	Mapper Mapper
}

CustomMapper is a Mapper that applies a function to a slice of fields, then applies a simple Mapper to the result of that, returning a rowID. This is a generic way to support mappings which span multiple fields. It is not supported by the importing config system.

func (m CustomMapper) ID(fields ...interface{}) (rowIDs []int64, err error)

type DayOfMonthMapper ¶

type DayOfMonthMapper struct {
}

func (m DayOfMonthMapper) ID(ti ...interface{}) (rowIDs []int64, err error)

type DayOfWeekMapper ¶

type DayOfWeekMapper struct {
}

DayOfWeekMapper is a Mapper for timestamps, mapping the day of week only

func (m DayOfWeekMapper) ID(ti ...interface{}) (rowIDs []int64, err error)

type Errors ¶

type Errors []error

func (errs Errors) Error() string

type FieldSpec ¶

type FieldSpec struct {
	Name string
	Min  int
	Max  int
}

type FileFragment ¶

type FileFragment struct {
	// contains filtered or unexported fields
}

func NewFileFragment(f *os.File, startLoc, endLoc int64) (*FileFragment, error)

func SplitFileLines(f *os.File, numParts int64) ([]*FileFragment, error)

func (ff *FileFragment) Close() error

func (ff *FileFragment) Read(b []byte) (n int, err error)

type FloatMapper ¶

type FloatMapper struct {
	Buckets []float64 // slice representing bucket intervals [left0 left1 ... leftN-1 rightN-1]
	// contains filtered or unexported fields
}

FloatMapper is a Mapper for float types, mapping to arbitrary buckets

func (m FloatMapper) ID(fi ...interface{}) (rowIDs []int64, err error)

type FloatParser ¶

type FloatParser struct {
}

FloatParser is a parser for float types

func (p FloatParser) Parse(field string) (result interface{}, err error)

type FrameSpec ¶

type FrameSpec struct {
	Name           string
	CacheType      pcli.CacheType
	CacheSize      uint
	InverseEnabled bool
	Fields         []FieldSpec
}

func NewFieldFrameSpec(name string, min int, max int) FrameSpec

func NewRankedFrameSpec(name string, size int) FrameSpec

type GridMapper ¶

type GridMapper struct {
	Xmin float64
	Xmax float64
	Xres int64
	Ymin float64
	Ymax float64
	Yres int64
	// contains filtered or unexported fields
}

GridMapper is a Mapper for a 2D grid (e.g. small-scale latitude/longitude)

func (m GridMapper) ID(xyi ...interface{}) (rowIDs []int64, err error)

type GridToFloatMapper ¶

type GridToFloatMapper struct {
	// contains filtered or unexported fields
}

func NewGridToFloatMapper(gm GridMapper, lfm LinearFloatMapper, gridVals []float64) GridToFloatMapper

func (m GridToFloatMapper) ID(vals ...interface{}) ([]int64, error)

type INexter ¶

type INexter interface {
	Next() uint64
	Last() uint64
}

type IPParser ¶

type IPParser struct {
}

IPParser is a parser for IP addresses

func (p IPParser) Parse(field string) (result interface{}, err error)

type ImportClient ¶

type ImportClient struct {
	BufferSize int
	// contains filtered or unexported fields
}

func NewImportClient(host, index string, frames []string, bufsize int) *ImportClient

func (ic *ImportClient) Close()

func (ic *ImportClient) SetBit(rowID, columnID uint64, frame string)

func (ic *ImportClient) SetBitTimestamp(rowID, columnID uint64, frame string, timestamp time.Time)

type Index ¶

type Index struct {
	// contains filtered or unexported fields
}

func NewIndex() *Index

func (i *Index) AddBit(frame string, col uint64, row uint64)

func (i *Index) AddValue(frame, field string, col uint64, val int64)

func (i *Index) Client() *pcli.Client

func (i *Index) Close() error

type Indexer ¶

type Indexer interface {
	AddBit(frame string, col uint64, row uint64)
	AddValue(frame, field string, col uint64, val int64)
	Close() error
	Client() *pcli.Client
}

func SetupPilosa(hosts []string, index string, frames []FrameSpec, batchsize uint) (Indexer, error)

type Ingester ¶

type Ingester struct {
	ParseConcurrency int
	// contains filtered or unexported fields
}

func NewIngester(source Source, parser Parrrser, mapper Mapppper, indexer Indexer) *Ingester

func (n *Ingester) Run() error

type IntMapper ¶

type IntMapper struct {
	Min int64
	Max int64
	Res int64 // number of bins
	// contains filtered or unexported fields
}

IntMapper is a Mapper for integer types, mapping each int in the range to a row

func (m IntMapper) ID(ii ...interface{}) (rowIDs []int64, err error)

type IntParser ¶

type IntParser struct {
}

IntParser is a parser for integer types

func (p IntParser) Parse(field string) (result interface{}, err error)

type KeyMapper ¶

type KeyMapper interface {
	MapRequest(body []byte) ([]byte, error)
	MapResult(frame string, res interface{}) (interface{}, error)
}

KeyMapper describes the functionality for mapping the keys contained in requests and responses.

type LevelTranslator ¶

type LevelTranslator struct {
	// contains filtered or unexported fields
}

LevelTranslator is a Translator which stores the two way val/id mapping in leveldb.

func NewLevelTranslator(dirname string, frames ...string) (lt *LevelTranslator, err error)

func (lt *LevelTranslator) Close() error

func (lt *LevelTranslator) Get(frame string, id uint64) (val interface{})

func (lt *LevelTranslator) GetID(frame string, val interface{}) (id uint64, err error)

type LinearFloatMapper ¶

type LinearFloatMapper struct {
	Min   float64
	Max   float64
	Res   float64
	Scale string // linear, logarithmic
	// contains filtered or unexported fields
}

LinearFloatMapper is a Mapper for float types, mapping to regularly spaced buckets TODO: consider defining this in terms of a linear mapping ID = floor(a*value + b)

func (m LinearFloatMapper) ID(fi ...interface{}) (rowIDs []int64, err error)

type Mapper ¶

type Mapper interface {
	ID(...interface{}) ([]int64, error)
}

Mapper represents a single method for mapping a specific data type to a slice of row IDs. A data type might be composed of multiple fields (e.g. a 2D point). A data type might use multiple mappers.

type Mapppper ¶

type Mapppper interface {
	Map(parsedRecord interface{}) (PilosaRecord, error)
}

Mapper is the interface for taking parsed records from the Parser and figuring out what bits and values to set in Pilosa. Mappers usually have a Translator and a Nexter for converting arbitrary values to monotonic integer ids and generating column ids respectively.

type MonthMapper ¶

type MonthMapper struct {
}

MonthMapper is a Mapper for timestamps, mapping the month only

func (m MonthMapper) ID(ti ...interface{}) (rowIDs []int64, err error)

type Nexter ¶

type Nexter struct {
	// contains filtered or unexported fields
}

Nexter is a threadsafe monotonic unique id generator

func NewNexter(opts ...NexterOption) *Nexter

func (n *Nexter) Last() (lastID uint64)

func (n *Nexter) Next() (nextID uint64)

type NexterOption ¶

type NexterOption func(n *Nexter)

NexterOption can be passed to NewNexter to modify the Nexter's behavior.

type Parrrser ¶

type Parrrser interface {
	Parse(record []byte) (interface{}, error)
}

Parser is the interface for turning raw records from Source into Go objects. Implementations of Parser should be thread safe. The current naming is a temporary workaround until the previous Parser interface can be deprecated.

type Parser ¶

type Parser interface {
	Parse(string) (interface{}, error)
}

Parser represents a single method for parsing a string field to a value

type PilosaImporter ¶

type PilosaImporter interface {
	SetBit(rowID, columnID uint64, frame string)
	SetBitTimestamp(rowID, columnID uint64, frame string, timestamp time.Time)
	Close()
}

type PilosaKeyMapper ¶

type PilosaKeyMapper struct {
	// contains filtered or unexported fields
}

PilosaKeyMapper implements the KeyMapper interface.

func NewPilosaKeyMapper(t Translator) *PilosaKeyMapper

func (p *PilosaKeyMapper) MapRequest(body []byte) ([]byte, error)

func (p *PilosaKeyMapper) MapResult(frame string, res interface{}) (mappedRes interface{}, err error)

type PilosaRecord ¶

type PilosaRecord struct {
	Col  uint64
	Bits []SetBit
	Vals []Val
}

PilosaRecord represents a number of set bits and values in a single Column in Pilosa.

type Point ¶

type Point struct {
	X float64
	Y float64
}

Point is a point in a 2D space

type Proxy ¶

type Proxy interface {
	ProxyRequest(orig *http.Request, origbody []byte) (*http.Response, error)
}

Proxy describes the functionality for proxying requests.

type Region ¶

type Region struct {
	Vertices []Point
}

Region is a simple polygonal region of R2 space

type RegionMapper ¶

type RegionMapper struct {
	Regions []Region
	// contains filtered or unexported fields
}

RegionMapper is a Mapper for a set of geometric regions (e.g. neighborhoods or states) TODO: generate regions by reading shapefile

type SetBit ¶

type SetBit struct {
	Frame string
	Row   uint64
	Time  time.Time
}

SetBit represents a bit to set in Pilosa sans column id (which is held by the PilosaRecord containg the SetBit).

type SingleVLock ¶

type SingleVLock struct {
	// contains filtered or unexported fields
}

func NewSingleVLock() SingleVLock

func (s SingleVLock) Lock(val []byte)

func (s SingleVLock) Unlock(val []byte)

type Source ¶

type Source interface {
	Record() ([]byte, error)
}

Source is the interface for getting raw data one record at a time. Implementations of Source should be thread safe.

type SparseIntMapper ¶

type SparseIntMapper struct {
	Min int64
	Max int64
	Map map[int64]int64
	// contains filtered or unexported fields
}

SparseIntMapper is a Mapper for integer types, mapping only relevant ints

func (m SparseIntMapper) ID(ii ...interface{}) (rowIDs []int64, err error)

type StringContainsMapper ¶

type StringContainsMapper struct {
	Matches []string // slice of strings to check for containment
}

StringContainsMapper is a Mapper for string types...

type StringMatchesMapper ¶

type StringMatchesMapper struct {
	Matches []string // slice of strings to check for match
}

StringMatchesMapper is a Mapper for string types...

type StringParser ¶

type StringParser struct {
}

StringParser is a parser for string types

func (p StringParser) Parse(field string) (result interface{}, err error)

type TimeOfDayMapper ¶

type TimeOfDayMapper struct {
	Res int64
}

TimeOfDayMapper is a Mapper for timestamps, mapping the time component only TODO: consider putting all time buckets in same frame pros: single frame cons: would have to abandon the simple ID interface. also single frame may not be a good thing

func (m TimeOfDayMapper) ID(ti ...interface{}) (rowIDs []int64, err error)

type TimeParser ¶

type TimeParser struct {
	Layout string
}

TimeParser is a parser for timestamps

func (p TimeParser) Parse(field string) (result interface{}, err error)

type Translator ¶

type Translator interface {
	// Get must be safe for concurrent access
	Get(frame string, id uint64) interface{}
	GetID(frame string, val interface{}) (uint64, error)
}

Translator describes the functionality which the proxy server requires to translate row ids to what they were mapped from.

type Val ¶

type Val struct {
	Frame string
	Field string
	Value int64
}

Val represents a BSI value to set in a Pilosa field sans column id (which is held by the PilosaRecord containg the Val).

type ValueLocker ¶

type ValueLocker interface {
	Lock(val []byte)
	Unlock(val []byte)
}

type YearMapper ¶

type YearMapper struct {
	MinYear int64 // TODO? use this to eliminate empty rows for year < 2000 or whatever
}

YearMapper is a Mapper for timestamps, mapping the year only

func (m YearMapper) ID(ti ...interface{}) (rowIDs []int64, err error)