storage

package module
v0.0.0-...-b51351f Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 18, 2018 License: Apache-2.0 Imports: 13 Imported by: 0

README

WARNING: This is Alpha software and not intended for use until a stable release.

M3Storage GoDoc Build Status Coverage Status

Tiered storage manager. Supports time series queries across multiple storage clusters, with support for aggregating results across retention periods, smart resolution downsampling, and graceful migration of time series data between clusters as capacity requirements change.


This project is released under the Apache License, Version 2.0.

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Cluster

type Cluster interface {
	// Name is the name of the cluster
	Name() string

	// Type is the storage type for the cluster
	Type() Type

	// Config is the cluster's configuration
	Config() Config

	// Database is the name of the database to which the cluster belongs
	Database() string
}

A Cluster defines a cluster of nodes within a database

type ClusterMappingProvider

type ClusterMappingProvider interface {
	xclose.Closer

	// QueryMappings returns the active cluster mappings for the given query
	QueryMappings(shard uint32, start, end time.Time) (MappingRuleIter, error)

	// WatchCluster returns the config for a cluster as a watch that can
	// be used to listen for updates to that cluster.  Callers must wait
	// on the watch channel before attempting to access the Cluster
	WatchCluster(database, cluster string) (ClusterWatch, error)
}

A ClusterMappingProvider provides shard mapping rules and cluster watches

type ClusterWatch

type ClusterWatch interface {
	xclose.Closer

	// C is the channel receiving notifications of config changes
	C() <-chan struct{}

	// Get returns the current state of the cluster
	Get() Cluster
}

A ClusterWatch watches for config changes on a cluster

func NewClusterWatch

func NewClusterWatch(w xwatch.Watch) ClusterWatch

NewClusterWatch wraps a ClusterWatch around an existing Watch

type Config

type Config interface {
	// Version is the version of the configuration
	Version() int

	// Unmarshal unmarshals the configuration
	Unmarshal(c proto.Message) error
}

A Config is configuration for a cluster

type Connection

type Connection interface {
	xclose.Closer

	// Read reads datapoints for the given id at the given time range and resolution
	Read(id string, r retention.Resolution, start, end time.Time) (SeriesIter, error)

	// Write writes a datapoint for the given id at the given time range
	Write(id string, r retention.Resolution, t time.Time, value float64) error
}

A Connection is a connection to a storage cluster, which can be used to read and write datapoints to that cluster

type ConnectionManager

type ConnectionManager interface {
	xclose.Closer

	// GetConnection retrieves the connection to the given database and cluster
	GetConnection(database, cluster string) (Connection, error)
}

The ConnectionManager manages connections to tsdb clusters, creating them as needed and keeping them up to date with configuration changes

func NewConnectionManager

func NewConnectionManager(opts ConnectionManagerOptions) (ConnectionManager, error)

NewConnectionManager creates a new connection manager based on the provided options

type ConnectionManagerOptions

type ConnectionManagerOptions interface {
	// Logger is the logger to use
	Logger(log xlog.Logger) ConnectionManagerOptions
	GetLogger() xlog.Logger

	// Provider provides information about active clusters
	Provider(p ClusterMappingProvider) ConnectionManagerOptions
	GetProvider() ClusterMappingProvider

	// Drivers are the set of drivers used by this ConnectionManager
	Drivers(d []Driver) ConnectionManagerOptions
	GetDrivers() []Driver

	// Validate validates the options
	Validate() error
}

ConnectionManagerOptions are creation time options for the ConnectionManager

func NewConnectionManagerOptions

func NewConnectionManagerOptions() ConnectionManagerOptions

NewConnectionManagerOptions returns new empty ConnectionManagerOptions

type Database

type Database interface {
	// Name is the name of the database
	Name() string

	// MaxRetention is the maximum amount of type datapoints will be retained in
	// this database
	MaxRetention() time.Duration
}

A Database holds datapoints up to a certain amount of time

type Downsampler

type Downsampler interface {
	// Init initializes the downsampler to store results in the given values
	Init(vals SeriesValues)

	// AddSample adds a datapoint sample to the given interval
	AddSample(n int, v float64)

	// Finish tells the downsampler we're complete and the final values
	// computed (if they are not already)
	Finish()
}

A Downsampler combines multiple datapoints that appear within the same time interval to produce a single downsampled result

type Driver

type Driver interface {
	xclose.Closer

	// ConfigType returns an empty proto.Message representing the configuration
	// type used by the driver
	ConfigType() proto.Message

	// Type is the type of storage supported by the driver
	Type() Type

	// OpenConnection opens a connection with the provided config
	OpenConnection(config proto.Message) (Connection, error)

	// ReconfigureConnection applies a new configuration to an existing connection.
	// Connections could be heavyweight objects, and drivers may wish to optimize
	// reconfiguration to avoid creating and destroying them.  Drivers that do
	// not support dynamic reconfiguration can create a new Connection and dispose
	// of the old connection
	ReconfigureConnection(c Connection, newConfig proto.Message) (Connection, error)
}

A Driver is used to create connections to a cluster of a given storage class

type Float64SeriesValues

type Float64SeriesValues []float64

Float64SeriesValues is a SeriesValues implementation based on a float64 slice

func (Float64SeriesValues) Len

func (s Float64SeriesValues) Len() int

Len returns the number of values in the series

func (Float64SeriesValues) Reset

func (s Float64SeriesValues) Reset()

Reset fills the series with NaNs

func (Float64SeriesValues) SetValueAt

func (s Float64SeriesValues) SetValueAt(n int, v float64)

SetValueAt sets the value at the given position

func (Float64SeriesValues) ValueAt

func (s Float64SeriesValues) ValueAt(n int) float64

ValueAt returns the value at the given position

type Manager

type Manager interface {
	// Query reads datapoints for the id between two times
	Query(id string, start, end time.Time, ds Downsampler) (QueryResult, error)
}

The Manager is the main interface into the storage system, supporting queries against multiple storage clusters

type MappingRuleIter

type MappingRuleIter interface {
	xclose.Closer

	// Next moves to the next mapping, returning false if there are no more
	// mappings
	Next() bool

	// Current returns the current mapping
	Current() mapping.Rule
}

A MappingRuleIter is an iterator over Rules. Allows provider to control how these are stored internally

type QueryResult

type QueryResult interface {
	// Resolution is the resolution of the returned series
	Resolution() retention.Resolution

	// Series contains the returned data
	Series() Series
}

QueryResult is the result of doing a read

func NewQueryResult

func NewQueryResult(r retention.Resolution, s Series) QueryResult

NewQueryResult returns a new QueryResult for the given resolution and series

type Series

type Series interface {
	xclose.Closer

	Len() int                    // Len returns the number of datapoints in the series
	StepSize() time.Duration     // StepSize is the size of each "step" in the series
	StartTime() time.Time        // StartTime is the start time for the series
	EndTime() time.Time          // EndTime is the end time for the series
	ValueAt(n int) float64       // ValueAt returns the value at the given step
	StartTimeAt(n int) time.Time // StartTimeAt returns the start time for the given step
}

A Series is a series of datapoints, bucketed into fixed time intervals

func NewSeries

func NewSeries(start time.Time, step time.Duration, vals SeriesValues, pool SeriesValuesPool) Series

NewSeries returns a new series wrapped around a set of values

type SeriesIter

type SeriesIter interface {
	xclose.Closer

	// Next returns true if there is more data in the series
	Next() bool

	// Current returns the value and timestamp for the current datapoint
	Current() (float64, time.Time)
}

A SeriesIter is used to return datapoints from a series

type SeriesValues

type SeriesValues interface {
	Len() int                    // the number of values in the series
	SetValueAt(n int, v float64) // sets the value at the given location
	ValueAt(n int) float64       // returns the value at the given location
}

SeriesValues hold the underlying values in a series. Values can often be large, and benefit from direct pooling.

type SeriesValuesPool

type SeriesValuesPool interface {
	// New creates or reserves series values of the specified length
	New(len int) SeriesValues

	// Release releases previously allocated values
	Release(v SeriesValues)
}

A SeriesValuesPool allows for pooling of the underlying SeriesValues, which are typically large and benefit from being held in some form of pool

type Type

type Type interface {
	// Name is the name of the storage type
	Name() string
}

A Type defines a type of storage (m3db, hbase, etc)

Directories

Path Synopsis
generated
proto/configtest
Package configtest is a generated protocol buffer package.
Package configtest is a generated protocol buffer package.
proto/schema
Package schema is a generated protocol buffer package.
Package schema is a generated protocol buffer package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL