server

package
v3.35.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 30, 2023 License: Apache-2.0, Apache-2.0 Imports: 65 Imported by: 0

Documentation

Overview

Copyright 2022 Molecula Corp. (DBA FeatureBase). SPDX-License-Identifier: Apache-2.0

Copyright 2022 Molecula Corp. (DBA FeatureBase). SPDX-License-Identifier: Apache-2.0

Copyright 2022 Molecula Corp. (DBA FeatureBase). SPDX-License-Identifier: Apache-2.0

Index

Constants

View Source
const (
	PartitionToNodeJmp     string = "jmp-hash"
	PartitionToNodeModulus string = "modulus"
)

Variables

This section is empty.

Functions

func ChainStreamInterceptors

func ChainStreamInterceptors(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor

Chains together multiple stream interceptors.

func ChainUnaryInterceptor

func ChainUnaryInterceptor(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor

Chains together multiple unary interceptors.

func GetTLSConfig

func GetTLSConfig(tlsConfig *TLSConfig, logger logger.Logger) (TLSConfig *tls.Config, err error)

func LogQuery

func LogQuery(ctx context.Context, method string, req interface{}, logger logger.Logger)

LogQuery logs requests

func NewGRPCServer

func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error)

func NewKeypairReloader

func NewKeypairReloader(certPath, keyPath string, logger logger.Logger) (*keypairReloader, error)

func OptGRPCServerAPI

func OptGRPCServerAPI(api *pilosa.API) grpcServerOption

func OptGRPCServerAuth

func OptGRPCServerAuth(authn *authn.Auth) grpcServerOption

func OptGRPCServerListener

func OptGRPCServerListener(ln net.Listener) grpcServerOption

func OptGRPCServerLogger

func OptGRPCServerLogger(logger logger.Logger) grpcServerOption

func OptGRPCServerPerm

func OptGRPCServerPerm(gp *authz.GroupPermissions) grpcServerOption

func OptGRPCServerQueryLogger

func OptGRPCServerQueryLogger(logger logger.Logger) grpcServerOption

func OptGRPCServerTLSConfig

func OptGRPCServerTLSConfig(tlsConfig *tls.Config) grpcServerOption

func ToRowserWrapper

func ToRowserWrapper(result interface{}) (pb.ToRowser, error)

Normally we wouldn't need this wrapper, but since pilosa returns some concrete types for which we can't implement the ToRowser interface, we have to check for those here and then wrap them with a custom type.

func ToTablerWrapper

func ToTablerWrapper(result interface{}) (pb.ToTabler, error)

Normally we wouldn't need this wrapper, but since pilosa returns some concrete types for which we can't implement the ToTabler interface, we have to check for those here and then wrap them with a custom type.

func Valid

func Valid(ctx context.Context, auth *authn.Auth) (context.Context, error)

Types

type Auth

type Auth struct {
	// Enable AuthZ/AuthN for featurebase server
	Enable bool `toml:"enable"`

	ClientId         string   `toml:"client-id"`
	ClientSecret     string   `toml:"client-secret"`
	AuthorizeURL     string   `toml:"authorize-url"`
	TokenURL         string   `toml:"token-url"`
	GroupEndpointURL string   `toml:"group-endpoint-url"`
	RedirectBaseURL  string   `toml:"redirect-base-url"`
	LogoutURL        string   `toml:"logout-url"`
	Scopes           []string `toml:"scopes"`
	SecretKey        string   `toml:"secret-key"`
	PermissionsFile  string   `toml:"permissions"`
	QueryLogPath     string   `toml:"query-log-path"`
	ConfiguredIPs    []string `toml:"configured-ips"`
}

type Command

type Command struct {
	Server *pilosa.Server

	// Configuration.
	Config *Config

	// Started will be closed once Command.Start is finished.
	Started chan struct{}

	Registrar computer.Registrar

	Handler pilosa.HandlerI

	API *pilosa.API
	// contains filtered or unexported fields
}

Command represents the state of the pilosa server command.

func NewCommand

func NewCommand(stderr io.Writer, opts ...CommandOption) *Command

NewCommand returns a new instance of Main.

func (*Command) Close

func (m *Command) Close() error

Close shuts down the server.

func (*Command) HTTPHandler

func (m *Command) HTTPHandler() http.Handler

HTTPHandler was added for the case where we want to get the full http.Handler, and not just those methods which satisfy the pilosa.HandlerI interface.

func (*Command) Logger

func (cmd *Command) Logger() logger.Logger

Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility

func (*Command) Start

func (m *Command) Start() (err error)

Start starts the pilosa server - it returns once the server is running.

func (*Command) StartNoServe

func (m *Command) StartNoServe(addr dax.Address) (err error)

StartNoServe starts the pilosa server, but doesn't serve on the http handler.

func (*Command) UpAndDown

func (m *Command) UpAndDown() (err error)

func (*Command) Wait

func (m *Command) Wait() error

Wait waits for the server to be closed or interrupted.

type CommandOption

type CommandOption func(c *Command) error

func OptCommandCloseTimeout

func OptCommandCloseTimeout(d time.Duration) CommandOption

func OptCommandConfig

func OptCommandConfig(config *Config) CommandOption

func OptCommandInjections

func OptCommandInjections(inj Injections) CommandOption

OptCommandInjections injects the interface implementations.

func OptCommandServerOptions

func OptCommandServerOptions(opts ...pilosa.ServerOption) CommandOption

func OptCommandSetConfig

func OptCommandSetConfig(config *Config) CommandOption

OptCommandSetConfig was added because OptCommandConfig only sets a small sub-set of the config options (it doesn't seem to be used for anything but tests). We need a functional option which sets the full Config.

type Config

type Config struct {
	// Name a unique name for this node in the cluster.
	Name string `toml:"name"`

	// ControllerAddress is the location at which this node should register
	// itself and retrieve its instructions. For example, after registring, the
	// Controller service might tell this node that it is responsible for
	// specific shards for a particular index.
	ControllerAddress string `toml:"controller-address"`

	// WriteloggerDir is the location at which this node should
	// read/write change logs. Typically a network mounted filesystem
	// for availability/durability.
	WriteloggerDir string `toml:"writelogger-dir"`

	// SnapshotterDir is the location at which this node should
	// read/write snapshots. Typically a network mounted filesystem
	// for availability/durability.
	SnapshotterDir string `toml:"snapshotter-dir"`

	// DataDir is the directory where Pilosa stores both indexed data and
	// running state such as cluster topology information.
	DataDir string `toml:"data-dir"`

	// Bind is the host:port on which Pilosa will listen.
	Bind string `toml:"bind"`

	// BindGRPC is the host:port on which Pilosa will bind for gRPC.
	BindGRPC string `toml:"bind-grpc"`

	// Listener is an already-bound listener to use for http.
	// Listener *net.TCPListener
	Listener net.Listener

	// GRPCListener is an already-bound listener to use for gRPC.
	// This is for use by test infrastructure, where it's useful to
	// be able to dynamically generate the bindings by actually binding
	// to :0, and avoid "address already in use" errors.
	GRPCListener net.Listener

	// Advertise is the address advertised by the server to other nodes
	// in the cluster. It should be reachable by all other nodes and should
	// route to an interface that Bind is listening on.
	Advertise string `toml:"advertise"`

	// AdvertiseGRPC is the address advertised by the server to other nodes
	// in the cluster. It should be reachable by all other nodes and should
	// route to an interface that BindGRPC is listening on.
	AdvertiseGRPC string `toml:"advertise-grpc"`

	// MaxWritesPerRequest limits the number of mutating commands that can be in
	// a single request to the server. This includes Set, Clear, ClearRow, Store, and SetBit.
	MaxWritesPerRequest int `toml:"max-writes-per-request"`

	// LogPath configures where Pilosa will write logs.
	LogPath string `toml:"log-path"`

	// Verbose toggles verbose logging which can be useful for debugging.
	Verbose bool `toml:"verbose"`

	// HTTP Handler options
	Handler struct {
		// CORS Allowed Origins
		AllowedOrigins []string `toml:"allowed-origins"`
	} `toml:"handler"`

	// MaxMapCount puts an in-process limit on the number of mmaps. After this
	// is exhausted, Pilosa will fall back to reading the file into memory
	// normally.
	MaxMapCount uint64 `toml:"max-map-count"`

	// MaxFileCount puts a soft, in-process limit on the number of open fragment
	// files. Once this limit is passed, Pilosa will only keep files open while
	// actively working with them, and will close them afterward. This has a
	// negative effect on performance for workloads which make small appends to
	// lots of fragments.
	MaxFileCount uint64 `toml:"max-file-count"`

	// TLS
	TLS TLSConfig `toml:"tls"`

	// WorkerPoolSize controls how many goroutines are created for
	// processing queries. Defaults to runtime.NumCPU(). It is
	// intentionally not defined as a flag... only exposed here so
	// that we can limit the size while running tests in CI so we
	// don't exhaust the goroutine limit.
	WorkerPoolSize int `toml:"-"`

	// ImportWorkerPoolSize controls how many goroutines are created for
	// processing importRoaring jobs. Defaults to runtime.NumCPU(). It is
	// intentionally not defined as a flag... only exposed here so
	// that we can limit the size while running tests in CI so we
	// don't exhaust the goroutine limit.
	ImportWorkerPoolSize int `toml:"-"`

	// DirectiveWorkerPoolSize controls how many goroutines are created for
	// processing a Directive (i.e. concurrently loading key/partition/shard
	// data from shapshotter and writelogger) on a compute node. Defaults to
	// runtime.NumCPU().
	DirectiveWorkerPoolSize int `toml:"-"`

	// Limits the total amount of memory to be used by Extract() & SELECT queries.
	MaxQueryMemory int64 `toml:"max-query-memory"`

	// On startup, featurebase server contacts a web server to check the latest version.
	// This stores the address for that check
	VerChkAddress string `toml:"verchk-address"`

	// When checking version, server sends a UUID so that we can keep track of
	// how many unique Featurebase installs are out there. The file is stored in
	// the data directory; this stores the filename to use.
	UUIDFile string `toml:"uuid-file"`

	Cluster struct {
		ReplicaN int    `toml:"replicas"`
		Name     string `toml:"name"`
		// This LongQueryTime is deprecated but still exists for backward compatibility
		LongQueryTime             toml.Duration `toml:"long-query-time"`
		PartitionToNodeAssignment string        `toml:"partition-to-node-assignment"`
	} `toml:"cluster"`

	// Etcd config is based on embedded etcd.
	Etcd petcd.Options `toml:"etcd"`

	LongQueryTime toml.Duration `toml:"long-query-time"`

	Translation struct {
		MapSize int `toml:"map-size"`
		// DEPRECATED: Translation config supports translation store replication.
		PrimaryURL string `toml:"primary-url"`
	} `toml:"translation"`

	// AntiEntropy config is now deprecated
	AntiEntropy struct {
		Interval toml.Duration `toml:"interval"`
	} `toml:"anti-entropy"`

	Metric struct {
		// Service can be statsd, prometheus, expvar, or none.
		Service string `toml:"service"`
		// Host tells the statsd client where to write.
		Host         string        `toml:"host"`
		PollInterval toml.Duration `toml:"poll-interval"`
		// Diagnostics toggles sending some limited diagnostic information to
		// Pilosa's developers.
		Diagnostics bool `toml:"diagnostics"`
	} `toml:"metric"`

	Tracing struct {
		// SamplerType is the type of sampler to use.
		SamplerType string `toml:"sampler-type"`
		// SamplerParam is the parameter passed to the tracing sampler.
		// Its meaning is dependent on the type of sampler.
		SamplerParam float64 `toml:"sampler-param"`
		// AgentHostPort is the host:port of the local agent.
		AgentHostPort string `toml:"agent-host-port"`
	} `toml:"tracing"`

	Profile struct {
		// BlockRate is passed directly to runtime.SetBlockProfileRate
		BlockRate int `toml:"block-rate"`
		// MutexFraction is passed directly to runtime.SetMutexProfileFraction
		MutexFraction int `toml:"mutex-fraction"`
	} `toml:"profile"`

	// CheckInTimeout is the amount of time between compute node check-ins to
	// Controller.
	CheckInInterval time.Duration `toml:"check-in-interval"`

	// Storage.Backend determines which Tx implementation the holder/Index will
	// use; one of the available transactional-storage engines. Choices are
	// listed in the string constants below. Should be one of "roaring" or
	// "rbf".
	Storage *storage.Config `toml:"storage"`

	// RBFConfig defines all externally configurable RBF flags.
	RBFConfig *rbfcfg.Config `toml:"rbf"`

	// QueryHistoryLength sets the maximum number of queries that are maintained
	// for the /query-history endpoint. This parameter is per-node, and the
	// result combines the history from all nodes.
	QueryHistoryLength int `toml:"query-history-length"`

	// LookupDBDSN is an external database to connect to for `ExternalLookup` queries.
	LookupDBDSN string `toml:"lookup-db-dsn"`

	// Future flags are used to represent features or functionality which is not
	// yet the default behavior, but will be in a future release.
	Future struct {
		// Rename, if true, will outwardly present the name of the application
		// as FeatureBase instead of Pilosa.
		Rename bool `toml:"rename"`
	} `toml:"future"`

	DataDog struct {
		Enable           bool   `toml:"enable"`
		EnableTracing    bool   `toml:"enable-tracing"`
		Service          string `toml:"service"`
		Env              string `toml:"env"`
		Version          string `toml:"version"`
		Tags             string `toml:"tags"`
		CPUProfile       bool   `toml:"cpu-profile"`
		MutexProfile     bool   `toml:"mutex-profile"`
		GoroutineProfile bool   `toml:"goroutine-profile"`
		BlockProfile     bool   `toml:"block-profile"`
		HeapProfile      bool   `toml:"heap-profile"`
	} `toml:"datadog"`

	Auth Auth

	Dataframe struct {
		Enable     bool `toml:"enable"`
		UseParquet bool `toml:"use-parquet"`
	} `toml:"dataframe"`
}

Config represents the configuration for the command.

func NewConfig

func NewConfig() *Config

NewConfig returns an instance of Config with default options.

func ParseConfig

func ParseConfig(s string) (Config, error)

ParseConfig parses s into a Config.

func (*Config) MustValidate

func (c *Config) MustValidate()

MustValidate checks that all ports in a Config are unique and not zero. We disallow zero because the tests need to be using from the pre-allocated block of ports maintained by the pilosa/test/port port-mapper.

func (*Config) MustValidateAuth

func (c *Config) MustValidateAuth()

func (*Config) Namespace

func (c *Config) Namespace() string

Namespace returns the namespace to use based on the Future flag.

func (*Config) ValidateAuth

func (c *Config) ValidateAuth() (errors []error)

func (*Config) ValidatePermissions

func (c *Config) ValidatePermissions(permsFile io.Reader) (errors []error)

func (*Config) ValidatePermissionsFile

func (c *Config) ValidatePermissionsFile() (err error)

type GRPCHandler

type GRPCHandler struct {
	// contains filtered or unexported fields
}

GRPCHandler contains methods which handle the various gRPC requests.

func NewGRPCHandler

func NewGRPCHandler(api *pilosa.API) *GRPCHandler

func (*GRPCHandler) CreateIndex

CreateIndex creates a new Index

func (*GRPCHandler) DeleteIndex

DeleteIndex deletes an Index

func (*GRPCHandler) GetIndex

GetIndex returns a single Index given a name

func (*GRPCHandler) GetIndexes

GetIndexes returns a list of all Indexes

func (*GRPCHandler) Inspect

func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error

Inspect handles the inspect request and sends an InspectResponse to the stream.

func (*GRPCHandler) QueryPQL

func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQLServer) error

QueryPQL handles the PQL request and sends RowResponses to the stream.

func (*GRPCHandler) QueryPQLUnary

func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest) (*pb.TableResponse, error)

QueryPQLUnary is a unary-response (non-streaming) version of QueryPQL, returning a TableResponse.

Note comment above QuerySQLUnary describing the need for the *Unary methods.

func (*GRPCHandler) QuerySQL

func (h *GRPCHandler) QuerySQL(req *pb.QuerySQLRequest, stream pb.Pilosa_QuerySQLServer) error

QuerySQL handles the SQL request and sends RowResponses to the stream.

func (*GRPCHandler) QuerySQLUnary

func (h *GRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLRequest) (*pb.TableResponse, error)

QuerySQLUnary is a unary-response (non-streaming) version of QuerySQL, returning a TableResponse.

Note regarding QuerySQLUnary and QueryPQLUnary: These methods are not ideal, as gRPC responses are payload-length limited to 4MB, so in most cases, we would recommend users use the QuerySQL and QueryPQL methods, as they stream the response as several small RowResponses. The response size limit is configurable on the client size, but we really only recommend these methods in the case that the payload is known to be quite small (e.g. single counts). These are provided mostly to support gRPC Futures, which are used by python-molecula to perform multiple queries concurrently. There is additional discussion and historical context here: https://github.com/molecula/pilosa/pull/644

func (*GRPCHandler) WithLogger

func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler

func (*GRPCHandler) WithPerms

func (h *GRPCHandler) WithPerms(perms *authz.GroupPermissions) *GRPCHandler

func (*GRPCHandler) WithQueryLogger

func (h *GRPCHandler) WithQueryLogger(logger logger.Logger) *GRPCHandler

type Injections

type Injections struct {
	Writelogger   computer.WritelogService
	Snapshotter   computer.SnapshotService
	IsComputeNode bool
}

type ResultBool

type ResultBool bool

ResultBool is a wrapper around a bool result type so that we can implement the ToTabler and ToRowser interfaces.

func (ResultBool) ToRows

func (r ResultBool) ToRows(callback func(*pb.RowResponse) error) error

ToRows implements the ToRowser interface.

func (ResultBool) ToTable

func (r ResultBool) ToTable() (*pb.TableResponse, error)

ToTable implements the ToTabler interface.

type ResultUint64

type ResultUint64 uint64

ResultUint64 is a wrapper around a uint64 result type so that we can implement the ToTabler and ToRowser interfaces.

func (ResultUint64) ToRows

func (r ResultUint64) ToRows(callback func(*pb.RowResponse) error) error

ToRows implements the ToRowser interface.

func (ResultUint64) ToTable

func (r ResultUint64) ToTable() (*pb.TableResponse, error)

ToTable implements the ToTabler interface.

type TLSConfig

type TLSConfig struct {
	// CertificatePath contains the path to the certificate (.crt or .pem file)
	CertificatePath string `toml:"certificate"`
	// CertificateKeyPath contains the path to the certificate key (.key file)
	CertificateKeyPath string `toml:"key"`
	// CACertPath is the path to a CA certificate (.crt or .pem file)
	CACertPath string `toml:"ca-certificate"`
	// SkipVerify disables verification of server certificates when connecting to another Pilosa node
	SkipVerify bool `toml:"skip-verify"`
	// EnableClientVerification enables verification of client TLS certificates (Mutual TLS)
	EnableClientVerification bool `toml:"enable-client-verification"`
}

TLSConfig contains TLS configuration

type VDSMGRPCHandler

type VDSMGRPCHandler struct {
	// contains filtered or unexported fields
}

VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM.

func NewVDSMGRPCHandler

func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler

func (*VDSMGRPCHandler) DeleteVDS

DeleteVDS deletes a VDS

func (*VDSMGRPCHandler) GetVDS

GetVDSs returns a single VDS given a name

func (*VDSMGRPCHandler) GetVDSs

GetVDSs returns a list of all VDSs

func (*VDSMGRPCHandler) Inspect

func (*VDSMGRPCHandler) PostVDS

PostVDS creates a new VDS

func (*VDSMGRPCHandler) QueryPQL

func (*VDSMGRPCHandler) QueryPQLUnary

func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*pb.TableResponse, error)

func (*VDSMGRPCHandler) QuerySQL

func (*VDSMGRPCHandler) QuerySQLUnary

func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLRequest) (*pb.TableResponse, error)

func (*VDSMGRPCHandler) WithLogger

func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL