chevalier

package module
v1.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 29, 2014 License: BSD-3-Clause Imports: 15 Imported by: 0

README

Chevalier

A search engine for Vaultaire data sources.

Requirements

Chevalier requires Go 1.2 or greater, ZeroMQ 4.0 or greater, and Protocol Buffers. (It also requires other things, but go get should be able to find them.)

It talks to a Vaultaire cluster of version 2.0 or greater, so you'll need one of those also.

Documentation

Overview

Chevalier is a search engine package for Vaultaire sources. It is backed to Elasticsearch.

Index

Constants

View Source 
const (
	Version = "1.1.3"
)

Variables

This section is empty.

Functions

func FmtResult 

func FmtResult(result *es.SearchResult) []string

FmtResult returns a string from a SearchResult by interpreting it in the most naive manner possible. For debugging.

func MarshalSourceBurst 

func MarshalSourceBurst(burst *DataSourceBurst) ([]byte, error)

func MarshalSourceRequest 

func MarshalSourceRequest(req *SourceRequest) ([]byte, error)

func MarshalStatusResponse added in v0.2.3 

func MarshalStatusResponse(s *StatusResponse) ([]byte, error)

Types

type ContentsEntry added in v1.1.2 

type ContentsEntry struct {
	// contains filtered or unexported fields
}

ContentsEntry is one address->source-dict mapping.

type ContentsResponse added in v1.1.2 

type ContentsResponse struct {
	// contains filtered or unexported fields
}

ContentsResponse is a single message received from the contents daemon.

type DataSource added in v0.0.2 

type DataSource struct {
	// Source tags. There can be an arbitrary number of these.
	// Tags which affect presentation rather than identity should be
	// underscore-prefixed.
	Source []*DataSource_Tag `protobuf:"bytes,1,rep,name=source" json:"source,omitempty"`
	// Unique identifier for this data source within Vaultaire.
	Address          *uint64 `protobuf:"fixed64,3,req,name=address" json:"address,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func NewDataSource added in v0.0.2 

func NewDataSource(tags []*DataSource_Tag) *DataSource

func UnmarshalSource added in v0.0.2 

func UnmarshalSource(packet []byte) (*DataSource, error)

func (*DataSource) Empty added in v1.1.2 

func (s *DataSource) Empty() bool

func (*DataSource) GetAddress added in v1.1.2 

func (m *DataSource) GetAddress() uint64

func (*DataSource) GetSource added in v0.0.2 

func (m *DataSource) GetSource() []*DataSource_Tag

func (*DataSource) ProtoMessage added in v0.0.2 

func (*DataSource) ProtoMessage()

func (*DataSource) Reset added in v0.0.2 

func (m *DataSource) Reset()

func (*DataSource) String added in v0.0.2 

func (m *DataSource) String() string

type DataSourceBurst added in v0.0.2 

type DataSourceBurst struct {
	// All sources that matched the query received (paginated
	// according to `start_page` and `sources_per_page` if they are
	// set in the request).
	Sources []*DataSource `protobuf:"bytes,1,rep,name=sources" json:"sources,omitempty"`
	// Error message - if present, some aspect of the request
	// failed.
	Error            *string `protobuf:"bytes,2,opt,name=error" json:"error,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

Returned as a response to the chevalier client.

func BuildSourceBurst added in v0.0.2 

func BuildSourceBurst(sources []*DataSource) *DataSourceBurst

func UnmarshalSourceBurst added in v0.0.2 

func UnmarshalSourceBurst(packet []byte) (*DataSourceBurst, error)

func (*DataSourceBurst) GetError added in v0.0.2 

func (m *DataSourceBurst) GetError() string

func (*DataSourceBurst) GetSources added in v0.0.2 

func (m *DataSourceBurst) GetSources() []*DataSource

func (*DataSourceBurst) ProtoMessage added in v0.0.2 

func (*DataSourceBurst) ProtoMessage()

func (*DataSourceBurst) Reset added in v0.0.2 

func (m *DataSourceBurst) Reset()

func (*DataSourceBurst) String added in v0.0.2 

func (m *DataSourceBurst) String() string

type DataSource_Tag added in v0.0.2 

type DataSource_Tag struct {
	Field            *string `protobuf:"bytes,1,req,name=field" json:"field,omitempty"`
	Value            *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func NewDataSourceTag added in v0.0.2 

func NewDataSourceTag(field, value string) *DataSource_Tag

func (*DataSource_Tag) GetField added in v0.0.2 

func (m *DataSource_Tag) GetField() string

func (*DataSource_Tag) GetValue added in v0.0.2 

func (m *DataSource_Tag) GetValue() string

func (*DataSource_Tag) ProtoMessage added in v0.0.2 

func (*DataSource_Tag) ProtoMessage()

func (*DataSource_Tag) Reset added in v0.0.2 

func (m *DataSource_Tag) Reset()

func (*DataSource_Tag) String added in v0.0.2 

func (m *DataSource_Tag) String() string

type ElasticsearchOrigin added in v0.2.3 

type ElasticsearchOrigin struct {
	Origin      string    `json:"origin"`
	Count       uint64    `json:"count"`
	Address     uint64    `json:"address"`
	LastUpdated time.Time `json:"last_updated"`
}

ElasticsearchOrigin stores metadata for each origin.

func NewElasticsearchOrigin added in v0.2.3 

func NewElasticsearchOrigin(origin string, count uint64, updated time.Time) *ElasticsearchOrigin

type ElasticsearchSource 

type ElasticsearchSource struct {
	Origin string
	// Address in Vaultaire.
	Address string
	Source  map[string]string `json:"source"`
}

ElasticsearchSource is the type used to serialize sources for indexing.

func GetContents added in v0.0.3 

func GetContents(endpoint, origin string) ([]*ElasticsearchSource, error)

GetContents list for origin from a Vaultaire readerd listening on endpoint, returning it as a DataSourceBurst.

func (*ElasticsearchSource) GetID 

func (s *ElasticsearchSource) GetID() string

GetID returns a (probably) unique ID for an ElasticsearchSource, in the form of a sha1 hash of underscore-separated field-value pairs separated by newlines.

func (*ElasticsearchSource) Unmarshal 

func (s *ElasticsearchSource) Unmarshal() (*DataSource, error)

Unmarshal turns an ElasticsearchSource (presumably itself unmarshaled from a JSON object stored in Elasticsearch) into the equivalent DataSource.

type ElasticsearchStatus added in v0.3.0 

type ElasticsearchStatus struct {
	IndicesInitialized bool `json:"indices_initialized"`
}

func NewElasticsearchStatus added in v0.3.0 

func NewElasticsearchStatus() *ElasticsearchStatus

type ElasticsearchWriter 

type ElasticsearchWriter struct {
	// contains filtered or unexported fields
}

ElasticsearchWriter maintains context for writes to the index.

func NewElasticsearchWriter 

func NewElasticsearchWriter(host string, maxConns int, retrySeconds int, index, metaIndex, dataType string) *ElasticsearchWriter

NewElasticsearchWriter builds a new Writer. retrySeconds is for the bulk indexer. index and dataType can be anything as long as they're consistent.

func (*ElasticsearchWriter) GetErrorChan 

func (w *ElasticsearchWriter) GetErrorChan() chan *es.ErrorBuffer

GetErrorChan returns the channel the bulk indexer writes errors to.

func (*ElasticsearchWriter) Shutdown 

func (w *ElasticsearchWriter) Shutdown()

Shutdown signals the bulk indexer to flush all pending writes.

func (*ElasticsearchWriter) UpdateOrigin added in v0.2.3 

func (w *ElasticsearchWriter) UpdateOrigin(origin string, count uint64) error

func (*ElasticsearchWriter) Write 

func (w *ElasticsearchWriter) Write(origin string, source *ElasticsearchSource) error

Write queues a DataSource for writing by the bulk indexer. Non-blocking.

type QueryEngine 

type QueryEngine struct {
	// contains filtered or unexported fields
}

QueryEngine presents an interface for running queries for sources against Elasticsearch.

func NewQueryEngine 

func NewQueryEngine(host, indexName, dataType, metaIndex string) *QueryEngine

NewQueryEngine initializes a QueryEngine with the supplied Elasticsearch metadata. indexName and dataType can be anything as long as they're consistent.

func (*QueryEngine) BuildQuery 

func (e *QueryEngine) BuildQuery(origin string, req *SourceRequest) (SourceQuery, error)

BuildQuery takes a SourceRequest and turns it into a multi-level map suitable for marshalling to JSON and sending to Elasticsearch.

func (*QueryEngine) GetOriginMetadata added in v0.2.3 

func (e *QueryEngine) GetOriginMetadata(origin string) (*ElasticsearchOrigin, error)

GetOriginMetadata returns an ElasticsearchOrigin object for the specified origin.

func (*QueryEngine) GetSourceCount added in v0.2.3 

func (e *QueryEngine) GetSourceCount() int64

GetSourceCount returns the number of sources we currently think exist in the index.

func (*QueryEngine) GetSources 

func (e *QueryEngine) GetSources(origin string, req *SourceRequest) (*DataSourceBurst, error)

GetSources takes a request object and returns the DataSourceBurst of the sources it gets back from Elasticsearch. If error is not nil, then a valid DataSourceBurst will still be returned, with the Error field set.

func (*QueryEngine) GetStatus added in v0.2.3 

func (e *QueryEngine) GetStatus(origins []string) *StatusResponse

type RequestOpCode added in v1.1.2 

type RequestOpCode byte
const (
	ContentsListRequest RequestOpCode = iota
	GenerateNewAddress
	UpdateSourceTag
	RemoveSourceTag
)

type ResponseOpCode added in v1.1.2 

type ResponseOpCode byte
const (
	RandomAddress ResponseOpCode = iota
	InvalidContentsOrigin
	ContentsListEntry
	EndOfContentsList
	UpdateSuccess
	RemoveSuccess
)

type SourceQuery 

type SourceQuery map[string]interface{}

SourceQuery is a multi-level map type representing an Elasticsearch query-string-type query. Suitable for marshalling as JSON and feeding to Elasticsearch.

type SourceRequest added in v0.0.2 

type SourceRequest struct {
	// Tags to use in search (as an 'and' query). If `query_string`
	// is specified, the content of this field is ignored.
	Tags []*SourceRequest_Tag `protobuf:"bytes,1,rep,name=tags" json:"tags,omitempty"`
	// Page to return results from. If not specified, 0 is assumed.
	StartPage *int64 `protobuf:"varint,2,opt,name=start_page" json:"start_page,omitempty"`
	// Page to return results from. If not specified, all results
	// are returned in one page.
	SourcesPerPage *int64 `protobuf:"varint,3,opt,name=sources_per_page" json:"sources_per_page,omitempty"`
	// Elasticsearch query string to use. If specified, the content
	// of `tags` will be ignored.
	QueryString *string `protobuf:"bytes,5,opt,name=query_string" json:"query_string,omitempty"`
	// Vaultaire address to look up. If specified, the `tags` and
	// `query_string` fields will be ignored.
	Address *uint64 `protobuf:"fixed64,6,opt,name=address" json:"address,omitempty"`
	// If true, do not filter empty sources (addresses with no tags
	// associated) from the result set. Defaults to false.
	IncludeEmpty     *bool  `protobuf:"varint,7,opt,name=include_empty" json:"include_empty,omitempty"`
	XXX_unrecognized []byte `json:"-"`
}

This message is sent by Chevalier clients; a DataSourceBurst is sent in response.

func GenTestSourceRequest added in v0.0.2 

func GenTestSourceRequest() *SourceRequest

GenTestSourceRequest returns a SourceRequest filled with random data.

func NewSourceRequest added in v0.0.2 

func NewSourceRequest(tags []*SourceRequest_Tag) *SourceRequest

NewSourceRequest builds a SourceRequest from a slice of tags.

func UnmarshalSourceRequest added in v0.0.2 

func UnmarshalSourceRequest(packet []byte) (*SourceRequest, error)

func (*SourceRequest) GetAddress added in v1.1.2 

func (m *SourceRequest) GetAddress() uint64

func (*SourceRequest) GetIncludeEmpty added in v1.1.2 

func (m *SourceRequest) GetIncludeEmpty() bool

func (*SourceRequest) GetQueryString added in v0.3.0 

func (m *SourceRequest) GetQueryString() string

func (*SourceRequest) GetSourcesPerPage added in v0.0.2 

func (m *SourceRequest) GetSourcesPerPage() int64

func (*SourceRequest) GetStartPage added in v0.0.2 

func (m *SourceRequest) GetStartPage() int64

func (*SourceRequest) GetTags added in v0.0.2 

func (m *SourceRequest) GetTags() []*SourceRequest_Tag

func (*SourceRequest) ProtoMessage added in v0.0.2 

func (*SourceRequest) ProtoMessage()

func (*SourceRequest) Reset added in v0.0.2 

func (m *SourceRequest) Reset()

func (*SourceRequest) String added in v0.0.2 

func (m *SourceRequest) String() string

type SourceRequest_Tag added in v0.0.2 

type SourceRequest_Tag struct {
	Field            *string `protobuf:"bytes,1,req,name=field" json:"field,omitempty"`
	Value            *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func NewSourceRequestTag added in v0.0.2 

func NewSourceRequestTag(field, value string) *SourceRequest_Tag

NewSourceRequestTag builds a SourceRequest_Tag from a field and a value.

func (*SourceRequest_Tag) GetField added in v0.0.2 

func (m *SourceRequest_Tag) GetField() string

func (*SourceRequest_Tag) GetValue added in v0.0.2 

func (m *SourceRequest_Tag) GetValue() string

func (*SourceRequest_Tag) ProtoMessage added in v0.0.2 

func (*SourceRequest_Tag) ProtoMessage()

func (*SourceRequest_Tag) Reset added in v0.0.2 

func (m *SourceRequest_Tag) Reset()

func (*SourceRequest_Tag) String added in v0.0.2 

func (m *SourceRequest_Tag) String() string

type StatusResponse added in v0.2.3 

type StatusResponse struct {
	// All origins currently in the index.
	Origins          []*StatusResponse_Origin `protobuf:"bytes,1,rep,name=origins" json:"origins,omitempty"`
	Errors           []string                 `protobuf:"bytes,2,rep,name=errors" json:"errors,omitempty"`
	XXX_unrecognized []byte                   `json:"-"`
}

Sent as a response to a status request.

func NewStatusResponse added in v0.2.3 

func NewStatusResponse() *StatusResponse

func UnmarshalStatusResponse added in v0.2.3 

func UnmarshalStatusResponse(b []byte) (*StatusResponse, error)

func (*StatusResponse) GetErrors added in v0.2.3 

func (m *StatusResponse) GetErrors() []string

func (*StatusResponse) GetOrigins added in v0.2.3 

func (m *StatusResponse) GetOrigins() []*StatusResponse_Origin

func (*StatusResponse) ProtoMessage added in v0.2.3 

func (*StatusResponse) ProtoMessage()

func (*StatusResponse) Reset added in v0.2.3 

func (m *StatusResponse) Reset()

func (*StatusResponse) String added in v0.2.3 

func (m *StatusResponse) String() string

func (*StatusResponse) ToJSON added in v0.2.3 

func (s *StatusResponse) ToJSON() ([]byte, error)

type StatusResponse_Origin added in v0.2.3 

type StatusResponse_Origin struct {
	// Origin name.
	Origin *string `protobuf:"bytes,1,req,name=origin" json:"origin,omitempty"`
	// Number of sources for this origin.
	Sources *uint64 `protobuf:"varint,2,req,name=sources" json:"sources,omitempty"`
	// Nanosecond-precision timestamp of last update.
	LastUpdated      *uint64 `protobuf:"fixed64,3,opt,name=last_updated" json:"last_updated,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func NewStatusResponse_Origin added in v0.2.3 

func NewStatusResponse_Origin(origin string, sources uint64, updated time.Time) *StatusResponse_Origin

func (*StatusResponse_Origin) GetLastUpdated added in v0.2.3 

func (m *StatusResponse_Origin) GetLastUpdated() uint64

func (*StatusResponse_Origin) GetOrigin added in v0.2.3 

func (m *StatusResponse_Origin) GetOrigin() string

func (*StatusResponse_Origin) GetSources added in v0.2.3 

func (m *StatusResponse_Origin) GetSources() uint64

func (*StatusResponse_Origin) ProtoMessage added in v0.2.3 

func (*StatusResponse_Origin) ProtoMessage()

func (*StatusResponse_Origin) Reset added in v0.2.3 

func (m *StatusResponse_Origin) Reset()

func (*StatusResponse_Origin) String added in v0.2.3 

func (m *StatusResponse_Origin) String() string

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.