core: go.gazette.dev/core/examples/word-count Index | Files | Directories

package word_count

import "go.gazette.dev/core/examples/word-count"

Package word_count is an example application which provides a gRPC API for publishing texts and querying running counts of NGrams extracted from previously published texts.

Index

Package Files

word_count.go word_count.pb.go

Variables

var (
    ErrInvalidLengthWordCount        = fmt.Errorf("proto: negative length found during unmarshaling")
    ErrIntOverflowWordCount          = fmt.Errorf("proto: integer overflow")
    ErrUnexpectedEndOfGroupWordCount = fmt.Errorf("proto: unexpected end of group")
)

func RegisterNGramServer Uses

func RegisterNGramServer(s *grpc.Server, srv NGramServer)

type Counter Uses

type Counter struct {
    // contains filtered or unexported fields
}

Counter consumes NGramCount messages and aggregates total counts of each NGram. It also provides gRPC APIs for publishing text and querying NGram counts. It implements the following interfaces: - runconsumer.Application - NGramServer (generated gRPC service stub).

func (Counter) ConsumeMessage Uses

func (Counter) ConsumeMessage(_ consumer.Shard, store consumer.Store, env message.Envelope, _ *message.Publisher) error

ConsumeMessage folds an NGramCount into its respective running NGram count. Implements consumer.Application.

func (Counter) FinalizeTxn Uses

func (Counter) FinalizeTxn(_ consumer.Shard, store consumer.Store, _ *message.Publisher) error

FinalizeTxn marshals in-memory NGram counts to the |store|, ensuring persistence across consumer transactions. Implements consumer.Application.

func (*Counter) InitApplication Uses

func (counter *Counter) InitApplication(args runconsumer.InitArgs) error

InitApplication initializes the application to serve the NGram gRPC service.

func (Counter) NewConfig Uses

func (Counter) NewConfig() runconsumer.Config

NewConfig returns a new configuration instance.

func (Counter) NewMessage Uses

func (Counter) NewMessage(*pb.JournalSpec) (message.Message, error)

NewMessage returns an NGramCount message. Implements consumer.Application.

func (Counter) NewStore Uses

func (Counter) NewStore(shard consumer.Shard, rec *recoverylog.Recorder) (consumer.Store, error)

NewStore builds a RocksDB store for the Shard. Implements consumer.Application.

func (*Counter) Publish Uses

func (counter *Counter) Publish(ctx context.Context, req *PublishRequest) (*PublishResponse, error)

Publish extracts NGrams of the configured length from the PublishRequest, and publishes an NGramCount message for each. It returns after all published messages have committed to their respective journals.

func (*Counter) Query Uses

func (counter *Counter) Query(ctx context.Context, req *QueryRequest) (resp *QueryResponse, err error)

Query a count for an NGram count (or counts for a prefix thereof). If the requested or imputed Shard does not resolve locally, Query will proxy the request to the responsible process.

type NGram Uses

type NGram string

NGram is a string of N space-delimited tokens, where N is fixed.

type NGramClient Uses

type NGramClient interface {
    // Publish text to the word-count example. The published text is tokenized
    // into NGrams, indexed, and aggregated into total NGram counts.
    Publish(ctx context.Context, in *PublishRequest, opts ...grpc.CallOption) (*PublishResponse, error)
    // Query for a specific NGram, or NGram prefixes.
    Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (*QueryResponse, error)
}

NGramClient is the client API for NGram service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.

func NewNGramClient Uses

func NewNGramClient(cc *grpc.ClientConn) NGramClient

type NGramCount Uses

type NGramCount struct {
    Uuid                 []byte   `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"`
    NGram                NGram    `protobuf:"bytes,2,opt,name=n_gram,json=nGram,proto3,casttype=NGram" json:"n_gram,omitempty"`
    Count                uint64   `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*NGramCount) Descriptor Uses

func (*NGramCount) Descriptor() ([]byte, []int)

func (*NGramCount) GetCount Uses

func (m *NGramCount) GetCount() uint64

func (*NGramCount) GetNGram Uses

func (m *NGramCount) GetNGram() NGram

func (*NGramCount) GetUUID Uses

func (c *NGramCount) GetUUID() (uuid message.UUID)

func (*NGramCount) GetUuid Uses

func (m *NGramCount) GetUuid() []byte

func (*NGramCount) Marshal Uses

func (m *NGramCount) Marshal() (dAtA []byte, err error)

func (*NGramCount) MarshalTo Uses

func (m *NGramCount) MarshalTo(dAtA []byte) (int, error)

func (*NGramCount) MarshalToSizedBuffer Uses

func (m *NGramCount) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*NGramCount) NewAcknowledgement Uses

func (c *NGramCount) NewAcknowledgement(pb.Journal) message.Message

func (*NGramCount) ProtoMessage Uses

func (*NGramCount) ProtoMessage()

func (*NGramCount) ProtoSize Uses

func (m *NGramCount) ProtoSize() (n int)

func (*NGramCount) Reset Uses

func (m *NGramCount) Reset()

func (*NGramCount) SetUUID Uses

func (c *NGramCount) SetUUID(uuid message.UUID)

func (*NGramCount) String Uses

func (m *NGramCount) String() string

func (*NGramCount) Unmarshal Uses

func (m *NGramCount) Unmarshal(dAtA []byte) error

func (*NGramCount) XXX_DiscardUnknown Uses

func (m *NGramCount) XXX_DiscardUnknown()

func (*NGramCount) XXX_Marshal Uses

func (m *NGramCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*NGramCount) XXX_Merge Uses

func (m *NGramCount) XXX_Merge(src proto.Message)

func (*NGramCount) XXX_Size Uses

func (m *NGramCount) XXX_Size() int

func (*NGramCount) XXX_Unmarshal Uses

func (m *NGramCount) XXX_Unmarshal(b []byte) error

type NGramServer Uses

type NGramServer interface {
    // Publish text to the word-count example. The published text is tokenized
    // into NGrams, indexed, and aggregated into total NGram counts.
    Publish(context.Context, *PublishRequest) (*PublishResponse, error)
    // Query for a specific NGram, or NGram prefixes.
    Query(context.Context, *QueryRequest) (*QueryResponse, error)
}

NGramServer is the server API for NGram service.

type PublishRequest Uses

type PublishRequest struct {
    Text                 string   `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*PublishRequest) Descriptor Uses

func (*PublishRequest) Descriptor() ([]byte, []int)

func (*PublishRequest) GetText Uses

func (m *PublishRequest) GetText() string

func (*PublishRequest) Marshal Uses

func (m *PublishRequest) Marshal() (dAtA []byte, err error)

func (*PublishRequest) MarshalTo Uses

func (m *PublishRequest) MarshalTo(dAtA []byte) (int, error)

func (*PublishRequest) MarshalToSizedBuffer Uses

func (m *PublishRequest) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PublishRequest) ProtoMessage Uses

func (*PublishRequest) ProtoMessage()

func (*PublishRequest) ProtoSize Uses

func (m *PublishRequest) ProtoSize() (n int)

func (*PublishRequest) Reset Uses

func (m *PublishRequest) Reset()

func (*PublishRequest) String Uses

func (m *PublishRequest) String() string

func (*PublishRequest) Unmarshal Uses

func (m *PublishRequest) Unmarshal(dAtA []byte) error

func (*PublishRequest) XXX_DiscardUnknown Uses

func (m *PublishRequest) XXX_DiscardUnknown()

func (*PublishRequest) XXX_Marshal Uses

func (m *PublishRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PublishRequest) XXX_Merge Uses

func (m *PublishRequest) XXX_Merge(src proto.Message)

func (*PublishRequest) XXX_Size Uses

func (m *PublishRequest) XXX_Size() int

func (*PublishRequest) XXX_Unmarshal Uses

func (m *PublishRequest) XXX_Unmarshal(b []byte) error

type PublishResponse Uses

type PublishResponse struct {
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*PublishResponse) Descriptor Uses

func (*PublishResponse) Descriptor() ([]byte, []int)

func (*PublishResponse) Marshal Uses

func (m *PublishResponse) Marshal() (dAtA []byte, err error)

func (*PublishResponse) MarshalTo Uses

func (m *PublishResponse) MarshalTo(dAtA []byte) (int, error)

func (*PublishResponse) MarshalToSizedBuffer Uses

func (m *PublishResponse) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PublishResponse) ProtoMessage Uses

func (*PublishResponse) ProtoMessage()

func (*PublishResponse) ProtoSize Uses

func (m *PublishResponse) ProtoSize() (n int)

func (*PublishResponse) Reset Uses

func (m *PublishResponse) Reset()

func (*PublishResponse) String Uses

func (m *PublishResponse) String() string

func (*PublishResponse) Unmarshal Uses

func (m *PublishResponse) Unmarshal(dAtA []byte) error

func (*PublishResponse) XXX_DiscardUnknown Uses

func (m *PublishResponse) XXX_DiscardUnknown()

func (*PublishResponse) XXX_Marshal Uses

func (m *PublishResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PublishResponse) XXX_Merge Uses

func (m *PublishResponse) XXX_Merge(src proto.Message)

func (*PublishResponse) XXX_Size Uses

func (m *PublishResponse) XXX_Size() int

func (*PublishResponse) XXX_Unmarshal Uses

func (m *PublishResponse) XXX_Unmarshal(b []byte) error

type QueryRequest Uses

type QueryRequest struct {
    // Header attached by a proxy-ing peer. Not directly set by clients.
    Header *protocol.Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"`
    // NGram prefix to query.
    Prefix NGram `protobuf:"bytes,2,opt,name=prefix,proto3,casttype=NGram" json:"prefix,omitempty"`
    // Shard to query. Optional; if not set, shard is inferred from |prefix|'s current mapping.
    Shard                go_gazette_dev_core_consumer_protocol.ShardID `protobuf:"bytes,3,opt,name=shard,proto3,casttype=go.gazette.dev/core/consumer/protocol.ShardID" json:"shard,omitempty"`
    XXX_NoUnkeyedLiteral struct{}                                      `json:"-"`
    XXX_unrecognized     []byte                                        `json:"-"`
    XXX_sizecache        int32                                         `json:"-"`
}

func (*QueryRequest) Descriptor Uses

func (*QueryRequest) Descriptor() ([]byte, []int)

func (*QueryRequest) GetHeader Uses

func (m *QueryRequest) GetHeader() *protocol.Header

func (*QueryRequest) GetPrefix Uses

func (m *QueryRequest) GetPrefix() NGram

func (*QueryRequest) GetShard Uses

func (m *QueryRequest) GetShard() go_gazette_dev_core_consumer_protocol.ShardID

func (*QueryRequest) Marshal Uses

func (m *QueryRequest) Marshal() (dAtA []byte, err error)

func (*QueryRequest) MarshalTo Uses

func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error)

func (*QueryRequest) MarshalToSizedBuffer Uses

func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*QueryRequest) ProtoMessage Uses

func (*QueryRequest) ProtoMessage()

func (*QueryRequest) ProtoSize Uses

func (m *QueryRequest) ProtoSize() (n int)

func (*QueryRequest) Reset Uses

func (m *QueryRequest) Reset()

func (*QueryRequest) String Uses

func (m *QueryRequest) String() string

func (*QueryRequest) Unmarshal Uses

func (m *QueryRequest) Unmarshal(dAtA []byte) error

func (*QueryRequest) XXX_DiscardUnknown Uses

func (m *QueryRequest) XXX_DiscardUnknown()

func (*QueryRequest) XXX_Marshal Uses

func (m *QueryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*QueryRequest) XXX_Merge Uses

func (m *QueryRequest) XXX_Merge(src proto.Message)

func (*QueryRequest) XXX_Size Uses

func (m *QueryRequest) XXX_Size() int

func (*QueryRequest) XXX_Unmarshal Uses

func (m *QueryRequest) XXX_Unmarshal(b []byte) error

type QueryResponse Uses

type QueryResponse struct {
    Grams                []NGramCount `protobuf:"bytes,1,rep,name=grams,proto3" json:"grams"`
    XXX_NoUnkeyedLiteral struct{}     `json:"-"`
    XXX_unrecognized     []byte       `json:"-"`
    XXX_sizecache        int32        `json:"-"`
}

func (*QueryResponse) Descriptor Uses

func (*QueryResponse) Descriptor() ([]byte, []int)

func (*QueryResponse) GetGrams Uses

func (m *QueryResponse) GetGrams() []NGramCount

func (*QueryResponse) Marshal Uses

func (m *QueryResponse) Marshal() (dAtA []byte, err error)

func (*QueryResponse) MarshalTo Uses

func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error)

func (*QueryResponse) MarshalToSizedBuffer Uses

func (m *QueryResponse) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*QueryResponse) ProtoMessage Uses

func (*QueryResponse) ProtoMessage()

func (*QueryResponse) ProtoSize Uses

func (m *QueryResponse) ProtoSize() (n int)

func (*QueryResponse) Reset Uses

func (m *QueryResponse) Reset()

func (*QueryResponse) String Uses

func (m *QueryResponse) String() string

func (*QueryResponse) Unmarshal Uses

func (m *QueryResponse) Unmarshal(dAtA []byte) error

func (*QueryResponse) XXX_DiscardUnknown Uses

func (m *QueryResponse) XXX_DiscardUnknown()

func (*QueryResponse) XXX_Marshal Uses

func (m *QueryResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*QueryResponse) XXX_Merge Uses

func (m *QueryResponse) XXX_Merge(src proto.Message)

func (*QueryResponse) XXX_Size Uses

func (m *QueryResponse) XXX_Size() int

func (*QueryResponse) XXX_Unmarshal Uses

func (m *QueryResponse) XXX_Unmarshal(b []byte) error

type UnimplementedNGramServer Uses

type UnimplementedNGramServer struct {
}

UnimplementedNGramServer can be embedded to have forward compatible implementations.

func (*UnimplementedNGramServer) Publish Uses

func (*UnimplementedNGramServer) Publish(ctx context.Context, req *PublishRequest) (*PublishResponse, error)

func (*UnimplementedNGramServer) Query Uses

func (*UnimplementedNGramServer) Query(ctx context.Context, req *QueryRequest) (*QueryResponse, error)

Directories

PathSynopsis
counterPackage counter runs the word_count.Counter consumer.
wordcountctl

Package word_count imports 26 packages (graph) and is imported by 6 packages. Updated 2020-06-11. Refresh now. Tools for package owners.