planner

package
v0.0.0-...-48dec30 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 12, 2018 License: MIT Imports: 24 Imported by: 0

Documentation

Overview

Package planner is a generated protocol buffer package.

It is generated from these files:

msgs.proto

It has these top-level messages:

Message
TaskResponse
SqlTask

Index

Constants

This section is empty.

Variables

View Source
var (
	// DFA States
	Starting = dfa.State("starting")
	Running  = dfa.State("running")
	Exiting  = dfa.State("exiting")
	// Letters
	Failure  = dfa.Letter("failure")
	Started  = dfa.Letter("started")
	Finished = dfa.Letter("finished")
	Exit     = dfa.Letter("exit")
)
View Source
var (
	// GridConf BuiltIn Default Conf, used for testing but real runtime swaps this out
	// for a real config
	GridConf = &Conf{
		GridName:    "dataux",
		Address:     "localhost:0",
		EtcdServers: strings.Split("http://127.0.0.1:2379", ","),
	}
)

Functions

func LeaderCreate

func LeaderCreate(client *grid.Client) grid.MakeActor

LeaderCreate factory function to create the Leader

func NextId

func NextId() (uint64, error)

NextId() get id, error

func NextIdUnsafe

func NextIdUnsafe() uint64

NextIdUnsafe create a unique id from sonyflake, unsafe because we ignore errors.

func NodeName

func NodeName(id uint64) string

NodeName given a unique id create name

func NodeName2

func NodeName2(id1, id2 uint64) string

func WorkerFactory

func WorkerFactory(conf *Conf, client *grid.Client, server *grid.Server) grid.MakeActor

WorkerFactory factory function to create the Leader

Types

type Conf

type Conf struct {
	JobMaker       JobMaker
	SchemaLoader   plan.SchemaLoader
	SupressRecover bool
	NodeCt         int
	MailboxCount   int
	Address        string
	GridName       string
	Hostname       string
	EtcdServers    []string
	NatsServers    []string
}

func (*Conf) Clone

func (c *Conf) Clone() *Conf

type GridTask

type GridTask struct {
	*exec.JobExecutor

	GridServer *PlannerGrid
	// contains filtered or unexported fields
}

GridTask a Sql job that can be distributed. Contains ref to the shared GridServer which has info to distribute tasks across servers

func BuildExecutorUnPlanned

func BuildExecutorUnPlanned(ctx *plan.Context, pg *PlannerGrid) (*GridTask, error)

BuildExecutorUnPlanned Build a Sql Job which has already been planned so this is just execution runner

func BuildSqlJob

func BuildSqlJob(ctx *plan.Context, pg *PlannerGrid) (*GridTask, error)

Build a Sql Job which may be a Grid/Distributed job

func (*GridTask) Finalize

func (m *GridTask) Finalize(resultWriter exec.Task) error

Finalize is after the Dag of Relational-algebra tasks have been assembled and just before we run them.

func (*GridTask) WalkGroupBy

func (m *GridTask) WalkGroupBy(p *plan.GroupBy) (exec.Task, error)

WalkGroupBy walks the GroupBy tasks for any re-plan due to being distributed

func (*GridTask) WalkSelect

func (m *GridTask) WalkSelect(p *plan.Select) (exec.Task, error)

func (*GridTask) WalkSelectPartition

func (m *GridTask) WalkSelectPartition(p *plan.Select, part *schema.Partition) (exec.Task, error)

WalkSelectPartition is ONLY called by child-dag's, ie the remote end of a distributed

sql query, to allow setup before walking

func (*GridTask) WalkSource

func (m *GridTask) WalkSource(p *plan.Source) (exec.Task, error)

type JobMaker

type JobMaker func(ctx *plan.Context) (*GridTask, error)

type LeaderActor

type LeaderActor struct {
	// contains filtered or unexported fields
}

LeaderActor is the scheduler to create and watch the workers.

func (*LeaderActor) Act

func (a *LeaderActor) Act(ctx context.Context)

Act checks for peers, ie: other processes running this code, in the same namespace and start the sqlworker actor on each of them.

type Message

type Message struct {
	Type string `protobuf:"bytes,1,opt,name=type" json:"type,omitempty"`
	Msg  []byte `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"`
}

func (*Message) Descriptor

func (*Message) Descriptor() ([]byte, []int)

func (*Message) GetMsg

func (m *Message) GetMsg() []byte

func (*Message) GetType

func (m *Message) GetType() string

func (*Message) ProtoMessage

func (*Message) ProtoMessage()

func (*Message) Reset

func (m *Message) Reset()

func (*Message) String

func (m *Message) String() string

type NewPeer

type NewPeer func(p *peerEntry)

type PlannerGrid

type PlannerGrid struct {
	Conf *Conf

	GridServer *grid.Server
	// contains filtered or unexported fields
}

PlannerGrid Is a singleton service context per process that manages access to registry, and other singleton resources. It starts the workers, grid processes, watch to ensure it knows about the rest of the peers in the system.

func NewPlannerGrid

func NewPlannerGrid(nodeCt int, r *schema.Registry) *PlannerGrid

func (*PlannerGrid) CheckinMailbox

func (m *PlannerGrid) CheckinMailbox(mb *grid.Mailbox)

CheckinMailbox return mailbox

func (*PlannerGrid) GetMailbox

func (m *PlannerGrid) GetMailbox() (*grid.Mailbox, error)

GetMailbox get next available mailbox, throttled

func (*PlannerGrid) Run

func (m *PlannerGrid) Run(quit chan bool) error

Run this planner grid server.

type Sink

type Sink struct {
	*exec.TaskBase
	// contains filtered or unexported fields
}

Sink task that receives messages that optionally may have been hashed to be sent via nats to a nats source consumer.

taska-1 ->  hash-key -> sink--> \                 / --> source -->
                                 \               /
                                   --> grid  -->
                                 /               \
taska-2 ->  hash-key -> sink--> /                 \ --> source -->

func NewSink

func NewSink(ctx *plan.Context, destination string, send SinkSend) *Sink

NewSink grid sink to route messages via gnatsd

func (*Sink) Close

func (m *Sink) Close() error

Close cleanup and coalesce

func (*Sink) CloseFinal

func (m *Sink) CloseFinal() error

CloseFinal after shutdown cleanup the rest of channels

func (*Sink) Run

func (m *Sink) Run() error

Run blocking runner

type SinkSend

type SinkSend func(msg interface{}) (interface{}, error)

SinkSend is func to mock the Grid Client Request

type Source

type Source struct {
	*exec.TaskBase
	// contains filtered or unexported fields
}

Source task is injected into a SQL dag task pipeline in order to recieve messages from another server via Grid Mailbox, for distribution across multiple workers. These messages optionally may have been hash routed to this node, ie partition-key routed.

taska-1 ->  hash-sink  \                        / --> source -->
                        \                      /
                          --route-by-key-->   --> source -->
                        /                      \
taska-2 ->  hash-sink  /                        \ --> source -->

func NewSource

func NewSource(ctx *plan.Context, mboxid string, c <-chan grid.Request) *Source

Source, the plan already provided info to the nats listener about which key/topic to listen to, Planner holds routing info not here.

func (*Source) Close

func (m *Source) Close() error

Close cleans up and closes channels

func (*Source) CloseFinal

func (m *Source) CloseFinal() error

CloseFinal after exit, cleanup some more

func (*Source) MailboxId

func (m *Source) MailboxId() string

func (*Source) Run

func (m *Source) Run() error

Run blocking runner

func (*Source) Setup

func (m *Source) Setup(depth int) error

type SqlActor

type SqlActor struct {
	// contains filtered or unexported fields
}

SqlActor a worker/actor that runs in distributed grid nodes and receives messages from the planner to fulfill parts of sql dag of tasks in order to complete a query.

func (*SqlActor) Act

func (m *SqlActor) Act(ctx context.Context)

func (*SqlActor) Exiting

func (m *SqlActor) Exiting()

func (*SqlActor) Finishing

func (m *SqlActor) Finishing() dfa.Letter

func (*SqlActor) ID

func (m *SqlActor) ID() string

func (*SqlActor) Running

func (m *SqlActor) Running() dfa.Letter

func (*SqlActor) Starting

func (m *SqlActor) Starting() dfa.Letter

func (*SqlActor) String

func (m *SqlActor) String() string

type SqlTask

type SqlTask struct {
	Id         string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
	Pb         []byte `protobuf:"bytes,2,opt,name=pb,proto3" json:"pb,omitempty"`
	Partition  string `protobuf:"bytes,3,opt,name=partition" json:"partition,omitempty"`
	Source     string `protobuf:"bytes,4,opt,name=source" json:"source,omitempty"`
	ActorCount int32  `protobuf:"varint,5,opt,name=actorCount" json:"actorCount,omitempty"`
	// The name of the mailbox that contains
	// the master task process to send results to
	Master string `protobuf:"bytes,6,opt,name=master" json:"master,omitempty"`
}

func (*SqlTask) Descriptor

func (*SqlTask) Descriptor() ([]byte, []int)

func (*SqlTask) GetActorCount

func (m *SqlTask) GetActorCount() int32

func (*SqlTask) GetId

func (m *SqlTask) GetId() string

func (*SqlTask) GetMaster

func (m *SqlTask) GetMaster() string

func (*SqlTask) GetPartition

func (m *SqlTask) GetPartition() string

func (*SqlTask) GetPb

func (m *SqlTask) GetPb() []byte

func (*SqlTask) GetSource

func (m *SqlTask) GetSource() string

func (*SqlTask) ProtoMessage

func (*SqlTask) ProtoMessage()

func (*SqlTask) Reset

func (m *SqlTask) Reset()

func (*SqlTask) String

func (m *SqlTask) String() string

type TaskResponse

type TaskResponse struct {
	Id  string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
	Msg string `protobuf:"bytes,2,opt,name=msg" json:"msg,omitempty"`
}

func (*TaskResponse) Descriptor

func (*TaskResponse) Descriptor() ([]byte, []int)

func (*TaskResponse) GetId

func (m *TaskResponse) GetId() string

func (*TaskResponse) GetMsg

func (m *TaskResponse) GetMsg() string

func (*TaskResponse) ProtoMessage

func (*TaskResponse) ProtoMessage()

func (*TaskResponse) Reset

func (m *TaskResponse) Reset()

func (*TaskResponse) String

func (m *TaskResponse) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL