pachyderm: github.com/pachyderm/pachyderm/src/server/pkg/work Index | Files

package work

import "github.com/pachyderm/pachyderm/src/server/pkg/work"

Index

Package Files

task_queue.go work.go work.pb.go

Variables

var (
    ErrInvalidLengthWork        = fmt.Errorf("proto: negative length found during unmarshaling")
    ErrIntOverflowWork          = fmt.Errorf("proto: integer overflow")
    ErrUnexpectedEndOfGroupWork = fmt.Errorf("proto: unexpected end of group")
)
var State_name = map[int32]string{
    0:  "RUNNING",
    1:  "SUCCESS",
    2:  "FAILURE",
}
var State_value = map[string]int32{
    "RUNNING": 0,
    "SUCCESS": 1,
    "FAILURE": 2,
}

type Claim Uses

type Claim struct {
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*Claim) Descriptor Uses

func (*Claim) Descriptor() ([]byte, []int)

func (*Claim) Marshal Uses

func (m *Claim) Marshal() (dAtA []byte, err error)

func (*Claim) MarshalTo Uses

func (m *Claim) MarshalTo(dAtA []byte) (int, error)

func (*Claim) MarshalToSizedBuffer Uses

func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Claim) ProtoMessage Uses

func (*Claim) ProtoMessage()

func (*Claim) Reset Uses

func (m *Claim) Reset()

func (*Claim) Size Uses

func (m *Claim) Size() (n int)

func (*Claim) String Uses

func (m *Claim) String() string

func (*Claim) Unmarshal Uses

func (m *Claim) Unmarshal(dAtA []byte) error

func (*Claim) XXX_DiscardUnknown Uses

func (m *Claim) XXX_DiscardUnknown()

func (*Claim) XXX_Marshal Uses

func (m *Claim) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Claim) XXX_Merge Uses

func (m *Claim) XXX_Merge(src proto.Message)

func (*Claim) XXX_Size Uses

func (m *Claim) XXX_Size() int

func (*Claim) XXX_Unmarshal Uses

func (m *Claim) XXX_Unmarshal(b []byte) error

type CollectFunc Uses

type CollectFunc func(context.Context, *TaskInfo) error

CollectFunc is a callback that is used for collecting the results from a subtask that has been processed.

type Master Uses

type Master struct {
    // contains filtered or unexported fields
}

Master manages subtasks in the task queue, and provides an interface for running subtasks.

func (*Master) Ctx Uses

func (m *Master) Ctx() context.Context

Ctx returns the context for the master.

func (*Master) RunSubtasks Uses

func (m *Master) RunSubtasks(subtasks []*Task, collectFunc CollectFunc) (retErr error)

RunSubtasks runs a set of subtasks and collects the results with the passed in callback.

func (*Master) RunSubtasksChan Uses

func (m *Master) RunSubtasksChan(subtaskChan chan *Task, collectFunc CollectFunc) (retErr error)

RunSubtasksChan runs a set of subtasks (provided through a channel) and collects the results with the passed in callback.

type ProcessFunc Uses

type ProcessFunc func(context.Context, *Task) error

ProcessFunc is a callback that is used for processing a subtask in a task.

type State Uses

type State int32
const (
    State_RUNNING State = 0
    State_SUCCESS State = 1
    State_FAILURE State = 2
)

func (State) EnumDescriptor Uses

func (State) EnumDescriptor() ([]byte, []int)

func (State) String Uses

func (x State) String() string

type Task Uses

type Task struct {
    ID                   string     `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
    Data                 *types.Any `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
    XXX_NoUnkeyedLiteral struct{}   `json:"-"`
    XXX_unrecognized     []byte     `json:"-"`
    XXX_sizecache        int32      `json:"-"`
}

func (*Task) Descriptor Uses

func (*Task) Descriptor() ([]byte, []int)

func (*Task) GetData Uses

func (m *Task) GetData() *types.Any

func (*Task) GetID Uses

func (m *Task) GetID() string

func (*Task) Marshal Uses

func (m *Task) Marshal() (dAtA []byte, err error)

func (*Task) MarshalTo Uses

func (m *Task) MarshalTo(dAtA []byte) (int, error)

func (*Task) MarshalToSizedBuffer Uses

func (m *Task) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Task) ProtoMessage Uses

func (*Task) ProtoMessage()

func (*Task) Reset Uses

func (m *Task) Reset()

func (*Task) Size Uses

func (m *Task) Size() (n int)

func (*Task) String Uses

func (m *Task) String() string

func (*Task) Unmarshal Uses

func (m *Task) Unmarshal(dAtA []byte) error

func (*Task) XXX_DiscardUnknown Uses

func (m *Task) XXX_DiscardUnknown()

func (*Task) XXX_Marshal Uses

func (m *Task) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Task) XXX_Merge Uses

func (m *Task) XXX_Merge(src proto.Message)

func (*Task) XXX_Size Uses

func (m *Task) XXX_Size() int

func (*Task) XXX_Unmarshal Uses

func (m *Task) XXX_Unmarshal(b []byte) error

type TaskInfo Uses

type TaskInfo struct {
    Task                 *Task    `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"`
    State                State    `protobuf:"varint,2,opt,name=state,proto3,enum=work.State" json:"state,omitempty"`
    Reason               string   `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*TaskInfo) Descriptor Uses

func (*TaskInfo) Descriptor() ([]byte, []int)

func (*TaskInfo) GetReason Uses

func (m *TaskInfo) GetReason() string

func (*TaskInfo) GetState Uses

func (m *TaskInfo) GetState() State

func (*TaskInfo) GetTask Uses

func (m *TaskInfo) GetTask() *Task

func (*TaskInfo) Marshal Uses

func (m *TaskInfo) Marshal() (dAtA []byte, err error)

func (*TaskInfo) MarshalTo Uses

func (m *TaskInfo) MarshalTo(dAtA []byte) (int, error)

func (*TaskInfo) MarshalToSizedBuffer Uses

func (m *TaskInfo) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*TaskInfo) ProtoMessage Uses

func (*TaskInfo) ProtoMessage()

func (*TaskInfo) Reset Uses

func (m *TaskInfo) Reset()

func (*TaskInfo) Size Uses

func (m *TaskInfo) Size() (n int)

func (*TaskInfo) String Uses

func (m *TaskInfo) String() string

func (*TaskInfo) Unmarshal Uses

func (m *TaskInfo) Unmarshal(dAtA []byte) error

func (*TaskInfo) XXX_DiscardUnknown Uses

func (m *TaskInfo) XXX_DiscardUnknown()

func (*TaskInfo) XXX_Marshal Uses

func (m *TaskInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*TaskInfo) XXX_Merge Uses

func (m *TaskInfo) XXX_Merge(src proto.Message)

func (*TaskInfo) XXX_Size Uses

func (m *TaskInfo) XXX_Size() int

func (*TaskInfo) XXX_Unmarshal Uses

func (m *TaskInfo) XXX_Unmarshal(b []byte) error

type TaskQueue Uses

type TaskQueue struct {
    // contains filtered or unexported fields
}

TaskQueue manages a set of parallel tasks, and provides an interface for running tasks. Priority of tasks (and therefore subtasks) is based on task creation time, so tasks created earlier will be prioritized over tasks that were created later.

func NewTaskQueue Uses

func NewTaskQueue(ctx context.Context, etcdClient *etcd.Client, etcdPrefix string, taskNamespace string) (*TaskQueue, error)

NewTaskQueue sets up a new task queue.

func (*TaskQueue) RunTask Uses

func (tq *TaskQueue) RunTask(ctx context.Context, f func(*Master)) (retErr error)

RunTask runs a task in the task queue. The task code should be contained within the passed in callback. The callback will receive a Master, which should be used for running subtasks in the task queue. The task state will be cleaned up upon return of the callback.

func (*TaskQueue) RunTaskBlock Uses

func (tq *TaskQueue) RunTaskBlock(ctx context.Context, f func(*Master) error) error

RunTaskBlock is similar to RunTask, but blocks on the callback.

type TestData Uses

type TestData struct {
    Processed            bool     `protobuf:"varint,1,opt,name=processed,proto3" json:"processed,omitempty"`
    XXX_NoUnkeyedLiteral struct{} `json:"-"`
    XXX_unrecognized     []byte   `json:"-"`
    XXX_sizecache        int32    `json:"-"`
}

func (*TestData) Descriptor Uses

func (*TestData) Descriptor() ([]byte, []int)

func (*TestData) GetProcessed Uses

func (m *TestData) GetProcessed() bool

func (*TestData) Marshal Uses

func (m *TestData) Marshal() (dAtA []byte, err error)

func (*TestData) MarshalTo Uses

func (m *TestData) MarshalTo(dAtA []byte) (int, error)

func (*TestData) MarshalToSizedBuffer Uses

func (m *TestData) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*TestData) ProtoMessage Uses

func (*TestData) ProtoMessage()

func (*TestData) Reset Uses

func (m *TestData) Reset()

func (*TestData) Size Uses

func (m *TestData) Size() (n int)

func (*TestData) String Uses

func (m *TestData) String() string

func (*TestData) Unmarshal Uses

func (m *TestData) Unmarshal(dAtA []byte) error

func (*TestData) XXX_DiscardUnknown Uses

func (m *TestData) XXX_DiscardUnknown()

func (*TestData) XXX_Marshal Uses

func (m *TestData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*TestData) XXX_Merge Uses

func (m *TestData) XXX_Merge(src proto.Message)

func (*TestData) XXX_Size Uses

func (m *TestData) XXX_Size() int

func (*TestData) XXX_Unmarshal Uses

func (m *TestData) XXX_Unmarshal(b []byte) error

type Worker Uses

type Worker struct {
    // contains filtered or unexported fields
}

Worker is a worker that will process subtasks in a task. A worker watches the task collection for tasks to be created / deleted and appropriately runs / deletes tasks in the internal task queue with a function that watches the subtask and claim collections for subtasks that need to be processed. The processFunc callback will be called for each subtask that needs to be processed in the task.

func NewWorker Uses

func NewWorker(etcdClient *etcd.Client, etcdPrefix string, taskNamespace string) *Worker

NewWorker creates a new worker.

func (*Worker) Run Uses

func (w *Worker) Run(ctx context.Context, processFunc ProcessFunc) error

Run runs the worker with the given context. The worker will continue to watch the task collection until the context is canceled.

Package work imports 20 packages (graph) and is imported by 5 packages. Updated 2020-08-13. Refresh now. Tools for package owners.