Documentation ¶
Overview ¶
Package gocelery is Celery Distributed Task Queue in Go
Celery distributed tasks are used heavily in many python web applications and this library allows you to implement celery workers in Go as well as being able to submit celery tasks in Go.
This package can also be used as pure go distributed task queue.
Supported brokers/backends
- Redis (broker/backend)
- AMQP (broker/backend)
Celery must be configured to use json instead of default pickle encoding. This is because Go currently has no stable support for decoding pickle objects. Pass below configuration parameters to use json.
CELERY_TASK_SERIALIZER='json' CELERY_ACCEPT_CONTENT=['json'] # Ignore other content CELERY_RESULT_SERIALIZER='json' CELERY_ENABLE_UTC=True
Example (Client) ¶
connectOpions, err := redis.ParseURL("redis://") if err != nil { panic(err) } client := redis.NewClient(connectOpions) // initialize celery client cli, _ := NewCeleryClient( NewRedisBroker(client), &RedisCeleryBackend{UniversalClient: client}, 1, ) // prepare arguments taskName := "worker.add" argA := rand.Intn(10) argB := rand.Intn(10) task := GetTaskMessage(taskName) task.Args = append(task.Args, argA, argB) // run task asyncResult, err := cli.Delay(context.Background(), task) if err != nil { panic(err) } // get results from backend with timeout res, err := asyncResult.Get(context.Background(), 10*time.Second) if err != nil { panic(err) } log.Printf("result: %+v of type %+v", res, reflect.TypeOf(res))
Output:
Example (ClientWithNamedArguments) ¶
connectOpions, err := redis.ParseURL("redis://") if err != nil { panic(err) } client := redis.NewClient(connectOpions) // initialize celery client cli, _ := NewCeleryClient( NewRedisBroker(client), &RedisCeleryBackend{UniversalClient: client}, 1, ) // prepare arguments taskName := "worker.add" argA := rand.Intn(10) argB := rand.Intn(10) // run task asyncResult, err := cli.DelayKwargs( context.Background(), taskName, map[string]interface{}{ "a": argA, "b": argB, }, ) if err != nil { panic(err) } // get results from backend with timeout res, err := asyncResult.Get(context.Background(), 10*time.Second) if err != nil { panic(err) } log.Printf("result: %+v of type %+v", res, reflect.TypeOf(res))
Output:
Example (Worker) ¶
connectOpions, err := redis.ParseURL("redis://") if err != nil { panic(err) } client := redis.NewClient(connectOpions) // initialize celery client cli, _ := NewCeleryClient( NewRedisBroker(client), &RedisCeleryBackend{UniversalClient: client}, 5, // number of workers ) // task add := func(a, b int) int { return a + b } // register task cli.Register("add", add) // start workers (non-blocking call) cli.StartWorker() // wait for client request time.Sleep(10 * time.Second) // stop workers gracefully (blocking call) cli.StopWorker()
Output:
Example (WorkerWithContext) ¶
connectOpions, err := redis.ParseURL("redis://") if err != nil { panic(err) } client := redis.NewClient(connectOpions) // initialize celery client cli, _ := NewCeleryClient( NewRedisBroker(client), &RedisCeleryBackend{UniversalClient: client}, 1, ) // task add := func(a, b int) int { return a + b } // register task cli.Register("add", add) // context with cancelFunc to handle exit gracefully ctx, cancel := context.WithCancel(context.Background()) // start workers (non-blocking call) cli.StartWorkerWithContext(ctx) // wait for client request time.Sleep(10 * time.Second) // stop workers by cancelling context cancel() // optional: wait for all workers to terminate cli.WaitForStopWorker()
Output:
Example (WorkerWithNamedArguments) ¶
package main import ( "fmt" "time" "github.com/redis/go-redis/v9" ) // exampleAddTask is integer addition task // with named arguments type exampleAddTask struct { a int b int } func (a *exampleAddTask) ParseKwargs(kwargs map[string]interface{}) error { kwargA, ok := kwargs["a"] if !ok { return fmt.Errorf("undefined kwarg a") } kwargAFloat, ok := kwargA.(float64) if !ok { return fmt.Errorf("malformed kwarg a") } a.a = int(kwargAFloat) kwargB, ok := kwargs["b"] if !ok { return fmt.Errorf("undefined kwarg b") } kwargBFloat, ok := kwargB.(float64) if !ok { return fmt.Errorf("malformed kwarg b") } a.b = int(kwargBFloat) return nil } func (a *exampleAddTask) RunTask() (interface{}, error) { result := a.a + a.b return result, nil } func main() { connectOpions, err := redis.ParseURL("redis://") if err != nil { panic(err) } client := redis.NewClient(connectOpions) // initialize celery client cli, _ := NewCeleryClient( NewRedisBroker(client), &RedisCeleryBackend{UniversalClient: client}, 5, // number of workers ) // register task cli.Register("add", &exampleAddTask{}) // start workers (non-blocking call) cli.StartWorker() // wait for client request time.Sleep(10 * time.Second) // stop workers gracefully (blocking call) cli.StopWorker() }
Output:
Index ¶
- func GetRealValue(val *reflect.Value) interface{}
- func NewAMQPConnection(host string) (*amqp.Connection, *amqp.Channel)
- type AMQPCeleryBackend
- func (cb *AMQPCeleryBackend) DeleteResult(ctx context.Context, taskID string) error
- func (b *AMQPCeleryBackend) GetResult(ctx context.Context, taskID string) (*ResultMessage, error)
- func (b *AMQPCeleryBackend) Reconnect()
- func (b *AMQPCeleryBackend) SetResult(ctx context.Context, taskID string, result *ResultMessage) error
- type AMQPCeleryBroker
- func (b *AMQPCeleryBroker) CreateExchange() error
- func (b *AMQPCeleryBroker) CreateQueue() error
- func (b *AMQPCeleryBroker) GetTaskMessage(ctx context.Context) (*TaskMessage, error)
- func (b *AMQPCeleryBroker) SendCeleryMessage(ctx context.Context, message *CeleryMessage) error
- func (b *AMQPCeleryBroker) StartConsumingChannel() error
- type AMQPExchange
- type AMQPQueue
- type AsyncResult
- type CeleryBackend
- type CeleryBroker
- type CeleryClient
- func (cc *CeleryClient) Delay(ctx context.Context, task *TaskMessage) (*AsyncResult, error)
- func (cc *CeleryClient) DelayArgs(ctx context.Context, task string, queue string, args ...interface{}) (*AsyncResult, error)
- func (cc *CeleryClient) DelayKwargs(ctx context.Context, task string, queue string, args map[string]interface{}) (*AsyncResult, error)
- func (cc *CeleryClient) Register(name string, task interface{})
- func (cc *CeleryClient) StartWorker()
- func (cc *CeleryClient) StartWorkerWithContext(ctx context.Context)
- func (cc *CeleryClient) StopWorker()
- func (cc *CeleryClient) WaitForStopWorker()
- type CeleryDeliveryInfo
- type CeleryMessage
- type CeleryProperties
- type CeleryTask
- type CeleryWorker
- func (w *CeleryWorker) GetNumWorkers() int
- func (w *CeleryWorker) GetTask(name string) interface{}
- func (w *CeleryWorker) Register(name string, task interface{})
- func (w *CeleryWorker) RunTask(message *TaskMessage) (*ResultMessage, error)
- func (w *CeleryWorker) StartWorker()
- func (w *CeleryWorker) StartWorkerWithContext(ctx context.Context)
- func (w *CeleryWorker) StopWait()
- func (w *CeleryWorker) StopWorker()
- type PropagatedError
- type RedisCeleryBackend
- type RedisCeleryBroker
- type ResultMessage
- type TaskMessage
- type TaskMessageTimeStamp
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetRealValue ¶
GetRealValue returns real value of reflect.Value Required for JSON Marshalling
func NewAMQPConnection ¶
func NewAMQPConnection(host string) (*amqp.Connection, *amqp.Channel)
NewAMQPConnection creates new AMQP channel
Types ¶
type AMQPCeleryBackend ¶
type AMQPCeleryBackend struct { *amqp.Channel Connection *amqp.Connection Host string }
AMQPCeleryBackend CeleryBackend for AMQP
func NewAMQPCeleryBackend ¶
func NewAMQPCeleryBackend(host string) *AMQPCeleryBackend
NewAMQPCeleryBackend creates new AMQPCeleryBackend
func NewAMQPCeleryBackendByConnAndChannel ¶
func NewAMQPCeleryBackendByConnAndChannel(conn *amqp.Connection, channel *amqp.Channel) *AMQPCeleryBackend
NewAMQPCeleryBackendByConnAndChannel creates new AMQPCeleryBackend by AMQP connection and channel
func (*AMQPCeleryBackend) DeleteResult ¶
func (cb *AMQPCeleryBackend) DeleteResult(ctx context.Context, taskID string) error
DeleteResult deletes the result from the AMQP queue
func (*AMQPCeleryBackend) GetResult ¶
func (b *AMQPCeleryBackend) GetResult(ctx context.Context, taskID string) (*ResultMessage, error)
GetResult retrieves result from AMQP queue
func (*AMQPCeleryBackend) Reconnect ¶
func (b *AMQPCeleryBackend) Reconnect()
Reconnect reconnects to AMQP server
func (*AMQPCeleryBackend) SetResult ¶
func (b *AMQPCeleryBackend) SetResult(ctx context.Context, taskID string, result *ResultMessage) error
SetResult sets result back to AMQP queue
type AMQPCeleryBroker ¶
type AMQPCeleryBroker struct { *amqp.Channel Connection *amqp.Connection Exchange *AMQPExchange Queue *AMQPQueue Rate int // contains filtered or unexported fields }
AMQPCeleryBroker is RedisBroker for AMQP
func NewAMQPCeleryBroker ¶
func NewAMQPCeleryBroker(host string) *AMQPCeleryBroker
NewAMQPCeleryBroker creates new AMQPCeleryBroker
func NewAMQPCeleryBrokerByConnAndChannel ¶
func NewAMQPCeleryBrokerByConnAndChannel(conn *amqp.Connection, channel *amqp.Channel) *AMQPCeleryBroker
NewAMQPCeleryBrokerByConnAndChannel creates new AMQPCeleryBroker using AMQP conn and channel
func (*AMQPCeleryBroker) CreateExchange ¶
func (b *AMQPCeleryBroker) CreateExchange() error
CreateExchange declares AMQP exchange with stored configuration
func (*AMQPCeleryBroker) CreateQueue ¶
func (b *AMQPCeleryBroker) CreateQueue() error
CreateQueue declares AMQP Queue with stored configuration
func (*AMQPCeleryBroker) GetTaskMessage ¶
func (b *AMQPCeleryBroker) GetTaskMessage(ctx context.Context) (*TaskMessage, error)
GetTaskMessage retrieves task message from AMQP queue
func (*AMQPCeleryBroker) SendCeleryMessage ¶
func (b *AMQPCeleryBroker) SendCeleryMessage(ctx context.Context, message *CeleryMessage) error
SendCeleryMessage sends CeleryMessage to broker
func (*AMQPCeleryBroker) StartConsumingChannel ¶
func (b *AMQPCeleryBroker) StartConsumingChannel() error
StartConsumingChannel spawns receiving channel on AMQP queue
type AMQPExchange ¶
AMQPExchange stores AMQP Exchange configuration
func NewAMQPExchange ¶
func NewAMQPExchange(name string) *AMQPExchange
NewAMQPExchange creates new AMQPExchange
type AsyncResult ¶
type AsyncResult struct { TaskID string // contains filtered or unexported fields }
AsyncResult represents pending result
func (*AsyncResult) AsyncGet ¶
func (ar *AsyncResult) AsyncGet(ctx context.Context) (interface{}, error)
AsyncGet gets actual result from backend and returns nil if not available
func (*AsyncResult) Delete ¶
func (ar *AsyncResult) Delete(ctx context.Context) error
Delete deletes the result from the backend
type CeleryBackend ¶
type CeleryBackend interface { GetResult(context.Context, string) (*ResultMessage, error) // must be non-blocking SetResult(ctx context.Context, taskID string, result *ResultMessage) error DeleteResult(ctx context.Context, taskID string) error }
CeleryBackend is interface for celery backend database
type CeleryBroker ¶
type CeleryBroker interface { SendCeleryMessage(context.Context, *CeleryMessage) error GetTaskMessage(context.Context) (*TaskMessage, error) // must be non-blocking }
CeleryBroker is interface for celery broker database
type CeleryClient ¶
type CeleryClient struct {
// contains filtered or unexported fields
}
CeleryClient provides API for sending celery tasks
func NewCeleryClient ¶
func NewCeleryClient(broker CeleryBroker, backend CeleryBackend, numWorkers int) (*CeleryClient, error)
NewCeleryClient creates new celery client
func (*CeleryClient) Delay ¶
func (cc *CeleryClient) Delay(ctx context.Context, task *TaskMessage) (*AsyncResult, error)
Delay adds the given task to the queue
func (*CeleryClient) DelayArgs ¶
func (cc *CeleryClient) DelayArgs(ctx context.Context, task string, queue string, args ...interface{}) (*AsyncResult, error)
DelayArgs gets asynchronous result with an array of arguments
func (*CeleryClient) DelayKwargs ¶
func (cc *CeleryClient) DelayKwargs(ctx context.Context, task string, queue string, args map[string]interface{}) (*AsyncResult, error)
DelayKwargs gets asynchronous results with argument map
func (*CeleryClient) Register ¶
func (cc *CeleryClient) Register(name string, task interface{})
Register task
func (*CeleryClient) StartWorker ¶
func (cc *CeleryClient) StartWorker()
StartWorker starts celery workers
func (*CeleryClient) StartWorkerWithContext ¶
func (cc *CeleryClient) StartWorkerWithContext(ctx context.Context)
StartWorkerWithContext starts celery workers with given parent context
func (*CeleryClient) StopWorker ¶
func (cc *CeleryClient) StopWorker()
StopWorker stops celery workers
func (*CeleryClient) WaitForStopWorker ¶
func (cc *CeleryClient) WaitForStopWorker()
WaitForStopWorker waits for celery workers to terminate
type CeleryDeliveryInfo ¶
type CeleryDeliveryInfo struct { Priority int `json:"priority"` RoutingKey string `json:"routing_key"` Exchange string `json:"exchange"` }
CeleryDeliveryInfo represents deliveryinfo json
type CeleryMessage ¶
type CeleryMessage struct { Body string `json:"body"` Headers map[string]interface{} `json:"headers,omitempty"` ContentType string `json:"content-type"` Properties CeleryProperties `json:"properties"` ContentEncoding string `json:"content-encoding"` }
CeleryMessage is actual message to be sent to Redis
func (*CeleryMessage) GetTaskMessage ¶
func (cm *CeleryMessage) GetTaskMessage() *TaskMessage
GetTaskMessage retrieve and decode task messages from broker
type CeleryProperties ¶
type CeleryProperties struct { BodyEncoding string `json:"body_encoding"` CorrelationID string `json:"correlation_id"` ReplyTo string `json:"reply_to"` DeliveryInfo CeleryDeliveryInfo `json:"delivery_info"` DeliveryMode int `json:"delivery_mode"` DeliveryTag string `json:"delivery_tag"` }
CeleryProperties represents properties json
type CeleryTask ¶
type CeleryTask interface { // ParseKwargs - define a method to parse kwargs ParseKwargs(map[string]interface{}) error // RunTask - define a method for execution RunTask() (interface{}, error) }
CeleryTask is an interface that represents actual task Passing CeleryTask interface instead of function pointer avoids reflection and may have performance gain. ResultMessage must be obtained using GetResultMessage()
type CeleryWorker ¶
type CeleryWorker struct {
// contains filtered or unexported fields
}
CeleryWorker represents distributed task worker
func NewCeleryWorker ¶
func NewCeleryWorker(broker CeleryBroker, backend CeleryBackend, numWorkers int) *CeleryWorker
NewCeleryWorker returns new celery worker
func (*CeleryWorker) GetNumWorkers ¶
func (w *CeleryWorker) GetNumWorkers() int
GetNumWorkers returns number of currently running workers
func (*CeleryWorker) GetTask ¶
func (w *CeleryWorker) GetTask(name string) interface{}
GetTask retrieves registered task
func (*CeleryWorker) Register ¶
func (w *CeleryWorker) Register(name string, task interface{})
Register registers tasks (functions)
func (*CeleryWorker) RunTask ¶
func (w *CeleryWorker) RunTask(message *TaskMessage) (*ResultMessage, error)
RunTask runs celery task
func (*CeleryWorker) StartWorker ¶
func (w *CeleryWorker) StartWorker()
StartWorker starts celery workers
func (*CeleryWorker) StartWorkerWithContext ¶
func (w *CeleryWorker) StartWorkerWithContext(ctx context.Context)
StartWorkerWithContext starts celery worker(s) with given parent context
func (*CeleryWorker) StopWait ¶
func (w *CeleryWorker) StopWait()
StopWait waits for celery workers to terminate
func (*CeleryWorker) StopWorker ¶
func (w *CeleryWorker) StopWorker()
StopWorker stops celery workers
type PropagatedError ¶
type PropagatedError struct {
ResMsg ResultMessage // The Celery ResultMessage containing the error
}
PropagatedError is an error that is returned when an error occurs in a Celery worker
func (*PropagatedError) Error ¶
func (e *PropagatedError) Error() string
type RedisCeleryBackend ¶
type RedisCeleryBackend struct {
redis.UniversalClient
}
RedisCeleryBackend is celery backend for redis
func NewRedisBackend ¶
func NewRedisBackend(conn redis.UniversalClient) *RedisCeleryBackend
NewRedisBackend creates new RedisCeleryBackend with given redis pool. RedisCeleryBackend can be initialized manually as well.
func NewRedisCeleryBackend
deprecated
func NewRedisCeleryBackend(uri string) *RedisCeleryBackend
NewRedisCeleryBackend creates new RedisCeleryBackend
Deprecated: NewRedisCeleryBackend exists for historical compatibility and should not be used. Pool should be initialized outside of gocelery package.
func (*RedisCeleryBackend) DeleteResult ¶
func (cb *RedisCeleryBackend) DeleteResult(ctx context.Context, taskID string) error
DeleteResult deletes the result from the redis backend
func (*RedisCeleryBackend) GetResult ¶
func (cb *RedisCeleryBackend) GetResult(ctx context.Context, taskID string) (*ResultMessage, error)
GetResult queries redis backend to get asynchronous result
func (*RedisCeleryBackend) SetResult ¶
func (cb *RedisCeleryBackend) SetResult(ctx context.Context, taskID string, result *ResultMessage) error
SetResult pushes result back into redis backend
type RedisCeleryBroker ¶
type RedisCeleryBroker struct { redis.UniversalClient QueueName string }
RedisCeleryBroker is celery broker for redis
func NewRedisBroker ¶
func NewRedisBroker(conn redis.UniversalClient) *RedisCeleryBroker
NewRedisBroker creates new RedisCeleryBroker with given redis connection pool
func NewRedisCeleryBroker
deprecated
func NewRedisCeleryBroker(uri string) *RedisCeleryBroker
NewRedisCeleryBroker creates new RedisCeleryBroker based on given uri
Deprecated: NewRedisCeleryBroker exists for historical compatibility and should not be used. Use NewRedisBroker instead to create new RedisCeleryBroker.
func (*RedisCeleryBroker) GetCeleryMessage ¶
func (cb *RedisCeleryBroker) GetCeleryMessage(ctx context.Context) (*CeleryMessage, error)
GetCeleryMessage retrieves celery message from redis queue
func (*RedisCeleryBroker) GetTaskMessage ¶
func (cb *RedisCeleryBroker) GetTaskMessage(ctx context.Context) (*TaskMessage, error)
GetTaskMessage retrieves task message from redis queue
func (*RedisCeleryBroker) SendCeleryMessage ¶
func (cb *RedisCeleryBroker) SendCeleryMessage(ctx context.Context, message *CeleryMessage) error
SendCeleryMessage sends CeleryMessage to redis queue
type ResultMessage ¶
type ResultMessage struct { ID string `json:"task_id"` Status string `json:"status"` Traceback interface{} `json:"traceback"` Result interface{} `json:"result"` Children []interface{} `json:"children"` }
ResultMessage is return message received from broker
type TaskMessage ¶
type TaskMessage struct { ID string `json:"id"` Task string `json:"task"` Args []interface{} `json:"args"` Kwargs map[string]interface{} `json:"kwargs"` Headers map[string]interface{} `json:"headers"` Retries int `json:"retries"` ETA *string `json:"eta"` Expires *TaskMessageTimeStamp `json:"expires"` RootID *string `json:"root_id"` ParentID *string `json:"parent_id"` GroupID *string `json:"group_id"` TimeLimit []float64 `json:"timelimit"` // Non Celery properties. Queue *string `json:"-"` // This allows you to use a different queue from this task. }
TaskMessage is celery-compatible message
func DecodeTaskMessage ¶
func DecodeTaskMessage(encodedBody string) (*TaskMessage, error)
DecodeTaskMessage decodes base64 encrypted body and return TaskMessage object
func GetTaskMessage ¶
func GetTaskMessage(task string) *TaskMessage
func (*TaskMessage) Encode ¶
func (tm *TaskMessage) Encode() (string, error)
Encode returns base64 json encoded string
type TaskMessageTimeStamp ¶
func (TaskMessageTimeStamp) MarshalJSON ¶
func (ct TaskMessageTimeStamp) MarshalJSON() ([]byte, error)
MarshalJSON writes a quoted string in the custom format
func (*TaskMessageTimeStamp) String ¶
func (ct *TaskMessageTimeStamp) String() string
String returns the time in the custom format
func (*TaskMessageTimeStamp) Time ¶
func (ct *TaskMessageTimeStamp) Time() time.Time
String returns the time in the custom format
func (*TaskMessageTimeStamp) UnmarshalJSON ¶
func (t *TaskMessageTimeStamp) UnmarshalJSON(b []byte) (err error)
UnmarshalJSON Parses the json string in the custom format