dag

DAG

Overview

DAG (Directed Acyclic Graph) based job runner.

Statuses

Job statuses

DAG statuses

Examples

Dag with a soft error

package main

// DAG job 'Task 2' finishes with a soft error => following tasks will succeed
// As we only have succssesful tasks or soft errors, DAG succeeds

// Using LOGRUS as logger. See example 1 and 2 how to use ZAP

import (
  "errors"
  "fmt"
  "os"
  "time"

  "github.com/sirupsen/logrus"
  "gitlab.com/lesql/dag"
)

func main() {

  mylogger := logrus.New()
  myloggerDag := mylogger.WithFields(logrus.Fields{
    "xyz": "xyz",
  })

  dagHooks := dag.Hooks{
    DagCreated: func(dag *dag.Dag) {
      mylogger.Infof("DagCreated: %s - %s ", dag.Name, dag.ID)
    },
    DagStarted: func(dag *dag.Dag) {
      mylogger.Infoln("DagStarted")
    },
    DagFinishedSuccess: func(dag *dag.Dag) {
      mylogger.Infoln("DagFinishedSuccess")
    },
    DagFinishedFailed: func(dag *dag.Dag) {
      mylogger.Errorln("DagFinishedFailed")
    },
    JobCreated: func(dagJob *dag.Job) {
      mylogger.Infoln("JobCreated")
    },
    JobStarted: func(dagJob *dag.Job) {
      mylogger.Infof("JobStarted: %s - %s", dagJob.Name, dagJob.ID)
    },
    JobFinishedSuccess: func(dagJob *dag.Job) {
      mylogger.Infof("JobFinishedSuccess: %s - %s", dagJob.Name, dagJob.ID)
    },
    JobFinishedFailed: func(dagJob *dag.Job) {
      mylogger.Errorf("JobFinishedFailed: %s - %s", dagJob.Name, dagJob.ID)
    },
  }

  dag, err := dag.New("Dag 1", myloggerDag, dagHooks)
  if err != nil {
    mylogger.Errorln(err)
    os.Exit(1)
  }

  job1, err := dag.CreateJob("Task 1", func() (map[string]string, error) {
    fmt.Println("Running task 1: sleep 12s")
    time.Sleep(12 * time.Second)
    ret := make(map[string]string)
    return ret, nil
  })

  if err != nil {
    logrus.Errorln(err)
    os.Exit(1)
  }

  job2, err := dag.CreateJob("Task 2", func() (map[string]string, error) {
    fmt.Println("Running task 2: sleep 10s")
    time.Sleep(10 * time.Second)
    ret := make(map[string]string)
    ret["SOFT_ERROR"] = "SOFT_ERROR"
    return ret, errors.New("Task 2 failed")
  })

  if err != nil {
    logrus.Errorln(err)
    os.Exit(1)
  }

  job3, err := dag.CreateJob("Task 3", func() (map[string]string, error) {
    fmt.Println("Running task 3: sleep 5s")
    time.Sleep(5 * time.Second)
    ret := make(map[string]string)
    return ret, nil
  })

  if err != nil {
    logrus.Errorln(err)
    os.Exit(1)
  }

  err = job1.Then(job3)
  if err != nil {
    mylogger.Errorln(err)
    os.Exit(1)
  }

  err = job2.Then(job3)
  if err != nil {
    mylogger.Errorln(err)
    os.Exit(1)
  }

  err = dag.Run()
  if err != nil {
    mylogger.Errorln(err)
    os.Exit(1)
  }
}

For more examples, see folder examples

Development

Run tests

go test --race -cover ./...  -v