luci: go.chromium.org/luci/client/internal/common Index | Examples | Files

package common

import "go.chromium.org/luci/client/internal/common"

Package common implements code and utilities shared across all packages in client/.

Index

Examples

Package Files

concurrent.go doc.go filesystem_view.go flags.go types.go utils.go

func IsTerminal Uses

func IsTerminal(out io.Writer) bool

IsTerminal returns true if the specified io.Writer is a terminal.

func WalkFuncSkipFile Uses

func WalkFuncSkipFile(file os.FileInfo) error

WalkFuncSkipFile is a helper for implementations of filepath.WalkFunc. The value that it returns may in turn be returned by the WalkFunc implementatiton to indicate that file should be skipped.

type FilesystemView Uses

type FilesystemView struct {
    // contains filtered or unexported fields
}

FilesystemView provides a filtered "view" of a filesystem. It translates absolute paths to relative paths based on its configured root path.

func NewFilesystemView Uses

func NewFilesystemView(root string, ignoredPathRe string) (FilesystemView, error)

NewFilesystemView returns a FilesystemView based on the supplied root, or an error if ignoredPathRe contains a bad pattern.

root is the the base path used by RelativePath to calculate relative paths.

ignoredPathRe is a regular expression string. Note that this is NOT a full string match, so "foo/.*" may match "bar/foo/xyz". Prepend ^ explicitly if you need to match a path that starts with the pattern. Similarly, append $ if necessary.

func (FilesystemView) NewSymlinkedView Uses

func (ff FilesystemView) NewSymlinkedView(source, linkname string) FilesystemView

NewSymlinkedView returns a filesystem view from a symlinked directory within itself.

func (FilesystemView) RelativePath Uses

func (ff FilesystemView) RelativePath(path string) (string, error)

RelativePath returns a version of path which is relative to the FilesystemView root or an empty string if path matches a ignored path filter.

type Flags Uses

type Flags struct {
    Quiet   bool
    Verbose bool
}

Flags contains values parsed from command line arguments.

func (*Flags) Init Uses

func (d *Flags) Init(f *flag.FlagSet)

Init registers flags in a given flag set.

func (*Flags) MakeLoggingContext Uses

func (d *Flags) MakeLoggingContext(out io.Writer) context.Context

MakeLoggingContext makes a luci-go/common/logging compatible context using gologger onto the given writer.

The default logging level will be Info, with Warning and Debug corresponding to quiet/verbose respectively.

func (*Flags) Parse Uses

func (d *Flags) Parse() error

Parse applies changes specified by command line flags.

type GoroutinePool Uses

type GoroutinePool struct {
    // contains filtered or unexported fields
}

GoroutinePool executes at a limited number of jobs concurrently, queueing others.

func NewGoroutinePool Uses

func NewGoroutinePool(ctx context.Context, maxConcurrentJobs int) *GoroutinePool

NewGoroutinePool creates a new GoroutinePool running at most maxConcurrentJobs concurrent operations.

Code:

package main

import (
    "context"
    "fmt"
    "sync"
    "testing"
    "time"

    . "github.com/smartystreets/goconvey/convey"
)

func main() {
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()
    pool := NewGoroutinePool(ctx, 2)
    for _, s := range []string{"knock!", "knock!"} {
        s := s // Create a new s for closure.
        pool.Schedule(func() { fmt.Print(s) }, nil)
    }
    if pool.Wait() == nil {
        fmt.Printf("\n")
    }
    cancel()
    pool.Schedule(func() {}, func() {
        fmt.Printf("canceled because %s\n", ctx.Err())
    })
    err := pool.Wait()
    fmt.Printf("all jobs either executed or canceled (%s)\n", err)
}

type goroutinePool interface {
    Schedule(job, onCanceled func())
    Wait() error
}

type newPoolFunc func(ctx context.Context, p int) goroutinePool

func TestGoroutinePool(t *testing.T) {
    testGoroutinePool(t, func(ctx context.Context, p int) goroutinePool {
        return NewGoroutinePool(ctx, p)
    })
}

func testGoroutinePool(t *testing.T, newPool newPoolFunc) {
    t.Parallel()
    Convey(`A goroutine pool should execute tasks.`, t, func() {

        const MAX = 10
        const J = 200
        pool := newPool(context.Background(), MAX)
        logs := make(chan int)
        for i := 1; i <= J; i++ {
            i := i
            pool.Schedule(func() {
                logs <- -i
                time.Sleep(time.Millisecond)
                logs <- i
            }, nil)
        }
        var fail error
        go func() {
            defer close(logs)
            fail = pool.Wait()
        }()
        currentJobs := map[int]bool{}
        doneJobs := map[int]bool{}
        for j := range logs {
            if j < 0 {
                t.Logf("Job %d started, %d others running", -j, len(currentJobs))
                So(len(currentJobs) < MAX, ShouldBeTrue)
                currentJobs[-j] = true
            } else {
                t.Logf("Job %d ended", j)
                delete(currentJobs, j)
                doneJobs[j] = true
            }
        }
        So(len(currentJobs), ShouldResemble, 0)
        So(len(doneJobs), ShouldResemble, J)
        So(fail, ShouldBeNil)
    })
}

func TestGoroutinePoolCancel(t *testing.T) {
    testGoroutinePoolCancel(t, func(ctx context.Context, p int) goroutinePool {
        return NewGoroutinePool(ctx, p)
    })
}

func testGoroutinePoolCancel(t *testing.T, newPool newPoolFunc) {
    t.Parallel()
    Convey(`A goroutine pool should handle a cancel request.`, t, func() {
        const MAX = 10
        const J = 11 * MAX
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()
        pool := newPool(ctx, MAX)
        logs := make(chan int, 2*J) // Avoid job blocking when writing to log.
        for i := 1; i <= J; i++ {
            i := i
            pool.Schedule(func() {
                if i == 1 {
                    cancel()
                }
                logs <- i
            }, func() {
                // Push negative on canceled tasks.
                logs <- -i
            })
        }
        var wait1 error
        var wait2 error
        go func() {
            defer close(logs)
            wait1 = pool.Wait()
            // Schedule more jobs, all of which must be cancelled.
            for i := J + 1; i <= 2*J; i++ {
                i := i
                pool.Schedule(func() {
                    logs <- i
                }, func() {
                    logs <- -i
                })
            }
            wait2 = pool.Wait()
        }()
        // At most MAX-1 could have been executing concurrently with job[i=1]
        // after it had cancelled the pool and before it wrote to log.
        // No new job should have started after that. So, log must have at most MAX-1
        // jobs after 1.
        finishedPre := 0
        finishedAfter := 0
        canceled := 0
        for i := range logs {
            if i == 1 {
                finishedAfter++
                break
            }
            finishedPre++
        }
        for i := range logs {
            So(i, ShouldBeLessThanOrEqualTo, J)
            if i > 0 {
                finishedAfter++
            } else {
                canceled++
            }
        }
        t.Logf("JOBS: Total %d Pre %d After %d Cancelled %d", 2*J, finishedPre, finishedAfter, canceled)
        So(finishedPre+finishedAfter+canceled, ShouldResemble, 2*J)
        Convey(fmt.Sprintf("%d (expect < %d MAX) jobs started after cancellation, and at least %d J canceled jobs.", finishedAfter, MAX, J), func() {
            So(finishedAfter, ShouldBeLessThan, MAX)
            So(canceled, ShouldBeGreaterThanOrEqualTo, J)
        })
        So(wait1, ShouldResemble, context.Canceled)
        So(wait2, ShouldResemble, context.Canceled)
    })
}

func TestGoroutinePoolCancelFuncCalled(t *testing.T) {
    testGoroutinePoolCancelFuncCalled(t, func(ctx context.Context, p int) goroutinePool {
        return NewGoroutinePool(ctx, p)
    })
}

func testGoroutinePoolCancelFuncCalled(t *testing.T, newPool newPoolFunc) {
    t.Parallel()

    Convey(`A goroutine pool should handle an onCancel call.`, t, func() {
        // Simulate deterministically when the semaphore returns immediately
        // because of cancellation, as opposed to actually having a resource.
        pipe := make(chan string) // must be unbuffered.
        logs := make(chan string, 3)
        slow := func() {
            item := <-pipe // block here until the main Goroutine pushes into pipe.
            logs <- item
            item = <-pipe // block here until the main Goroutine pushes 2nd time.
        }
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()
        pool := newPool(ctx, 1)
        pool.Schedule(slow, slow)
        pipe <- "ensure slow actually started s.t. new job can't *yet* acquire semaphore"
        pool.Schedule(func() { logs <- "job executed" }, func() { logs <- "job cancelled" })
        cancel()
        // Canceling should result in onCancel of last job.
        // In case it's a bug, don't wait forever for slow, but unblock slow().
        pipe <- "let slow job finish *after* ctx is cancelled"
        So(pool.Wait(), ShouldResemble, context.Canceled)
        close(pipe)
        close(logs)
        for l := range logs {
            if l == "job cancelled" {
                return
            }
        }
        t.Fatalf("job wasn't cancelled.")
    })
}

// Purpose: re-use GoroutinePool tests for GoroutinePriorityPool.
type goroutinePriorityPoolforTest struct {
    *GoroutinePriorityPool
}

func (c *goroutinePriorityPoolforTest) Schedule(job func(), onCanceled func()) {
    c.GoroutinePriorityPool.Schedule(0, job, onCanceled)
}

func TestGoroutinePriorityPool(t *testing.T) {
    testGoroutinePool(t, func(ctx context.Context, p int) goroutinePool {
        return &goroutinePriorityPoolforTest{NewGoroutinePriorityPool(ctx, p)}
    })
}

func TestGoroutinePriorityPoolCancel(t *testing.T) {
    testGoroutinePoolCancel(t, func(ctx context.Context, p int) goroutinePool {
        return &goroutinePriorityPoolforTest{NewGoroutinePriorityPool(ctx, p)}
    })
}

func TestGoroutinePriorityPoolCancelFuncCalled(t *testing.T) {
    testGoroutinePoolCancelFuncCalled(t, func(ctx context.Context, p int) goroutinePool {
        return &goroutinePriorityPoolforTest{NewGoroutinePriorityPool(ctx, p)}
    })
}

func TestGoroutinePriorityPoolWithPriority(t *testing.T) {
    t.Parallel()
    Convey(`A goroutine pool should execute high priority jobs first.`, t, func(ctx C) {

        const maxPriorities = 15
        pool := NewGoroutinePriorityPool(context.Background(), 1)
        logs := make(chan int)
        wg := sync.WaitGroup{}
        for i := 0; i < maxPriorities; i++ {
            wg.Add(1)
            i := i
            go func() {
                pool.Schedule(int64(i), func() { logs <- i }, nil)
                wg.Done()
            }()
        }
        wg.Wait()
        var fail error
        go func() {
            defer close(logs)
            fail = pool.Wait()
            ctx.So(fail, ShouldBeNil)
        }()
        doneJobs := make([]bool, maxPriorities)
        // First job can be any, the rest must be in order.
        prio := <-logs
        doneJobs[prio] = true
        for prio := range logs {
            So(doneJobs[prio], ShouldBeFalse)
            doneJobs[prio] = true
            // All higher priority jobs must be finished.
            for before := 0; before < prio; before++ {
                So(doneJobs[before], ShouldBeTrue)
            }
        }
        So(len(doneJobs), ShouldResemble, maxPriorities)
        for _, d := range doneJobs {
            So(d, ShouldBeTrue)
        }
    })
}

func (*GoroutinePool) Schedule Uses

func (g *GoroutinePool) Schedule(job, onCanceled func())

Schedule adds a new job for execution as a separate goroutine.

If the GoroutinePool context is canceled, onCanceled is called instead. It is fine to pass nil as onCanceled.

func (*GoroutinePool) Wait Uses

func (g *GoroutinePool) Wait() error

Wait blocks until all started jobs are done, or the context is canceled.

Returns nil if all jobs have been executed, or the error if the associated context is canceled.

type GoroutinePriorityPool Uses

type GoroutinePriorityPool struct {
    // contains filtered or unexported fields
}

GoroutinePriorityPool executes a limited number of jobs concurrently, queueing others.

func NewGoroutinePriorityPool Uses

func NewGoroutinePriorityPool(ctx context.Context, maxConcurrentJobs int) *GoroutinePriorityPool

NewGoroutinePriorityPool creates a new goroutine pool with at most maxConcurrentJobs.

Each task is run according to the priority of each item.

func (*GoroutinePriorityPool) Schedule Uses

func (g *GoroutinePriorityPool) Schedule(priority int64, job, onCanceled func())

Schedule adds a new job for execution as a separate goroutine.

If the GoroutinePriorityPool is canceled, onCanceled is called instead. It is fine to pass nil as onCanceled. Smaller values of priority imply earlier execution.

The lower the priority value, the higher the priority of the item.

func (*GoroutinePriorityPool) Wait Uses

func (g *GoroutinePriorityPool) Wait() error

Wait blocks until all started jobs are done, or the context is canceled.

Returns nil if all jobs have been executed, or the error if the associated context is canceled.

type Strings Uses

type Strings []string

Strings accumulates string values from repeated flags.

Use with flag.Var to accumulate values from "-flag s1 -flag s2".

func (*Strings) Set Uses

func (c *Strings) Set(value string) error

Set is needed to implements flag.Var interface.

func (*Strings) String Uses

func (c *Strings) String() string

Package common imports 16 packages (graph) and is imported by 16 packages. Updated 2020-11-26. Refresh now. Tools for package owners.