go_os_io/

back to contents

Go: os, io

• Reference
• package os
• package os/exec
• package flag
• package io
• io.Pipe
• package io/ioutil
• stdout, stdin, stderr
• exist, readDir, recover
• create/open/write: files, directories
• io/ioutil, file
• bufio, file
• copy: files, directories
• csv
• tsv
• compress/gzip
• temporary file
• walk
• http.Flusher
• os.Signal
• importDeps
• directory
• streaming OS command
• streaming long running
• java

↑ top

---

Reference

• CORE TECHNOLOGY: SIGNALS(Linux)

↑ top

---

package os

Package os provides a platform-independent interface to operating system functionality. Note that os.File implements Read(b []byte) (n int, err error) and Write(b []byte) (n int, err error) methods, therefore satisfying io.Reader and io.Writer interface.

Try this code:

package main

import (
	"fmt"
	"os"
	"os/user"
	"time"
)

func main() {
	fmt.Println("TempDir:", os.TempDir())

	pwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	// get the current user
	usr, err := user.Current()
	if err != nil {
		panic(err)
	}
	homePath := usr.HomeDir

	// change the directory
	if err := os.Chdir(homePath); err != nil {
		panic(err)
	}

	// get current working directory
	if twd, err := os.Getwd(); err != nil {
		panic(err)
	} else {
		fmt.Println("home pwd:", twd)
	}
	// home pwd: /home/ubuntu

	if err := os.Chdir(pwd); err != nil {
		panic(err)
	}
	// get current working directory
	if twd, err := os.Getwd(); err != nil {
		panic(err)
	} else {
		fmt.Println("home pwd:", twd)
	}

	fpath := "temp.txt"

	// this errors for non-existing file.
	if err := os.Remove(fpath); err != nil {
		// panic(err)
		fmt.Println(err)
	}

	// this errors for non-existing file.
	file, err := os.Open(fpath)
	if err != nil {
		// panic(err)
		fmt.Println(err)
		// open temp.txt: no such file or directory

		// THIS DOES NOT GET CALLED
		// BECAUSE the process will be killed
		// in the next lines
		defer func() {
			fmt.Println("Deleting", fpath)
			if err := os.Remove(fpath); err != nil {
				// panic(err)
				fmt.Println(err)
			}
		}()

		fmt.Println("Creating", fpath)
		file, err = os.Create(fpath)
		if err != nil {
			panic(err)
		}
		fmt.Println(file)
	}

	fmt.Println("Deleting", fpath)
	if err := os.Remove(fpath); err != nil {
		// panic(err)
		fmt.Println(err)
	}

	// get process id
	pid := os.Getppid()
	fmt.Println("pid:", pid)

	// find the process
	p, err := os.FindProcess(pid)
	if err != nil {
		panic(err)
	}
	go func() {
		fmt.Println("goroutine: killing the process in 1 second...")
		time.Sleep(time.Second)
		if err := p.Kill(); err != nil {
			panic(err)
		}
	}()
	fmt.Println("Sleeping 1,000 hours in main function...")
	time.Sleep(1000 * time.Hour)
}

/*
remove temp.txt: no such file or directory
open temp.txt: no such file or directory
Creating temp.txt
&{0xc8200164b0}
Deleting temp.txt
pid: 16635
Sleeping 1,000 hours in main function...
goroutine: killing the process in 1 second...
Killed
*/

↑ top

---

package os/exec

Package os/exec runs external commands.

Try this:

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
)

func main() {
	pwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	fmt.Println("pwd:", pwd)

	lsCmd := exec.Command("ls", "-a")

	lsOutput1, err := lsCmd.Output()
	if err != nil {
		panic(err)
	}
	fmt.Println(string(lsOutput1))

	if err := exec.Command("touch", "temp.txt").Run(); err != nil {
		panic(err)
	}

	// lsOutput2, err := lsCmd.Output()
	// if err != nil {
	// 	panic(err)
	// }
	// panic: exec: Stdout already set
	// SHOULD NOT REUSE IT

	lsCmd2 := exec.Command("ls", "-a")
	lsOutput2, err := lsCmd2.Output()
	if err != nil {
		panic(err)
	}
	fmt.Println(string(lsOutput2))

	if err := os.Remove("temp.txt"); err != nil {
		panic(err)
	}

	lsCmd3 := exec.Command("ls", "-a")
	lsOutput3, err := lsCmd3.Output()
	if err != nil {
		panic(err)
	}
	fmt.Println(string(lsOutput3))

	// http://golang.org/pkg/os/exec/#example_Cmd_Start
	slCmd := exec.Command("sleep", "1")
	if err := slCmd.Start(); err != nil {
		panic(err)
	}
	fmt.Println("Waiting for command to finish...")
	err = slCmd.Wait()
	fmt.Println("Command finished with error:", err)

	// http://golang.org/pkg/os/exec/#Cmd.Output
	cmd := exec.Command("echo", "-n", `{"Name": "Bob", "Age": 32}`)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	var person struct {
		Name string
		Age  int
	}
	if err := json.NewDecoder(stdout).Decode(&person); err != nil {
		panic(err)
	}
	if err := cmd.Wait(); err != nil {
		panic(err)
	}
	fmt.Printf("%s is %d years old\n", person.Name, person.Age)
	// Bob is 32 years old
}

/*
pwd: /home/ubuntu/go/src/github.com/gyuho/learn/doc/go_os_io/code
.
..
00_os.go
01_os_exec.go
02_io.go
03_io_pipe.go
04_io_ioutil.go
05_stdin.go
06_stdout_stdin_stderr.go
07_stdout_stdin_stderr_os.go
08_exist.go
09_open_create.go
10_ioutil_string.go
11_bufio.go
12_copy.go
13_csv.go
14_tsv.go
15_gzip.go
16_walk.go
17_flush.go
stderr.txt
stdin.txt
stdout.txt
testdata

.
..
00_os.go
01_os_exec.go
02_io.go
03_io_pipe.go
04_io_ioutil.go
05_stdin.go
06_stdout_stdin_stderr.go
07_stdout_stdin_stderr_os.go
08_exist.go
09_open_create.go
10_ioutil_string.go
11_bufio.go
12_copy.go
13_csv.go
14_tsv.go
15_gzip.go
16_walk.go
17_flush.go
stderr.txt
stdin.txt
stdout.txt
temp.txt
testdata

.
..
00_os.go
01_os_exec.go
02_io.go
03_io_pipe.go
04_io_ioutil.go
05_stdin.go
06_stdout_stdin_stderr.go
07_stdout_stdin_stderr_os.go
08_exist.go
09_open_create.go
10_ioutil_string.go
11_bufio.go
12_copy.go
13_csv.go
14_tsv.go
15_gzip.go
16_walk.go
17_flush.go
stderr.txt
stdin.txt
stdout.txt
testdata

Waiting for command to finish...
Command finished with error: <nil>
Bob is 32 years old
*/

↑ top

---

package flag

package main

import (
	"flag"
	"fmt"
)

func main() {
	idxPtr := flag.Int(
		"index",
		0,
		"Specify the index.",
	)
	dscPtr := flag.String(
		"description",
		"None",
		"Describe the argument.",
	)
	flag.Parse()
	fmt.Println("index:", *idxPtr)
	fmt.Println("description:", *dscPtr)
}

/*
You can:

-description aaa
-description 'aaa'
-description "aaa"
-description=aaa
-description='aaa'
-description="aaa"

--description aaa
--description 'aaa'
--description "aaa"
--description=aaa
--description='aaa'
--description="aaa"
*/

↑ top

---

package io

Package io provides basic interfaces to I/O primitives. When other packages, other than io, implement methods and satisfy interfaces in io package, those different packages can interact based on the shared io interfaces.

type Reader interface {
	Read(p []byte) (n int, err error)
}

type Writer interface {
	Write(p []byte) (n int, err error)
}

• io.Reader interface has method Read.
• Any type that implements Read(p []byte) (n int, err error) method satisfies io.Reader interface.
• io.Writer interface has method Write.
• Any type that implements Write(p []byte) (n int, err error) method satisfies io.Writer interface.
• As long as a type satisfies the interface, it can be used as an function argument.
• As long as a type satisfies the interface, it can contain data.

For example,

• os.File implements Read and Write methods.
• Therefore, it satisfies io.Reader and io.Writer interfaces.
• And json.NewDecoder takes io.Reader as an argument.
• Since os.File satisfies io.Reader interface, we can pass os.File as an argument to json.NewDecoder.

func (f *File) Read(b []byte) (n int, err error) {
    if f == nil {
        return 0, ErrInvalid
    }
    n, e := f.read(b)
    if n < 0 {
        n = 0
    }
    if n == 0 && len(b) > 0 && e == nil {
        return 0, io.EOF
    }
    if e != nil {
        err = &PathError("read", f.name, e}
    }
    return n, err
}

Try this:

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

// And os.File implements Read and Write method
// therefore satisfies io.Reader and io.Writer method

func main() {
	fpath := "testdata/sample.json"

	file, err := os.Open(fpath)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	tbytes, err := ioutil.ReadAll(file)
	if err != nil {
		panic(err)
	}
	jsonStream := string(tbytes)
	decodeString(jsonStream)
	// map[Go:Gopher Hello:World]

	decodeFile(file)
	// map[]

	// need to open again
	file2, err := os.Open(fpath)
	if err != nil {
		panic(err)
	}
	decodeFile(file2)
	// map[Go:Gopher Hello:World]
}

func decodeFile(file *os.File) {
	rmap := map[string]string{}
	dec := json.NewDecoder(file)
	for {
		if err := dec.Decode(&rmap); err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
	}
	fmt.Printf("%+v\n", rmap)
}

func decodeString(jsonStream string) {
	rmap := map[string]string{}
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	for {
		if err := dec.Decode(&rmap); err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
	}
	fmt.Printf("%+v\n", rmap)
}

↑ top

---

io.Pipe

io.Pipe creates a synchronous in-memory pipe.

package main

import (
	"fmt"
	"io"
)

func main() {
	done := make(chan struct{})
	r, w := io.Pipe()

	go func() {
		data := []byte("Hello World!")
		n, err := w.Write(data)
		if err != nil {
			panic(err)
		}
		if n != len(data) {
			panic(data)
		}
		done <- struct{}{}
	}()

	buf := make([]byte, 64)
	n, err := r.Read(buf)
	if err != nil {
		panic(err)
	}

	fmt.Println("wrote:", n)         // 12
	fmt.Println("buf:", string(buf)) // Hello World!

	<-done

	r.Close()
	w.Close()
}

↑ top

---

package io/ioutil

package io/ioutil implements some I/O utility functions.

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	resp, err := http.Get("http://google.com/")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	bts, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	fmt.Println(string(bts))
	// <div id=gbar><nobr><b class=gb1>Search</b>...
}

ioutil.ReadAll takes io.Reader as an argument:

func ReadAll(r io.Reader) ([]byte, error)

http.Response struct embeds io.ReadCloser interface.:

type Response struct {
	...
	Body io.ReadCloser
}

io.ReadCloser interface embeds io.Reader interface:

type ReadCloser interface {
	Reader
	Closer
}

Therefore, http.Response.Body of type io.ReadCloser is also type io.Reader. Any type that satisfies io.Reader interface (implements Read method) can be passed to ioutil.ReadAll.

ioutil.ReadAll(resp.Body)

Then does http.Response.Body type implement Read method? No. http.Response.Body is used as io.Reader interface argument. Then you might think http.Response.Body type should implement Read method to satisfy the io.Reader interface but it doesn not. Neither does Close. http.Response.Body can hold any type that implements Read and Close, but the actual type depends on the server that you are getting response from, as here:

	// Prepare body reader.  ContentLength < 0 means chunked encoding
	// or close connection when finished, since multipart is not supported yet
	switch {
	case chunked(t.TransferEncoding):
		if noBodyExpected(t.RequestMethod) {
			t.Body = eofReader
		} else {
			t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close}
		}
	case realLength == 0:
		t.Body = eofReader
	case realLength > 0:
		t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close}
	default:
		// realLength < 0, i.e. "Content-Length" not mentioned in header
		if t.Close {
			// Close semantics (i.e. HTTP/1.0)
			t.Body = &body{src: r, closing: t.Close}
		} else {
			// Persistent connection (i.e. HTTP/1.1)
			t.Body = eofReader
		}
	}

↑ top

---

stdout, stdin, stderr

To get the input from terminal:

package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	fmt.Print("Enter text: ")
	var input string
	fmt.Scanln(&input)
	fmt.Print("input:", input)

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		fmt.Println("scanner:", scanner.Text())
	}
	// Ctrl + D
}

/*
$ go run 05_stdin.go
Enter text: Hello
input:Hello
scanner:
1
scanner: 1
2
scanner: 2
3
scanner: 3
4
scanner: 4
5
scanner: 5
*/

And note that log output goes to standard err (file descriptor 2):

package main

import (
	"fmt"
	"log"
)

func main() {
	fmt.Println("fmt.Println")
	log.Println("log.Println")
	// log.Fatal("log.Fatal")
	panic("panic")
}

/*
go run 06_stdout_stdin_stderr.go Hello 0>>stdin.txt 1>>stdout.txt 2>>stderr.txt


stdin.txt
<empty>

stdout.txt
fmt.Println




log.Println goes to standard err

stderr.txt
2015/08/05 06:09:32 log.Println
2015/08/05 06:09:32 log.Fatal
exit status 1

or

2015/08/05 06:10:42 log.Println
panic: panic

goroutine 1 [running]:
main.main()
	/home/ubuntu/go/src/github.com/gyuho/learn/doc/go_os_io/code/05_stdout_stdin_stderr.go:12 +0x1e4
exit status 2
*/

And here's how you interact with OS standard output, input, error:

package main

import (
	"fmt"
	"os"
)

func main() {
	fmt.Fprintln(os.Stdout, "Hello World!")
	// Hello World!

	fmt.Fprintln(os.Stdin, "Input")

	fmt.Fprintln(os.Stderr, "Error")
}

// go run 07_stdout_stdin_stderr_os.go 0>>stdin.txt 1>>stdout.txt 2>>stderr.txt

↑ top

---

exist, readDir, recover

package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
)

// exist returns true if the file or directory exists.
func exist(fpath string) bool {
	_, err := os.Stat(name)
	return err == nil
}

// existDir returns true if the specified path points to a directory.
// It returns false and error if the directory does not exist.
func existDir(fpath string) bool {
	st, err := os.Stat(fpath)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return st.IsDir()
}

// readDir lists files in a directory.
func readDir(dir string) ([]string, error) {
	d, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer d.Close()

	ns, err := d.Readdirnames(-1)
	if err != nil {
		return nil, err
	}
	sort.Strings(ns)

	return ns, nil
}

func main() {
	fmt.Println(exist("00_os.go"))    // true
	fmt.Println(exist("aaaaa.go"))    // false
	fmt.Println(exist("testdata"))    // true
	fmt.Println(existDir("testdata")) // true
	ns, err := readDir("./testdata")
	if err != nil {
		panic(err)
	}
	fmt.Println(ns) // [sample.csv sample.json sample.txt sample_copy.csv sub]

	if err := copyDir("testdata", "copy_test"); err != nil {
		panic(err)
	}
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("recovered:", err)
		}
		fmt.Println("deleting...")
		os.RemoveAll("copy_test")
	}()
	panic(111)
}

func copyDir(src, dst string) error {
	si, err := os.Stat(src)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(dst, si.Mode()); err != nil {
		return err
	}

	dir, err := os.Open(src)
	if err != nil {
		return err
	}
	defer dir.Close()

	fis, err := dir.Readdir(-1)
	for _, fi := range fis {
		sp := src + "/" + fi.Name()
		dp := dst + "/" + fi.Name()
		if fi.IsDir() {
			if err := copyDir(sp, dp); err != nil {
				// create sub-directories - recursively
				return err
			}
		} else {
			if err := copy(sp, dp); err != nil {
				return err
			}
		}
	}

	return nil
}

/*
0777    full access for everyone
0700    only private access
0755    private read/write access, others only read access
0750    private read/write access, group read access, others no access
*/
func copy(src, dst string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return fmt.Errorf("copy: mkdirall: %v", err)
	}

	r, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("copy: open(%q): %v", src, err)
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("copy: create(%q): %v", dst, err)
	}
	defer w.Close()

	// func Copy(dst Writer, src Reader) (written int64, err error)
	if _, err = io.Copy(w, r); err != nil {
		return err
	}
	if err := w.Sync(); err != nil {
		return err
	}
	if _, err := w.Seek(0, 0); err != nil {
		return err
	}
	return nil
}

↑ top

---

create/open/write: files, directories

package main

import (
	"fmt"
	"io/ioutil"
	"os"
)

const (
	// privateFileMode grants owner to read/write a file.
	privateFileMode = 0600

	// privateDirMode grants owner to make/remove files inside the directory.
	privateDirMode = 0700
)

func openToRead(fpath string) (*os.File, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, privateFileMode)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func openToOverwrite(fpath string) (*os.File, error) {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, privateFileMode)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func openToAppend(fpath string) (*os.File, error) {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_APPEND|os.O_CREATE, privateFileMode)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func main() {
	fpath := "./testdata/sample.txt"

	func() {
		f, err := openToRead(fpath)
		if err != nil {
			panic(err)
		}
		defer func() {
			fmt.Println("Closing", f.Name())
			f.Close()
		}()
		if f.Name() != fpath {
			panic(f.Name())
		}
		tbytes, err := ioutil.ReadAll(f)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(tbytes))
	}()
	/*
	   Hello World!
	   Closing ./testdata/sample.txt
	*/

	fmt.Println()
	fmt.Println()

	func() {
		fpath := "test.txt"
		for range []int{0, 1} {
			f, err := openToOverwrite(fpath)
			if err != nil {
				panic(err)
			}
			if _, err := f.WriteString("Hello World!"); err != nil {
				panic(err)
			}
			f.Close()
		}
		f, err := openToRead(fpath)
		defer f.Close()
		if err != nil {
			panic(err)
		}
		tbytes, err := ioutil.ReadAll(f)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(tbytes))
		if err := os.Remove(fpath); err != nil {
			panic(err)
		}
	}()
	// Hello World!

	fmt.Println()
	fmt.Println()

	func() {
		fpath := "test.txt"
		for _, k := range []int{0, 1} {
			f, err := openToAppend(fpath)
			if err != nil {
				panic(err)
			}
			if _, err := f.WriteString(fmt.Sprintf("Hello World! %d\n", k)); err != nil {
				panic(err)
			}
			f.Close()
		}
		f, err := openToRead(fpath)
		defer f.Close()
		if err != nil {
			panic(err)
		}
		tbytes, err := ioutil.ReadAll(f)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(tbytes))
		if err := os.Remove(fpath); err != nil {
			panic(err)
		}
	}()
	/*
	   Hello World! 0
	   Hello World! 1
	*/
}

↑ top

---

io/ioutil, file

Package ioutil implements some I/O utility functions.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"syscall"
	"unsafe"
)

func main() {
	func() {
		fpath := "temp.txt"
		txt := "Hello World!"
		if err := toFileWriteString(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenReadAll(fpath); err != nil {
			panic(err)
		} else {
			fmt.Println("fromFileOpenReadAll:", s)
		}
	}()
	// fromFileOpenReadAll: Hello World!

	func() {
		fpath := "temp.txt"
		txt := "Hello World!"
		if err := toFileIO(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenReadAll(fpath); err != nil {
			panic(err)
		} else {
			fmt.Println("fromFileOpenReadAll:", s)
		}
	}()
	// fromFileOpenReadAll: Hello World!

	func() {
		fpath := "temp.txt"
		txt := "Hello World!"
		if err := toFileWrite(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenReadAll(fpath); err != nil {
			panic(err)
		} else {
			fmt.Println("fromFileOpenReadAll:", s)
		}
	}()
	// fromFileOpenReadAll: Hello World!

	func() {
		fpath := "toFileWriteSyscall.txt"
		txt := "Hello World!"
		if err := toFileWriteSyscall(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenReadAll(fpath); err != nil {
			panic(err)
		} else {
			fmt.Println("toFileWriteSyscall and fromFileOpenReadAll:", s)
		}
	}()
	// toFileWriteSyscall and fromFileOpenReadAll: Hello World!

	func() {
		fpath := "temp.txt"
		txt := "Hello World!"
		if err := toFileWriteString(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenFileReadAll(fpath); err != nil {
			panic(err)
		} else {
			fmt.Println("fromFileOpenFileReadAll:", s)
		}
	}()
	// fromFileOpenFileReadAll: Hello World!

	func() {
		fpath := "temp.txt"
		txt := "Hello World!"
		if err := toFileWriteString(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		if s, err := fromFileOpenFileReadFull(fpath, len(txt)); err != nil {
			panic(err)
		} else {
			fmt.Println("fromFileOpenFileReadFull:", s)
		}
	}()
	// fromFileOpenFileReadFull: Hello World!

	func() {
		fpath := "temp.txt"
		txt := strings.Repeat("Hello World!", 10000)
		if err := toFileWriteString(txt, fpath); err != nil {
			panic(err)
		}
		defer os.Remove(fpath)
		isSupported := isDirectIOSupported(fpath)
		fmt.Println("isDirectIOSupported:", isSupported)
		if isSupported {
			if s, err := fromFileDirectIO(fpath); err != nil {
				panic(err)
			} else {
				fmt.Println("fromFileDirectIO:", s[:10], "...")
			}
		}
	}()
	// isDirectIOSupported: true
	// fromFileDirectIO: Hello Worl ...
}

func toFileWriteString(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		// OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.WriteString(txt); err != nil {
		return err
	}
	return nil
}

func toFileIO(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		// OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := io.WriteString(f, txt); err != nil {
		return err
	}
	return nil
}

func toFileWrite(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		// OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.Write([]byte(txt)); err != nil {
		return err
	}
	return nil
}

func toFileWriteSyscall(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|syscall.MAP_POPULATE, 0777)
	if err != nil {
		// OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.Write([]byte(txt)); err != nil {
		return err
	}
	return nil
}

func fromFileOpenReadAll(fpath string) (string, error) {
	f, err := os.Open(fpath)
	if err != nil {
		// NOT retur nil, err
		// []byte can be null but not string
		return "", err
	}
	defer f.Close()
	// func ReadAll(r io.Reader) ([]byte, error)
	tbytes, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(tbytes), nil
}

func fromFileOpenFileReadAll(fpath string) (string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0777)
	if err != nil {
		// NOT retur nil, err
		// []byte can be null but not string
		return "", err
	}
	defer f.Close()
	// func ReadAll(r io.Reader) ([]byte, error)
	tbytes, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(tbytes), nil
}

func fromFileOpenFileReadFull(fpath string, length int) (string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0777)
	if err != nil {
		// NOT retur nil, err
		// []byte can be null but not string
		return "", err
	}
	defer f.Close()
	buf := make([]byte, length)
	if _, err := io.ReadFull(f, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

func isDirectIOSupported(fpath string) bool {
	f, err := os.OpenFile(fpath, syscall.O_DIRECT, 0)
	defer f.Close()
	return err == nil
}

func fromFileDirectIO(fpath string) (string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY|syscall.O_DIRECT, 0777)
	if err != nil {
		return "", err
	}
	defer f.Close()
	block := AlignedBlock(BlockSize)
	if _, err := io.ReadFull(f, block); err != nil {
		return "", err
	}
	return string(block), nil
}

/*****************************************************/

// Copied from https://github.com/ncw/directio

// alignment returns alignment of the block in memory
// with reference to AlignSize
//
// Can't check alignment of a zero sized block as &block[0] is invalid
func alignment(block []byte, AlignSize int) int {
	return int(uintptr(unsafe.Pointer(&block[0])) & uintptr(AlignSize-1))
}

// AlignedBlock returns []byte of size BlockSize aligned to a multiple
// of AlignSize in memory (must be power of two)
func AlignedBlock(BlockSize int) []byte {
	block := make([]byte, BlockSize+AlignSize)
	if AlignSize == 0 {
		return block
	}
	a := alignment(block, AlignSize)
	offset := 0
	if a != 0 {
		offset = AlignSize - a
	}
	block = block[offset : offset+BlockSize]
	// Can't check alignment of a zero sized block
	if BlockSize != 0 {
		a = alignment(block, AlignSize)
		if a != 0 {
			log.Fatal("Failed to align block")
		}
	}
	return block
}

const (
	// Size to align the buffer to
	AlignSize = 4096

	// Minimum block size
	BlockSize = 4096
)

// OpenFile is a modified version of os.OpenFile which sets O_DIRECT
func OpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) {
	return os.OpenFile(name, syscall.O_DIRECT|flag, perm)
}

/*****************************************************/

↑ top

---

bufio, file

Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer object, creating another object:

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
)

func main() {
	func() {
		if err := fromLines([]string{"A", "B", "C"}, "./tmp.txt"); err != nil {
			panic(err)
		}
		defer os.RemoveAll("./tmp.txt")
		lines, err := toLines1("./tmp.txt")
		if err != nil {
			panic(err)
		}
		if len(lines) != 3 {
			panic(fmt.Errorf("expected 3 but %v", lines))
		}
	}()

	func() {
		if err := fromLines([]string{"A", "B", "C"}, "./tmp.txt"); err != nil {
			panic(err)
		}
		defer os.RemoveAll("./tmp.txt")
		lines, err := toLines2("./tmp.txt")
		if err != nil {
			panic(err)
		}
		if len(lines) != 3 {
			panic(fmt.Errorf("expected 3 but %v", lines))
		}
	}()

	func() {
		if err := fromLines([]string{"aaa bbb ccc"}, "./tmp.txt"); err != nil {
			panic(err)
		}
		defer os.RemoveAll("./tmp.txt")
		words, err := toWords("./tmp.txt")
		if err != nil {
			panic(err)
		}
		if len(words) != 3 {
			panic(fmt.Errorf("expected 3 but %v", words))
		}
	}()

	func() {
		fpath := "stdout.txt"
		d, err := toBytes(fpath)
		if err != nil {
			panic(err)
		}
		fmt.Println("toBytes:", string(d))
		/*
		   toBytes: Enter text: input:fmt.Println
		   Enter text: input:fmt.Println
		   Hello World!
		*/
	}()
}

func fromLines(lines []string, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()

	// func NewWriter(w io.Writer) *Writer
	wr := bufio.NewWriter(f)

	for _, line := range lines {
		// func Fprintln(w io.Writer, a ...interface{}) (n int, err error)
		fmt.Fprintln(wr, line)
	}

	if err := wr.Flush(); err != nil {
		return err
	}
	return nil
}

func toLines1(fpath string) ([]string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	lines := []string{}

	// func NewScanner(r io.Reader) *Scanner
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return lines, err
	}
	return lines, nil
}

func toLines2(fpath string) ([]string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	rs := []string{}
	br := bufio.NewReader(f)
	for {
		l, err := br.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		rs = append(rs, l)
	}
	return rs, nil
}

func toWords(fpath string) ([]string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	lines := []string{}

	// func NewScanner(r io.Reader) *Scanner
	scanner := bufio.NewScanner(f)

	// This must be called before Scan.
	// The default split function is bufio.ScanLines.
	scanner.Split(bufio.ScanWords)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return lines, err
	}
	return lines, nil
}

func toBytes(fpath string) ([]byte, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	rs := []byte{}
	br := bufio.NewReader(f)
	for {
		c, err := br.ReadByte()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		rs = append(rs, c)
	}
	return rs, nil
}

↑ top

---

copy: files, directories

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

/*
0777	full access for everyone
0700	only private access
0755	private read/write access, others only read access
0750	private read/write access, group read access, others no access
*/
func copy(src, dst string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return fmt.Errorf("copy: mkdirall: %v", err)
	}

	r, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("copy: open(%q): %v", src, err)
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("copy: create(%q): %v", dst, err)
	}
	defer w.Close()

	// func Copy(dst Writer, src Reader) (written int64, err error)
	if _, err = io.Copy(w, r); err != nil {
		return err
	}
	if err := w.Sync(); err != nil {
		return err
	}
	if _, err := w.Seek(0, 0); err != nil {
		return err
	}
	return nil
}

func copyToTempFile(src, tempPrefix string) (string, error) {
	r, err := os.Open(src)
	if err != nil {
		return "", fmt.Errorf("copy: open(%q): %v", src, err)
	}
	defer r.Close()

	w, err := ioutil.TempFile("", tempPrefix)
	if err != nil {
		return "", fmt.Errorf("ioutil.TempFile error: %+v", err)
	}
	defer w.Close()

	if _, err = io.Copy(w, r); err != nil {
		return "", err
	}
	if err := w.Sync(); err != nil {
		return "", err
	}
	if _, err := w.Seek(0, 0); err != nil {
		return "", err
	}
	return w.Name(), nil
}

func copyDir(src, dst string) error {
	si, err := os.Stat(src)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(dst, si.Mode()); err != nil {
		return err
	}

	dir, err := os.Open(src)
	if err != nil {
		return err
	}
	defer dir.Close()

	fis, err := dir.Readdir(-1)
	for _, fi := range fis {
		sp := src + "/" + fi.Name()
		dp := dst + "/" + fi.Name()
		if fi.IsDir() {
			if err := copyDir(sp, dp); err != nil {
				// create sub-directories - recursively
				return err
			}
		} else {
			if err := copy(sp, dp); err != nil {
				return err
			}
		}
	}

	return nil
}

func main() {
	func() {
		fpath := "test.txt"
		defer os.Remove(fpath)
		for _, k := range []int{0, 1} {
			f, err := openToAppend(fpath)
			if err != nil {
				panic(err)
			}
			if _, err := f.WriteString(fmt.Sprintf("Hello World! %d\n", k)); err != nil {
				panic(err)
			}
			f.Close()
		}
		f, err := openToRead(fpath)
		defer f.Close()
		if err != nil {
			panic(err)
		}
		tbytes, err := ioutil.ReadAll(f)
		if err != nil {
			panic(err)
		}
		fmt.Println("fpath:", string(tbytes))
		/*
		   fpath: Hello World! 0
		   Hello World! 1
		*/

		fpathCopy := "test_copy.txt"
		defer os.Remove(fpathCopy)
		if err := copy(fpath, fpathCopy); err != nil {
			panic(err)
		}
		fc, err := openToRead(fpathCopy)
		defer f.Close()
		if err != nil {
			panic(err)
		}
		tbc, err := ioutil.ReadAll(fc)
		if err != nil {
			panic(err)
		}
		fmt.Println("fpathCopy:", string(tbc))
		/*
		   fpathCopy: Hello World! 0
		   Hello World! 1
		*/
	}()

	func() {
		fpath := "test.txt"
		defer os.Remove(fpath)
		for _, k := range []int{0, 1} {
			f, err := openToAppend(fpath)
			if err != nil {
				panic(err)
			}
			if _, err := f.WriteString(fmt.Sprintf("Hello World! %d\n", k)); err != nil {
				panic(err)
			}
			f.Close()
		}
		tempPath, err := copyToTempFile(fpath, "temp_prefix_")
		if err != nil {
			panic(err)
		}
		fmt.Println("tempPath:", tempPath)
	}()

	func() {
		if err := copyDir("testdata", "testdata2"); err != nil {
			panic(err)
		}
		os.RemoveAll("testdata2")
	}()
}

func openToRead(fpath string) (*os.File, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return f, err
	}
	return f, nil
}

func openToAppend(fpath string) (*os.File, error) {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_APPEND, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return f, err
		}
	}
	return f, nil
}

↑ top

---

csv

package main

import (
	"encoding/csv"
	"log"
	"os"
)

func main() {
	fpath := "test.csv"
	if err := toCSV([]string{"col1", "col2", "col3"}, [][]string{{"A", "B", "C"}, {"D", "E", "F"}}, fpath); err != nil {
		panic(err)
	}
	rows, err := fromCSV(fpath)
	if err != nil {
		panic(err)
	}
	if len(rows) != 3 {
		log.Fatal("must be 3 rows")
	}
	os.Remove(fpath)
}

func toCSV(header []string, rows [][]string, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()

	// func NewWriter(w io.Writer) *Writer
	wr := csv.NewWriter(f)

	if err := wr.Write(header); err != nil {
		return err
	}

	if err := wr.WriteAll(rows); err != nil {
		return err
	}

	wr.Flush()
	return wr.Error()
}

func fromCSV(fpath string) ([][]string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	// func NewReader(r io.Reader) *Reader
	rd := csv.NewReader(f)
	// Reading does not require `Flush`

	// in case that rows have different number of fields
	rd.FieldsPerRecord = -1

	// rd.TrailingComma = true
	// rd.TrimLeadingSpace = true
	// rd.LazyQuotes = true

	rows, err := rd.ReadAll()
	if err != nil {
		return rows, err
	}

	return rows, nil
}

↑ top

---

tsv

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
)

func main() {
	fpath := "test.tsv"
	if err := toTSV([][]string{{"A", "B", "C"}, {"D", "E", "F"}}, fpath); err != nil {
		panic(err)
	}
	rows, err := fromTSV(fpath)
	if err != nil {
		panic(err)
	}
	if len(rows) != 2 {
		panic(err)
	}
	os.Remove(fpath)
}

func toTSV(rows [][]string, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()

	// func NewWriter(w io.Writer) *Writer
	wr := bufio.NewWriter(f)
	for _, row := range rows {
		for idx, elem := range row {

			// func Fprintln(w io.Writer, a ...interface{}) (n int, err error)
			fmt.Fprint(wr, elem)

			if len(row)-1 != idx {
				fmt.Fprint(wr, "\t")
			}
		}
		fmt.Fprint(wr, "\n")
	}
	if err := wr.Flush(); err != nil {
		return err
	}
	return nil
}

func fromTSV(fpath string) ([][]string, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	rows := [][]string{}

	// func NewScanner(r io.Reader) *Scanner
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		rows = append(rows, strings.Split(scanner.Text(), "\t"))
	}
	if err := scanner.Err(); err != nil {
		return rows, err
	}
	return rows, f.Close()
}

↑ top

---

compress/gzip

Make sure to be effective when reading from gzip as explained:

• Crossing Streams: a Love Letter to io.Reader by Jason Moiron

package main

import (
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"os"
)

func main() {
	fpath := "test.tar.gz"
	if err := toGzip("Hello World!", fpath); err != nil {
		panic(err)
	}
	if tb, err := toBytes(fpath); err != nil {
		panic(err)
	} else {
		fmt.Println(fpath, ":", string(tb))
		// test.tar.gz : Hello World!
	}
	os.Remove(fpath)
}

// exec.Command("gzip", "-f", fpath).Run()
func toGzip(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	gw := gzip.NewWriter(f)
	if _, err := gw.Write([]byte(txt)); err != nil {
		return err
	}
	gw.Close()
	gw.Flush()
	return nil
}

func toBytes(fpath string) ([]byte, error) {
	f, err := os.OpenFile(fpath, os.O_RDONLY, 0444)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fz, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer fz.Close()

	// or JSON
	// http://jmoiron.net/blog/crossing-streams-a-love-letter-to-ioreader/
	s, err := ioutil.ReadAll(fz)
	if err != nil {
		return nil, err
	}
	return s, nil
}

↑ top

---

temporary file

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func main() {
	func() {
		wd, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		cf, err := ioutil.TempFile(wd, "hello")
		if err != nil {
			panic(err)
		}
		fmt.Println(cf.Name())
		os.Remove(cf.Name())
	}()

	func() {
		wd, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		cf, err := ioutil.TempFile(wd, "hello")
		if err != nil {
			panic(err)
		}
		op := cf.Name()
		os.Rename(op, "new_name")
		fmt.Println(op, "to new_name")
		os.Remove("new_name")
	}()

	func() {
		tmp := os.TempDir()
		f, err := ioutil.TempFile(tmp, "hello")
		if err != nil {
			panic(err)
		}
		fpath, err := filepath.Abs(f.Name())
		if err != nil {
			panic(err)
		}
		fmt.Println(fpath)

		wd, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		npath := filepath.Join(wd, "hello")
		if err := copy(fpath, npath); err != nil {
			panic(err)
		}

		os.Remove(fpath)
		os.Remove(npath)
	}()
}

/*
0777	full access for everyone
0700	only private access
0755	private read/write access, others only read access
0750	private read/write access, group read access, others no access
*/
func copy(src, dst string) error {
	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
		return fmt.Errorf("copy: mkdirall: %v", err)
	}

	r, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("copy: open(%q): %v", src, err)
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("copy: create(%q): %v", dst, err)
	}
	defer w.Close()

	// func Copy(dst Writer, src Reader) (written int64, err error)
	if _, err = io.Copy(w, r); err != nil {
		return err
	}
	if err := w.Sync(); err != nil {
		return err
	}
	if _, err := w.Seek(0, 0); err != nil {
		return err
	}
	return nil
}

↑ top

---

walk

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	func() {
		// recursively walk
		rmap, err := walk(".")
		if err != nil {
			panic(err)
		}
		for _, v := range rmap {
			fmt.Println("walk:", v)
		}
	}()

	fmt.Println()
	fmt.Println()

	func() {
		// recursively walk
		rmap, err := walkExt(".", ".txt")
		if err != nil {
			panic(err)
		}
		for _, v := range rmap {
			fmt.Println("walkExt:", v)
		}
	}()

	fmt.Println()
	fmt.Println()

	func() {
		// only the present working directory
		rmap, err := walkExtCurrentDir(".", ".txt")
		if err != nil {
			panic(err)
		}
		for _, v := range rmap {
			fmt.Println("walkExtCurrentDir:", v)
		}
	}()

	fmt.Println()
	fmt.Println()

	func() {
		// walk only directories
		rmap, err := walkDir(".")
		if err != nil {
			panic(err)
		}
		for _, v := range rmap {
			fmt.Println("walkDir:", v)
		}
	}()
}

// walk returns all FileInfos with recursive Walk in the target directory.
// It does not include the directories but include the files inside each sub-directories.
// It does not follow the symbolic links. And excludes hidden files.
// It returns the map from os.FileInfo to its absolute path.
func walk(targetDir string) (map[os.FileInfo]string, error) {
	rmap := make(map[os.FileInfo]string)
	visit := func(path string, f os.FileInfo, err error) error {
		if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
			if _, ok := rmap[f]; !ok {
				wd, err := os.Getwd()
				if err != nil {
					return err
				}
				rmap[f] = filepath.Join(wd, path)
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rmap, nil
}

// walkExt returns all FileInfos with specific extension.
// Make sure to prefix the extension name with dot.
// For example, to find all go files, pass ".go".
func walkExt(targetDir, ext string) (map[os.FileInfo]string, error) {
	rmap := make(map[os.FileInfo]string)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				if filepath.Ext(path) == ext {
					if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
						if _, ok := rmap[f]; !ok {
							wd, err := os.Getwd()
							if err != nil {
								return err
							}
							thepath := filepath.Join(wd, strings.Replace(path, wd, "", -1))
							rmap[f] = thepath
						}
					}
				}
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rmap, nil
}

// walkExtCurrentDir only walks the current directory, not sub-directories.
func walkExtCurrentDir(targetDir, ext string) (map[os.FileInfo]string, error) {
	rmap := make(map[os.FileInfo]string)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				if filepath.Ext(path) == ext {
					if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
						if _, ok := rmap[f]; !ok {
							wd, err := os.Getwd()
							if err != nil {
								return err
							}
							thepath := filepath.Join(wd, strings.Replace(path, wd, "", -1))
							if wd == filepath.Dir(thepath) {
								rmap[f] = thepath
							}
						}
					}
				}
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rmap, nil
}

// walkDir returns all directories.
func walkDir(targetDir string) (map[os.FileInfo]string, error) {
	rmap := make(map[os.FileInfo]string)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if f.IsDir() {
				if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
					if _, ok := rmap[f]; !ok {
						rmap[f] = filepath.Join(targetDir, path)
					}
				}
			}
		}
		return nil
	}
	if err := filepath.Walk(targetDir, visit); err != nil {
		return nil, err
	}
	return rmap, nil
}

/*
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/02_flag.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/07_stdout_stdin_stderr.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/18_walk.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sample.csv
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sample_copy.csv
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/09_exist.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/22_importdeps.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub/sample_copy.csv
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sample.txt
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub/sample.csv
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/03_io.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/13_bufio.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/15_csv.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/21_temp_file.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stderr.txt
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdin.txt
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/17_gzip.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/01_os_exec.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/05_io_ioutil.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/06_stdin.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/11_io_ioutil_file.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/14_copy.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/12_temp_file.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/16_tsv.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub/sample.txt
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sample.json
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/00_os.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/04_io_pipe.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/08_stdout_stdin_stderr_os.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/10_open_create.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/19_flush.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/20_signal.go
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdout.txt
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata
walk: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub/sample.json


walkExt: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stderr.txt
walkExt: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdin.txt
walkExt: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdout.txt
walkExt: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sample.txt
walkExt: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/testdata/sub/sample.txt


walkExtCurrentDir: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stderr.txt
walkExtCurrentDir: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdin.txt
walkExtCurrentDir: /home/gyuho/go/src/github.com/gyuho/learn/doc/go_os_io/code/stdout.txt


walkDir: testdata
walkDir: testdata/sub

*/

↑ top

---

http.Flusher

package main

import (
	"io"
	"log"
	"net/http"
	"os/exec"
)

type flusherWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flusherWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

func handler(w http.ResponseWriter, r *http.Request) {
	fw := flusherWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	cmd := exec.Command("ls")
	cmd.Stdout = &fw
	cmd.Stderr = &fw
	cmd.Run()
}

func main() {
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(":8080", nil))
}

/*
00_os.go
01_os_exec.go
02_io.go
03_io_pipe.go
04_io_ioutil.go
05_stdin.go
06_stdout_stdin_stderr.go
07_stdout_stdin_stderr_os.go
08_exist.go
09_open_create.go
10_bufio.go
11_copy.go
12_flush.go
stderr.txt
stdin.txt
stdout.txt
testdata
*/

↑ top

---

os.Signal

package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

func main() {
	for _, sig := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} {
		c := make(chan os.Signal, 2)
		// Notify causes package signal to relay incoming signals to c. If
		// no signals are provided, all incoming signals will be relayed to c.
		// Otherwise, just the provided signals will.
		signal.Notify(c, sig)

		handleInterrupts()
		p := syscall.Getpid()
		syscall.Kill(p, sig)

		time.Sleep(time.Second)
	}
}

// https://github.com/coreos/etcd/blob/master/pkg/osutil/interrupt_unix.go
func handleInterrupts() {
	notifier := make(chan os.Signal, 1)
	signal.Notify(notifier, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		sig := <-notifier
		log.Printf("Received %v signal, shutting down...", sig)
		signal.Stop(notifier)
		pid := syscall.Getpid()
		// exit directly if it is the "init" process, since the kernel will not help to kill pid 1.
		if pid == 1 {
			os.Exit(0)
		}
		syscall.Kill(pid, sig.(syscall.Signal))
	}()
}

↑ top

---

importDeps

package main

import (
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	pathpkg "path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

// https://github.com/golang/go/blob/master/src/go/build/build.go#L320
func envOr(name, def string) string {
	s := os.Getenv(name)
	if s == "" {
		return def
	}
	return s
}

func main() {
	goRoot := pathpkg.Clean(runtime.GOROOT())
	fmt.Println("GOROOT:", goRoot)
	goPath := envOr("GOPATH", "")
	fmt.Println("GOPATH:", goPath)

	pwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	projectPath, err := filepath.Rel(filepath.Join(goPath, "src"), pwd)
	if err != nil {
		panic(err)
	}

	wm, err := walkExt(".", ".go")
	if err != nil {
		panic(err)
	}
	fpaths := []string{}
	for _, v := range wm {
		fpaths = append(fpaths, filepath.Base(v))
	}

	func() {
		rmap, err := importDeps(pwd)
		if err != nil {
			panic(err)
		}
		for k := range rmap {
			fmt.Println("importDeps:", k)
		}
	}()

	func() {
		rmap, err := importDepsWithProjectPath(pwd, projectPath, fpaths...)
		if err != nil {
			panic(err)
		}
		for k := range rmap {
			fmt.Println("importDepsWithProjectPath:", k)
		}
	}()
}

// https://github.com/golang/go/blob/master/src/go/build/syslist.go#L7
const goosList = "android darwin dragonfly freebsd linux nacl netbsd openbsd plan9 solaris windows "
const goarchList = "386 amd64 amd64p32 arm armbe arm64 arm64be ppc64 ppc64le mips mipsle mips64 mips64le mips64p32 mips64p32le ppc s390 s390x sparc sparc64 "
const appengineList = "appengine appenginevm"

func importDeps(dir string) (map[string]struct{}, error) {
	tm, err := walkExt(dir, ".go")
	if err != nil {
		return nil, err
	}
	wm := make(map[string]struct{})
	for _, v := range tm {
		wm[v] = struct{}{}
	}
	fSize := len(wm)
	if fSize == 0 {
		return nil, nil
	}
	var mu sync.Mutex // guards the map
	fmap := make(map[string]struct{})
	done, errCh := make(chan struct{}), make(chan error)
	for fpath := range wm {
		go func(fpath string) {
			fset := token.NewFileSet()
			f, err := parser.ParseFile(fset, fpath, nil, parser.ImportsOnly|parser.ParseComments)
			if err != nil {
				errCh <- err
				return
			}
			ignore := false
			for _, cc := range f.Comments {
				for _, v := range cc.List {
					if strings.HasPrefix(v.Text, "// +build ignore") {
						ignore = true
						break
					}
					if strings.HasPrefix(v.Text, "// +build") {
						p := strings.Replace(v.Text, "// +build ", "", -1)
						if !strings.Contains(goosList, p) && !strings.Contains(goarchList, p) && !strings.Contains(appengineList, p) {
							ignore = true
							break
						}
					}
				}
				if ignore {
					break
				}
			}
			if !ignore {
				for _, elem := range f.Imports {
					pv := strings.TrimSpace(strings.Replace(elem.Path.Value, `"`, "", -1))
					if pv == "C" || build.IsLocalImport(pv) || strings.HasPrefix(pv, ".") {
						continue
					}
					mu.Lock()
					fmap[pv] = struct{}{}
					mu.Unlock()
				}
			}
			done <- struct{}{}
		}(fpath)
	}
	i := 0
	for {
		select {
		case e := <-errCh:
			return nil, e
		case <-done:
			i++
			if i == fSize {
				close(done)
				return fmap, nil
			}
		}
	}
}

func importDepsWithProjectPath(dir string, importPath string, fpaths ...string) (map[string]struct{}, error) {
	fSize := len(fpaths)
	if fSize == 0 {
		return nil, nil
	}
	projectPath := importPath
	il := strings.Split(importPath, "/")
	if len(il) > 2 {
		// get github.com/boltdb/bolt
		// if given 'github.com/boltdb/bolt/subpkg'
		projectPath = strings.Join(il[:3], "/")
	}
	var mu sync.Mutex // guards the map
	fmap := make(map[string]struct{})
	done, errCh := make(chan struct{}), make(chan error)
	for _, fs := range fpaths {
		go func(fs string) {
			fset := token.NewFileSet()
			fpath := filepath.Join(dir, fs)
			f, err := parser.ParseFile(fset, fpath, nil, parser.ImportsOnly)
			if err != nil {
				errCh <- err
				return
			}
			for _, elem := range f.Imports {
				pv := strings.TrimSpace(strings.Replace(elem.Path.Value, `"`, "", -1))
				mu.Lock()
				if !strings.HasPrefix(pv, projectPath) {
					fmap[pv] = struct{}{}
				}
				mu.Unlock()
			}
			done <- struct{}{}
		}(fs)
	}
	i := 0
	for {
		select {
		case e := <-errCh:
			return nil, e
		case <-done:
			i++
			if i == fSize {
				close(done)
				return fmap, nil
			}
		}
	}
}

// walkExt returns all FileInfos with specific extension.
// Make sure to prefix the extension name with dot.
// For example, to find all go files, pass ".go".
func walkExt(targetDir, ext string) (map[os.FileInfo]string, error) {
	rmap := make(map[os.FileInfo]string)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				if filepath.Ext(path) == ext {
					if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
						if _, ok := rmap[f]; !ok {
							wd, err := os.Getwd()
							if err != nil {
								return err
							}
							thepath := filepath.Join(wd, strings.Replace(path, wd, "", -1))
							rmap[f] = thepath
						}
					}
				}
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rmap, nil
}

/*
GOROOT: /usr/local/go
GOPATH: /home/gyuho/go
importDeps: log
importDeps: net/http
importDeps: time
importDeps: runtime
importDeps: sync
importDeps: io
importDeps: os/exec
importDeps: os/user
importDeps: go/token
importDeps: os/signal
importDeps: flag
importDeps: go/parser
importDeps: fmt
importDeps: path/filepath
importDeps: encoding/csv
importDeps: encoding/json
importDeps: bufio
importDeps: syscall
importDeps: path
importDeps: os
importDeps: io/ioutil
importDeps: strings
importDeps: compress/gzip
importDeps: unsafe
importDeps: go/build
importDepsWithProjectPath: flag
importDepsWithProjectPath: os/exec
importDepsWithProjectPath: go/build
importDepsWithProjectPath: io/ioutil
importDepsWithProjectPath: syscall
importDepsWithProjectPath: io
importDepsWithProjectPath: path/filepath
importDepsWithProjectPath: go/parser
importDepsWithProjectPath: path
importDepsWithProjectPath: runtime
importDepsWithProjectPath: encoding/json
importDepsWithProjectPath: fmt
importDepsWithProjectPath: os
importDepsWithProjectPath: os/signal
importDepsWithProjectPath: unsafe
importDepsWithProjectPath: sync
importDepsWithProjectPath: os/user
importDepsWithProjectPath: net/http
importDepsWithProjectPath: strings
importDepsWithProjectPath: time
importDepsWithProjectPath: compress/gzip
importDepsWithProjectPath: bufio
importDepsWithProjectPath: log
importDepsWithProjectPath: encoding/csv
importDepsWithProjectPath: go/token

*/

↑ top

---

directory

package main

import (
	"fmt"
	"os"
	"path"
)

var (
	path0 = "temporary"
	path1 = "member"
	path2 = "file.txt"

	fpath0 = path0
	fpath1 = path.Join(path0, path1)
	fpath2 = path.Join(path.Join(path0, path1), path2)
)

var isDelete bool

func init() {
	if err := os.RemoveAll(fpath0); err != nil {
		panic(err)
	}
	fmt.Println("fpath0:", fpath0)
	fmt.Println("fpath1:", fpath1)
	fmt.Println("fpath2:", fpath2)
}

func main() {
	defer func() {
		if isDelete {
			os.RemoveAll(fpath0)
		}
	}()

	if err := os.MkdirAll(fpath0, 0700); err != nil {
		panic(err)
	}

	if existDir(fpath1) {
		fmt.Println(fpath1, "already exists... skipping...")
		return
	}

	if err := os.MkdirAll(fpath1, 0700); err != nil {
		panic(err)
	}

	if err := toFileWriteString("hello world!", fpath2); err != nil {
		panic(err)
	}

	fmt.Println("Done")
	isDelete = true
}

/*
fpath0: temporary
fpath1: temporary/member
fpath2: temporary/member/file.txt
Done
*/

// existDir returns true if the specified path points to a directory.
// It returns false and error if the directory does not exist.
func existDir(fpath string) bool {
	st, err := os.Stat(fpath)
	if err != nil {
		return false
	}
	return st.IsDir()
}

func toFileWriteString(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		// OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.WriteString(txt); err != nil {
		return err
	}
	return nil
}

↑ top

---

streaming OS command

package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/exec"
	"time"
)

func main() {
	var (
		writer   = os.Stdout
		doneWait = make(chan struct{})
		errChan  = make(chan error)
		waitSig  = make(chan bool)
		// cmd      = exec.Command(filepath.Join(os.Getenv("GOPATH"), "bin/etcd"))
		cmd = exec.Command("echo", "hello")
	)

	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	defer cmdOut.Close()
	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	defer cmdErr.Close()

	go func() {
		fmt.Println("cmd.Start:", cmd.Path, cmd.Args)
		if err := cmd.Start(); err != nil {
			errChan <- err
			close(waitSig)
			return
		}
		waitSig <- true
	}()

	go func() {
		scanner := bufio.NewScanner(cmdOut)
		for scanner.Scan() {
			fmt.Fprintln(writer, scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			errChan <- err
		}
	}()

	go func() {
		scanner := bufio.NewScanner(cmdErr)
		for scanner.Scan() {
			fmt.Fprintln(writer, scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			errChan <- err
		}
	}()

	go func() {
		ready, ok := <-waitSig
		if !ready && !ok {
			log.Fatal("something wrong with cmd.Start!")
		}

		fmt.Println("cmd.Wait")
		if err := cmd.Wait(); err != nil {
			errChan <- err
			return
		}
		doneWait <- struct{}{}
	}()

	select {
	case <-doneWait:
		fmt.Println("cmd done!")

	case err := <-errChan:
		fmt.Println("error:", err)

	case <-time.After(10 * time.Second):
		fmt.Println("timed out and cmd.Process.Kill")
		if err := cmd.Process.Kill(); err != nil {
			fmt.Println("error when cmd.Process.Kill:", err)
		}
	}
}

/*
cmd.Start: /bin/echo [echo hello]
cmd.Wait
hello
cmd done!
*/

↑ top

---

streaming long running

package main

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
)

type Flag struct {
	EtcdOld string
	EtcdNew string
}

var (
	rootCommand = &cobra.Command{
		Use:        "migration",
		Short:      "migration handles etcd migration.",
		SuggestFor: []string{"migration", "miation", "miration"},
	}
)

func init() {
	cobra.EnablePrefixMatching = true
}

func init() {
	rootCommand.AddCommand(releaseCommand)
}

func main() {
	if err := rootCommand.Execute(); err != nil {
		fmt.Fprintln(os.Stdout, err)
		os.Exit(1)
	}
}

var (
	releaseCommand = &cobra.Command{
		Use:   "release",
		Short: "release checks etcd migration between two releases.",
		Run:   CommandFunc,
	}

	cmdFlag = Flag{}
)

func init() {
	cobra.EnablePrefixMatching = true
}

func init() {
	releaseCommand.PersistentFlags().StringVarP(&cmdFlag.EtcdOld, "etcd-binary-old", "a", "~/etcd_old", "Path of executable etcd binary to migrate from.")
	releaseCommand.PersistentFlags().StringVarP(&cmdFlag.EtcdNew, "etcd-binary-new", "b", "~/etcd_new", "Path of executable etcd binary to migrate to.")
}

var (
	defaultFlags1 = []string{
		"--name", "infra1",
		"--listen-client-urls", "http://localhost:12379",
		"--advertise-client-urls", "http://localhost:12379",
		"--listen-peer-urls", "http://localhost:12380",
		"--initial-advertise-peer-urls", "http://localhost:12380",
		"--initial-cluster-token", "etcd-cluster-1",
		"--initial-cluster", "infra1=http://localhost:12380,infra2=http://localhost:22380,infra3=http://localhost:32380",
		"--initial-cluster-state", "new",
	}
	defaultFlags2 = []string{
		"--name", "infra2",
		"--listen-client-urls", "http://localhost:22379",
		"--advertise-client-urls", "http://localhost:22379",
		"--listen-peer-urls", "http://localhost:22380",
		"--initial-advertise-peer-urls", "http://localhost:22380",
		"--initial-cluster-token", "etcd-cluster-1",
		"--initial-cluster", "infra1=http://localhost:12380,infra2=http://localhost:22380,infra3=http://localhost:32380",
		"--initial-cluster-state", "new",
	}
	defaultFlags3 = []string{
		"--name", "infra3",
		"--listen-client-urls", "http://localhost:32379",
		"--advertise-client-urls", "http://localhost:32379",
		"--listen-peer-urls", "http://localhost:32380",
		"--initial-advertise-peer-urls", "http://localhost:32380",
		"--initial-cluster-token", "etcd-cluster-1",
		"--initial-cluster", "infra1=http://localhost:12380,infra2=http://localhost:22380,infra3=http://localhost:32380",
		"--initial-cluster-state", "new",
	}
	memberStartReadyString   = "etcdserver: set the initial cluster version to "
	memberReStartReadySuffix = " became active"
)

func getInfraFlags(i int) []string {
	switch i {
	case 1:
		return defaultFlags1
	case 2:
		return defaultFlags2
	case 3:
		return defaultFlags3
	default:
		panic(fmt.Sprintf("%d is not defined", i))
	}
}

// half-mega-bytes
// i == 50, then stress 50MB
var putSize = 1 << (10 * 2) / 2

func stress(mb int) error {
	time.Sleep(5 * time.Second)

	cfg := client.Config{
		Endpoints: []string{"http://localhost:12379", "http://localhost:22379", "http://localhost:32379"},
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		return err
	}
	kapi := client.NewKeysAPI(c)

	for i := 0; i < mb*2; i++ {
		fmt.Println("stressing", i)
		k := make([]byte, 100)
		binary.PutVarint(k, int64(rand.Intn(putSize)))
		_, err = kapi.Set(context.Background(), string(k), "", nil)
		if err != nil {
			if i < 2 {
				return err
			}
		}
		time.Sleep(500 * time.Millisecond)
	}

	return nil
}

var (
	mu         sync.Mutex
	nodeStatus = map[string]string{
		"infra1": "none",
		"infra2": "none",
		"infra3": "none",
	}
)

func CommandFunc(cmd *cobra.Command, args []string) {
	defer func() {
		fmt.Println("deleting...")
		os.RemoveAll("infra1.etcd")
		os.RemoveAll("infra2.etcd")
		os.RemoveAll("infra3.etcd")
	}()

	oldCmds := make([]*exec.Cmd, 3)
	oldOutputs := make([]io.ReadCloser, 3)
	newCmds := make([]*exec.Cmd, 3)
	newOutputs := make([]io.ReadCloser, 3)
	for i := range oldCmds {
		oldCmd := exec.Command(cmdFlag.EtcdOld, getInfraFlags(i+1)...)
		oldCmds[i] = oldCmd
		oldOutput, err := oldCmd.StderrPipe()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		oldOutputs[i] = oldOutput

		newCmd := exec.Command(cmdFlag.EtcdNew, getInfraFlags(i+1)...)
		newCmds[i] = newCmd
		newOutput, err := newCmd.StderrPipe()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		newOutputs[i] = newOutput
	}

	errChan := make(chan error)
	done := make(chan struct{})
	for i := range oldCmds {
		cmd := oldCmds[i]
		go func(i int, cmd *exec.Cmd) {
			if err := cmd.Start(); err != nil {
				errChan <- err
				return
			}
			done <- struct{}{}
		}(i, cmd)
	}
	cn := 0
	for cn != 3 {
		cn++
		select {
		case err := <-errChan:
			fmt.Fprintln(os.Stderr, err)
			return
		case <-done:
		}
	}

	becameActiveCount := 0
	for i, o := range oldOutputs {
		go func(i int, reader io.ReadCloser) {
			scanner := bufio.NewScanner(reader)
			for {
				for scanner.Scan() {
					txt := scanner.Text()
					fmt.Printf("[old infra%d] %s\n", i+1, txt)
					if strings.Contains(txt, memberStartReadyString) {
						mu.Lock()
						nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE"
						mu.Unlock()
						fmt.Printf("[old infra%d] %s  READY!!!!!!!!!!!!!\n", i+1, txt)
						done <- struct{}{}
					}
					if strings.HasSuffix(txt, memberReStartReadySuffix) {
						fmt.Printf("[old infra%d] reconnected!\n", i+1)
						mu.Lock()
						nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE"
						mu.Unlock()
						becameActiveCount++
					}
				}
			}
			if err := scanner.Err(); err != nil {
				errChan <- err
				return
			}
		}(i, o)
	}
	for i, o := range newOutputs {
		go func(i int, reader io.ReadCloser) {
			scanner := bufio.NewScanner(reader)
			for {
				for scanner.Scan() {
					txt := scanner.Text()
					fmt.Printf("[new infra%d] %s\n", i+1, txt)
					if strings.HasSuffix(txt, memberReStartReadySuffix) {
						fmt.Printf("[new infra%d] reconnected!\n", i+1)
						mu.Lock()
						nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE"
						mu.Unlock()
						becameActiveCount++
					}
				}
			}
			if err := scanner.Err(); err != nil {
				errChan <- err
				return
			}
		}(i, o)
	}
	cn = 0
	for cn != 3 {
		cn++
		select {
		case err := <-errChan:
			fmt.Fprintln(os.Stderr, err)
			return
		case <-done:
		}
	}

	es := stress(10)
	if es != nil {
		log.Println(es)
		return
	}
	go func() {
		es := stress(50)
		if es != nil {
			log.Println(es)
			return
		}
	}()

	for i := 0; i < 3; i++ {
		fmt.Printf("[old infra%d] killing...\n", i+1)
		mu.Lock()
		nodeStatus[fmt.Sprintf("infra%d", i+1)] = "KILLED"
		mu.Unlock()
		if err := syscall.Kill(oldCmds[i].Process.Pid, syscall.SIGKILL); err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		fmt.Printf("[old infra%d] killed!\n", i+1)
		time.Sleep(10 * time.Second)

		fmt.Printf("[new infra%d] restarting...\n", i+1)
		if err := newCmds[i].Start(); err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		mu.Lock()
		nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE"
		mu.Unlock()
		fmt.Printf("[new infra%d] restarted!\n", i+1)
		time.Sleep(10 * time.Second)
	}

	// 6(2 per node) at the beginning of cluster + 12(4 per kill) during migration = 18
	if becameActiveCount >= 18 {
		fmt.Printf("migration successful from %s to %s (node status %v)\n", cmdFlag.EtcdOld, cmdFlag.EtcdNew, nodeStatus)
	} else {
		fmt.Printf("migration failed from %s to %s (becameActiveCount %d, node status %v)\n", cmdFlag.EtcdOld, cmdFlag.EtcdNew, becameActiveCount, nodeStatus)
	}
}

↑ top

---

java

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
)

func main() {
	cmd := exec.Command("bash", "-c", "java HelloWorld")
	err := cmd.Start()
	fmt.Printf("PID: %d\n", cmd.Process.Pid)
	if err != nil {
		log.Fatal(err)
	}

	c := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)

	go func() {
		<-c
		fmt.Printf("Got signal to %d\n", cmd.Process.Pid)
		syscall.Kill(cmd.Process.Pid, syscall.SIGHUP)
		done <- true
	}()
	<-done
}

↑ top

---