filehashes

package module
v0.0.0-...-64dd94d Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 20, 2020 License: MIT Imports: 10 Imported by: 1

README

filehashes

filehashes is a Golang packages which computes file hashes based on standard crypto package.

Documentation

Index

Examples

Constants

This section is empty.

Variables

View Source
var (
	DefaultConcurrency = 4
	DefaultBufferSize  = 8 * 1024 * 1024

	ErrNoFileToHash         = errors.New("no file to hash")
	ErrNoHashFuncs          = errors.New("no hash functions")
	ErrHashFuncNotAvailable = errors.New("hash function is not available")
	ErrInvalidHashState     = errors.New("invalid hash state")
	ErrFileIsDir            = errors.New("file is dir")
)

Functions

This section is empty.

Types

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

Manager controls the number of concurrent worker goroutines which compute files hashes.

func NewManager

func NewManager(concurrency int, bufferSize int) (*Manager, <-chan *Message)

NewManager creates a new manager and returns a channel to receive the messages. The channel will not be closed.

func (*Manager) Start

func (m *Manager) Start(ctx context.Context, req *Request)

Start starts to sum file by given request. Caller should import hash function packages for the hash functions. e.g. import (_ "crypto/md5") It'll start a new goroutine for each request.

Example
package main

import (
	"context"
	"crypto"
	"log"
	"math/rand"
	"time"

	_ "crypto/md5"
	_ "crypto/sha1"
	"github.com/northbright/filehashes"
)

func main() {
	// You may set the number of concurrent worker goroutines.
	// It's filehashes.DefaultConcurrency by default.
	concurrency := 2
	bufferSize := filehashes.DefaultBufferSize

	man, ch := filehashes.NewManager(concurrency, bufferSize)

	// Create requests.
	reqs := []*filehashes.Request{
		filehashes.NewRequest("README.md", []crypto.Hash{crypto.MD5}, nil),
		filehashes.NewRequest("manager.go", []crypto.Hash{crypto.SHA1}, nil),
		filehashes.NewRequest("manager_test.go", []crypto.Hash{crypto.MD5, crypto.SHA1}, nil),
		filehashes.NewRequest("go.sum", []crypto.Hash{crypto.MD5, crypto.SHA1}, nil),
	}

	// Use a timeout to exit the program.
	tm := time.After(5 * time.Second)

	// Record the cancel func for each request's file.
	cancelMap := map[string]context.CancelFunc{}

	for _, req := range reqs {
		ctx, cancel := context.WithCancel(context.Background())
		cancelMap[req.File] = cancel
		// Start to sum files.
		man.Start(ctx, req)
	}

	// Stop 1 task randomly.
	// The stopped task will be resume with previous progress in message loop.
	go func() {
		time.Sleep(1 * time.Millisecond)
		i := rand.Intn(len(reqs))
		cancel := cancelMap[reqs[i].File]
		cancel()
	}()

	// Consume messages.
	for {
		select {
		case <-tm:
			log.Printf("timeout")
			return
		case m := <-ch:
			switch m.Type {
			case filehashes.ERROR,
				filehashes.SCHEDULED,
				filehashes.STARTED,
				filehashes.RESTORED,
				filehashes.PROGRESS_UPDATED,
				filehashes.DONE,
				filehashes.EXITED:
				// All messages can be marshaled to JSON.
				buf, _ := m.JSON()
				log.Printf("message: %v", string(buf))

			case filehashes.STOPPED:
				buf, _ := m.JSON()
				log.Printf("message: %v", string(buf))

				// Resume the task if it's stopped.
				resumeReq := m.Data.(*filehashes.Request)
				ctx, cancel := context.WithCancel(context.Background())
				cancelMap[resumeReq.File] = cancel

				man.Start(ctx, resumeReq)

			default:
				log.Printf("unknown message: %v", m)
			}
		}
	}

}
Output:

type Message

type Message struct {
	// Type is the type code(uint) of message.
	Type MessageType `json:"type"`
	// TypeStr is the type in string.
	TypeStr string `json:"type_str"`
	// Req is the request of sum a file.
	Req *Request `json:"request,omitempty"`
	// Data stores the data of message.
	// Each type has its own data type.
	//
	// ERROR: data is a string to store error message.
	// SCHEDULED: data is nil.
	// STARTED: data is nil.
	// STOPPED, RESTORED: data is a updated *Request with *State.
	// It can be used to pause / resume hashing.
	// PROGRESS_UPDATED: data is a int to store the percent(0 - 100).
	// DONE: data is a map[crypto.Hash]string to store the checksums.
	// EXITED: data is nil.
	Data interface{} `json:"data,omitempty"`
}

Message represents the messages.

func (*Message) JSON

func (m *Message) JSON() ([]byte, error)

JSON serializes a message as JSON. All types of messages are JSON-friendly. To get the meaning of Message.Data for different types of messages, check comments of Message struct.

type MessageType

type MessageType uint

MessageType represents the type of messages, which are generated when computing file hashes.

const (
	// An error occured.
	ERROR MessageType = iota
	// Task is scheduled.
	SCHEDULED
	// Task is started.
	STARTED
	// Task is stopped.
	STOPPED
	// Task is restored.
	RESTORED
	// Progress of task is updated.
	PROGRESS_UPDATED
	// Task is done.
	DONE
	// Goroutine of the task exited.
	// It's send after one of ERROR / STOPPED / DONE message.
	EXITED
	// Unknown message type.
	UNKNOWN
)

type Request

type Request struct {
	File      string        `json:"file"`
	HashFuncs []crypto.Hash `json:"hash_funcs"`
	Stat      *State        `json:"stat"`
}

Request represents the request of sum a single file.

func NewRequest

func NewRequest(file string, hashFuncs []crypto.Hash, stat *State) *Request

NewRequest returns a new request. file is the path of file to be compute hash checksum. hashFuncs contains the hash functions. stat contains the previous hash states.

If stat is nil, it starts to read the beginning of the file and compute checksums. otherwise, it'll restore saved hash states and resume reading file from the given offset.

func (*Request) JSON

func (req *Request) JSON() ([]byte, error)

JSON marshals a request as a JSON.

func (*Request) StateIsValid

func (req *Request) StateIsValid() bool

StateIsValid validates the saved hash states in the request.

type State

type State struct {
	// SummedSize is summed size of file.
	SummedSize int64 `json:"summed_size,string"`
	// Progress stores the last progress.
	Progress int `json:"progress"`
	// Data contains binary datas marshaled from hashes.
	Datas map[crypto.Hash][]byte `json:"datas"`
}

State represents the hashes states.

Directories

Path Synopsis
examples

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL