diskmgr

diskmgr

DiskMgr manages the files and space on the disk.

Modules

It consists of the following modules:

• sqlite db: record cache file info and download file info
• local cache: cache file info and header info
• folderMgr: manages folders and disk space
• downloadMgr: download files

Provide folder

You can set several folders(at least one folder) for file storage. Files will be saved in these folders. If you have multiple folders, files will be stored in these folders proportionally according to the size of the folders.

Background jobs

There are some background jobs: 1 scan leak files: scan disk and delete file which on disk but not in db. It will run every 12 hours, and sleep 5 seconds after scan 100 files.

2 delete empty folder: check path(which from a channel) is empty folder or not, and delete empty folder. A file path will be push into a channel when the file is deleted, this bgjob will check these path and delete empty folder

3 retry download notify: onDownloadSuccess or onDownloadFailed will be called when download finish(success or failed) to notify server. If notify failed, this job will retry the notification till server receive the message.

Interface

The main interface of diskMgr is SaveFile, DeleteFile, and GetFile

GetSpaceInfo will used to monitor space usage.

GetFilesBeforeAccessTime and GetFilesByAccessTime is used to get files which have not been accessed for a long time.

static function tool

diskmgr.CheckFolder is a static tool to check folder size. This function will generate the space holder files to make sure you have enough permissions and space. Those files will be deleted after check.

Example

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/meson-network/diskMgr"
	"github.com/meson-network/diskMgr/folderMgr"
	"github.com/meson-network/downloadMgr"
	"github.com/coreservice-io/LogrusULog"
	"github.com/coreservice-io/ULog"
)

func main() {
	//logger is needed
	logger, _ := LogrusULog.New("/Users/bruce/workspace/go/project/diskmgr/testfolder/logs", 2, 20, 30)
	logger.SetLevel(ULog.DebugLevel)

	//some callback
	onSuccess := func(nameHash string, size int64) error {
		logger.Debugln("success", "name:", nameHash, "size:", size)
		return nil
	}
	onFail := func(nameHash string, failReason downloadmgr.FailReasonType) error {
		logger.Debugln("fail", "name:", nameHash)
		switch failReason {
		case downloadmgr.Fail_NoSpace:
			logger.Debugln("fail reason", "no space")
		case downloadmgr.Fail_Other, downloadmgr.Fail_RequestError:
			logger.Debugln("fail reason", "request error")
		case downloadmgr.Fail_SizeLimit:
			logger.Debugln("fail reason", "file too big")
		default:
			logger.Debugln("fail reason", "other")

		}
		return nil
	}
	onCancel := func(task *downloadmgr.Task) error {
		logger.Debugln("cancel", task.Id, task.NameHash)
		return nil
	}
	onDownloading := func(task *downloadmgr.Task) error {
		logger.Debugln("downloading", task.Id, task.NameHash)
		return nil
	}
	onLowSpeed := func(task *downloadmgr.Task) error {
		logger.Debugln("lowSpeed", task.Id, task.NameHash)
		return nil
	}

	onCachedFileDeleted := func(deletedFile []string) error {
		if len(deletedFile) <= 0 {
			return nil
		}
		logger.Debugln("deletedCachedFile:")
		for _, v := range deletedFile {
			logger.Debugln(v)
		}
		return nil
	}
	onDownloadingFileDeleted := func(deletedFile []string) error {
		if len(deletedFile) <= 0 {
			return nil
		}
		logger.Debugln("deletedDownloadingFile:")
		for _, v := range deletedFile {
			logger.Debugln(v)
		}
		return nil
	}
	onFileMissing := func(missingFile []string) error {
		if len(missingFile) <= 0 {
			return nil
		}
		logger.Debugln("missingFile:")
		for _, v := range missingFile {
			logger.Debugln(v)
		}
		return nil
	}

	//provide folder
	//form config file in real project
	folderInfo := []folderMgr.FolderConfig{
		{"/Users/bruce/workspace/go/project/diskmgr/testfolder/1", 4},
		{"/Users/bruce/workspace/go/project/diskmgr/testfolder/2", 4},
	}

	for _, v := range folderInfo {
		err := diskmgr.CheckFolder(v.AbsPath, v.SizeGB, 40, 4)
		if err != nil {
			logger.Fatalln(err)
		}
	}

        //new diskMgr
	diskMgr := diskmgr.New(
		logger,
		func(panic interface{}) {
			logger.Errorln(panic)
		},
	)
	err := diskMgr.StartUp(
		folderInfo,
		"/Users/bruce/workspace/go/project/diskmgr/testfolder",
		onSuccess,
		onFail,
		onCancel,
		onDownloading,
		onLowSpeed,
		onCachedFileDeleted,
		onDownloadingFileDeleted,
		onFileMissing,
	)
	if err != nil {
		logger.Fatalln(err)
	}
  
	//get space info
	log.Println(diskMgr.GetSpaceInfo())

	//cache some files
	for i := 0; i < 10; i++ {
		nameHash := fmt.Sprintf("123456789012%04d", i)
		err := diskMgr.CacheFile(
			nameHash,
			[]string{
	            "https://images-na.ssl-images-amazon.com/images/I/41eXG7xY09L._SY291_BO1,204,203,200_QL40_FMwebp_.jpg",
				},
			false,
			500*1024*1024)
		if err != nil {
			log.Println(err)
		}
	}

	//try to get some files
	go func() {
		files, _ := diskMgr.GetFilesByAccessTime(200)
		for _, v := range files {
			err := diskMgr.DeleteFile([]string{v.NameHash})
			if err != nil {
				log.Println("delete file err:", err)
			}
		}
	}()

	time.Sleep(1 * time.Hour)
}