qshell

package
v2.2.0+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 10, 2018 License: MIT Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source
const (
	BLOCK_BITS = 22 // Indicate that the blocksize is 4M
	BLOCK_SIZE = 1 << BLOCK_BITS
)
View Source
const (
	STATUS_OK = iota
	//process error
	STATUS_ERROR
	//local error
	STATUS_HALT
)
View Source
const (
	MIN_DOWNLOAD_THREAD_COUNT = 1
	MAX_DOWNLOAD_THREAD_COUNT = 2000
)
View Source
const (
	DEFAULT_PUT_THRESHOLD   int64 = 10 * 1024 * 1024 //10MB
	MIN_UPLOAD_THREAD_COUNT int64 = 1
	MAX_UPLOAD_THREAD_COUNT int64 = 2000
)
View Source
const (
	RETRY_MAX_TIMES = 5
	RETRY_INTERVAL  = time.Second * 1
	HTTP_TIMEOUT    = time.Second * 10
)
View Source
const (
	ZoneNB  = "z0"
	ZoneBC  = "z1"
	ZoneHN  = "z2"
	ZoneNA0 = "na0"
	ZoneAS0 = "as0"
)

Variables

View Source
var (
	BUCKET_RS_HOST  = "http://rs.qiniu.com"
	BUCKET_API_HOST = "http://api.qiniu.com"
)
View Source
var QAccountFile string
View Source
var QAccountName string
View Source
var QShellRootPath string

dir to store some cached files for qshell, like ak, sk

View Source
var ZoneBCConfig = ZoneConfig{
	UpHost:    "http://upload-z1.qiniu.com",
	IovipHost: "http://iovip-z1.qbox.me",
	RsHost:    "http://rs-z1.qiniu.com",
	RsfHost:   "http://rsf-z1.qiniu.com",
	ApiHost:   "http://api-z1.qiniu.com",
}
View Source
var ZoneHNConfig = ZoneConfig{
	UpHost:    "http://upload-z2.qiniu.com",
	IovipHost: "http://iovip-z2.qbox.me",
	RsHost:    "http://rs-z2.qiniu.com",
	RsfHost:   "http://rsf-z2.qiniu.com",
	ApiHost:   "http://api-z2.qiniu.com",
}
View Source
var ZoneNA0Config = ZoneConfig{
	UpHost:    "http://upload-na0.qiniu.com",
	IovipHost: "http://iovip-na0.qbox.me",
	RsHost:    "http://rs-na0.qiniu.com",
	RsfHost:   "http://rsf-na0.qiniu.com",
	ApiHost:   "http://api-na0.qiniu.com",
}
View Source
var ZoneNBConfig = ZoneConfig{
	UpHost:    "http://upload.qiniu.com",
	IovipHost: "http://iovip.qbox.me",
	RsHost:    "http://rs.qiniu.com",
	RsfHost:   "http://rsf.qiniu.com",
	ApiHost:   "http://api.qiniu.com",
}
View Source
var ZoneSA0Config = ZoneConfig{
	UpHost:    "http://upload-as0.qiniu.com",
	IovipHost: "http://iovip-as0.qbox.me",
	RsHost:    "http://rs-as0.qiniu.com",
	RsfHost:   "http://rsf-as0.qiniu.com",
	ApiHost:   "http://api-as0.qiniu.com",
}

Functions

func AesDecrypt

func AesDecrypt(crypted, key []byte) ([]byte, error)

func AesEncrypt

func AesEncrypt(origData, key []byte) ([]byte, error)

func BlockCount added in v1.4.1

func BlockCount(fsize int64) int

func CalSha1 added in v1.4.1

func CalSha1(b []byte, r io.Reader) ([]byte, error)

func DirCache

func DirCache(cacheRootPath string, cacheResultFile string) (fileCount int64, retErr error)

generate the file list for the specified directory

@param cacheRootPath - dir to generate cache file @param cacheResultFile - cache result file path

@return (fileCount, retErr) - total file count and any error meets

func GetBuckets added in v1.4.6

func GetBuckets(mac *digest.Mac) (buckets []string, err error)

get all the buckets, including the authorized access buckets

func GetEtag added in v1.4.1

func GetEtag(filename string) (etag string, err error)

func GetFileFromBucket

func GetFileFromBucket(mac *digest.Mac, bucket, key, localFile string) (err error)

GetFileFromBucket check domains check src domains proxy download

func GetFileLineCount

func GetFileLineCount(filePath string) (totalCount int64)

func IsValidZone added in v1.6.0

func IsValidZone(zone string) (valid bool)

func ListBucket

func ListBucket(mac *digest.Mac, bucket, prefix, marker, listResultFile string) (retErr error)

*@param bucket *@param prefix *@param marker *@param listResultFile *@return listError

func ListBucketV2

func ListBucketV2(mac *digest.Mac, bucket, prefix, marker, listResultFile string) (nextMarker string, err error)

ListFilesV2 改进版本的 ListFiles 以解决 ListFiles 的超时问题 https://github.com/qbox/product/blob/master/kodo/rsf.md#v2list-%E5%88%97%E5%87%BA%E5%86%85%E5%AE%B9 在这个方法中,需要注意的是即使函数返回的 err 不为 nil,entries 也有可能有值,另外 nextMarker 也有可能不为空,所以 正确的逻辑是检查 hasNext 是否为 true,如果有表示还可以继续使用 nextMarker 来进行list,另外保存下 entries 的记录 在这些逻辑处理完毕之后,检查下是否 err 不为 nil,如果不为 nil,应该打印一个 Warnning 的日志表示 list 曾经出现过错误 另外如果你希望全量列举空间的话,limit 参数设置为 0 即可。

func M3u8FileList added in v1.4.1

func M3u8FileList(mac *digest.Mac, bucket string, m3u8Key string) (slicesToDelete []rs.EntryPath, err error)

func M3u8ReplaceDomain added in v1.7.8

func M3u8ReplaceDomain(mac *digest.Mac, bucket string, m3u8Key string, newDomain string) (err error)

replace and upload

func Md5Hex added in v1.6.3

func Md5Hex(from string) string

func PKCS5Padding

func PKCS5Padding(ciphertext []byte, blockSize int) []byte

func PKCS5UnPadding

func PKCS5UnPadding(origData []byte) []byte

func Prefetch

func Prefetch(mac *digest.Mac, bucket, key string) (err error)

func Prefop

func Prefop(persistentId string, fopRet *FopRet) (err error)

func PrivateUrl added in v1.4.1

func PrivateUrl(mac *digest.Mac, publicUrl string, deadline int64) (finalUrl string, err error)

func QiniuDownload

func QiniuDownload(threadCount int, downConfig *DownloadConfig)

func QiniuUpload

func QiniuUpload(threadCount int, uploadConfig *UploadConfig, exporter *FileExporter)

QiniuUpload

func Saveas added in v1.4.1

func Saveas(mac *digest.Mac, publicUrl string, saveBucket string, saveKey string) (string, error)

func SetAccount

func SetAccount(accessKey string, secretKey string) (err error)

func SetZone added in v1.6.0

func SetZone(zone string)

func Unzip added in v1.4.1

func Unzip(zipFilePath string, unzipPath string) (err error)

Types

type Account

type Account struct {
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
}

func GetAccount

func GetAccount() (account Account, err error)

func (*Account) String

func (acc *Account) String() string

func (*Account) ToJson

func (acc *Account) ToJson() (jsonStr string, err error)

type AliListBucket

type AliListBucket struct {
	DataCenter      string
	AccessKeyId     string
	AccessKeySecret string
	Bucket          string
	Prefix          string
}

func (*AliListBucket) ListBucket

func (this *AliListBucket) ListBucket(listResultFile string) (err error)

type BatchItemRet added in v1.4.1

type BatchItemRet struct {
	Code int              `json:"code"`
	Data BatchItemRetData `json:"data"`
}

func BatchChgm added in v1.4.1

func BatchChgm(client rs.Client, entries []ChgmEntryPath) (ret []BatchItemRet, err error)

func BatchChtype

func BatchChtype(client rs.Client, entries []ChtypeEntryPath) (ret []BatchItemRet, err error)

func BatchCopy added in v1.4.1

func BatchCopy(client rs.Client, entries []CopyEntryPath, force bool) (ret []BatchItemRet, err error)

func BatchDelete added in v1.4.1

func BatchDelete(client rs.Client, entries []rs.EntryPath) (ret []BatchItemRet, err error)

func BatchDeleteAfterDays

func BatchDeleteAfterDays(client rs.Client, entries []DeleteAfterDaysEntryPath) (ret []BatchItemRet, err error)

func BatchMove added in v1.4.1

func BatchMove(client rs.Client, entries []MoveEntryPath, force bool) (ret []BatchItemRet, err error)

func BatchRename added in v1.4.1

func BatchRename(client rs.Client, entries []RenameEntryPath, force bool) (ret []BatchItemRet, err error)

func BatchStat added in v1.4.4

func BatchStat(client rs.Client, entries []rs.EntryPath) (ret []BatchItemRet, err error)

type BatchItemRetData added in v1.4.1

type BatchItemRetData struct {
	Fsize    int    `json:"fsize,omitempty"`
	Hash     string `json:"hash,omitempty"`
	MimeType string `json:"mimeType,omitempty"`
	PutTime  int64  `json:"putTime,omitempty"`
	Error    string `json:"error,omitempty"`
	FileType int    `json:"type"`
}

type BatchPrefetchRequest added in v1.8.5

type BatchPrefetchRequest struct {
	Urls []string `json:"urls"`
}

type BatchPrefetchResponse added in v1.8.5

type BatchPrefetchResponse struct {
	Code        int      `json:"code"`
	Error       string   `json:"error"`
	RequestId   string   `json:"requestId"`
	InvalidUrls []string `json:"invalidUrls"`
	QuotaDay    int      `json:"quotaDay"`
	SurplusDay  int      `json:"surplusDay"`
}

func BatchPrefetch added in v1.8.5

func BatchPrefetch(client *rs.Client, urls []string) (batchPrefetchResponse BatchPrefetchResponse, err error)

type BatchRefreshRequest added in v1.7.9

type BatchRefreshRequest struct {
	Urls []string `json:"urls,omitempty"`
	Dirs []string `json:"dirs,omitempty"`
}

type BatchRefreshResponse added in v1.8.5

type BatchRefreshResponse struct {
	Code          int      `json:"code"`
	Error         string   `json:"error"`
	RequestId     string   `json:"requestId"`
	InvalidUrls   []string `json:"invalidUrls"`
	InvalidDirs   []string `json:"invalidDirs"`
	UrlQuotaDay   int      `json:"urlQuotaDay"`
	UrlSurplusDay int      `json:"urlSurplusDay"`
	DirQuotaDay   int      `json:"dirQuotaDay"`
	DirSurplusDay int      `json:"dirSurplusDay"`
}

func BatchRefresh added in v1.5.2

func BatchRefresh(client *rs.Client, urls []string, dirs []string) (batchRefreshResp BatchRefreshResponse, err error)

type BeeLogConfig

type BeeLogConfig struct {
	Filename string `json:"filename"`
	Level    int    `json:"level"`
	Daily    bool   `json:"daily"`
	MaxDays  int    `json:"maxdays"`
}

func (*BeeLogConfig) ToJson

func (c *BeeLogConfig) ToJson() string

type BucketDomain added in v1.4.1

type BucketDomain struct {
	Domain string `json:"domain"`
	Owner  int    `json:"owner"`
}

func GetDomainsOfBucket added in v1.4.6

func GetDomainsOfBucket(mac *digest.Mac, bucket string) (domains []BucketDomain, err error)

type BucketInfo

type BucketInfo struct {
	Region string `json:"region"`
}

func GetBucketInfo

func GetBucketInfo(mac *digest.Mac, bucket string) (bucketInfo BucketInfo, err error)

get bucket info

@param mac @param bucket - bucket name

@return bucketInfo, err

type ChgmEntryPath added in v1.4.1

type ChgmEntryPath struct {
	Bucket   string
	Key      string
	MimeType string
}

type ChtypeEntryPath

type ChtypeEntryPath struct {
	Bucket   string
	Key      string
	FileType int
}

type CopyEntryPath added in v1.4.1

type CopyEntryPath struct {
	SrcBucket  string
	DestBucket string
	SrcKey     string
	DestKey    string
}

type DeleteAfterDaysEntryPath

type DeleteAfterDaysEntryPath struct {
	Bucket          string
	Key             string
	DeleteAfterDays int
}

type DownloadConfig

type DownloadConfig struct {
	DestDir  string `json:"dest_dir"`
	Bucket   string `json:"bucket"`
	Prefix   string `json:"prefix,omitempty"`
	Suffixes string `json:"suffixes,omitempty"`
	//down from cdn
	Referer   string `json:"referer,omitempty"`
	CdnDomain string `json:"cdn_domain,omitempty"`
	//log settings
	LogLevel  string `json:"log_level,omitempty"`
	LogFile   string `json:"log_file,omitempty"`
	LogRotate int    `json:"log_rotate,omitempty"`
	LogStdout bool   `json:"log_stdout,omitempty"`

	IsHostFileSpecified bool `json:"-"`
}

type FetchResult added in v1.4.1

type FetchResult struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

func Fetch

func Fetch(mac *digest.Mac, remoteResUrl, bucket, key string) (fetchResult FetchResult, err error)

type FileExporter

type FileExporter struct {
	SuccessFname    string
	SuccessLock     sync.RWMutex
	SuccessWriter   *bufio.Writer
	FailureFname    string
	FailureLock     sync.RWMutex
	FailureWriter   *bufio.Writer
	OverwriteFname  string
	OverwriteLock   sync.RWMutex
	OverwriteWriter *bufio.Writer
}

FileExporter

type FopResult

type FopResult struct {
	Cmd   string   `json:"cmd"`
	Code  int      `json:"code"`
	Desc  string   `json:"desc"`
	Error string   `json:"error,omitempty"`
	Hash  string   `json:"hash,omitempty"`
	Key   string   `json:"key,omitempty"`
	Keys  []string `json:"keys,omitempty"`
}

type FopRet

type FopRet struct {
	Id          string `json:"id"`
	Code        int    `json:"code"`
	Desc        string `json:"desc"`
	InputBucket string `json:"inputBucket,omitempty"`
	InputKey    string `json:"inputKey,omitempty"`
	Pipeline    string `json:"pipeline,omitempty"`
	Reqid       string `json:"reqid,omitempty"`
	Items       []FopResult
}

func (*FopRet) String

func (this *FopRet) String() string

type MoveEntryPath added in v1.4.1

type MoveEntryPath struct {
	SrcBucket  string
	DestBucket string
	SrcKey     string
	DestKey    string
}

type PutRet

type PutRet struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

func Sync added in v1.6.0

func Sync(mac *digest.Mac, srcResUrl, bucket, key, upHostIp string) (putRet PutRet, err error)

type RenameEntryPath added in v1.4.1

type RenameEntryPath struct {
	Bucket string
	OldKey string
	NewKey string
}

type SyncProgress added in v1.6.0

type SyncProgress struct {
	BlkCtxs   []rio.BlkputRet `json:"blk_ctxs"`
	Offset    int64           `json:"offset"`
	TotalSize int64           `json:"total_size"`
}

type UploadConfig

type UploadConfig struct {
	//basic config
	SrcDir string `json:"src_dir"`
	Bucket string `json:"bucket"`

	//optional config
	FileList         string `json:"file_list,omitempty"`
	PutThreshold     int64  `json:"put_threshold,omitempty"`
	KeyPrefix        string `json:"key_prefix,omitempty"`
	IgnoreDir        bool   `json:"ignore_dir,omitempty"`
	Overwrite        bool   `json:"overwrite,omitempty"`
	CheckExists      bool   `json:"check_exists,omitempty"`
	CheckHash        bool   `json:"check_hash,omitempty"`
	CheckSize        bool   `json:"check_size,omitempty"`
	SkipFilePrefixes string `json:"skip_file_prefixes,omitempty"`
	SkipPathPrefixes string `json:"skip_path_prefixes,omitempty"`
	SkipFixedStrings string `json:"skip_fixed_strings,omitempty"`
	SkipSuffixes     string `json:"skip_suffixes,omitempty"`
	RescanLocal      bool   `json:"rescan_local,omitempty"`
	FileType         int    `json:"file_type,omitempty"`

	//advanced config
	UpHost string `json:"up_host,omitempty"`

	BindUpIp string `json:"bind_up_ip,omitempty"`
	BindRsIp string `json:"bind_rs_ip,omitempty"`
	//local network interface card config
	BindNicIp string `json:"bind_nic_ip,omitempty"`

	//log settings
	LogLevel  string `json:"log_level,omitempty"`
	LogFile   string `json:"log_file,omitempty"`
	LogRotate int    `json:"log_rotate,omitempty"`
	LogStdout bool   `json:"log_stdout,omitempty"`

	//more settings
	DeleteOnSuccess bool `json:"delete_on_success,omitempty"`
	DisableResume   bool `json:"disable_resume,omitempty"`

	IsHostFileSpecified bool `json:"-"`
}

type UploadInfo added in v1.7.2

type UploadInfo struct {
	TotalFileCount int64 `json:"total_file_count"`
}

type ZoneConfig added in v1.6.0

type ZoneConfig struct {
	UpHost    string
	RsHost    string
	RsfHost   string
	IovipHost string
	ApiHost   string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL