util

package

v0.0.0-...-5c6c1e7 Latest Latest Go to latest Published: Jul 29, 2022 License: Apache-2.0 Imports: 32 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/chrislusf/seaweedfs

Documentation ¶

Rendered for

Index ¶

Constants
Variables
func Base64Encode(data []byte) string
func Base64Md5(data []byte) string
func Base64Md5ToBytes(contentMd5 string) []byte
func BytesToHumanReadable(b uint64) string
func BytesToUint16(b []byte) (v uint16)
func BytesToUint32(b []byte) (v uint32)
func BytesToUint64(b []byte) (v uint64)
func CanonicalizeETag(etag string) string
func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)
func CloseRequest(req *http.Request)
func CloseResponse(resp *http.Response)
func DecompressData(input []byte) ([]byte, error)
func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)
func Delete(url string, jwt string) error
func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error)
func DetectedHostAddress() string
func Do(req *http.Request) (resp *http.Response, err error)
func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error)
func EmptyTo(s, to string) string
func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)
func FileExists(filename string) bool
func FileNameBase(filename string) string
func Get(url string) ([]byte, bool, error)
func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error
func GetFileSize(file *os.File) (size int64, err error)
func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)
func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error
func GunzipStream(w io.Writer, r io.Reader) (int64, error)
func GzipData(input []byte) ([]byte, error)
func GzipStream(w io.Writer, r io.Reader) (int64, error)
func HashStringToLong(dir string) (v int64)
func HashToInt32(data []byte) (v int32)
func Head(url string) (http.Header, error)
func HumanReadableInts(ids ...int) string
func HumanReadableIntsMax(max int, ids ...int) string
func IfElse(b bool, this, that string) string
func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)
func IsEmpty(given interface{}) bool
func IsGzippedContent(data []byte) bool
func IsNotEmpty(given interface{}) bool
func Join(names ...string) string
func JoinHostPort(host string, port int) string
func LoadConfiguration(configFileName string, required bool) (loaded bool)
func MaybeDecompressData(input []byte) []byte
func MaybeGzipData(input []byte) []byte
func Md5(data []byte) []byte
func Md5String(data []byte) string
func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipListner net.Listener, localListener net.Listener, err error)
func NewListener(addr string, timeout time.Duration) (ipListner net.Listener, err error)
func NormalizeUrl(url string) string
func Nvl(values ...string) string
func ParseBytes(s string) (uint64, error)
func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)
func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)
func ParseInt(text string, defaultValue int) int
func ParseUint64(text string, defaultValue uint64) uint64
func Post(url string, values url.Values) ([]byte, error)
func RandomBytes(byteCount int) []byte
func RandomInt32() int32
func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, ...) (int64, error)
func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (io.ReadCloser, error)
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, ...) (retryable bool, err error)
func ResolvePath(path string) string
func Retry(name string, job func() error) (err error)
func RetryForever(name string, job func() error, onErrFn func(err error) (shouldContinue bool))
func TestFolderWritable(folder string) (err error)
func Uint16toBytes(b []byte, v uint16)
func Uint32toBytes(b []byte, v uint32)
func Uint64toBytes(b []byte, v uint64)
func Uint8toBytes(b []byte, v uint8)
func Version() string
func WriteFile(name string, data []byte, perm os.FileMode) error
type BytesReader
- func NewBytesReader(b []byte) *BytesReader
type CipherKey
- func GenCipherKey() CipherKey
type ConcurrentReadMap
- func NewConcurrentReadMap() *ConcurrentReadMap
- func (m *ConcurrentReadMap) Delete(key string)
- func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)
- func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}
- func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})
type Configuration
type Conn
- func (c *Conn) Close() error
- func (c *Conn) Read(b []byte) (count int, e error)
- func (c *Conn) Write(b []byte) (count int, e error)
type CountingReader
- func (r *CountingReader) Read(p []byte) (n int, err error)
type DirectoryValueType
- func (s *DirectoryValueType) Set(value string) error
- func (s *DirectoryValueType) String() string
type FullPath
- func JoinPath(names ...string) FullPath
- func NewFullPath(dir, name string) FullPath
- func (fp FullPath) AsInode(unixTime int64) uint64
- func (fp FullPath) Child(name string) FullPath
- func (fp FullPath) DirAndName() (string, string)
- func (fp FullPath) Name() string
- func (fp FullPath) Split() []string
type LimitedConcurrentExecutor
- func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor
- func (c *LimitedConcurrentExecutor) Execute(job func())
type LimitedOutOfOrderProcessor
- func NewLimitedOutOfOrderProcessor(limit int32) (c *LimitedOutOfOrderProcessor)
- func (c *LimitedOutOfOrderProcessor) Execute(request OperationRequest)
type Listener
- func (l *Listener) Accept() (net.Conn, error)
type MinFreeSpace
- func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace)
- func ParseMinFreeSpace(s string) (*MinFreeSpace, error)
- func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string)
- func (s MinFreeSpace) String() string
type MinFreeSpaceType
type OperationRequest
type Queue
- func NewQueue() *Queue
- func (q *Queue) Dequeue() interface{}
- func (q *Queue) Enqueue(item interface{})
- func (q *Queue) Len() int
type UnboundedQueue
- func NewUnboundedQueue() *UnboundedQueue
- func (q *UnboundedQueue) Consume(fn func([]string))
- func (q *UnboundedQueue) EnQueue(items ...string)
type ViperProxy
- func GetViper() *ViperProxy
- func (vp *ViperProxy) GetBool(key string) bool
- func (vp *ViperProxy) GetInt(key string) int
- func (vp *ViperProxy) GetString(key string) string
- func (vp *ViperProxy) GetStringSlice(key string) []string
- func (vp *ViperProxy) SetDefault(key string, value interface{})
type WriteThrottler
- func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler
- func (wt *WriteThrottler) MaybeSlowdown(delta int64)

Constants ¶

View Source

const (
	Byte = 1 << (iota * 10)
	KiByte
	MiByte
	GiByte
	TiByte
	PiByte
	EiByte
)

IEC Sizes. kibis of bits

View Source

const (
	IByte = 1
	KByte = IByte * 1000
	MByte = KByte * 1000
	GByte = MByte * 1000
	TByte = GByte * 1000
	PByte = TByte * 1000
	EByte = PByte * 1000
)

SI Sizes.

View Source

const (
	VolumeSizeLimitGB = 30
)

Variables ¶

View Source

var (
	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.18)
	VERSION        = sizeLimit + " " + VERSION_NUMBER
	COMMIT         = ""
)

View Source

var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")

View Source

var RetryWaitTime = 6 * time.Second

View Source

var (
	Transport *http.Transport
)

View Source

var (
	UnsupportedCompression = fmt.Errorf("unsupported compression")
)

Functions ¶

func Base64Encode ¶

func Base64Encode(data []byte) string

func Base64Md5 ¶

func Base64Md5(data []byte) string

func Base64Md5ToBytes ¶

func Base64Md5ToBytes(contentMd5 string) []byte

func BytesToHumanReadable ¶

func BytesToHumanReadable(b uint64) string

BytesToHumanReadable returns the converted human readable representation of the bytes.

func BytesToUint16 ¶

func BytesToUint16(b []byte) (v uint16)

func BytesToUint32 ¶

func BytesToUint32(b []byte) (v uint32)

func BytesToUint64 ¶

func BytesToUint64(b []byte) (v uint64)

func CanonicalizeETag ¶

func CanonicalizeETag(etag string) string

func CheckFile ¶

func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)

func CloseRequest ¶

func CloseRequest(req *http.Request)

func CloseResponse ¶

func CloseResponse(resp *http.Response)

func DecompressData ¶

func DecompressData(input []byte) ([]byte, error)

func Decrypt ¶

func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)

func Delete ¶

func Delete(url string, jwt string) error

func DeleteProxied ¶

func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error)

func DetectedHostAddress ¶

func DetectedHostAddress() string

func Do ¶

func Do(req *http.Request) (resp *http.Response, err error)

func DownloadFile ¶

func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error)

func EmptyTo ¶

func EmptyTo(s, to string) string

EmptyTo returns to if s is empty.

func Encrypt ¶

func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)

func FileExists ¶

func FileExists(filename string) bool

func FileNameBase ¶

func FileNameBase(filename string) string

func Get ¶

func Get(url string) ([]byte, bool, error)

github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go may need increasing http.Client.Timeout

func GetBufferStream ¶

func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error

func GetFileSize ¶

func GetFileSize(file *os.File) (size int64, err error)

func GetFileUidGid ¶

func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)

func GetUrlStream ¶

func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error

func GunzipStream ¶

func GunzipStream(w io.Writer, r io.Reader) (int64, error)

func GzipData ¶

func GzipData(input []byte) ([]byte, error)

func GzipStream ¶

func GzipStream(w io.Writer, r io.Reader) (int64, error)

func HashStringToLong ¶

func HashStringToLong(dir string) (v int64)

returns a 64 bit big int

func HashToInt32 ¶

func HashToInt32(data []byte) (v int32)

func Head ¶

func Head(url string) (http.Header, error)

func HumanReadableInts ¶

func HumanReadableInts(ids ...int) string

HumanReadableInts joins a serials of inits into a smart one like 1-3 5 7-10 for human readable.

func HumanReadableIntsMax ¶

func HumanReadableIntsMax(max int, ids ...int) string

HumanReadableIntsMax joins a serials of inits into a smart one like 1-3 5 ... for human readable.

func IfElse ¶

func IfElse(b bool, this, that string) string

IfElse works like b ? this : that.

func IsCompressableFileType ¶

func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)

func IsEmpty ¶

func IsEmpty(given interface{}) bool

IsEmpty returns true if the given value has the zero value for its type.

func IsGzippedContent ¶

func IsGzippedContent(data []byte) bool

func IsNotEmpty ¶

func IsNotEmpty(given interface{}) bool

IsNotEmpty returns true if the given value is not zero or empty.

func Join ¶

func Join(names ...string) string

func JoinHostPort ¶

func JoinHostPort(host string, port int) string

func LoadConfiguration ¶

func LoadConfiguration(configFileName string, required bool) (loaded bool)

func MaybeDecompressData ¶

func MaybeDecompressData(input []byte) []byte

func MaybeGzipData ¶

func MaybeGzipData(input []byte) []byte

func Md5 ¶

func Md5(data []byte) []byte

func Md5String ¶

func Md5String(data []byte) string

func NewIpAndLocalListeners ¶

func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipListner net.Listener, localListener net.Listener, err error)

func NewListener ¶

func NewListener(addr string, timeout time.Duration) (ipListner net.Listener, err error)

func NormalizeUrl ¶

func NormalizeUrl(url string) string

func Nvl ¶

func Nvl(values ...string) string

return the first non empty string

func ParseBytes ¶

func ParseBytes(s string) (uint64, error)

ParseBytes parses a string representation of bytes into the number of bytes it represents.

func ParseFilerUrl ¶

func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)

func ParseHostPort ¶

func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)

func ParseInt ¶

func ParseInt(text string, defaultValue int) int

func ParseUint64 ¶

func ParseUint64(text string, defaultValue uint64) uint64

func Post ¶

func Post(url string, values url.Values) ([]byte, error)

func RandomBytes ¶

func RandomBytes(byteCount int) []byte

func RandomInt32 ¶

func RandomInt32() int32

func ReadUrl ¶

func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error)

func ReadUrlAsReaderCloser ¶

func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (io.ReadCloser, error)

func ReadUrlAsStream ¶

func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error)

func ResolvePath ¶

func ResolvePath(path string) string

func Retry ¶

func Retry(name string, job func() error) (err error)

func RetryForever ¶

func RetryForever(name string, job func() error, onErrFn func(err error) (shouldContinue bool))

func TestFolderWritable ¶

func TestFolderWritable(folder string) (err error)

func Uint16toBytes ¶

func Uint16toBytes(b []byte, v uint16)

func Uint32toBytes ¶

func Uint32toBytes(b []byte, v uint32)

func Uint64toBytes ¶

func Uint64toBytes(b []byte, v uint64)

func Uint8toBytes ¶

func Uint8toBytes(b []byte, v uint8)

func Version ¶

func Version() string

func WriteFile ¶

func WriteFile(name string, data []byte, perm os.FileMode) error

Copied from os.WriteFile(), adding file sync. see https://github.com/golang/go/issues/20599

Types ¶

type BytesReader ¶

type BytesReader struct {
	Bytes []byte
	*bytes.Reader
}

func NewBytesReader ¶

func NewBytesReader(b []byte) *BytesReader

type CipherKey ¶

type CipherKey []byte

func GenCipherKey ¶

func GenCipherKey() CipherKey

type ConcurrentReadMap ¶

type ConcurrentReadMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

A mostly for read map, which can thread-safely initialize the map entries.

func NewConcurrentReadMap ¶

func NewConcurrentReadMap() *ConcurrentReadMap

func (*ConcurrentReadMap) Delete ¶

func (m *ConcurrentReadMap) Delete(key string)

func (*ConcurrentReadMap) Find ¶

func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)

func (*ConcurrentReadMap) Get ¶

func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}

func (*ConcurrentReadMap) Items ¶

func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})

type Configuration ¶

type Configuration interface {
	GetString(key string) string
	GetBool(key string) bool
	GetInt(key string) int
	GetStringSlice(key string) []string
	SetDefault(key string, value interface{})
}

type Conn ¶

type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
	// contains filtered or unexported fields
}

Conn wraps a net.Conn, and sets a deadline for every read and write operation.

func (*Conn) Close ¶

func (c *Conn) Close() error

func (*Conn) Read ¶

func (c *Conn) Read(b []byte) (count int, e error)

func (*Conn) Write ¶

func (c *Conn) Write(b []byte) (count int, e error)

type CountingReader ¶

type CountingReader struct {
	BytesRead int
	// contains filtered or unexported fields
}

func (*CountingReader) Read ¶

func (r *CountingReader) Read(p []byte) (n int, err error)

type DirectoryValueType ¶

type DirectoryValueType string

var (
	ConfigurationFileDirectory DirectoryValueType
)

func (*DirectoryValueType) Set ¶

func (s *DirectoryValueType) Set(value string) error

func (*DirectoryValueType) String ¶

func (s *DirectoryValueType) String() string

type FullPath ¶

type FullPath string

func JoinPath ¶

func JoinPath(names ...string) FullPath

func NewFullPath ¶

func NewFullPath(dir, name string) FullPath

func (FullPath) AsInode ¶

func (fp FullPath) AsInode(unixTime int64) uint64

AsInode an in-memory only inode representation

func (FullPath) Child ¶

func (fp FullPath) Child(name string) FullPath

func (FullPath) DirAndName ¶

func (fp FullPath) DirAndName() (string, string)

func (FullPath) Name ¶

func (fp FullPath) Name() string

func (FullPath) Split ¶

func (fp FullPath) Split() []string

split, but skipping the root

type LimitedConcurrentExecutor ¶

type LimitedConcurrentExecutor struct {
	// contains filtered or unexported fields
}

LimitedConcurrentExecutor object

func NewLimitedConcurrentExecutor ¶

func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor

func (*LimitedConcurrentExecutor) Execute ¶

func (c *LimitedConcurrentExecutor) Execute(job func())

Execute adds a function to the execution queue. if num of go routines allocated by this instance is < limit launch a new go routine to execute job else wait until a go routine becomes available

type LimitedOutOfOrderProcessor ¶

type LimitedOutOfOrderProcessor struct {
	// contains filtered or unexported fields
}

func NewLimitedOutOfOrderProcessor ¶

func NewLimitedOutOfOrderProcessor(limit int32) (c *LimitedOutOfOrderProcessor)

func (*LimitedOutOfOrderProcessor) Execute ¶

func (c *LimitedOutOfOrderProcessor) Execute(request OperationRequest)

type Listener ¶

type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

Listener wraps a net.Listener, and gives a place to store the timeout parameters. On Accept, it will wrap the net.Conn with our own Conn for us.

func (*Listener) Accept ¶

func (l *Listener) Accept() (net.Conn, error)

type MinFreeSpace ¶

type MinFreeSpace struct {
	Type    MinFreeSpaceType
	Bytes   uint64
	Percent float32
	Raw     string
}

MinFreeSpace is type that defines the limit for the minimum free space.

func MustParseMinFreeSpace ¶

func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace)

MustParseMinFreeSpace parses comma-separated argument for min free space setting. minFreeSpace has the high priority than minFreeSpacePercent if it is set.

func ParseMinFreeSpace ¶

func ParseMinFreeSpace(s string) (*MinFreeSpace, error)

ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G

func (MinFreeSpace) IsLow ¶

func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string)

IsLow tells whether the free space is low or not.

func (MinFreeSpace) String ¶

func (s MinFreeSpace) String() string

String returns a string representation of MinFreeSpace.

type MinFreeSpaceType ¶

type MinFreeSpaceType int

MinFreeSpaceType is the type of MinFreeSpace.

const (
	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
	AsPercent MinFreeSpaceType = iota
	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
	AsBytes
)

type OperationRequest ¶

type OperationRequest func()

a different implementation, but somehow more "conservative"

type Queue ¶

type Queue struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewQueue ¶

func NewQueue() *Queue

func (*Queue) Dequeue ¶

func (q *Queue) Dequeue() interface{}

func (*Queue) Enqueue ¶

func (q *Queue) Enqueue(item interface{})

func (*Queue) Len ¶

func (q *Queue) Len() int

type UnboundedQueue ¶

type UnboundedQueue struct {
	// contains filtered or unexported fields
}

func NewUnboundedQueue ¶

func NewUnboundedQueue() *UnboundedQueue

func (*UnboundedQueue) Consume ¶

func (q *UnboundedQueue) Consume(fn func([]string))

func (*UnboundedQueue) EnQueue ¶

func (q *UnboundedQueue) EnQueue(items ...string)

type ViperProxy ¶

type ViperProxy struct {
	*viper.Viper
	sync.Mutex
}

func GetViper ¶

func GetViper() *ViperProxy

func (*ViperProxy) GetBool ¶

func (vp *ViperProxy) GetBool(key string) bool

func (*ViperProxy) GetInt ¶

func (vp *ViperProxy) GetInt(key string) int

func (*ViperProxy) GetString ¶

func (vp *ViperProxy) GetString(key string) string

func (*ViperProxy) GetStringSlice ¶

func (vp *ViperProxy) GetStringSlice(key string) []string

func (*ViperProxy) SetDefault ¶

func (vp *ViperProxy) SetDefault(key string, value interface{})

type WriteThrottler ¶

type WriteThrottler struct {
	// contains filtered or unexported fields
}

func NewWriteThrottler ¶

func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler

func (*WriteThrottler) MaybeSlowdown ¶

func (wt *WriteThrottler) MaybeSlowdown(delta int64)

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
buffered_writer
chunk_cache
fla9 Package flag implements command-line flag parsing.	Package flag implements command-line flag parsing.
grace
httpdown Package httpdown provides http.ConnState enabled graceful termination of http.Server.	Package httpdown provides http.ConnState enabled graceful termination of http.Server.
log_buffer
mem
skiplist

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL