needle

package
v0.0.0-...-935c85c Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 17, 2020 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const (
	NeedleChecksumSize = 4
	PairNamePrefix     = "Seaweed-"
)
View Source
const (
	FlagIsCompressed        = 0x01
	FlagHasName             = 0x02
	FlagHasMime             = 0x04
	FlagHasLastModifiedDate = 0x08
	FlagHasTtl              = 0x10
	FlagHasPairs            = 0x20
	FlagIsChunkManifest     = 0x80
	LastModifiedBytesLength = 5
	TtlBytesLength          = 2
)
View Source
const (
	// stored unit types
	Empty byte = iota
	Minute
	Hour
	Day
	Week
	Month
	Year
)
View Source
const (
	Version1       = Version(1)
	Version2       = Version(2)
	Version3       = Version(3)
	CurrentVersion = Version3
)

Variables

View Source
var EMPTY_TTL = &TTL{}

Functions

func GetActualSize

func GetActualSize(size Size, version Version) int64

func NeedleBodyLength

func NeedleBodyLength(needleSize Size, version Version) int64

func PaddingLength

func PaddingLength(needleSize Size, version Version) Size

func ParseNeedleIdCookie

func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error)

func ReadNeedleBlob

func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error)

func SecondsToTTL

func SecondsToTTL(seconds int32) string

Types

type AsyncRequest

type AsyncRequest struct {
	N              *Needle
	IsWriteRequest bool
	ActualSize     int64
	// contains filtered or unexported fields
}

func NewAsyncRequest

func NewAsyncRequest(n *Needle, isWriteRequest bool) *AsyncRequest

func (*AsyncRequest) Complete

func (r *AsyncRequest) Complete(offset uint64, size uint64, isUnchanged bool, err error)

func (*AsyncRequest) IsSucceed

func (r *AsyncRequest) IsSucceed() bool

func (*AsyncRequest) Submit

func (r *AsyncRequest) Submit()

func (*AsyncRequest) UpdateResult

func (r *AsyncRequest) UpdateResult(offset uint64, size uint64, isUnchanged bool, err error)

func (*AsyncRequest) WaitComplete

func (r *AsyncRequest) WaitComplete() (uint64, uint64, bool, error)

type CRC

type CRC uint32

func NewCRC

func NewCRC(b []byte) CRC

func (CRC) Update

func (c CRC) Update(b []byte) CRC

func (CRC) Value

func (c CRC) Value() uint32

type FileId

type FileId struct {
	VolumeId VolumeId
	Key      NeedleId
	Cookie   Cookie
}

func NewFileId

func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId

func NewFileIdFromNeedle

func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId

func ParseFileIdFromString

func ParseFileIdFromString(fid string) (*FileId, error)

Deserialize the file id

func (*FileId) GetCookie

func (n *FileId) GetCookie() Cookie

func (*FileId) GetNeedleId

func (n *FileId) GetNeedleId() NeedleId

func (*FileId) GetNeedleIdCookie

func (n *FileId) GetNeedleIdCookie() string

func (*FileId) GetVolumeId

func (n *FileId) GetVolumeId() VolumeId

func (*FileId) String

func (n *FileId) String() string

type Needle

type Needle struct {
	Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
	Id     NeedleId `comment:"needle id"`
	Size   Size     `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`

	DataSize     uint32 `comment:"Data size"` //version2
	Data         []byte `comment:"The actual file data"`
	Flags        byte   `comment:"boolean flags"` //version2
	NameSize     uint8  //version2
	Name         []byte `comment:"maximum 256 characters"` //version2
	MimeSize     uint8  //version2
	Mime         []byte `comment:"maximum 256 characters"` //version2
	PairsSize    uint16 //version2
	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
	Ttl          *TTL

	Checksum   CRC    `comment:"CRC32 to check integrity"`
	AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
	Padding    []byte `comment:"Aligned to 8 bytes"`
}

* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.

func CreateNeedleFromRequest

func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, contentMd5 string, e error)

func ReadNeedleHeader

func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error)

func (*Needle) Append

func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error)

func (*Needle) DiskSize

func (n *Needle) DiskSize(version Version) int64

func (*Needle) Etag

func (n *Needle) Etag() string

func (*Needle) HasLastModifiedDate

func (n *Needle) HasLastModifiedDate() bool

func (*Needle) HasMime

func (n *Needle) HasMime() bool

func (*Needle) HasName

func (n *Needle) HasName() bool

func (*Needle) HasPairs

func (n *Needle) HasPairs() bool

func (*Needle) HasTtl

func (n *Needle) HasTtl() bool

func (*Needle) IsChunkedManifest

func (n *Needle) IsChunkedManifest() bool

func (*Needle) IsCompressed

func (n *Needle) IsCompressed() bool

func (*Needle) LastModifiedString

func (n *Needle) LastModifiedString() string

func (*Needle) ParseNeedleHeader

func (n *Needle) ParseNeedleHeader(bytes []byte)

func (*Needle) ParsePath

func (n *Needle) ParsePath(fid string) (err error)

func (*Needle) ReadBytes

func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error)

ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.

func (*Needle) ReadData

func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error)

ReadData hydrates the needle from the file, with only n.Id is set.

func (*Needle) ReadNeedleBody

func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error)

n should be a needle already read the header the input stream will read until next file entry

func (*Needle) ReadNeedleBodyBytes

func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error)

func (*Needle) SetHasLastModifiedDate

func (n *Needle) SetHasLastModifiedDate()

func (*Needle) SetHasMime

func (n *Needle) SetHasMime()

func (*Needle) SetHasName

func (n *Needle) SetHasName()

func (*Needle) SetHasPairs

func (n *Needle) SetHasPairs()

func (*Needle) SetHasTtl

func (n *Needle) SetHasTtl()

func (*Needle) SetIsChunkManifest

func (n *Needle) SetIsChunkManifest()

func (*Needle) SetIsCompressed

func (n *Needle) SetIsCompressed()

func (*Needle) String

func (n *Needle) String() (str string)

type ParsedUpload

type ParsedUpload struct {
	FileName         string
	Data             []byte
	MimeType         string
	PairMap          map[string]string
	IsGzipped        bool
	IsZstd           bool
	OriginalDataSize int
	ModifiedTime     uint64
	Ttl              *TTL
	IsChunkedFile    bool
	UncompressedData []byte
	ContentMd5       string
}

func ParseUpload

func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error)

type TTL

type TTL struct {
	Count byte
	Unit  byte
}

func LoadTTLFromBytes

func LoadTTLFromBytes(input []byte) (t *TTL)

read stored bytes to a ttl

func LoadTTLFromUint32

func LoadTTLFromUint32(ttl uint32) (t *TTL)

read stored bytes to a ttl

func ReadTTL

func ReadTTL(ttlString string) (*TTL, error)

translate a readable ttl to internal ttl Supports format example: 3m: 3 minutes 4h: 4 hours 5d: 5 days 6w: 6 weeks 7M: 7 months 8y: 8 years

func (TTL) Minutes

func (t TTL) Minutes() uint32

func (*TTL) String

func (t *TTL) String() string

func (*TTL) ToBytes

func (t *TTL) ToBytes(output []byte)

save stored bytes to an output with 2 bytes

func (*TTL) ToUint32

func (t *TTL) ToUint32() (output uint32)

type Version

type Version uint8

type VolumeId

type VolumeId uint32

func NewVolumeId

func NewVolumeId(vid string) (VolumeId, error)

func (VolumeId) Next

func (vid VolumeId) Next() VolumeId

func (VolumeId) String

func (vid VolumeId) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL