needle

package
v0.0.0-...-f1e43fe Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 24, 2019 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	NeedleChecksumSize = 4
	PairNamePrefix     = "Seaweed-"
)
View Source
const (
	FlagGzip                = 0x01
	FlagHasName             = 0x02
	FlagHasMime             = 0x04
	FlagHasLastModifiedDate = 0x08
	FlagHasTtl              = 0x10
	FlagHasPairs            = 0x20
	FlagIsChunkManifest     = 0x80
	LastModifiedBytesLength = 5
	TtlBytesLength          = 2
)
View Source
const (
	//stored unit types
	Empty byte = iota
	Minute
	Hour
	Day
	Week
	Month
	Year
)
View Source
const (
	Version1       = Version(1)
	Version2       = Version(2)
	Version3       = Version(3)
	CurrentVersion = Version3
)

Variables

View Source
var EMPTY_TTL = &TTL{}

Functions

func GetActualSize

func GetActualSize(size uint32, version Version) int64

func NeedleBodyLength

func NeedleBodyLength(needleSize uint32, version Version) int64

func PaddingLength

func PaddingLength(needleSize uint32, version Version) uint32

func ParseNeedleIdCookie

func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error)

func ReadNeedleBlob

func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dataSlice []byte, err error)

Types

type CRC

type CRC uint32

func NewCRC

func NewCRC(b []byte) CRC

func (CRC) Update

func (c CRC) Update(b []byte) CRC

func (CRC) Value

func (c CRC) Value() uint32

type FileId

type FileId struct {
	VolumeId VolumeId
	Key      NeedleId
	Cookie   Cookie
}

func NewFileId

func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId

func NewFileIdFromNeedle

func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId

func ParseFileIdFromString

func ParseFileIdFromString(fid string) (*FileId, error)

Deserialize the file id

func (*FileId) GetCookie

func (n *FileId) GetCookie() Cookie

func (*FileId) GetNeedleId

func (n *FileId) GetNeedleId() NeedleId

func (*FileId) GetNeedleIdCookie

func (n *FileId) GetNeedleIdCookie() string

func (*FileId) GetVolumeId

func (n *FileId) GetVolumeId() VolumeId

func (*FileId) String

func (n *FileId) String() string

type Needle

type Needle struct {
	Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
	Id     NeedleId `comment:"needle id"`
	Size   uint32   `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`

	DataSize     uint32 `comment:"Data size"` //version2
	Data         []byte `comment:"The actual file data"`
	Flags        byte   `comment:"boolean flags"` //version2
	NameSize     uint8  //version2
	Name         []byte `comment:"maximum 256 characters"` //version2
	MimeSize     uint8  //version2
	Mime         []byte `comment:"maximum 256 characters"` //version2
	PairsSize    uint16 //version2
	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
	Ttl          *TTL

	Checksum   CRC    `comment:"CRC32 to check integrity"`
	AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
	Padding    []byte `comment:"Aligned to 8 bytes"`
}

* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.

func CreateNeedleFromRequest

func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool) (n *Needle, originalSize int, e error)

func ReadNeedleHeader

func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error)

func (*Needle) Append

func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error)

func (*Needle) DiskSize

func (n *Needle) DiskSize(version Version) int64

func (*Needle) Etag

func (n *Needle) Etag() string

func (*Needle) HasLastModifiedDate

func (n *Needle) HasLastModifiedDate() bool

func (*Needle) HasMime

func (n *Needle) HasMime() bool

func (*Needle) HasName

func (n *Needle) HasName() bool

func (*Needle) HasPairs

func (n *Needle) HasPairs() bool

func (*Needle) HasTtl

func (n *Needle) HasTtl() bool

func (*Needle) IsChunkedManifest

func (n *Needle) IsChunkedManifest() bool

func (*Needle) IsGzipped

func (n *Needle) IsGzipped() bool

func (*Needle) LastModifiedString

func (n *Needle) LastModifiedString() string

func (*Needle) MD5

func (n *Needle) MD5() string

func (*Needle) ParseNeedleHeader

func (n *Needle) ParseNeedleHeader(bytes []byte)

func (*Needle) ParsePath

func (n *Needle) ParsePath(fid string) (err error)

func (*Needle) ReadBytes

func (n *Needle) ReadBytes(bytes []byte, offset int64, size uint32, version Version) (err error)

ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.

func (*Needle) ReadData

func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error)

ReadData hydrates the needle from the file, with only n.Id is set.

func (*Needle) ReadNeedleBody

func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength int64) (bytes []byte, err error)

n should be a needle already read the header the input stream will read until next file entry

func (*Needle) ReadNeedleBodyBytes

func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error)

func (*Needle) SetGzipped

func (n *Needle) SetGzipped()

func (*Needle) SetHasLastModifiedDate

func (n *Needle) SetHasLastModifiedDate()

func (*Needle) SetHasMime

func (n *Needle) SetHasMime()

func (*Needle) SetHasName

func (n *Needle) SetHasName()

func (*Needle) SetHasPairs

func (n *Needle) SetHasPairs()

func (*Needle) SetHasTtl

func (n *Needle) SetHasTtl()

func (*Needle) SetIsChunkManifest

func (n *Needle) SetIsChunkManifest()

func (*Needle) String

func (n *Needle) String() (str string)

type TTL

type TTL struct {
	Count byte
	Unit  byte
}

func LoadTTLFromBytes

func LoadTTLFromBytes(input []byte) (t *TTL)

read stored bytes to a ttl

func LoadTTLFromUint32

func LoadTTLFromUint32(ttl uint32) (t *TTL)

read stored bytes to a ttl

func ParseUpload

func ParseUpload(r *http.Request) (
	fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
	modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error)

func ReadTTL

func ReadTTL(ttlString string) (*TTL, error)

translate a readable ttl to internal ttl Supports format example: 3m: 3 minutes 4h: 4 hours 5d: 5 days 6w: 6 weeks 7M: 7 months 8y: 8 years

func (TTL) Minutes

func (t TTL) Minutes() uint32

func (*TTL) String

func (t *TTL) String() string

func (*TTL) ToBytes

func (t *TTL) ToBytes(output []byte)

save stored bytes to an output with 2 bytes

func (*TTL) ToUint32

func (t *TTL) ToUint32() (output uint32)

type Version

type Version uint8

type VolumeId

type VolumeId uint32

func NewVolumeId

func NewVolumeId(vid string) (VolumeId, error)

func (VolumeId) Next

func (vid VolumeId) Next() VolumeId

func (VolumeId) String

func (vid VolumeId) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL