Documentation ¶
Index ¶
- Constants
- Variables
- func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error
- func ParseKeyHash(key_hash_string string) (uint64, uint32, error)
- func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, err error)
- func ScanVolumeFile(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, ...) (err error)
- func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error
- type BoltDbNeedleMap
- func (m *BoltDbNeedleMap) Close()
- func (m *BoltDbNeedleMap) Delete(key uint64, offset uint32) error
- func (m *BoltDbNeedleMap) Destroy() error
- func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool)
- func (nm *BoltDbNeedleMap) IndexFileContent() ([]byte, error)
- func (nm *BoltDbNeedleMap) IndexFileName() string
- func (nm *BoltDbNeedleMap) IndexFileSize() uint64
- func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error
- type ByOffset
- type CRC
- type CompactMap
- type CompactSection
- type DiskLocation
- func (l *DiskLocation) Close()
- func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error)
- func (l *DiskLocation) DeleteVolume(vid VolumeId) error
- func (l *DiskLocation) FindVolume(vid VolumeId) (*Volume, bool)
- func (l *DiskLocation) LoadVolume(vid VolumeId, needleMapKind NeedleMapType) bool
- func (l *DiskLocation) SetVolume(vid VolumeId, volume *Volume)
- func (l *DiskLocation) UnloadVolume(vid VolumeId) error
- func (l *DiskLocation) VolumesLen() int
- type FileId
- type Key
- type LevelDbNeedleMap
- func (m *LevelDbNeedleMap) Close()
- func (m *LevelDbNeedleMap) Delete(key uint64, offset uint32) error
- func (m *LevelDbNeedleMap) Destroy() error
- func (m *LevelDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool)
- func (nm *LevelDbNeedleMap) IndexFileContent() ([]byte, error)
- func (nm *LevelDbNeedleMap) IndexFileName() string
- func (nm *LevelDbNeedleMap) IndexFileSize() uint64
- func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error
- type MasterNodes
- type Needle
- func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize int64, err error)
- func (n *Needle) DiskSize() int64
- func (n *Needle) Etag() string
- func (n *Needle) HasLastModifiedDate() bool
- func (n *Needle) HasMime() bool
- func (n *Needle) HasName() bool
- func (n *Needle) HasPairs() bool
- func (n *Needle) HasTtl() bool
- func (n *Needle) IsChunkedManifest() bool
- func (n *Needle) IsGzipped() bool
- func (n *Needle) ParseNeedleHeader(bytes []byte)
- func (n *Needle) ParsePath(fid string) (err error)
- func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error)
- func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error)
- func (n *Needle) SetGzipped()
- func (n *Needle) SetHasLastModifiedDate()
- func (n *Needle) SetHasMime()
- func (n *Needle) SetHasName()
- func (n *Needle) SetHasPairs()
- func (n *Needle) SetHasTtl()
- func (n *Needle) SetIsChunkManifest()
- func (n *Needle) String() (str string)
- type NeedleMap
- func (nm *NeedleMap) Close()
- func (nm *NeedleMap) Delete(key uint64, offset uint32) error
- func (nm *NeedleMap) Destroy() error
- func (nm *NeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool)
- func (nm *NeedleMap) IndexFileContent() ([]byte, error)
- func (nm *NeedleMap) IndexFileName() string
- func (nm *NeedleMap) IndexFileSize() uint64
- func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error
- type NeedleMapType
- type NeedleMapper
- type NeedleValue
- type ReplicaPlacement
- type Store
- func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, ...) error
- func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool)
- func (s *Store) Close()
- func (s *Store) CollectHeartbeat() *pb.Heartbeat
- func (s *Store) CommitCleanupVolume(volumeIdString string) error
- func (s *Store) CommitCompactVolume(volumeIdString string) error
- func (s *Store) CompactVolume(volumeIdString string, preallocate int64) error
- func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error)
- func (s *Store) DeleteCollection(collection string) (e error)
- func (s *Store) DeleteVolume(i VolumeId) error
- func (s *Store) GetVolume(i VolumeId) *Volume
- func (s *Store) HasVolume(i VolumeId) bool
- func (s *Store) MountVolume(i VolumeId) error
- func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error)
- func (s *Store) SetDataCenter(dataCenter string)
- func (s *Store) SetRack(rack string)
- func (s *Store) Status() []*VolumeInfo
- func (s *Store) String() (str string)
- func (s *Store) UnmountVolume(i VolumeId) error
- func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error)
- type SuperBlock
- type TTL
- type Version
- type Volume
- func (v *Volume) AppendBlob(b []byte) (offset int64, err error)
- func (v *Volume) Close()
- func (v *Volume) Compact(preallocate int64) error
- func (v *Volume) Compact2() error
- func (v *Volume) ContentSize() uint64
- func (v *Volume) DataFile() *os.File
- func (v *Volume) Destroy() (err error)
- func (v *Volume) FileName() (fileName string)
- func (v *Volume) GetVolumeSyncStatus() operation.SyncVolumeResponse
- func (v *Volume) IndexFileContent() ([]byte, error)
- func (v *Volume) NeedToReplicate() bool
- func (v *Volume) Size() int64
- func (v *Volume) String() string
- func (v *Volume) Synchronize(volumeServer string) (err error)
- func (v *Volume) Version() Version
- type VolumeId
- type VolumeInfo
Constants ¶
const ( NeedleHeaderSize = 16 //should never change this NeedlePaddingSize = 8 NeedleChecksumSize = 4 MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 TombstoneFileSize = math.MaxUint32 PairNamePrefix = "Seaweed-" )
const ( FlagGzip = 0x01 FlagHasName = 0x02 FlagHasMime = 0x04 FlagHasLastModifiedDate = 0x08 FlagHasTtl = 0x10 FlagHasPairs = 0x20 FlagIsChunkManifest = 0x80 LastModifiedBytesLength = 5 TtlBytesLength = 2 )
const ( //stored unit types Empty byte = iota Minute Hour Day Week Month Year )
const ( Version1 = Version(1) Version2 = Version(2) CurrentVersion = Version2 )
const (
MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
)
const (
NeedleIndexSize = 16
)
const (
RowsToRead = 1024
)
const (
SuperBlockSize = 8
)
Variables ¶
var EMPTY_TTL = &TTL{}
Functions ¶
func ReadNeedleBlob ¶
func ScanVolumeFile ¶
Types ¶
type BoltDbNeedleMap ¶
type BoltDbNeedleMap struct {
// contains filtered or unexported fields
}
func NewBoltDbNeedleMap ¶
func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error)
func (*BoltDbNeedleMap) Close ¶
func (m *BoltDbNeedleMap) Close()
func (*BoltDbNeedleMap) Destroy ¶
func (m *BoltDbNeedleMap) Destroy() error
func (*BoltDbNeedleMap) Get ¶
func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool)
func (*BoltDbNeedleMap) IndexFileContent ¶
func (*BoltDbNeedleMap) IndexFileName ¶
func (nm *BoltDbNeedleMap) IndexFileName() string
func (*BoltDbNeedleMap) IndexFileSize ¶
func (nm *BoltDbNeedleMap) IndexFileSize() uint64
type ByOffset ¶
type ByOffset []needle.NeedleValue
type CompactMap ¶
type CompactMap struct {
// contains filtered or unexported fields
}
This map assumes mostly inserting increasing keys
func NewCompactMap ¶
func NewCompactMap() CompactMap
func (*CompactMap) Delete ¶
func (cm *CompactMap) Delete(key Key) uint32
func (*CompactMap) Get ¶
func (cm *CompactMap) Get(key Key) (*NeedleValue, bool)
func (*CompactMap) Set ¶
func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)
func (*CompactMap) Visit ¶
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error
Visit visits all entries or stop if any error when visiting
type CompactSection ¶
func NewCompactSection ¶
func NewCompactSection(start Key) *CompactSection
func (*CompactSection) Delete ¶
func (cs *CompactSection) Delete(key Key) uint32
return old entry size
func (*CompactSection) Get ¶
func (cs *CompactSection) Get(key Key) (*NeedleValue, bool)
type DiskLocation ¶
type DiskLocation struct { Directory string MaxVolumeCount int sync.RWMutex // contains filtered or unexported fields }
func NewDiskLocation ¶
func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation
func (*DiskLocation) Close ¶
func (l *DiskLocation) Close()
func (*DiskLocation) DeleteCollectionFromDiskLocation ¶
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error)
func (*DiskLocation) DeleteVolume ¶
func (l *DiskLocation) DeleteVolume(vid VolumeId) error
func (*DiskLocation) FindVolume ¶
func (l *DiskLocation) FindVolume(vid VolumeId) (*Volume, bool)
func (*DiskLocation) LoadVolume ¶
func (l *DiskLocation) LoadVolume(vid VolumeId, needleMapKind NeedleMapType) bool
func (*DiskLocation) SetVolume ¶
func (l *DiskLocation) SetVolume(vid VolumeId, volume *Volume)
func (*DiskLocation) UnloadVolume ¶
func (l *DiskLocation) UnloadVolume(vid VolumeId) error
func (*DiskLocation) VolumesLen ¶
func (l *DiskLocation) VolumesLen() int
type LevelDbNeedleMap ¶
type LevelDbNeedleMap struct {
// contains filtered or unexported fields
}
func NewLevelDbNeedleMap ¶
func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error)
func (*LevelDbNeedleMap) Close ¶
func (m *LevelDbNeedleMap) Close()
func (*LevelDbNeedleMap) Delete ¶
func (m *LevelDbNeedleMap) Delete(key uint64, offset uint32) error
func (*LevelDbNeedleMap) Destroy ¶
func (m *LevelDbNeedleMap) Destroy() error
func (*LevelDbNeedleMap) Get ¶
func (m *LevelDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool)
func (*LevelDbNeedleMap) IndexFileContent ¶
func (*LevelDbNeedleMap) IndexFileName ¶
func (nm *LevelDbNeedleMap) IndexFileName() string
func (*LevelDbNeedleMap) IndexFileSize ¶
func (nm *LevelDbNeedleMap) IndexFileSize() uint64
type MasterNodes ¶
type MasterNodes struct {
// contains filtered or unexported fields
}
func NewMasterNodes ¶
func NewMasterNodes(bootstrapNode string) (mn *MasterNodes)
func (*MasterNodes) FindMaster ¶
func (mn *MasterNodes) FindMaster() (leader string, err error)
func (*MasterNodes) Reset ¶
func (mn *MasterNodes) Reset()
func (*MasterNodes) SetPossibleLeader ¶
func (mn *MasterNodes) SetPossibleLeader(possibleLeader string)
func (*MasterNodes) String ¶
func (mn *MasterNodes) String() string
type Needle ¶
type Needle struct { Cookie uint32 `comment:"random number to mitigate brute force lookups"` Id uint64 `comment:"needle id"` Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"` DataSize uint32 `comment:"Data size"` //version2 Data []byte `comment:"The actual file data"` Flags byte `comment:"boolean flags"` //version2 NameSize uint8 //version2 Name []byte `comment:"maximum 256 characters"` //version2 MimeSize uint8 //version2 Mime []byte `comment:"maximum 256 characters"` //version2 PairsSize uint16 //version2 Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"` LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk Ttl *TTL Checksum CRC `comment:"CRC32 to check integrity"` Padding []byte `comment:"Aligned to 8 bytes"` }
* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.
func ReadNeedleHeader ¶
func (*Needle) HasLastModifiedDate ¶
func (*Needle) IsChunkedManifest ¶
func (*Needle) ParseNeedleHeader ¶
func (*Needle) ReadNeedleBody ¶
func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error)
n should be a needle already read the header the input stream will read until next file entry
func (*Needle) SetGzipped ¶
func (n *Needle) SetGzipped()
func (*Needle) SetHasLastModifiedDate ¶
func (n *Needle) SetHasLastModifiedDate()
func (*Needle) SetHasMime ¶
func (n *Needle) SetHasMime()
func (*Needle) SetHasName ¶
func (n *Needle) SetHasName()
func (*Needle) SetHasPairs ¶
func (n *Needle) SetHasPairs()
func (*Needle) SetIsChunkManifest ¶
func (n *Needle) SetIsChunkManifest()
type NeedleMap ¶
type NeedleMap struct {
// contains filtered or unexported fields
}
func NewBtreeNeedleMap ¶
func NewCompactNeedleMap ¶
func (*NeedleMap) IndexFileContent ¶
func (*NeedleMap) IndexFileName ¶
func (nm *NeedleMap) IndexFileName() string
func (*NeedleMap) IndexFileSize ¶
func (nm *NeedleMap) IndexFileSize() uint64
type NeedleMapType ¶
type NeedleMapType int
const ( NeedleMapInMemory NeedleMapType = iota NeedleMapLevelDb NeedleMapBoltDb NeedleMapBtree )
type NeedleMapper ¶
type NeedleMapper interface { Put(key uint64, offset uint32, size uint32) error Get(key uint64) (element *needle.NeedleValue, ok bool) Delete(key uint64, offset uint32) error Close() Destroy() error ContentSize() uint64 DeletedSize() uint64 FileCount() int DeletedCount() int MaxFileKey() uint64 IndexFileSize() uint64 IndexFileContent() ([]byte, error) IndexFileName() string }
type NeedleValue ¶
type ReplicaPlacement ¶
func NewReplicaPlacementFromByte ¶
func NewReplicaPlacementFromByte(b byte) (*ReplicaPlacement, error)
func NewReplicaPlacementFromString ¶
func NewReplicaPlacementFromString(t string) (*ReplicaPlacement, error)
func (*ReplicaPlacement) Byte ¶
func (rp *ReplicaPlacement) Byte() byte
func (*ReplicaPlacement) GetCopyCount ¶
func (rp *ReplicaPlacement) GetCopyCount() int
func (*ReplicaPlacement) String ¶
func (rp *ReplicaPlacement) String() string
type Store ¶
type Store struct { Ip string Port int PublicUrl string Locations []*DiskLocation VolumeSizeLimit uint64 //read from the master Client pb.Seaweed_SendHeartbeatClient NeedleMapType NeedleMapType // contains filtered or unexported fields }
* A VolumeServer contains one Store
func (*Store) CheckCompactVolume ¶
func (*Store) CollectHeartbeat ¶
func (*Store) CommitCleanupVolume ¶
func (*Store) CommitCompactVolume ¶
func (*Store) CompactVolume ¶
func (*Store) DeleteCollection ¶
func (*Store) DeleteVolume ¶
func (*Store) MountVolume ¶
func (*Store) ReadVolumeNeedle ¶
func (*Store) SetDataCenter ¶
func (*Store) Status ¶
func (s *Store) Status() []*VolumeInfo
func (*Store) UnmountVolume ¶
type SuperBlock ¶
type SuperBlock struct { ReplicaPlacement *ReplicaPlacement Ttl *TTL CompactRevision uint16 // contains filtered or unexported fields }
* Super block currently has 8 bytes allocated for each volume. * Byte 0: version, 1 or 2 * Byte 1: Replica Placement strategy, 000, 001, 002, 010, etc * Byte 2 and byte 3: Time to live. See TTL for definition * Byte 4 and byte 5: The number of times the volume has been compacted. * Rest bytes: Reserved
func ParseSuperBlock ¶
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error)
func (*SuperBlock) Bytes ¶
func (s *SuperBlock) Bytes() []byte
func (*SuperBlock) Version ¶
func (s *SuperBlock) Version() Version
type TTL ¶
type TTL struct {
// contains filtered or unexported fields
}
func ParseUpload ¶
func ReadTTL ¶
translate a readable ttl to internal ttl Supports format example: 3m: 3 minutes 4h: 4 hours 5d: 5 days 6w: 6 weeks 7M: 7 months 8y: 8 years
type Volume ¶
type Volume struct { Id VolumeId Collection string SuperBlock // contains filtered or unexported fields }
func NewVolume ¶
func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) (v *Volume, e error)
func (*Volume) AppendBlob ¶
AppendBlob append a blob to end of the data file, used in replication
func (*Volume) ContentSize ¶
func (*Volume) GetVolumeSyncStatus ¶
func (v *Volume) GetVolumeSyncStatus() operation.SyncVolumeResponse
func (*Volume) IndexFileContent ¶
func (*Volume) NeedToReplicate ¶
func (*Volume) Synchronize ¶
type VolumeInfo ¶
type VolumeInfo struct { Id VolumeId Size uint64 ReplicaPlacement *ReplicaPlacement Ttl *TTL Collection string Version Version FileCount int DeleteCount int DeletedByteCount uint64 ReadOnly bool }
func NewVolumeInfo ¶
func NewVolumeInfo(m *pb.VolumeInformationMessage) (vi VolumeInfo, err error)
func (VolumeInfo) String ¶
func (vi VolumeInfo) String() string
Source Files ¶
- compact_map.go
- crc.go
- disk_location.go
- file_id.go
- needle.go
- needle_byte_cache.go
- needle_map.go
- needle_map_boltdb.go
- needle_map_leveldb.go
- needle_map_memory.go
- needle_read_write.go
- replica_placement.go
- store.go
- store_vacuum.go
- volume.go
- volume_checking.go
- volume_create_linux.go
- volume_id.go
- volume_info.go
- volume_loading.go
- volume_read_write.go
- volume_super_block.go
- volume_sync.go
- volume_ttl.go
- volume_vacuum.go
- volume_version.go