Documentation ¶
Overview ¶
Hashbox core, version 0.1
Hashbox core, version 0.1 ¶
Hashbox core, version 0.1 ¶
Hashbox core, version 0.1 ¶
Hashbox core client routines ¶
Hashbox core, version 0.1 ¶
Hashbox core, version 0.1
Index ¶
- Constants
- Variables
- func BytesInt64(bytes []byte) (v int64)
- func CopyNOrPanic(dst io.Writer, src io.Reader, n int) int
- func CopyOrPanic(dst io.Writer, src io.Reader) int
- func DecryptDataInPlace(cipherdata []byte, key Byte128)
- func EncryptDataInPlace(data []byte, key Byte128)
- func ExpandEnv(s string) string
- func FreeSpace(path string) (int64, error)
- func HumanSize(size int64) string
- func LimitInt(big int64) (v int)
- func Log(level int, format string, a ...interface{})
- func MemoryStats() string
- func ReadBytes(r io.Reader, data []byte) int
- func ReadInt64(r io.Reader, data *int64) int
- func ReadUint16(r io.Reader, data *uint16) int
- func ReadUint32(r io.Reader, data *uint32) int
- func ReadUint8(r io.Reader, data *uint8) int
- func ShortHumanSize(size int64) string
- func SplitPath(path string) []string
- func WriteBytes(w io.Writer, data []byte) int
- func WriteInt64(w io.Writer, data int64) int
- func WriteMessage(w io.Writer, msg *ProtocolMessage)
- func WriteUint16(w io.Writer, data uint16) int
- func WriteUint32(w io.Writer, data uint32) int
- func WriteUint8(w io.Writer, data uint8) int
- func ZlibCompress(src bytearray.ByteArray) (dst bytearray.ByteArray)
- func ZlibUncompress(src bytearray.ByteArray) (dst bytearray.ByteArray)
- type BufferedFile
- type BufferedReader
- type BufferedSerializer
- type BufferedUnserializer
- type BufferedWriter
- type Byte128
- func DeepHmac(depth int, data []byte, key Byte128) Byte128
- func GenerateAccessKey(account string, password string) Byte128
- func GenerateBackupKey(account string, password string) Byte128
- func GenerateDataEncryptionKey() Byte128
- func Hash(data []byte) Byte128
- func Hmac(data []byte, key Byte128) Byte128
- type Client
- func (c *Client) AddDatasetState(datasetName string, state DatasetState)
- func (c *Client) Authorize(connection *TimeoutConn)
- func (c *Client) Close(polite bool)
- func (c *Client) Commit()
- func (c *Client) Connect()
- func (c *Client) Done() bool
- func (c *Client) GetAccountInfo() *MsgServerAccountInfo
- func (c *Client) GetStats() (tranismitted int32, skipped int32, queued int32, queuesize int64)
- func (c *Client) ListDataset(datasetName string) *MsgServerListDataset
- func (c *Client) Paint(what string)
- func (c *Client) ReadBlock(blockID Byte128) *HashboxBlock
- func (c *Client) RemoveDatasetState(datasetName string, stateID Byte128)
- func (c *Client) StoreBlock(block *HashboxBlock) Byte128
- func (c *Client) StoreData(dataType byte, data bytearray.ByteArray, links []Byte128) Byte128
- func (c *Client) VerifyBlock(blockID Byte128) bool
- type Dataset
- type DatasetArray
- type DatasetState
- type DatasetStateArray
- type DatasetStateEntry
- type HashboxBlock
- func (b *HashboxBlock) CompressData()
- func (b *HashboxBlock) HashData() (BlockID Byte128)
- func (b *HashboxBlock) Release()
- func (b *HashboxBlock) Serialize(w io.Writer) (size int)
- func (b *HashboxBlock) SerializeLinks(w io.Writer) (size int)
- func (b *HashboxBlock) UncompressData()
- func (b *HashboxBlock) Unserialize(r io.Reader) (size int)
- func (b *HashboxBlock) UnserializeHeader(r io.Reader) (size int)
- func (b *HashboxBlock) VerifyBlock() bool
- type MsgClientAccountInfo
- type MsgClientAddDatasetState
- type MsgClientAllocateBlock
- type MsgClientAuthenticate
- type MsgClientGreeting
- type MsgClientListDataset
- type MsgClientReadBlock
- type MsgClientRemoveDatasetState
- type MsgClientWriteBlock
- type MsgServerAccountInfo
- type MsgServerAcknowledgeBlock
- type MsgServerError
- type MsgServerGreeting
- type MsgServerListDataset
- type MsgServerReadBlock
- type MsgServerWriteBlock
- type ProtocolMessage
- type Serializer
- type Session
- type String
- type TimeoutConn
- type Uint32
- type Unserializer
Constants ¶
const ( BlockDataTypeRaw = 0xff BlockDataTypeZlib = 0x01 )
const ( ProtocolVersion Uint32 = 1 MsgTypeOldGreeting uint32 = 0x686F6C61 // = "hola" MsgTypeGreeting uint32 = 0x68616C6F // = "halo" MsgTypeAuthenticate uint32 = 0x61757468 // = "auth" MsgTypeGoodbye uint32 = 0x71756974 // = "quit" MsgTypeAllocateBlock uint32 = 0x616C6C6F // = "allo" MsgTypeReadBlock uint32 = 0x72656164 // = "read" MsgTypeWriteBlock uint32 = 0x77726974 // = "writ" MsgTypeAcknowledgeBlock uint32 = 0x61636B6E // = "ackn" MsgTypeAccountInfo uint32 = 0x696E666F // = "info" MsgTypeAddDatasetState uint32 = 0x61646473 // = "adds" MsgTypeListDataset uint32 = 0x6C697374 // = "list" MsgTypeRemoveDatasetState uint32 = 0x64656C73 // = "dels" MsgTypeError uint32 = 0x65727273 // = "ERRS" )
const ( LogError = iota LogWarning LogInfo LogDebug LogTrace )
const DEFAULT_CONNECTION_TIMEOUT time.Duration = 5 * time.Minute
const DEFAULT_QUEUE_SIZE int64 = 32 * 1024 * 1024 // 32 MiB max memory
const LOGTIMEFORMAT string = "20060102 15:04:05"
const MaxInt = int(MaxUint >> 1)
const MaxUint = ^uint(0)
const MinInt = -MaxInt - 1
const MinUint = 0
const MsgTypeClientMask uint32 = 0x20202020 // MsgType | MsgTypeClientMask
If we would have needed to convert a server message type to client message type, we could OR the bit in again
const MsgTypeServerMask uint32 = 0xDFDFDFDF // MsgType & MsgTypeClientMask
We use the same function for reading server and client messages but they do have different payloads so we need to differentiate between the two message types by using lowercase / uppercase. We do this simply by removing the lowercase bit in the ASCII table by using AND on 0xDf
const (
StateFlagInvalid = 1 << 7 // Dataset State is invalid
)
Variables ¶
var LogLevel int = LogInfo
var LogMutex sync.Mutex
Functions ¶
func BytesInt64 ¶
func DecryptDataInPlace ¶
func EncryptDataInPlace ¶
func MemoryStats ¶
func MemoryStats() string
func ShortHumanSize ¶
func WriteMessage ¶
func WriteMessage(w io.Writer, msg *ProtocolMessage)
Types ¶
type BufferedFile ¶
type BufferedFile struct { Path string BufferSize int Flag int Perm os.FileMode Reader *BufferedReader Writer *BufferedWriter }
func OpenBufferedFile ¶
func (*BufferedFile) Close ¶
func (b *BufferedFile) Close() (err error)
func (*BufferedFile) Size ¶
func (b *BufferedFile) Size() int64
func (*BufferedFile) Sync ¶
func (b *BufferedFile) Sync() (err error)
type BufferedReader ¶
func OpenBufferedReader ¶
func OpenBufferedReader(path string, buffersize int, flag int) (*BufferedReader, error)
type BufferedSerializer ¶
type BufferedSerializer interface {
Serialize(w *BufferedWriter) (size int)
}
type BufferedUnserializer ¶
type BufferedUnserializer interface {
Unserialize(r *BufferedReader) (size int)
}
type BufferedWriter ¶
func OpenBufferedWriter ¶
type Byte128 ¶
type Byte128 [16]byte
Byte128 is just an alias to a 16 byte array since we use a lot of 128-bit key and check values in Hashbox
func GenerateAccessKey ¶
func GenerateBackupKey ¶
func GenerateDataEncryptionKey ¶
func GenerateDataEncryptionKey() Byte128
type Client ¶
type Client struct { WriteData int64 // total data written WriteDataCompressed int64 // total compressed data written Session AccessKey Byte128 // = hmac^20000( AccountName "*ACCESS*KEY*PAD*", md5( password )) ServerAddress string EnablePaint bool QueueMax int64 // max size of the outgoing block queue (in bytes) ThreadMax int32 // maximum number of goroutines started by send queue (defaults to runtime.NumCPU) // contains filtered or unexported fields }
func (*Client) AddDatasetState ¶
func (c *Client) AddDatasetState(datasetName string, state DatasetState)
func (*Client) Authorize ¶
func (c *Client) Authorize(connection *TimeoutConn)
func (*Client) GetAccountInfo ¶
func (c *Client) GetAccountInfo() *MsgServerAccountInfo
func (*Client) ListDataset ¶
func (c *Client) ListDataset(datasetName string) *MsgServerListDataset
func (*Client) ReadBlock ¶
func (c *Client) ReadBlock(blockID Byte128) *HashboxBlock
func (*Client) RemoveDatasetState ¶
func (*Client) StoreBlock ¶
func (c *Client) StoreBlock(block *HashboxBlock) Byte128
StoreBlock is blocking if the blockbuffer is full
func (*Client) VerifyBlock ¶
type Dataset ¶
type Dataset struct { Name String // Name of the Dataset Size int64 // Size of all data referenced by this dataset ListH Byte128 // = md5(DatasetContentList) }
Dataset stores the information regarding a dataset. ListH is used so that the client can make sure it has the correct listing.
type DatasetArray ¶
type DatasetArray []Dataset
func (DatasetArray) Len ¶
func (a DatasetArray) Len() int
func (DatasetArray) Less ¶
func (a DatasetArray) Less(i, j int) bool
func (DatasetArray) Swap ¶
func (a DatasetArray) Swap(i, j int)
func (*DatasetArray) Unserialize ¶
func (a *DatasetArray) Unserialize(r io.Reader) (size int)
type DatasetState ¶
type DatasetState struct { StateID Byte128 // Unique ID of the state BlockID Byte128 // ID of the Block this Dataset is referring to // TODO: figure out how to calculate size and uniquesize for real, going through metadata tree when saving takes too long Size int64 // Size of all data referenced by this dataset state UniqueSize int64 // Size of unique data (added blocks) }
DatasetState stores a specific state (snapshot) of a Dataset.
func (*DatasetState) Unserialize ¶
func (d *DatasetState) Unserialize(r io.Reader) (size int)
type DatasetStateArray ¶
type DatasetStateArray []DatasetStateEntry
func (DatasetStateArray) Len ¶
func (a DatasetStateArray) Len() int
func (DatasetStateArray) Less ¶
func (a DatasetStateArray) Less(i, j int) bool
func (DatasetStateArray) Swap ¶
func (a DatasetStateArray) Swap(i, j int)
func (*DatasetStateArray) Unserialize ¶
func (a *DatasetStateArray) Unserialize(r io.Reader) (size int)
type DatasetStateEntry ¶
type DatasetStateEntry struct { StateFlags uint8 // 1 byte Dataset state flags State DatasetState }
func (*DatasetStateEntry) Unserialize ¶
func (e *DatasetStateEntry) Unserialize(r io.Reader) (size int)
type HashboxBlock ¶
type HashboxBlock struct { BlockID Byte128 // = md5( LinkLength Links DataLength Data ) Links []Byte128 // Array of BlockIDs DataType uint8 // 1 byte data type Data bytearray.ByteArray Compressed bool CompressedSize int UncompressedSize int }
HashboxBlock is serialized with counters before all arrays
func NewHashboxBlock ¶
func NewHashboxBlock(dataType byte, data bytearray.ByteArray, links []Byte128) *HashboxBlock
func (*HashboxBlock) CompressData ¶
func (b *HashboxBlock) CompressData()
func (*HashboxBlock) HashData ¶
func (b *HashboxBlock) HashData() (BlockID Byte128)
func (*HashboxBlock) Release ¶
func (b *HashboxBlock) Release()
func (*HashboxBlock) SerializeLinks ¶
func (b *HashboxBlock) SerializeLinks(w io.Writer) (size int)
func (*HashboxBlock) UncompressData ¶
func (b *HashboxBlock) UncompressData()
func (*HashboxBlock) Unserialize ¶
func (b *HashboxBlock) Unserialize(r io.Reader) (size int)
func (*HashboxBlock) UnserializeHeader ¶
func (b *HashboxBlock) UnserializeHeader(r io.Reader) (size int)
func (*HashboxBlock) VerifyBlock ¶
func (b *HashboxBlock) VerifyBlock() bool
type MsgClientAccountInfo ¶
type MsgClientAccountInfo struct {
AccountNameH Byte128 // = md5(accountName)
}
MsgClientAccountInfo is sent from the client to retrieve quota information and a list of all Datasets under the account.
type MsgClientAddDatasetState ¶
type MsgClientAddDatasetState struct { AccountNameH Byte128 // = md5(accountName) DatasetName String // Name of the Dataset to store under State DatasetState // Dataset state to add }
MsgClientAddDatasetState sent to the server to add a Dataset or a Dataset state. Server returns MsgServerError if the DatasetState refers to a BlockID that does not exist.
type MsgClientAllocateBlock ¶
type MsgClientAllocateBlock struct {
BlockID Byte128 // ID of Block to allocate
}
MsgClientAllocateBlock allocates a new Block on the server. The server responds with a MsgServerAcknowledgeBlock if the Block already exists or a MsgServerReadBlock if the server needs to read the data from the client.
type MsgClientAuthenticate ¶
type MsgClientAuthenticate struct { AccountNameH Byte128 // = md5(accountName) AuthenticationH Byte128 // = hmac(AccountNameH, Session.SessionKey) }
MsgClientAuthenticate sent from the client to authenticate the session.
type MsgClientGreeting ¶
type MsgClientGreeting struct {
Version Uint32 // Protocol version
}
MsgClientGreeting sent from client after connecting.
type MsgClientListDataset ¶
type MsgClientListDataset struct { AccountNameH Byte128 // = md5(accountName) DatasetName String // Name of the Dataset you wish to list }
MsgClientListDataset sent from the client to retrieve a list of all states for a Dataset
type MsgClientReadBlock ¶
type MsgClientReadBlock struct {
BlockID Byte128 // ID of Block to read
}
MsgClientReadBlock sent to the server to read a Block. Server responds with a MsgServerWriteBlock if the Block exists.
type MsgClientRemoveDatasetState ¶
type MsgClientRemoveDatasetState struct { AccountNameH Byte128 // = md5(accountName) DatasetName String // Name of the Dataset to remove under StateID Byte128 // ID of the state to remove }
MsgClientRemoveDatasetState sent by the client to remove a Dataset state.
type MsgClientWriteBlock ¶
type MsgClientWriteBlock struct {
Block *HashboxBlock // Block to be sent
}
MsgClientWriteBlock sent to the server to write a Block. Server calculates its own BlockID from the block to verify its integrity. Server responds with MsgServerAcknowledgeBlock if the Block was written successfully.
type MsgServerAccountInfo ¶
type MsgServerAccountInfo struct { // TODO: Add quota stuff DatasetList DatasetArray // List of all Datasets under the account }
MsgServerAccountInfo sent from the server as a response to MsgClientAccountInfo.
type MsgServerAcknowledgeBlock ¶
type MsgServerAcknowledgeBlock struct {
BlockID Byte128 // ID of Block that exists
}
MsgServerAcknowledgeBlock sent as a confirmation that the server has the Block.
type MsgServerError ¶
type MsgServerError struct {
ErrorMessage String // Error message
}
MsgServerError is sent on all kind of severe protocol / server errors. The ErrorMessage contains more details regarding the error.
type MsgServerGreeting ¶
type MsgServerGreeting struct {
SessionNonce Byte128 // Unique nonce for the session
}
MsgServerGreeting sent from server in response to client greeting.
type MsgServerListDataset ¶
type MsgServerListDataset struct { States DatasetStateArray // Array of all states under the Dataset ListH Byte128 // = md5(States) }
MsgServerListDataset sent from the server as a response to MsgClientListDataset. ListH is used so that the client can make sure it has the correct listing
type MsgServerReadBlock ¶
type MsgServerReadBlock struct {
BlockID Byte128 // ID of Block to read
}
MsgServerReadBlock sent from the server to retreive Block data from client.
type MsgServerWriteBlock ¶
type MsgServerWriteBlock struct {
Block *HashboxBlock // Block to be sent
}
MsgServerWriteBlock
type ProtocolMessage ¶
func ReadMessage ¶
func ReadMessage(r io.Reader) *ProtocolMessage
IMPORTANT messages containing HashboxBlock data will ByteArray allocate memory that needs to be freed manually
func (ProtocolMessage) Details ¶
func (m ProtocolMessage) Details() string
func (*ProtocolMessage) Release ¶
func (m *ProtocolMessage) Release()
func (ProtocolMessage) String ¶
func (m ProtocolMessage) String() string
func (*ProtocolMessage) Unserialize ¶
func (m *ProtocolMessage) Unserialize(r io.Reader) (size int)
type Serializer ¶
Serializer is our own form of BinaryMarshaler but it works directly off a stream to be compatible with Unserializer
type Session ¶
type Session struct { AccountNameH Byte128 // = md5( accountName ) SessionNonce Byte128 // Unique nonce for the session, based on uint64(servertime) + uint64(random) SessionKey Byte128 // = hmac^20000( AccountNameH SessionNonce, AccessKey) }
func (*Session) GenerateSessionKey ¶
type TimeoutConn ¶
type TimeoutConn struct {
// contains filtered or unexported fields
}
func NewTimeoutConn ¶
func NewTimeoutConn(c net.Conn, t time.Duration) *TimeoutConn
func (*TimeoutConn) Close ¶
func (t *TimeoutConn) Close() error
type Unserializer ¶
Unserializer is our own form of BinaryUnmarshaler but it works directly off a stream so we do not need to know the full size beforehand