core

package
v0.0.0-...-06ee8c7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 22, 2024 License: MIT Imports: 24 Imported by: 0

Documentation

Overview

Hashbox core, version 0.1

Hashbox core, version 0.1

Hashbox core, version 0.1

Hashbox core, version 0.1

Hashbox core client routines

Hashbox core, version 0.1

Hashbox core, version 0.1

Index

Constants

View Source
const (
	BlockDataTypeRaw  = 0xff
	BlockDataTypeZlib = 0x01
)
View Source
const (
	ProtocolVersion Uint32 = 1

	MsgTypeOldGreeting  uint32 = 0x686F6C61 // = "hola"
	MsgTypeGreeting     uint32 = 0x68616C6F // = "halo"
	MsgTypeAuthenticate uint32 = 0x61757468 // = "auth"
	MsgTypeGoodbye      uint32 = 0x71756974 // = "quit"

	MsgTypeAllocateBlock    uint32 = 0x616C6C6F // = "allo"
	MsgTypeReadBlock        uint32 = 0x72656164 // = "read"
	MsgTypeWriteBlock       uint32 = 0x77726974 // = "writ"
	MsgTypeAcknowledgeBlock uint32 = 0x61636B6E // = "ackn"

	MsgTypeAccountInfo        uint32 = 0x696E666F // = "info"
	MsgTypeAddDatasetState    uint32 = 0x61646473 // = "adds"
	MsgTypeListDataset        uint32 = 0x6C697374 // = "list"
	MsgTypeRemoveDatasetState uint32 = 0x64656C73 // = "dels"

	MsgTypeError uint32 = 0x65727273 // = "ERRS"
)
View Source
const (
	LogError = iota
	LogWarning
	LogInfo
	LogDebug
	LogTrace
)
View Source
const DEFAULT_CONNECTION_TIMEOUT time.Duration = 5 * time.Minute
View Source
const DEFAULT_QUEUE_SIZE int64 = 32 * 1024 * 1024 // 32 MiB max memory
View Source
const LOGTIMEFORMAT string = "20060102 15:04:05"
View Source
const MaxInt = int(MaxUint >> 1)
View Source
const MaxUint = ^uint(0)
View Source
const MinInt = -MaxInt - 1
View Source
const MinUint = 0
View Source
const MsgTypeClientMask uint32 = 0x20202020 // MsgType | MsgTypeClientMask

If we would have needed to convert a server message type to client message type, we could OR the bit in again

View Source
const MsgTypeServerMask uint32 = 0xDFDFDFDF // MsgType & MsgTypeClientMask

We use the same function for reading server and client messages but they do have different payloads so we need to differentiate between the two message types by using lowercase / uppercase. We do this simply by removing the lowercase bit in the ASCII table by using AND on 0xDf

View Source
const (
	StateFlagInvalid = 1 << 7 // Dataset State is invalid
)

Variables

View Source
var LogLevel int = LogInfo
View Source
var LogMutex sync.Mutex

Functions

func BytesInt64

func BytesInt64(bytes []byte) (v int64)

func CopyNOrPanic

func CopyNOrPanic(dst io.Writer, src io.Reader, n int) int

func CopyOrPanic

func CopyOrPanic(dst io.Writer, src io.Reader) int

func DecryptDataInPlace

func DecryptDataInPlace(cipherdata []byte, key Byte128)

func EncryptDataInPlace

func EncryptDataInPlace(data []byte, key Byte128)

func ExpandEnv

func ExpandEnv(s string) string

func FreeSpace

func FreeSpace(path string) (int64, error)

func HumanSize

func HumanSize(size int64) string

func LimitInt

func LimitInt(big int64) (v int)

func Log

func Log(level int, format string, a ...interface{})

func MemoryStats

func MemoryStats() string

func ReadBytes

func ReadBytes(r io.Reader, data []byte) int

func ReadInt64

func ReadInt64(r io.Reader, data *int64) int

func ReadUint16

func ReadUint16(r io.Reader, data *uint16) int

func ReadUint32

func ReadUint32(r io.Reader, data *uint32) int

func ReadUint8

func ReadUint8(r io.Reader, data *uint8) int

func ShortHumanSize

func ShortHumanSize(size int64) string

func SplitPath

func SplitPath(path string) []string

func WriteBytes

func WriteBytes(w io.Writer, data []byte) int

func WriteInt64

func WriteInt64(w io.Writer, data int64) int

func WriteMessage

func WriteMessage(w io.Writer, msg *ProtocolMessage)

func WriteUint16

func WriteUint16(w io.Writer, data uint16) int

func WriteUint32

func WriteUint32(w io.Writer, data uint32) int

func WriteUint8

func WriteUint8(w io.Writer, data uint8) int

func ZlibCompress

func ZlibCompress(src bytearray.ByteArray) (dst bytearray.ByteArray)

func ZlibUncompress

func ZlibUncompress(src bytearray.ByteArray) (dst bytearray.ByteArray)

Types

type BufferedFile

type BufferedFile struct {
	Path       string
	BufferSize int
	Flag       int
	Perm       os.FileMode
	Reader     *BufferedReader
	Writer     *BufferedWriter
}

func OpenBufferedFile

func OpenBufferedFile(path string, buffersize int, flag int, perm os.FileMode) (*BufferedFile, error)

func (*BufferedFile) Close

func (b *BufferedFile) Close() (err error)

func (*BufferedFile) Size

func (b *BufferedFile) Size() int64

func (*BufferedFile) Sync

func (b *BufferedFile) Sync() (err error)

type BufferedReader

type BufferedReader struct {
	File *os.File
	*bufio.Reader
}

func OpenBufferedReader

func OpenBufferedReader(path string, buffersize int, flag int) (*BufferedReader, error)

func (*BufferedReader) Seek

func (b *BufferedReader) Seek(offset int64, whence int) (ret int64, err error)

type BufferedSerializer

type BufferedSerializer interface {
	Serialize(w *BufferedWriter) (size int)
}

type BufferedUnserializer

type BufferedUnserializer interface {
	Unserialize(r *BufferedReader) (size int)
}

type BufferedWriter

type BufferedWriter struct {
	File *os.File
	*bufio.Writer
}

func OpenBufferedWriter

func OpenBufferedWriter(path string, buffersize int, flag int, perm os.FileMode) (*BufferedWriter, error)

func (*BufferedWriter) Seek

func (b *BufferedWriter) Seek(offset int64, whence int) (ret int64, err error)

type Byte128

type Byte128 [16]byte

Byte128 is just an alias to a 16 byte array since we use a lot of 128-bit key and check values in Hashbox

func DeepHmac

func DeepHmac(depth int, data []byte, key Byte128) Byte128

DeepHmac runs N number of Hmac on the data

func GenerateAccessKey

func GenerateAccessKey(account string, password string) Byte128

func GenerateBackupKey

func GenerateBackupKey(account string, password string) Byte128

func GenerateDataEncryptionKey

func GenerateDataEncryptionKey() Byte128

func Hash

func Hash(data []byte) Byte128

Hash wrapper that returns Byte128 type

func Hmac

func Hmac(data []byte, key Byte128) Byte128

Hmac is a standard HMAC-MD5 that runs on Byte128 types

func (*Byte128) Compare

func (b *Byte128) Compare(a Byte128) int

func (Byte128) Serialize

func (b Byte128) Serialize(w io.Writer) (size int)

func (*Byte128) Set

func (b *Byte128) Set(from []byte)

func (*Byte128) Unserialize

func (b *Byte128) Unserialize(r io.Reader) (size int)

type Client

type Client struct {
	WriteData           int64 // total data written
	WriteDataCompressed int64 // total compressed data written

	Session
	AccessKey Byte128 // = hmac^20000( AccountName "*ACCESS*KEY*PAD*", md5( password ))

	ServerAddress string

	EnablePaint bool

	QueueMax  int64 // max size of the outgoing block queue (in bytes)
	ThreadMax int32 // maximum number of goroutines started by send queue (defaults to runtime.NumCPU)
	// contains filtered or unexported fields
}

func NewClient

func NewClient(address string, account string, accesskey Byte128) *Client

func (*Client) AddDatasetState

func (c *Client) AddDatasetState(datasetName string, state DatasetState)

func (*Client) Authorize

func (c *Client) Authorize(connection *TimeoutConn)

func (*Client) Close

func (c *Client) Close(polite bool)

func (*Client) Commit

func (c *Client) Commit()

func (*Client) Connect

func (c *Client) Connect()

func (*Client) Done

func (c *Client) Done() bool

func (*Client) GetAccountInfo

func (c *Client) GetAccountInfo() *MsgServerAccountInfo

func (*Client) GetStats

func (c *Client) GetStats() (tranismitted int32, skipped int32, queued int32, queuesize int64)

func (*Client) ListDataset

func (c *Client) ListDataset(datasetName string) *MsgServerListDataset

func (*Client) Paint

func (c *Client) Paint(what string)

func (*Client) ReadBlock

func (c *Client) ReadBlock(blockID Byte128) *HashboxBlock

func (*Client) RemoveDatasetState

func (c *Client) RemoveDatasetState(datasetName string, stateID Byte128)

func (*Client) StoreBlock

func (c *Client) StoreBlock(block *HashboxBlock) Byte128

StoreBlock is blocking if the blockbuffer is full

func (*Client) StoreData

func (c *Client) StoreData(dataType byte, data bytearray.ByteArray, links []Byte128) Byte128

func (*Client) VerifyBlock

func (c *Client) VerifyBlock(blockID Byte128) bool

type Dataset

type Dataset struct {
	Name  String  // Name of the Dataset
	Size  int64   // Size of all data referenced by this dataset
	ListH Byte128 // = md5(DatasetContentList)
}

Dataset stores the information regarding a dataset. ListH is used so that the client can make sure it has the correct listing.

func (*Dataset) Serialize

func (d *Dataset) Serialize(w io.Writer) (size int)

func (*Dataset) Unserialize

func (d *Dataset) Unserialize(r io.Reader) (size int)

type DatasetArray

type DatasetArray []Dataset

func (DatasetArray) Len

func (a DatasetArray) Len() int

func (DatasetArray) Less

func (a DatasetArray) Less(i, j int) bool

func (DatasetArray) Serialize

func (a DatasetArray) Serialize(w io.Writer) (size int)

func (DatasetArray) Swap

func (a DatasetArray) Swap(i, j int)

func (*DatasetArray) Unserialize

func (a *DatasetArray) Unserialize(r io.Reader) (size int)

type DatasetState

type DatasetState struct {
	StateID Byte128 // Unique ID of the state
	BlockID Byte128 // ID of the Block this Dataset is referring to
	// TODO: figure out how to calculate size and uniquesize for real, going through metadata tree when saving takes too long
	Size       int64 // Size of all data referenced by this dataset state
	UniqueSize int64 // Size of unique data (added blocks)
}

DatasetState stores a specific state (snapshot) of a Dataset.

func (DatasetState) Serialize

func (d DatasetState) Serialize(w io.Writer) (size int)

func (*DatasetState) Unserialize

func (d *DatasetState) Unserialize(r io.Reader) (size int)

type DatasetStateArray

type DatasetStateArray []DatasetStateEntry

func (DatasetStateArray) Len

func (a DatasetStateArray) Len() int

func (DatasetStateArray) Less

func (a DatasetStateArray) Less(i, j int) bool

func (DatasetStateArray) Serialize

func (a DatasetStateArray) Serialize(w io.Writer) (size int)

func (DatasetStateArray) Swap

func (a DatasetStateArray) Swap(i, j int)

func (*DatasetStateArray) Unserialize

func (a *DatasetStateArray) Unserialize(r io.Reader) (size int)

type DatasetStateEntry

type DatasetStateEntry struct {
	StateFlags uint8 // 1 byte  Dataset state flags
	State      DatasetState
}

func (DatasetStateEntry) Serialize

func (e DatasetStateEntry) Serialize(w io.Writer) (size int)

func (*DatasetStateEntry) Unserialize

func (e *DatasetStateEntry) Unserialize(r io.Reader) (size int)

type HashboxBlock

type HashboxBlock struct {
	BlockID  Byte128   // = md5( LinkLength Links DataLength Data )
	Links    []Byte128 // Array of BlockIDs
	DataType uint8     // 1 byte data type

	Data bytearray.ByteArray

	Compressed       bool
	CompressedSize   int
	UncompressedSize int
}

HashboxBlock is serialized with counters before all arrays

func NewHashboxBlock

func NewHashboxBlock(dataType byte, data bytearray.ByteArray, links []Byte128) *HashboxBlock

func (*HashboxBlock) CompressData

func (b *HashboxBlock) CompressData()

func (*HashboxBlock) HashData

func (b *HashboxBlock) HashData() (BlockID Byte128)

func (*HashboxBlock) Release

func (b *HashboxBlock) Release()

func (*HashboxBlock) Serialize

func (b *HashboxBlock) Serialize(w io.Writer) (size int)
func (b *HashboxBlock) SerializeLinks(w io.Writer) (size int)

func (*HashboxBlock) UncompressData

func (b *HashboxBlock) UncompressData()

func (*HashboxBlock) Unserialize

func (b *HashboxBlock) Unserialize(r io.Reader) (size int)

func (*HashboxBlock) UnserializeHeader

func (b *HashboxBlock) UnserializeHeader(r io.Reader) (size int)

func (*HashboxBlock) VerifyBlock

func (b *HashboxBlock) VerifyBlock() bool

type MsgClientAccountInfo

type MsgClientAccountInfo struct {
	AccountNameH Byte128 // = md5(accountName)
}

MsgClientAccountInfo is sent from the client to retrieve quota information and a list of all Datasets under the account.

type MsgClientAddDatasetState

type MsgClientAddDatasetState struct {
	AccountNameH Byte128      // = md5(accountName)
	DatasetName  String       // Name of the Dataset to store under
	State        DatasetState // Dataset state to add
}

MsgClientAddDatasetState sent to the server to add a Dataset or a Dataset state. Server returns MsgServerError if the DatasetState refers to a BlockID that does not exist.

type MsgClientAllocateBlock

type MsgClientAllocateBlock struct {
	BlockID Byte128 // ID of Block to allocate
}

MsgClientAllocateBlock allocates a new Block on the server. The server responds with a MsgServerAcknowledgeBlock if the Block already exists or a MsgServerReadBlock if the server needs to read the data from the client.

type MsgClientAuthenticate

type MsgClientAuthenticate struct {
	AccountNameH    Byte128 // = md5(accountName)
	AuthenticationH Byte128 // = hmac(AccountNameH, Session.SessionKey)
}

MsgClientAuthenticate sent from the client to authenticate the session.

type MsgClientGreeting

type MsgClientGreeting struct {
	Version Uint32 // Protocol version
}

MsgClientGreeting sent from client after connecting.

type MsgClientListDataset

type MsgClientListDataset struct {
	AccountNameH Byte128 // = md5(accountName)
	DatasetName  String  // Name of the Dataset you wish to list
}

MsgClientListDataset sent from the client to retrieve a list of all states for a Dataset

type MsgClientReadBlock

type MsgClientReadBlock struct {
	BlockID Byte128 // ID of Block to read
}

MsgClientReadBlock sent to the server to read a Block. Server responds with a MsgServerWriteBlock if the Block exists.

type MsgClientRemoveDatasetState

type MsgClientRemoveDatasetState struct {
	AccountNameH Byte128 // = md5(accountName)
	DatasetName  String  // Name of the Dataset to remove under
	StateID      Byte128 // ID of the state to remove
}

MsgClientRemoveDatasetState sent by the client to remove a Dataset state.

type MsgClientWriteBlock

type MsgClientWriteBlock struct {
	Block *HashboxBlock // Block to be sent
}

MsgClientWriteBlock sent to the server to write a Block. Server calculates its own BlockID from the block to verify its integrity. Server responds with MsgServerAcknowledgeBlock if the Block was written successfully.

type MsgServerAccountInfo

type MsgServerAccountInfo struct {
	// TODO: Add quota stuff
	DatasetList DatasetArray // List of all Datasets under the account
}

MsgServerAccountInfo sent from the server as a response to MsgClientAccountInfo.

type MsgServerAcknowledgeBlock

type MsgServerAcknowledgeBlock struct {
	BlockID Byte128 // ID of Block that exists
}

MsgServerAcknowledgeBlock sent as a confirmation that the server has the Block.

type MsgServerError

type MsgServerError struct {
	ErrorMessage String // Error message
}

MsgServerError is sent on all kind of severe protocol / server errors. The ErrorMessage contains more details regarding the error.

type MsgServerGreeting

type MsgServerGreeting struct {
	SessionNonce Byte128 // Unique nonce for the session
}

MsgServerGreeting sent from server in response to client greeting.

type MsgServerListDataset

type MsgServerListDataset struct {
	States DatasetStateArray // Array of all states under the Dataset
	ListH  Byte128           // = md5(States)
}

MsgServerListDataset sent from the server as a response to MsgClientListDataset. ListH is used so that the client can make sure it has the correct listing

type MsgServerReadBlock

type MsgServerReadBlock struct {
	BlockID Byte128 // ID of Block to read
}

MsgServerReadBlock sent from the server to retreive Block data from client.

type MsgServerWriteBlock

type MsgServerWriteBlock struct {
	Block *HashboxBlock // Block to be sent
}

MsgServerWriteBlock

type ProtocolMessage

type ProtocolMessage struct {
	Num  uint16
	Type uint32
	Data interface{}
}

func ReadMessage

func ReadMessage(r io.Reader) *ProtocolMessage

IMPORTANT messages containing HashboxBlock data will ByteArray allocate memory that needs to be freed manually

func (ProtocolMessage) Details

func (m ProtocolMessage) Details() string

func (*ProtocolMessage) Release

func (m *ProtocolMessage) Release()

func (*ProtocolMessage) Serialize

func (m *ProtocolMessage) Serialize(w io.Writer) (size int)

func (ProtocolMessage) String

func (m ProtocolMessage) String() string

func (*ProtocolMessage) Unserialize

func (m *ProtocolMessage) Unserialize(r io.Reader) (size int)

type Serializer

type Serializer interface {
	Serialize(w io.Writer) (size int)
}

Serializer is our own form of BinaryMarshaler but it works directly off a stream to be compatible with Unserializer

type Session

type Session struct {
	AccountNameH Byte128 // = md5( accountName )
	SessionNonce Byte128 // Unique nonce for the session, based on uint64(servertime) + uint64(random)
	SessionKey   Byte128 // = hmac^20000( AccountNameH SessionNonce, AccessKey)
}

func (*Session) GenerateSessionKey

func (s *Session) GenerateSessionKey(AccessKey Byte128)

type String

type String string

String is serialized as uint32(length) + [length]byte arrays

func (String) Serialize

func (m String) Serialize(w io.Writer) (size int)

func (*String) Unserialize

func (m *String) Unserialize(r io.Reader) (size int)

type TimeoutConn

type TimeoutConn struct {
	// contains filtered or unexported fields
}

func NewTimeoutConn

func NewTimeoutConn(c net.Conn, t time.Duration) *TimeoutConn

func (*TimeoutConn) Close

func (t *TimeoutConn) Close() error

func (*TimeoutConn) Read

func (t *TimeoutConn) Read(b []byte) (n int, err error)

func (*TimeoutConn) Write

func (t *TimeoutConn) Write(b []byte) (n int, err error)

type Uint32

type Uint32 uint32

func (Uint32) Serialize

func (m Uint32) Serialize(w io.Writer) (size int)

func (*Uint32) Unserialize

func (m *Uint32) Unserialize(r io.Reader) (size int)

type Unserializer

type Unserializer interface {
	Unserialize(r io.Reader) (size int)
}

Unserializer is our own form of BinaryUnmarshaler but it works directly off a stream so we do not need to know the full size beforehand

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL