storage

package module
v0.0.0-...-8a6a3e7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 30, 2022 License: Apache-2.0 Imports: 18 Imported by: 0

README

storage

存储引擎，第一个版本依据海量小文件存储, 这是一个库，可以直接嵌入到程序中使用。目前版本不建议在生产中使用，也自带了一个很简易的服务端，方便压测性能

运行服务端

./storage server -d ./my-store -s 64GB

运行压测

./storage benchmark -p -b "hello world" -d 20s -s 127.0.0.1:8080
  Completed           99870 requests [2022-10-30 23:45:53.946218]
  Completed          197209 requests [2022-10-30 23:45:55.946235]
  Completed          297909 requests [2022-10-30 23:45:57.946233]
  Completed          399386 requests [2022-10-30 23:45:59.946219]
  Completed          501585 requests [2022-10-30 23:46:01.946443]
  Completed          588044 requests [2022-10-30 23:46:03.946283]
  Completed          687964 requests [2022-10-30 23:46:05.946247]
  Completed          786152 requests [2022-10-30 23:46:07.946261]
  Completed          883646 requests [2022-10-30 23:46:09.946316]
  Completed          979513 requests [2022-10-30 23:46:11.946272]
  Finished           979523 requests


Status Codes:            979523:200  [count:code]
Concurrency Level:      10
Time taken for tests:   20.000225667s
Complete requests:      979523
Failed requests:        0
Total Read Data:        170860963 bytes
Total Read body         50379634 bytes
Total Write Body        10774753 bytes
Requests per second:    48975.59739119318 [#/sec] (mean)
Time per request:       0.02041833184825675 [ms] (mean)
Time per request:       0.20418331848256754 [ms] (mean, across all concurrent requests)
Transfer rate:          8342.726325083295 [Kbytes/sec] received
Percentage of the requests served within a certain time (ms)
  50%    173.375µs
  66%    206.5µs
  75%    233.833µs
  80%    258.541µs
  90%    341.625µs
  95%    406.792µs
  98%    533.459µs
  99%    664.417µs
 100%    61.231584ms

Documentation

Index

Constants

View Source 
const (
	KB Size = 1024
	MB      = KB * 1024
	GB      = MB * 1024
	TB      = 1024 * GB
)

Variables

View Source 
var (
	ErrIllegalKey = errors.New("Illegal key")
	ErrDirName    = errors.New("dir name is empty")
)
View Source 
var (
	ErrFull = errors.New("The space is full")
)
View Source 
var File_storage_proto protoreflect.FileDescriptor

Functions

This section is empty.

Types

type Data 

type Data struct {
	Index
	Data []byte
}

type Group 

type Group struct {
	// contains filtered or unexported fields
}

一组里面有多个引擎，一个引擎只存储32GB

func (*Group) Close 

func (g *Group) Close() (err error)

关闭所有索引

func (*Group) Delete 

func (g *Group) Delete(key string) (err error)

func (*Group) Get 

func (g *Group) Get(key string) (element Data, ok bool, err error)

func (*Group) Put 

func (g *Group) Put(data []byte) (index string, err error)

type IdxVersion0 

type IdxVersion0 struct {
	Key     int64  `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"`         //返回给客户端的值
	Size    int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`       //大小
	Offset  int64  `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"`   //偏移量
	Timeout int64  `protobuf:"varint,4,opt,name=timeout,proto3" json:"timeout,omitempty"` //超时时间
	Crc32   uint32 `protobuf:"varint,5,opt,name=crc32,proto3" json:"crc32,omitempty"`     //crc32校验和
	// contains filtered or unexported fields
}

索引的数据结构 完整的索引，还有payload(放在第一位)和crc32(第在最后一位)

func (*IdxVersion0) Descriptor deprecated 

func (*IdxVersion0) Descriptor() ([]byte, []int)

Deprecated: Use IdxVersion0.ProtoReflect.Descriptor instead.

func (*IdxVersion0) GetCrc32 

func (x *IdxVersion0) GetCrc32() uint32

func (*IdxVersion0) GetKey 

func (x *IdxVersion0) GetKey() int64

func (*IdxVersion0) GetOffset 

func (x *IdxVersion0) GetOffset() int64

func (*IdxVersion0) GetSize 

func (x *IdxVersion0) GetSize() int32

func (*IdxVersion0) GetTimeout 

func (x *IdxVersion0) GetTimeout() int64

func (*IdxVersion0) ProtoMessage 

func (*IdxVersion0) ProtoMessage()

func (*IdxVersion0) ProtoReflect 

func (x *IdxVersion0) ProtoReflect() protoreflect.Message

func (*IdxVersion0) Reset 

func (x *IdxVersion0) Reset()

func (*IdxVersion0) String 

func (x *IdxVersion0) String() string

type Index 

type Index struct {
	Key     int32  `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"`         //返回给客户端的值
	Size    int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`       //大小
	Offset  int64  `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"`   //偏移量
	Timeout int64  `protobuf:"varint,4,opt,name=timeout,proto3" json:"timeout,omitempty"` //超时时间
	Crc32   uint32 `protobuf:"varint,5,opt,name=crc32,proto3" json:"crc32,omitempty"`     //crc32校验和
}

type IndexInMemory 

type IndexInMemory struct {
	// contains filtered or unexported fields
}

一个内存索引管理32GB文件

func (*IndexInMemory) Close 

func (i *IndexInMemory) Close() (err error)

close

func (*IndexInMemory) Delete 

func (i *IndexInMemory) Delete(key int64) error

删除

func (*IndexInMemory) Get 

func (i *IndexInMemory) Get(key int64) (element Data, ok bool, err error)

获取

func (*IndexInMemory) GetSeq 

func (i *IndexInMemory) GetSeq() (key int64)

func (*IndexInMemory) Put 

func (i *IndexInMemory) Put(key int64, data []byte) (err error)

保存

type Size 

type Size int64

type Storage 

type Storage struct {
	*Group
}

func Open 

func Open(name string, max Size) (s Storage, err error)

type Storager 

type Storager interface {
	Put(key int64, data []byte) (err error)
	Get(key int64) (element Data, ok bool, err error)
	GetSeq() int64
	Delete(key int64) error
	Close() error
}

Source Files

View all Source files

Directories

Path Synopsis
cmd
benchmark
storage

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.