research_online_redis_go

package module
v0.0.0-...-bd7e638 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 31, 2024 License: MIT Imports: 7 Imported by: 0

README ΒΆ

Efficiently store online with Redis and Go

Support Ukraine πŸ‡ΊπŸ‡¦

Databases

Name Stars Language
Redis 64300+ C
KeyDB 10100+ C++
DragonflyDB 23300+ C++
Garnet 7900+ C#

Data structure usage examples

Hash
docker exec research-online-redis-1 redis-cli monitor

docker exec research-online-redis-go-app go test ./... -v -run=TestRedisHashOnlineStorage -count=1

flushall
hset "h:online:main" "10000001" "1679800725"
hset "h:online:main" "10000002" "1679800730"
hset "h:online:main" "10000003" "1679800735"
hlen "h:online:main"
rename "h:online:main" "h:online:tmp"
hgetall "h:online:tmp"
Sorted Set
docker exec research-online-redis-1 redis-cli monitor

docker exec research-online-redis-go-app go test ./... -v -run=TestRedisSortedSetOnlineStorage -count=1

flushall
zadd "z:online:main" "1679800725" "10000001"
zadd "z:online:main" "1679800730" "10000002"
zadd "z:online:main" "1679800735" "10000003"
zcard "z:online:main"
rename "z:online:main" "z:online:tmp"
zrange "z:online:tmp" "0" "-1" "withscores"
Set
docker exec research-online-redis-1 redis-cli monitor

docker exec research-online-redis-go-app go test ./... -v -run=TestRedisSetOnlineStorage -count=1

flushall
sadd "s:online:main:1679800725" "10000001"
sadd "s:online:main:1679800730" "10000002"
sadd "s:online:main:1679800735" "10000003"
keys "s:online:main:*"
scard "s:online:main:1679800730"
scard "s:online:main:1679800725"
scard "s:online:main:1679800735"
keys "s:online:main:*"
rename "s:online:main:1679800730" "s:online:tmp"
smembers "s:online:tmp"
rename "s:online:main:1679800725" "s:online:tmp"
smembers "s:online:tmp"
rename "s:online:main:1679800735" "s:online:tmp"
smembers "s:online:tmp"

Testing

make env-up
make test
make env-down

=== RUN   TestGoOnlineStorage
--- PASS: TestGoOnlineStorage (0.00s)
=== RUN   TestRedisHashOnlineStorage
--- PASS: TestRedisHashOnlineStorage (0.00s)
=== RUN   TestKeydbHashOnlineStorage
--- PASS: TestKeydbHashOnlineStorage (0.00s)
=== RUN   TestDragonflydbHashOnlineStorage
--- PASS: TestDragonflydbHashOnlineStorage (0.00s)
=== RUN   TestGarnetHashOnlineStorage
--- PASS: TestGarnetHashOnlineStorage (0.00s)
=== RUN   TestRedisPing
--- PASS: TestRedisPing (0.00s)
=== RUN   TestKeydbPing
--- PASS: TestKeydbPing (0.00s)
=== RUN   TestDragonflydbPing
--- PASS: TestDragonflydbPing (0.00s)
=== RUN   TestRedisSetOnlineStorage
--- PASS: TestRedisSetOnlineStorage (0.00s)
=== RUN   TestKeydbSetOnlineStorage
--- PASS: TestKeydbSetOnlineStorage (0.00s)
=== RUN   TestDragonflydbSetOnlineStorage
--- PASS: TestDragonflydbSetOnlineStorage (0.01s)
=== RUN   TestGarnetSetOnlineStorage
--- PASS: TestGarnetSetOnlineStorage (0.11s)
=== RUN   TestRedisSortedSetOnlineStorage
--- PASS: TestRedisSortedSetOnlineStorage (0.00s)
=== RUN   TestKeydbSortedSetOnlineStorage
--- PASS: TestKeydbSortedSetOnlineStorage (0.00s)
=== RUN   TestDragonflydbSortedSetOnlineStorage
--- PASS: TestDragonflydbSortedSetOnlineStorage (0.02s)
=== RUN   TestGarnetSortedSetOnlineStorage
--- PASS: TestGarnetSortedSetOnlineStorage (0.00s)
PASS
ok  	github.com/doutivity/research-online-redis-go	0.154s

Benchmark

make bench
# MODE=sequence go test ./... -v -bench='Go'                         -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-go-1000000x-sequence.txt
# MODE=sequence go test ./... -v -bench='Redis(Hash|SortedSet|Set)'  -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-redis-1000000x-sequence.txt
# MODE=sequence go test ./... -v -bench='Keydb(Hash|SortedSet|Set)'  -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-keydb-1000000x-sequence.txt
# MODE=sequence go test ./... -v -bench='Dragonflydb(SortedSet|Set)' -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-dragonflydb-1000000x-sequence.txt
# MODE=parallel go test ./... -v -bench='Go'                         -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-go-1000000x-parallel.txt
# MODE=parallel go test ./... -v -bench='Redis(Hash|SortedSet|Set)'  -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-redis-1000000x-parallel.txt
# MODE=parallel go test ./... -v -bench='Keydb(Hash|SortedSet|Set)'  -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-keydb-1000000x-parallel.txt
# MODE=parallel go test ./... -v -bench='Dragonflydb(SortedSet|Set)' -benchmem -benchtime=1000000x -count=5 | tee ./output/bench-dragonflydb-1000000x-parallel.txt
# benchstat ./output/bench-go-1000000x-sequence.txt
# benchstat ./output/bench-redis-1000000x-sequence.txt
# benchstat ./output/bench-keydb-1000000x-sequence.txt
# benchstat ./output/bench-dragonflydb-1000000x-sequence.txt
# benchstat ./output/bench-go-1000000x-parallel.txt
# benchstat ./output/bench-redis-1000000x-parallel.txt
# benchstat ./output/bench-keydb-1000000x-parallel.txt
# benchstat ./output/bench-dragonflydb-1000000x-parallel.txt
Database name Data structure sequence time/op parallel time/op
Go map[int]int 515ns Β± 9% 696ns Β± 8%
Redis Hash 33.5Β΅s Β± 6% 13.9Β΅s Β±24%
KeyDB Hash 36.9Β΅s Β± 2% 14.5Β΅s Β±23%
DragonflyDB Hash 44.0Β΅s Β± 2% 13.6Β΅s Β±12%
Redis Sorted Set 34.4Β΅s Β± 1% 13.5Β΅s Β± 6%
KeyDB Sorted Set 38.6Β΅s Β± 1% 14.1Β΅s Β± 2%
DragonflyDB Sorted Set 52.9Β΅s Β±15% 16.3Β΅s Β± 8%
Redis Set 32.6Β΅s Β± 1% 12.4Β΅s Β± 2%
KeyDB Set 36.7Β΅s Β± 1% 13.7Β΅s Β± 3%
DragonflyDB Set 45.9Β΅s Β± 4% 14.4Β΅s Β±16%

Used memory

make bench-redis-memory-1m
make bench-keydb-memory-1m
make bench-dragonflydb-memory-1m
# ...
make bench-redis-memory-25m
make bench-keydb-memory-25m
make bench-dragonflydb-memory-25m

# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$^ -bench='Redis(Hash)' -benchmem -benchtime=25000000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-hash-25m.txt
#
# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$^ -bench='Redis(SortedSet)' -benchmem -benchtime=25000000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-sorted-set-25m.txt
#
# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$^ -bench='Redis(Set)' -benchmem -benchtime=25000000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-set-25m.txt
Database name Data structure Users Memory
Redis Hash 1 000 000 62.64 MB
KeyDB Hash 1 000 000 63.49 MB
DragonflyDB Hash 1 000 000 61.51 MB
Redis Hash 10 000 000 727.20 MB
KeyDB Hash 10 000 000 728.14 MB
DragonflyDB Hash 10 000 000 622.59 MB
Redis Hash 25 000 000 1592.14 MB
KeyDB Hash 25 000 000 1593.27 MB
DragonflyDB Hash 25 000 000 1481.70 MB
Redis Sorted Set 1 000 000 91.09 MB
KeyDB Sorted Set 1 000 000 91.93 MB
DragonflyDB Sorted Set 1 000 000 107.87 MB
Redis Sorted Set 10 000 000 1011.78 MB
KeyDB Sorted Set 10 000 000 1012.64 MB
DragonflyDB Sorted Set 10 000 000 1161.64 MB
Redis Sorted Set 25 000 000 2303.58 MB
KeyDB Sorted Set 25 000 000 2304.70 MB
DragonflyDB Sorted Set 25 000 000 2675.25 MB
Redis Set 1 000 000 48.14 MB
KeyDB Set 1 000 000 49.02 MB
DragonflyDB Set 1 000 000 32.60 MB
Redis Set 10 000 000 469.57 MB
KeyDB Set 10 000 000 471.44 MB
DragonflyDB Set 10 000 000 297.01 MB
Redis Set 25 000 000 1169.33 MB
KeyDB Set 25 000 000 1175.45 MB
DragonflyDB Set 25 000 000 unknown, cause store less then expected, 15276400 from 25000000

Batch insert 10k rows x 10k times benchmark

make bench-redis-memory-10k-batch-10k
make bench-keydb-memory-10k-batch-10k
make bench-dragonflydb-memory-10k-batch-10k

# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel -e BATCH=10000 research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash)' -benchmem -benchtime=10000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-hash-10k-batch-10k.txt
#
# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel -e BATCH=10000 research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(SortedSet)' -benchmem -benchtime=10000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-sorted-set-10k-batch-10k.txt
#
# docker exec research-online-redis-1 redis-cli flushall
# docker exec -e MODE=parallel -e BATCH=10000 research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Set)' -benchmem -benchtime=10000x -count=1
# docker exec research-online-redis-1 redis-cli info memory | tee ./output/redis-memory-set-10k-batch-10k.txt
#
# docker exec research-online-redis-1 redis-cli flushall
Database name Data structure parallel time/op
Redis Hash 8232276 ns/op
KeyDB Hash 21357358 ns/op
DragonflyDB Hash 6716157 ns/op
Redis Sorted Set 12016807 ns/op
KeyDB Sorted Set 15114051 ns/op
DragonflyDB Sorted Set 9535106 ns/op
Redis Set 3187424 ns/op
KeyDB Set 3233770 ns/op
DragonflyDB Set unknown, cause store less then expected, 15622200 from 100000000
Database name Data structure Memory
Redis Hash 6.72 GB
KeyDB Hash 6.22 GB
DragonflyDB Hash 5.77 GB
Redis Sorted Set 9.00 GB
KeyDB Sorted Set 9.00 GB
DragonflyDB Sorted Set 10.44 GB
Redis Set 4.58 GB
KeyDB Set 4.59 GB
DragonflyDB Set unknown, cause store less then expected, 15622200 from 100000000

Star history of Redis vs KeyDB vs DragonflyDB

Star History Chart

Versions

docker pull redis:latest
docker pull eqalpha/keydb:latest
docker pull docker.dragonflydb.io/dragonflydb/dragonfly
docker pull ghcr.io/microsoft/garnet

docker image inspect redis:latest --format '{{.RepoDigests}} {{.Size}}'
docker image inspect eqalpha/keydb:latest --format '{{.RepoDigests}} {{.Size}}'
docker image inspect docker.dragonflydb.io/dragonflydb/dragonfly --format '{{.RepoDigests}} {{.Size}}'
docker image inspect ghcr.io/microsoft/garnet --format '{{.RepoDigests}} {{.Size}}'
Database name Docker image size Docker image
Redis 131.32 MB sha256:3134997edb04277814aa51a4175a588d45eb4299272f8eff2307bbf8b39e4d43
KeyDB 130.39 MB sha256:6537505c42355ca1f571276bddf83f5b750f760f07b2a185a676481791e388ac
DragonflyDB 106.65 MB sha256:48d7f1679a895702262808c83689df94b14d40c07401fad90723ad164d271150
Garnet 196.50 MB sha256:0da3da0cd45d8a7084d670b7b1a96748cea40917ed054fc98840ce6e4036b97b

Documentation ΒΆ

Index ΒΆ

Constants ΒΆ

This section is empty.

Variables ΒΆ

This section is empty.

Functions ΒΆ

func Client ΒΆ 

func Client(ctx context.Context, addr string) (*redis.Client, error)

Types ΒΆ

type GoOnlineStorage ΒΆ 

type GoOnlineStorage struct {
	// contains filtered or unexported fields
}

func NewGoOnlineStorage ΒΆ 

func NewGoOnlineStorage() *GoOnlineStorage

func (*GoOnlineStorage) BatchStore ΒΆ 

func (s *GoOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error

func (*GoOnlineStorage) Count ΒΆ 

func (s *GoOnlineStorage) Count(ctx context.Context) (int64, error)

func (*GoOnlineStorage) GetAndClear ΒΆ 

func (s *GoOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error)

func (*GoOnlineStorage) Store ΒΆ 

func (s *GoOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error

type HashOnlineStorage ΒΆ 

type HashOnlineStorage struct {
	// contains filtered or unexported fields
}

func NewHashOnlineStorage ΒΆ 

func NewHashOnlineStorage(client *redis.Client) *HashOnlineStorage

func (*HashOnlineStorage) BatchStore ΒΆ 

func (s *HashOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error

func (*HashOnlineStorage) Count ΒΆ 

func (s *HashOnlineStorage) Count(ctx context.Context) (int64, error)

func (*HashOnlineStorage) GetAndClear ΒΆ 

func (s *HashOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error)

func (*HashOnlineStorage) Store ΒΆ 

func (s *HashOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error

type OnlineStorage ΒΆ 

type OnlineStorage interface {
	Store(ctx context.Context, pair UserOnlinePair) error
	BatchStore(ctx context.Context, pairs []UserOnlinePair) error
	Count(ctx context.Context) (int64, error)
	GetAndClear(ctx context.Context) ([]UserOnlinePair, error)
}

type SetOnlineStorage ΒΆ 

type SetOnlineStorage struct {
	// contains filtered or unexported fields
}

func NewSetOnlineStorage ΒΆ 

func NewSetOnlineStorage(client *redis.Client, group int64) *SetOnlineStorage

func (*SetOnlineStorage) BatchStore ΒΆ 

func (s *SetOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error

func (*SetOnlineStorage) Count ΒΆ 

func (s *SetOnlineStorage) Count(ctx context.Context) (int64, error)

func (*SetOnlineStorage) GetAndClear ΒΆ 

func (s *SetOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error)

func (*SetOnlineStorage) Store ΒΆ 

func (s *SetOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error

type SortedSetOnlineStorage ΒΆ 

type SortedSetOnlineStorage struct {
	// contains filtered or unexported fields
}

func NewSortedSetOnlineStorage ΒΆ 

func NewSortedSetOnlineStorage(client *redis.Client) *SortedSetOnlineStorage

func (*SortedSetOnlineStorage) BatchStore ΒΆ 

func (s *SortedSetOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error

func (*SortedSetOnlineStorage) Count ΒΆ 

func (s *SortedSetOnlineStorage) Count(ctx context.Context) (int64, error)

func (*SortedSetOnlineStorage) GetAndClear ΒΆ 

func (s *SortedSetOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error)

func (*SortedSetOnlineStorage) Store ΒΆ 

func (s *SortedSetOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error

type UserOnlinePair ΒΆ 

type UserOnlinePair struct {
	UserID    int64
	Timestamp int64
}

type UserOnlinePairs ΒΆ 

type UserOnlinePairs []UserOnlinePair

func (UserOnlinePairs) Len ΒΆ 

func (ps UserOnlinePairs) Len() int

func (UserOnlinePairs) Less ΒΆ 

func (ps UserOnlinePairs) Less(i, j int) bool

func (UserOnlinePairs) Swap ΒΆ 

func (ps UserOnlinePairs) Swap(i, j int)

Source Files ΒΆ

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.