simpledb

package module
v0.0.0-...-5a2c45c Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 4, 2018 License: MIT Imports: 7 Imported by: 0

README

simpledb

Build Status Go Report Card GoDoc

simpledb is a Golang package which provides CRUD and search operations on records stored in Redis. It's based on Redigo.

How it works

  • Record Buckets

    • simpledb stores a record in a Redis hash(we call it record bucket).
    • Hash key name: contains record bucket id generated by record id.
      • record bucket id = record id / hash-max-ziplist-entries. Ex: "student/bucket/23".
    • Hash field: record id.
    • Value of field: record data.
    • The number of hash entries is the same as "hash-max-ziplist-entries" of Redis settings to reduce memory usage.
      • User should also make sure record data length matches "hash-max-ziplist-values".

  • Index Buckets

    • simple db stores one more record data as reverse index in a Redis hash(we call it index bucket).
    • Hash key name: contains index bucket id generated by record data.
      • index bucket id = CRC32(record data) % Estimated Bucket Num. Ex: "student/idx/bucket/3".
      • Estimated Bucket Num = Estimated Max Record Num(1000000 by default) / ("hash-max-ziplist-entries" * 0.9").
    • Hash field: record data(duplicated).
    • Value of field: record id.

  • Search

    • Search() scans all index buckets(hashes) and use HSCAN command with pattern of Redis(Ex: '{"name":"Frank*"}*') on record data directly to find matched record ids.
    • RegexpSeach() scans all index buckets(hashes) and use HSCAN command to retrieve all fields and use Regexp pattern(Ex: '{"name":"Frank.+"}') on record data directly to find matched record ids.
Documentation
License

Documentation

Overview

Package simpledb is a Golang package which provides CRUD and search operations on records stored in Redis. It's based on Redigo.

Index

Examples

Constants

View Source 
const (
	// EstimatedMaxRecordNum is estimated max record number.
	EstimatedMaxRecordNum uint64 = 1000000
)

Variables

View Source 
var (
	// DEBUG represents debug mode. It'll output debug messages if it's true.
	DEBUG = true
)

Functions

func GetRedisConn 

func GetRedisConn(redisAddr, redisPassword string) (c redis.Conn, err error)

GetRedisConn gets the Redis connection.

Example 
package main

import (
	"log"

	"github.com/gomodule/redigo/redis"
	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var c redis.Conn

	log.Printf("\n")
	log.Printf("--------- GetRedisConn() Test Begin --------\n")

	if c, err = simpledb.GetRedisConn(":6379", ""); err != nil {
		goto end
	}
	defer c.Close()

	log.Printf("OK.\n")
end:
	if err != nil {
		log.Printf("error: %v\n", err)
	}

	log.Printf("--------- GetRedisConn() Test End --------\n")
}

Output:

func GetRedisHashMaxZiplistEntries 

func GetRedisHashMaxZiplistEntries(c redis.Conn) (redisHashMaxZiplistEntries uint64, err error)

GetRedisHashMaxZiplistEntries gets the Redis "hash-max-ziplist-entries" config value.

Example 
package main

import (
	"log"

	"github.com/gomodule/redigo/redis"
	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var c redis.Conn
	var redisHashMaxZiplistEntries uint64 = 0

	log.Printf("\n")
	log.Printf("--------- GetRedisHashMaxZiplistEntries() Test Begin --------\n")

	if c, err = redis.Dial("tcp", ":6379"); err != nil {
		goto end
	}
	defer c.Close()

	if redisHashMaxZiplistEntries, err = simpledb.GetRedisHashMaxZiplistEntries(c); err != nil {
		goto end
	}

	log.Printf("Redis hash-max-ziplist-entries: %v\n", redisHashMaxZiplistEntries)
end:
	if err != nil {
		log.Printf("error: %v\n", err)
	}

	log.Printf("--------- GetRedisHashMaxZiplistEntries() Test End --------\n")
}

Output:

Types

type DB 

type DB struct {
	// contains filtered or unexported fields
}

DB represents a record collection stored in Redis server.

func Open 

func Open(redisAddr, redisPassword, name string) (db *DB, err error)

Open returns an DB instance by given database name.

func (*DB) BatchCreate 

func (db *DB) BatchCreate(dataArr []string) (ids []string, err error)

BatchCreate creates records in database.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB
	ids := []string{}
	data := []string{
		`{"name":"Frank Xu","tel":"13700137000"}`,
		`{"name":"Frank Wang","tel":"13600136000"}`,
		`{"name":"张三","tel":"13800138001"}`,
		`{"name":"李四","tel":"13800138002"}`,
		`{"name":"王大宝","tel":"13800138003"}`,
		`{"name":"王小宝","tel":"13800138003"}`,
		`{"name":"王宝多","tel":"13700137077"}`,
	}
	records := []simpledb.Record{}

	log.Printf("\n")
	log.Printf("--------- BatchCreate() Test Begin --------\n")

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	if ids, err = db.BatchCreate(data); err != nil {
		goto end
	}

	log.Printf("Result:\n")
	if records, err = db.BatchGet(ids); err != nil {
		goto end
	}

	for _, r := range records {
		log.Printf("id: %v, data: %v\n", r.ID, r.Data)
	}

end:
	log.Printf("--------- BatchCreate() Test End --------\n")
}

Output:

func (*DB) BatchDelete 

func (db *DB) BatchDelete(ids []string) (err error)

BatchDelete deletes multiple records in database by given ids.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"name":"*"*`
	records := []simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- BatchDelete() Test Begin --------\n")
	// Get all records.
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	if records, err = db.BatchGet(ids); err != nil {
		goto end
	}

	log.Printf("Search pattern: %v, Result:\n", pattern)
	for _, r := range records {
		log.Printf("id: %v, data: %v\n", r.ID, r.Data)
	}

	// Delete all record.
	if err = db.BatchDelete(ids); err != nil {
		goto end
	}

	// Search again after record deleted.
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	log.Printf("Search again after record deleted. Result:\n")
	log.Printf("ids: %v\n", ids)

end:
	log.Printf("--------- BatchDelete() Test End --------\n")
}

Output:

func (*DB) BatchGet 

func (db *DB) BatchGet(ids []string) (records []Record, err error)

BatchGet returns multiple record data by given record ids. 

Params
    ids: record id array.
Return:
    records: record array.
Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"name":"王*"*`
	records := []simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- BatchGet() Test Begin --------\n")
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}
	if records, err = db.BatchGet(ids); err != nil {
		goto end
	}

	log.Printf("Search pattern: %v, Result:\n", pattern)
	for _, record := range records {
		log.Printf("id: %v, data: %v\n", record.ID, record.Data)
	}

end:
	log.Printf("--------- BatchGet() Test End --------\n")
}

Output:

func (*DB) BatchUpdate 

func (db *DB) BatchUpdate(records []Record) (err error)

BatchUpdate updates multiple records by given ids and new data. 

Params:
    records: record array to be updated.
Example 
package main

import (
	"log"
	"strings"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"tel":"138*"*`
	oldRecords := []simpledb.Record{}
	newRecords := []simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- BatchUpdate() Test Begin --------\n")
	log.Printf("Search pattern: %v, Result:\n", pattern)
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}
	if oldRecords, err = db.BatchGet(ids); err != nil {
		goto end
	}

	log.Printf("Result:\n")
	for _, r := range oldRecords {
		log.Printf("id: %v, data: %v\n", r.ID, r.Data)
		newRecords = append(newRecords, simpledb.Record{ID: r.ID, Data: strings.Replace(r.Data, "138", "186", -1)})
	}

	if err = db.BatchUpdate(newRecords); err != nil {
		goto end
	}

	if newRecords, err = db.BatchGet(ids); err != nil {
		goto end
	}

	log.Printf("Batch update phone number from 138xxx to 186xxx.")
	for _, r := range newRecords {
		log.Printf("id: %v, data: %v\n", r.ID, r.Data)
	}
end:
	log.Printf("--------- BatchUpdate() Test End --------\n")
}

Output:

func (*DB) Close 

func (db *DB) Close()

Close closes an DB instance after use.

func (*DB) Count 

func (db *DB) Count() (count uint64, err error)

Count returns record count stored in Redis.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	var count uint64

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Count() Test Begin --------\n")

	if count, err = db.Count(); err != nil {
		goto end
	}
	log.Printf("Record count: %v\n", count)
end:
	log.Printf("--------- Count() Test End --------\n")
}

Output:

func (*DB) Create 

func (db *DB) Create(data string) (id string, err error)

Create creates a new record in database.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB
	var record = simpledb.Record{}
	id := ""
	data := `{"name":"Bob Smith","tel":"13500135000"}`

	log.Printf("\n")
	log.Printf("--------- Create() Test Begin --------\n")

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	if id, err = db.Create(data); err != nil {
		goto end
	}

	if record, err = db.Get(id); err != nil {
		goto end
	}

	log.Printf("Result: id: %v, data: %v\n", record.ID, record.Data)
end:
	log.Printf("--------- Create() Test End --------\n")
}

Output:

func (*DB) Delete 

func (db *DB) Delete(id string) (err error)

Delete deletes the record in database by given id.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"name":"Frank Wang"*`
	record := simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Delete() Test Begin --------\n")

	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	if len(ids) != 1 {
		goto end
	}

	if record, err = db.Get(ids[0]); err != nil {
		goto end
	}

	log.Printf("Search pattern: %v, Result:\n", pattern)
	log.Printf("id: %v, data: %v\n", record.ID, record.Data)

	// Delete
	if err = db.Delete(ids[0]); err != nil {
		goto end
	}

	// Search again after record deleted.
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	log.Printf("Search again after record deleted. Result:\n")
	log.Printf("ids: %v\n", ids)

end:
	log.Printf("--------- Delete() Test End --------\n")
}

Output:

func (*DB) Exists 

func (db *DB) Exists(data string) (exists bool, err error)

Exists checks if given record exists in database.

func (*DB) Get 

func (db *DB) Get(id string) (r Record, err error)

Get returns record data by given record id.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"name":"Bob*"*`
	record := simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Get() Test Begin --------\n")
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	if len(ids) != 1 {
		goto end
	}

	if record, err = db.Get(ids[0]); err != nil {
		goto end
	}

	log.Printf("Search pattern: %v, Result:\n", pattern)
	log.Printf("id: %v, data: %v\n", record.ID, record.Data)

end:
	log.Printf("--------- Get() Test End --------\n")
}

Output:

func (*DB) GetMaxBucketID 

func (db *DB) GetMaxBucketID() (maxBucketID uint64, err error)

GetMaxBucketID gets the max record bucket id.

func (*DB) GetMaxID 

func (db *DB) GetMaxID() (maxID uint64, err error)

GetMaxID gets max record id.

func (*DB) IDExists 

func (db *DB) IDExists(id string) (exists bool, err error)

IDExists checks if record with given record id exists in database.

Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	exists := false
	id := "1"

	log.Printf("\n")
	log.Printf("--------- IDExists() Test Begin --------\n")

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	if exists, err = db.IDExists(id); err != nil {
		goto end
	}

	log.Printf("IdExsits(%v): %v\n", id, exists)

end:
	log.Printf("--------- IDExists() Test End --------\n")
}

Output:

func (*DB) Info 

func (db *DB) Info() (infoMap map[string]string, err error)

Info returns the information of current DB. 

Returns:
    infoMap: key: section, value: information.
Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	infoMap := make(map[string]string)

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Info() Test Begin --------\n")

	if infoMap, err = db.Info(); err != nil {
		goto end
	}
	log.Printf("Info():\n")
	for k, v := range infoMap {
		log.Printf("%v: %v\n", k, v)
	}

end:
	log.Printf("--------- Info() Test End --------\n")
}

Output:

func (*DB) RegexpSearch 

func (db *DB) RegexpSearch(patterns []string) (ids [][]string, err error)

RegexpSearch scans all indexes(record data) in database and use regexp patterns to find records which match the patterns. 

Params:
    patterns: regexp pattern array. Ex: {`{"name":"Frank.+"}`,`{"tel":"136\d{8}"}`}
Returns:
    ids: matched record ids arrays. Each array contains result IDs match the pattern.
Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB
	ids := [][]string{}

	patterns := []string{
		// Search UTF8 string.
		//`"name":"王.*宝"`,
		`王.+宝`,
		// Search name matches Frank* and tel matches 13700137000.
		`"Frank.*".*"tel":"13700137000"`,
	}

	records := []simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- RegexpSearch() Test Begin --------\n")

	if ids, err = db.RegexpSearch(patterns); err != nil {
		goto end
	}

	for i, p := range patterns {
		log.Printf("Regexp pattern: %v\n", p)
		if records, err = db.BatchGet(ids[i]); err != nil {
			goto end
		}

		log.Printf("Result:\n")
		for _, r := range records {
			log.Printf("id: %v, data: %v\n", r.ID, r.Data)
		}
	}
end:
	log.Printf("--------- RegexpSearch() Test End --------\n")
}

Output:

func (*DB) Search 

func (db *DB) Search(pattern string) (ids []string, err error)

Search scans all indexes(record data) in database and use the pattern of Redis "SCAN" command to find records which match the pattern. 

Params:
    pattern: pattern of Redis "SCAN" command.
    It'll return all record ID if pattern is empty.
    Ex: `{"name":"Frank*"}*`
Returns:
    ids: matched record ids.
Example 
package main

import (
	"log"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	patterns := []string{
		// Search UTF8 string.
		`*"name":"王*宝"*`,
		// Search name matches Frank* and tel matches 13700137000.
		`*"name":"Frank*"*"tel":"13700137000"*`,
		// Empty pattern: return all record ID.
		``,
	}

	records := []simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Search() Test Begin --------\n")

	for _, p := range patterns {
		log.Printf("Search pattern: %v\n", p)
		if ids, err = db.Search(p); err != nil {
			goto end
		}

		if records, err = db.BatchGet(ids); err != nil {
			goto end
		}

		log.Printf("Result:\n")
		for _, r := range records {
			log.Printf("id: %v, data: %v\n", r.ID, r.Data)
		}

		log.Printf("\n")
	}

end:
	log.Printf("--------- Search() Test End --------\n")
}

Output:

func (*DB) Update 

func (db *DB) Update(record Record) error

Update updates the record by given id and new data.

Example 
package main

import (
	"log"
	"strings"

	"github.com/northbright/simpledb"
)

func main() {
	var err error
	var db *simpledb.DB

	ids := []string{}
	pattern := `*"name":"Bob*"*`
	record := simpledb.Record{}

	db, _ = simpledb.Open(":6379", "", "student")
	defer db.Close()

	log.Printf("\n")
	log.Printf("--------- Update() Test Begin --------\n")
	if ids, err = db.Search(pattern); err != nil {
		goto end
	}

	if len(ids) != 1 {
		goto end
	}

	if record, err = db.Get(ids[0]); err != nil {
		goto end
	}

	log.Printf("Search pattern: %v, Result:\n", pattern)
	log.Printf("id: %v, data: %v\n", record.ID, record.Data)

	// Change phone number from 135xxx to 138xxx.
	record.Data = strings.Replace(record.Data, "135", "158", -1)
	if err = db.Update(record); err != nil {
		goto end
	}

	log.Printf("Change phone number from 135xxx to 158xxx.")
	if record, err = db.Get(ids[0]); err != nil {
		goto end
	}
	log.Printf("Updated. id: %v, data: %v\n", record.ID, record.Data)

end:
	log.Printf("--------- Update() Test End --------\n")
}

Output:

type Record 

type Record struct {
	// ID is record ID.
	ID string
	// data is record dta.
	Data string
}

Record contains record ID and data string.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.