apps/redis-go
0
0
Fork 0
mirror of https://github.com/HDT3213/godis.git synced 2025-10-05 08:46:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor project structure

Browse Source
This commit is contained in:
hdt3213
2021-05-02 14:54:42 +08:00
parent bb9c140653
commit f29298cc68
78 changed files with 140 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
datastruct/dict/concurrent.go Normal file
View File

@@ -0,0 +1,280 @@
package dict
import (
"math"
"math/rand"
"sync"
"sync/atomic"
)
type ConcurrentDict struct {
table []*Shard
count int32
}
type Shard struct {
m map[string]interface{}
mutex sync.RWMutex
}
func computeCapacity(param int) (size int) {
if param <= 16 {
return 16
}
n := param - 1
n |= n >> 1
n |= n >> 2
n |= n >> 4
n |= n >> 8
n |= n >> 16
if n < 0 {
return math.MaxInt32
} else {
return int(n + 1)
}
}
func MakeConcurrent(shardCount int) *ConcurrentDict {
shardCount = computeCapacity(shardCount)
table := make([]*Shard, shardCount)
for i := 0; i < shardCount; i++ {
table[i] = &Shard{
m: make(map[string]interface{}),
}
}
d := &ConcurrentDict{
count: 0,
table: table,
}
return d
}
const prime32 = uint32(16777619)
func fnv32(key string) uint32 {
hash := uint32(2166136261)
for i := 0; i < len(key); i++ {
hash *= prime32
hash ^= uint32(key[i])
}
return hash
}
func (dict *ConcurrentDict) spread(hashCode uint32) uint32 {
if dict == nil {
panic("dict is nil")
}
tableSize := uint32(len(dict.table))
return (tableSize - 1) & uint32(hashCode)
}
func (dict *ConcurrentDict) getShard(index uint32) *Shard {
if dict == nil {
panic("dict is nil")
}
return dict.table[index]
}
func (dict *ConcurrentDict) Get(key string) (val interface{}, exists bool) {
if dict == nil {
panic("dict is nil")
}
hashCode := fnv32(key)
index := dict.spread(hashCode)
shard := dict.getShard(index)
shard.mutex.RLock()
defer shard.mutex.RUnlock()
val, exists = shard.m[key]
return
}
func (dict *ConcurrentDict) Len() int {
if dict == nil {
panic("dict is nil")
}
return int(atomic.LoadInt32(&dict.count))
}
// return the number of new inserted key-value
func (dict *ConcurrentDict) Put(key string, val interface{}) (result int) {
if dict == nil {
panic("dict is nil")
}
hashCode := fnv32(key)
index := dict.spread(hashCode)
shard := dict.getShard(index)
shard.mutex.Lock()
defer shard.mutex.Unlock()
if _, ok := shard.m[key]; ok {
shard.m[key] = val
return 0
} else {
shard.m[key] = val
dict.addCount()
return 1
}
}
// return the number of updated key-value
func (dict *ConcurrentDict) PutIfAbsent(key string, val interface{}) (result int) {
if dict == nil {
panic("dict is nil")
}
hashCode := fnv32(key)
index := dict.spread(hashCode)
shard := dict.getShard(index)
shard.mutex.Lock()
defer shard.mutex.Unlock()
if _, ok := shard.m[key]; ok {
return 0
} else {
shard.m[key] = val
dict.addCount()
return 1
}
}
// return the number of updated key-value
func (dict *ConcurrentDict) PutIfExists(key string, val interface{}) (result int) {
if dict == nil {
panic("dict is nil")
}
hashCode := fnv32(key)
index := dict.spread(hashCode)
shard := dict.getShard(index)
shard.mutex.Lock()
defer shard.mutex.Unlock()
if _, ok := shard.m[key]; ok {
shard.m[key] = val
return 1
} else {
return 0
}
}
// return the number of deleted key-value
func (dict *ConcurrentDict) Remove(key string) (result int) {
if dict == nil {
panic("dict is nil")
}
hashCode := fnv32(key)
index := dict.spread(hashCode)
shard := dict.getShard(index)
shard.mutex.Lock()
defer shard.mutex.Unlock()
if _, ok := shard.m[key]; ok {
delete(shard.m, key)
return 1
} else {
return 0
}
return
}
func (dict *ConcurrentDict) addCount() int32 {
return atomic.AddInt32(&dict.count, 1)
}
/*
* may not contains new entry inserted during traversal
*/
func (dict *ConcurrentDict) ForEach(consumer Consumer) {
if dict == nil {
panic("dict is nil")
}
for _, shard := range dict.table {
shard.mutex.RLock()
func() {
defer shard.mutex.RUnlock()
for key, value := range shard.m {
continues := consumer(key, value)
if !continues {
return
}
}
}()
}
}
func (dict *ConcurrentDict) Keys() []string {
keys := make([]string, dict.Len())
i := 0
dict.ForEach(func(key string, val interface{}) bool {
if i < len(keys) {
keys[i] = key
i++
} else {
keys = append(keys, key)
}
return true
})
return keys
}
func (shard *Shard) RandomKey() string {
if shard == nil {
panic("shard is nil")
}
shard.mutex.RLock()
defer shard.mutex.RUnlock()
for key := range shard.m {
return key
}
return ""
}
func (dict *ConcurrentDict) RandomKeys(limit int) []string {
size := dict.Len()
if limit >= size {
return dict.Keys()
}
shardCount := len(dict.table)
result := make([]string, limit)
for i := 0; i < limit; {
shard := dict.getShard(uint32(rand.Intn(shardCount)))
if shard == nil {
continue
}
key := shard.RandomKey()
if key != "" {
result[i] = key
i++
}
}
return result
}
func (dict *ConcurrentDict) RandomDistinctKeys(limit int) []string {
size := dict.Len()
if limit >= size {
return dict.Keys()
}
shardCount := len(dict.table)
result := make(map[string]bool)
for len(result) < limit {
shardIndex := uint32(rand.Intn(shardCount))
shard := dict.getShard(shardIndex)
if shard == nil {
continue
}
key := shard.RandomKey()
if key != "" {
result[key] = true
}
}
arr := make([]string, limit)
i := 0
for k := range result {
arr[i] = k
i++
}
return arr
}

16
datastruct/dict/dict.go Normal file
View File

@@ -0,0 +1,16 @@
package dict
type Consumer func(key string, val interface{}) bool
type Dict interface {
Get(key string) (val interface{}, exists bool)
Len() int
Put(key string, val interface{}) (result int)
PutIfAbsent(key string, val interface{}) (result int)
PutIfExists(key string, val interface{}) (result int)
Remove(key string) (result int)
ForEach(consumer Consumer)
Keys() []string
RandomKeys(limit int) []string
RandomDistinctKeys(limit int) []string
}

244
datastruct/dict/dict_test.go Normal file
View File

@@ -0,0 +1,244 @@
package dict
import (
"strconv"
"sync"
"testing"
)
func TestPut(t *testing.T) {
d := MakeConcurrent(0)
count := 100
var wg sync.WaitGroup
wg.Add(count)
for i := 0; i < count; i++ {
go func(i int) {
// insert
key := "k" + strconv.Itoa(i)
ret := d.Put(key, i)
if ret != 1 { // insert 1
t.Error("put test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key: " + key)
}
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) + ", key: " + key)
}
} else {
_, ok := d.Get(key)
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
}
wg.Done()
}(i)
}
wg.Wait()
}
func TestPutIfAbsent(t *testing.T) {
d := MakeConcurrent(0)
count := 100
var wg sync.WaitGroup
wg.Add(count)
for i := 0; i < count; i++ {
go func(i int) {
// insert
key := "k" + strconv.Itoa(i)
ret := d.PutIfAbsent(key, i)
if ret != 1 { // insert 1
t.Error("put test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key: " + key)
}
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) +
", key: " + key)
}
} else {
_, ok := d.Get(key)
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
}
// update
ret = d.PutIfAbsent(key, i*10)
if ret != 0 { // no update
t.Error("put test failed: expected result 0, actual: " + strconv.Itoa(ret))
}
val, ok = d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) + ", key: " + key)
}
} else {
t.Error("put test failed: expected true, actual: false, key: " + key)
}
wg.Done()
}(i)
}
wg.Wait()
}
func TestPutIfExists(t *testing.T) {
d := MakeConcurrent(0)
count := 100
var wg sync.WaitGroup
wg.Add(count)
for i := 0; i < count; i++ {
go func(i int) {
// insert
key := "k" + strconv.Itoa(i)
// insert
ret := d.PutIfExists(key, i)
if ret != 0 { // insert
t.Error("put test failed: expected result 0, actual: " + strconv.Itoa(ret))
}
d.Put(key, i)
d.PutIfExists(key, 10*i)
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != 10*i {
t.Error("put test failed: expected " + strconv.Itoa(10*i) + ", actual: " + strconv.Itoa(intVal))
}
} else {
_, ok := d.Get(key)
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
}
wg.Done()
}(i)
}
wg.Wait()
}
func TestRemove(t *testing.T) {
d := MakeConcurrent(0)
// remove head node
for i := 0; i < 100; i++ {
// insert
key := "k" + strconv.Itoa(i)
d.Put(key, i)
}
for i := 0; i < 100; i++ {
key := "k" + strconv.Itoa(i)
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
} else {
t.Error("put test failed: expected true, actual: false")
}
ret := d.Remove(key)
if ret != 1 {
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key:" + key)
}
_, ok = d.Get(key)
if ok {
t.Error("remove test failed: expected true, actual false")
}
ret = d.Remove(key)
if ret != 0 {
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
}
}
// remove tail node
d = MakeConcurrent(0)
for i := 0; i < 100; i++ {
// insert
key := "k" + strconv.Itoa(i)
d.Put(key, i)
}
for i := 9; i >= 0; i-- {
key := "k" + strconv.Itoa(i)
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
} else {
t.Error("put test failed: expected true, actual: false")
}
ret := d.Remove(key)
if ret != 1 {
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret))
}
_, ok = d.Get(key)
if ok {
t.Error("remove test failed: expected true, actual false")
}
ret = d.Remove(key)
if ret != 0 {
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
}
}
// remove middle node
d = MakeConcurrent(0)
d.Put("head", 0)
for i := 0; i < 10; i++ {
// insert
key := "k" + strconv.Itoa(i)
d.Put(key, i)
}
d.Put("tail", 0)
for i := 9; i >= 0; i-- {
key := "k" + strconv.Itoa(i)
val, ok := d.Get(key)
if ok {
intVal, _ := val.(int)
if intVal != i {
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
} else {
t.Error("put test failed: expected true, actual: false")
}
ret := d.Remove(key)
if ret != 1 {
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret))
}
_, ok = d.Get(key)
if ok {
t.Error("remove test failed: expected true, actual false")
}
ret = d.Remove(key)
if ret != 0 {
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
}
}
}
func TestForEach(t *testing.T) {
d := MakeConcurrent(0)
size := 100
for i := 0; i < size; i++ {
// insert
key := "k" + strconv.Itoa(i)
d.Put(key, i)
}
i := 0
d.ForEach(func(key string, value interface{}) bool {
intVal, _ := value.(int)
expectedKey := "k" + strconv.Itoa(intVal)
if key != expectedKey {
t.Error("remove test failed: expected " + expectedKey + ", actual: " + key)
}
i++
return true
})
if i != size {
t.Error("remove test failed: expected " + strconv.Itoa(size) + ", actual: " + strconv.Itoa(i))
}
}

108
datastruct/dict/simple.go Normal file
View File

@@ -0,0 +1,108 @@
package dict
type SimpleDict struct {
m map[string]interface{}
}
func MakeSimple() *SimpleDict {
return &SimpleDict{
m: make(map[string]interface{}),
}
}
func (dict *SimpleDict) Get(key string) (val interface{}, exists bool) {
val, ok := dict.m[key]
return val, ok
}
func (dict *SimpleDict) Len() int {
if dict.m == nil {
panic("m is nil")
}
return len(dict.m)
}
func (dict *SimpleDict) Put(key string, val interface{}) (result int) {
_, existed := dict.m[key]
dict.m[key] = val
if existed {
return 0
} else {
return 1
}
}
func (dict *SimpleDict) PutIfAbsent(key string, val interface{}) (result int) {
_, existed := dict.m[key]
if existed {
return 0
} else {
dict.m[key] = val
return 1
}
}
func (dict *SimpleDict) PutIfExists(key string, val interface{}) (result int) {
_, existed := dict.m[key]
if existed {
dict.m[key] = val
return 1
} else {
return 0
}
}
func (dict *SimpleDict) Remove(key string) (result int) {
_, existed := dict.m[key]
delete(dict.m, key)
if existed {
return 1
} else {
return 0
}
}
func (dict *SimpleDict) Keys() []string {
result := make([]string, len(dict.m))
i := 0
for k := range dict.m {
result[i] = k
}
return result
}
func (dict *SimpleDict) ForEach(consumer Consumer) {
for k, v := range dict.m {
if !consumer(k, v) {
break
}
}
}
func (dict *SimpleDict) RandomKeys(limit int) []string {
result := make([]string, limit)
for i := 0; i < limit; i++ {
for k := range dict.m {
result[i] = k
break
}
}
return result
}
func (dict *SimpleDict) RandomDistinctKeys(limit int) []string {
size := limit
if size > len(dict.m) {
size = len(dict.m)
}
result := make([]string, size)
i := 0
for k := range dict.m {
if i == limit {
break
}
result[i] = k
i++
}
return result
}

326
datastruct/list/linked.go Normal file
View File

@@ -0,0 +1,326 @@
package list
import "github.com/hdt3213/godis/datastruct/utils"
type LinkedList struct {
first *node
last *node
size int
}
type node struct {
val interface{}
prev *node
next *node
}
func (list *LinkedList) Add(val interface{}) {
if list == nil {
panic("list is nil")
}
n := &node{
val: val,
}
if list.last == nil {
// empty list
list.first = n
list.last = n
} else {
n.prev = list.last
list.last.next = n
list.last = n
}
list.size++
}
func (list *LinkedList) find(index int) (n *node) {
if index < list.size/2 {
n := list.first
for i := 0; i < index; i++ {
n = n.next
}
return n
} else {
n := list.last
for i := list.size - 1; i > index; i-- {
n = n.prev
}
return n
}
}
func (list *LinkedList) Get(index int) (val interface{}) {
if list == nil {
panic("list is nil")
}
if index < 0 || index >= list.size {
panic("index out of bound")
}
return list.find(index).val
}
func (list *LinkedList) Set(index int, val interface{}) {
if list == nil {
panic("list is nil")
}
if index < 0 || index > list.size {
panic("index out of bound")
}
n := list.find(index)
n.val = val
}
func (list *LinkedList) Insert(index int, val interface{}) {
if list == nil {
panic("list is nil")
}
if index < 0 || index > list.size {
panic("index out of bound")
}
if index == list.size {
list.Add(val)
return
} else {
// list is not empty
pivot := list.find(index)
n := &node{
val: val,
prev: pivot.prev,
next: pivot,
}
if pivot.prev == nil {
list.first = n
} else {
pivot.prev.next = n
}
pivot.prev = n
list.size++
}
}
func (list *LinkedList) removeNode(n *node) {
if n.prev == nil {
list.first = n.next
} else {
n.prev.next = n.next
}
if n.next == nil {
list.last = n.prev
} else {
n.next.prev = n.prev
}
// for gc
n.prev = nil
n.next = nil
list.size--
}
func (list *LinkedList) Remove(index int) (val interface{}) {
if list == nil {
panic("list is nil")
}
if index < 0 || index >= list.size {
panic("index out of bound")
}
n := list.find(index)
list.removeNode(n)
return n.val
}
func (list *LinkedList) RemoveLast() (val interface{}) {
if list == nil {
panic("list is nil")
}
if list.last == nil {
// empty list
return nil
}
n := list.last
list.removeNode(n)
return n.val
}
func (list *LinkedList) RemoveAllByVal(val interface{}) int {
if list == nil {
panic("list is nil")
}
n := list.first
removed := 0
for n != nil {
var toRemoveNode *node
if utils.Equals(n.val, val) {
toRemoveNode = n
}
if n.next == nil {
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
break
} else {
n = n.next
}
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
}
return removed
}
/**
* remove at most `count` values of the specified value in this list
* scan from left to right
*/
func (list *LinkedList) RemoveByVal(val interface{}, count int) int {
if list == nil {
panic("list is nil")
}
n := list.first
removed := 0
for n != nil {
var toRemoveNode *node
if utils.Equals(n.val, val) {
toRemoveNode = n
}
if n.next == nil {
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
break
} else {
n = n.next
}
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
if removed == count {
break
}
}
return removed
}
func (list *LinkedList) ReverseRemoveByVal(val interface{}, count int) int {
if list == nil {
panic("list is nil")
}
n := list.last
removed := 0
for n != nil {
var toRemoveNode *node
if utils.Equals(n.val, val) {
toRemoveNode = n
}
if n.prev == nil {
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
break
} else {
n = n.prev
}
if toRemoveNode != nil {
removed++
list.removeNode(toRemoveNode)
}
if removed == count {
break
}
}
return removed
}
func (list *LinkedList) Len() int {
if list == nil {
panic("list is nil")
}
return list.size
}
func (list *LinkedList) ForEach(consumer func(int, interface{}) bool) {
if list == nil {
panic("list is nil")
}
n := list.first
i := 0
for n != nil {
goNext := consumer(i, n.val)
if !goNext || n.next == nil {
break
} else {
i++
n = n.next
}
}
}
func (list *LinkedList) Contains(val interface{}) bool {
contains := false
list.ForEach(func(i int, actual interface{}) bool {
if actual == val {
contains = true
return false
}
return true
})
return contains
}
func (list *LinkedList) Range(start int, stop int) []interface{} {
if list == nil {
panic("list is nil")
}
if start < 0 || start >= list.size {
panic("`start` out of range")
}
if stop < start || stop > list.size {
panic("`stop` out of range")
}
sliceSize := stop - start
slice := make([]interface{}, sliceSize)
n := list.first
i := 0
sliceIndex := 0
for n != nil {
if i >= start && i < stop {
slice[sliceIndex] = n.val
sliceIndex++
} else if i >= stop {
break
}
if n.next == nil {
break
} else {
i++
n = n.next
}
}
return slice
}
func Make(vals ...interface{}) *LinkedList {
list := LinkedList{}
for _, v := range vals {
list.Add(v)
}
return &list
}
func MakeBytesList(vals ...[]byte) *LinkedList {
list := LinkedList{}
for _, v := range vals {
list.Add(v)
}
return &list
}

215
datastruct/list/linked_test.go Normal file
View File

@@ -0,0 +1,215 @@
package list
import (
"strconv"
"strings"
"testing"
)
func ToString(list *LinkedList) string {
arr := make([]string, list.size)
list.ForEach(func(i int, v interface{}) bool {
integer, _ := v.(int)
arr[i] = strconv.Itoa(integer)
return true
})
return "[" + strings.Join(arr, ", ") + "]"
}
func TestAdd(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
}
list.ForEach(func(i int, v interface{}) bool {
intVal, _ := v.(int)
if intVal != i {
t.Error("add test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
return true
})
}
func TestGet(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
}
for i := 0; i < 10; i++ {
v := list.Get(i)
k, _ := v.(int)
if i != k {
t.Error("get test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(k))
}
}
}
func TestRemove(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
}
for i := 9; i >= 0; i-- {
list.Remove(i)
if i != list.Len() {
t.Error("remove test fail: expected size " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(list.Len()))
}
list.ForEach(func(i int, v interface{}) bool {
intVal, _ := v.(int)
if intVal != i {
t.Error("remove test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
return true
})
}
}
func TestRemoveVal(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
list.Add(i)
}
for index := 0; index < list.Len(); index++ {
list.RemoveAllByVal(index)
list.ForEach(func(i int, v interface{}) bool {
intVal, _ := v.(int)
if intVal == index {
t.Error("remove test fail: found " + strconv.Itoa(index) + " at index: " + strconv.Itoa(i))
}
return true
})
}
list = Make()
for i := 0; i < 10; i++ {
list.Add(i)
list.Add(i)
}
for i := 0; i < 10; i++ {
list.RemoveByVal(i, 1)
}
list.ForEach(func(i int, v interface{}) bool {
intVal, _ := v.(int)
if intVal != i {
t.Error("test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
return true
})
for i := 0; i < 10; i++ {
list.RemoveByVal(i, 1)
}
if list.Len() != 0 {
t.Error("test fail: expected 0, actual: " + strconv.Itoa(list.Len()))
}
list = Make()
for i := 0; i < 10; i++ {
list.Add(i)
list.Add(i)
}
for i := 0; i < 10; i++ {
list.ReverseRemoveByVal(i, 1)
}
list.ForEach(func(i int, v interface{}) bool {
intVal, _ := v.(int)
if intVal != i {
t.Error("test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
return true
})
for i := 0; i < 10; i++ {
list.ReverseRemoveByVal(i, 1)
}
if list.Len() != 0 {
t.Error("test fail: expected 0, actual: " + strconv.Itoa(list.Len()))
}
}
func TestInsert(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
}
for i := 0; i < 10; i++ {
list.Insert(i*2, i)
list.ForEach(func(j int, v interface{}) bool {
var expected int
if j < (i+1)*2 {
if j%2 == 0 {
expected = j / 2
} else {
expected = (j - 1) / 2
}
} else {
expected = j - i - 1
}
actual, _ := list.Get(j).(int)
if actual != expected {
t.Error("insert test fail: at i " + strconv.Itoa(i) + " expected " + strconv.Itoa(expected) + ", actual: " + strconv.Itoa(actual))
}
return true
})
for j := 0; j < list.Len(); j++ {
var expected int
if j < (i+1)*2 {
if j%2 == 0 {
expected = j / 2
} else {
expected = (j - 1) / 2
}
} else {
expected = j - i - 1
}
actual, _ := list.Get(j).(int)
if actual != expected {
t.Error("insert test fail: at i " + strconv.Itoa(i) + " expected " + strconv.Itoa(expected) + ", actual: " + strconv.Itoa(actual))
}
}
}
}
func TestRemoveLast(t *testing.T) {
list := Make()
for i := 0; i < 10; i++ {
list.Add(i)
}
for i := 9; i >= 0; i-- {
val := list.RemoveLast()
intVal, _ := val.(int)
if intVal != i {
t.Error("add test fail: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
}
}
}
func TestRange(t *testing.T) {
list := Make()
size := 10
for i := 0; i < size; i++ {
list.Add(i)
}
for start := 0; start < size; start++ {
for stop := start; stop < size; stop++ {
slice := list.Range(start, stop)
if len(slice) != stop-start {
t.Error("expected " + strconv.Itoa(stop-start) + ", get: " + strconv.Itoa(len(slice)) +
", range: [" + strconv.Itoa(start) + "," + strconv.Itoa(stop) + "]")
}
sliceIndex := 0
for i := start; i < stop; i++ {
val := slice[sliceIndex]
intVal, _ := val.(int)
if intVal != i {
t.Error("expected " + strconv.Itoa(i) + ", get: " + strconv.Itoa(intVal) +
", range: [" + strconv.Itoa(start) + "," + strconv.Itoa(stop) + "]")
}
sliceIndex++
}
}
}
}

152
datastruct/lock/lock_map.go Normal file
View File

@@ -0,0 +1,152 @@
package lock
import (
"fmt"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
)
const (
prime32 = uint32(16777619)
)
type Locks struct {
table []*sync.RWMutex
}
func Make(tableSize int) *Locks {
table := make([]*sync.RWMutex, tableSize)
for i := 0; i < tableSize; i++ {
table[i] = &sync.RWMutex{}
}
return &Locks{
table: table,
}
}
func fnv32(key string) uint32 {
hash := uint32(2166136261)
for i := 0; i < len(key); i++ {
hash *= prime32
hash ^= uint32(key[i])
}
return hash
}
func (locks *Locks) spread(hashCode uint32) uint32 {
if locks == nil {
panic("dict is nil")
}
tableSize := uint32(len(locks.table))
return (tableSize - 1) & uint32(hashCode)
}
func (locks *Locks) Lock(key string) {
index := locks.spread(fnv32(key))
mu := locks.table[index]
mu.Lock()
}
func (locks *Locks) RLock(key string) {
index := locks.spread(fnv32(key))
mu := locks.table[index]
mu.RLock()
}
func (locks *Locks) UnLock(key string) {
index := locks.spread(fnv32(key))
mu := locks.table[index]
mu.Unlock()
}
func (locks *Locks) RUnLock(key string) {
index := locks.spread(fnv32(key))
mu := locks.table[index]
mu.RUnlock()
}
func (locks *Locks) toLockIndices(keys []string, reverse bool) []uint32 {
indexMap := make(map[uint32]bool)
for _, key := range keys {
index := locks.spread(fnv32(key))
indexMap[index] = true
}
indices := make([]uint32, 0, len(indexMap))
for index := range indexMap {
indices = append(indices, index)
}
sort.Slice(indices, func(i, j int) bool {
if !reverse {
return indices[i] < indices[j]
} else {
return indices[i] > indices[j]
}
})
return indices
}
func (locks *Locks) Locks(keys ...string) {
indices := locks.toLockIndices(keys, false)
for _, index := range indices {
mu := locks.table[index]
mu.Lock()
}
}
func (locks *Locks) RLocks(keys ...string) {
indices := locks.toLockIndices(keys, false)
for _, index := range indices {
mu := locks.table[index]
mu.RLock()
}
}
func (locks *Locks) UnLocks(keys ...string) {
indices := locks.toLockIndices(keys, true)
for _, index := range indices {
mu := locks.table[index]
mu.Unlock()
}
}
func (locks *Locks) RUnLocks(keys ...string) {
indices := locks.toLockIndices(keys, true)
for _, index := range indices {
mu := locks.table[index]
mu.RUnlock()
}
}
func GoID() int {
var buf [64]byte
n := runtime.Stack(buf[:], false)
idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0]
id, err := strconv.Atoi(idField)
if err != nil {
panic(fmt.Sprintf("cannot get goroutine id: %v", err))
}
return id
}
func debug(testing.T) {
lm := Locks{}
size := 10
var wg sync.WaitGroup
wg.Add(size)
for i := 0; i < size; i++ {
go func(i int) {
lm.Locks("1", "2")
println("go: " + strconv.Itoa(GoID()))
time.Sleep(time.Second)
println("go: " + strconv.Itoa(GoID()))
lm.UnLocks("1", "2")
wg.Done()
}(i)
}
wg.Wait()
}

116
datastruct/set/set.go Normal file
View File

@@ -0,0 +1,116 @@
package set
import "github.com/hdt3213/godis/datastruct/dict"
type Set struct {
dict dict.Dict
}
func Make() *Set {
return &Set{
dict: dict.MakeSimple(),
}
}
func MakeFromVals(members ...string) *Set {
set := &Set{
dict: dict.MakeConcurrent(len(members)),
}
for _, member := range members {
set.Add(member)
}
return set
}
func (set *Set) Add(val string) int {
return set.dict.Put(val, nil)
}
func (set *Set) Remove(val string) int {
return set.dict.Remove(val)
}
func (set *Set) Has(val string) bool {
_, exists := set.dict.Get(val)
return exists
}
func (set *Set) Len() int {
return set.dict.Len()
}
func (set *Set) ToSlice() []string {
slice := make([]string, set.Len())
i := 0
set.dict.ForEach(func(key string, val interface{}) bool {
if i < len(slice) {
slice[i] = key
} else {
// set extended during traversal
slice = append(slice, key)
}
i++
return true
})
return slice
}
func (set *Set) ForEach(consumer func(member string) bool) {
set.dict.ForEach(func(key string, val interface{}) bool {
return consumer(key)
})
}
func (set *Set) Intersect(another *Set) *Set {
if set == nil {
panic("set is nil")
}
result := Make()
another.ForEach(func(member string) bool {
if set.Has(member) {
result.Add(member)
}
return true
})
return result
}
func (set *Set) Union(another *Set) *Set {
if set == nil {
panic("set is nil")
}
result := Make()
another.ForEach(func(member string) bool {
result.Add(member)
return true
})
set.ForEach(func(member string) bool {
result.Add(member)
return true
})
return result
}
func (set *Set) Diff(another *Set) *Set {
if set == nil {
panic("set is nil")
}
result := Make()
set.ForEach(func(member string) bool {
if !another.Has(member) {
result.Add(member)
}
return true
})
return result
}
func (set *Set) RandomMembers(limit int) []string {
return set.dict.RandomKeys(limit)
}
func (set *Set) RandomDistinctMembers(limit int) []string {
return set.dict.RandomDistinctKeys(limit)
}

32
datastruct/set/set_test.go Normal file
View File

@@ -0,0 +1,32 @@
package set
import (
"strconv"
"testing"
)
func TestSet(t *testing.T) {
size := 10
set := Make()
for i := 0; i < size; i++ {
set.Add(strconv.Itoa(i))
}
for i := 0; i < size; i++ {
ok := set.Has(strconv.Itoa(i))
if !ok {
t.Error("expected true actual false, key: " + strconv.Itoa(i))
}
}
for i := 0; i < size; i++ {
ok := set.Remove(strconv.Itoa(i))
if ok != 1 {
t.Error("expected true actual false, key: " + strconv.Itoa(i))
}
}
for i := 0; i < size; i++ {
ok := set.Has(strconv.Itoa(i))
if ok {
t.Error("expected false actual true, key: " + strconv.Itoa(i))
}
}
}

91
datastruct/sortedset/border.go Normal file
View File

@@ -0,0 +1,91 @@
package sortedset
import (
"errors"
"strconv"
)
/*
* ScoreBorder is a struct represents `min` `max` parameter of redis command `ZRANGEBYSCORE`
* can accept:
* int or float value, such as 2.718, 2, -2.718, -2 ...
* exclusive int or float value, such as (2.718, (2, (-2.718, (-2 ...
* infinity: +inf, -inf， inf(same as +inf)
*/
const (
negativeInf int8 = -1
positiveInf int8 = 1
)
type ScoreBorder struct {
Inf int8
Value float64
Exclude bool
}
// if max.greater(score) then the score is within the upper border
// do not use min.greater()
func (border *ScoreBorder) greater(value float64) bool {
if border.Inf == negativeInf {
return false
} else if border.Inf == positiveInf {
return true
}
if border.Exclude {
return border.Value > value
} else {
return border.Value >= value
}
}
func (border *ScoreBorder) less(value float64) bool {
if border.Inf == negativeInf {
return true
} else if border.Inf == positiveInf {
return false
}
if border.Exclude {
return border.Value < value
} else {
return border.Value <= value
}
}
var positiveInfBorder = &ScoreBorder{
Inf: positiveInf,
}
var negativeInfBorder = &ScoreBorder{
Inf: negativeInf,
}
func ParseScoreBorder(s string) (*ScoreBorder, error) {
if s == "inf" || s == "+inf" {
return positiveInfBorder, nil
}
if s == "-inf" {
return negativeInfBorder, nil
}
if s[0] == '(' {
value, err := strconv.ParseFloat(s[1:], 64)
if err != nil {
return nil, errors.New("ERR min or max is not a float")
}
return &ScoreBorder{
Inf: 0,
Value: value,
Exclude: true,
}, nil
} else {
value, err := strconv.ParseFloat(s, 64)
if err != nil {
return nil, errors.New("ERR min or max is not a float")
}
return &ScoreBorder{
Inf: 0,
Value: value,
Exclude: false,
}, nil
}
}

344
datastruct/sortedset/skiplist.go Normal file
View File

@@ -0,0 +1,344 @@
package sortedset
import "math/rand"
const (
maxLevel = 16
)
type Element struct {
Member string
Score float64
}
// level aspect of a Node
type Level struct {
forward *Node // forward node has greater score
span int64
}
type Node struct {
Element
backward *Node
level []*Level // level[0] is base level
}
type skiplist struct {
header *Node
tail *Node
length int64
level int16
}
func makeNode(level int16, score float64, member string) *Node {
n := &Node{
Element: Element{
Score: score,
Member: member,
},
level: make([]*Level, level),
}
for i := range n.level {
n.level[i] = new(Level)
}
return n
}
func makeSkiplist() *skiplist {
return &skiplist{
level: 1,
header: makeNode(maxLevel, 0, ""),
}
}
func randomLevel() int16 {
level := int16(1)
for float32(rand.Int31()&0xFFFF) < (0.25 * 0xFFFF) {
level++
}
if level < maxLevel {
return level
}
return maxLevel
}
func (skiplist *skiplist) insert(member string, score float64) *Node {
update := make([]*Node, maxLevel) // link new node with node in `update`
rank := make([]int64, maxLevel)
// find position to insert
node := skiplist.header
for i := skiplist.level - 1; i >= 0; i-- {
if i == skiplist.level-1 {
rank[i] = 0
} else {
rank[i] = rank[i+1] // store rank that is crossed to reach the insert position
}
if node.level[i] != nil {
// traverse the skip list
for node.level[i].forward != nil &&
(node.level[i].forward.Score < score ||
(node.level[i].forward.Score == score && node.level[i].forward.Member < member)) { // same score, different key
rank[i] += node.level[i].span
node = node.level[i].forward
}
}
update[i] = node
}
level := randomLevel()
// extend skiplist level
if level > skiplist.level {
for i := skiplist.level; i < level; i++ {
rank[i] = 0
update[i] = skiplist.header
update[i].level[i].span = skiplist.length
}
skiplist.level = level
}
// make node and link into skiplist
node = makeNode(level, score, member)
for i := int16(0); i < level; i++ {
node.level[i].forward = update[i].level[i].forward
update[i].level[i].forward = node
// update span covered by update[i] as node is inserted here
node.level[i].span = update[i].level[i].span - (rank[0] - rank[i])
update[i].level[i].span = (rank[0] - rank[i]) + 1
}
// increment span for untouched levels
for i := level; i < skiplist.level; i++ {
update[i].level[i].span++
}
// set backward node
if update[0] == skiplist.header {
node.backward = nil
} else {
node.backward = update[0]
}
if node.level[0].forward != nil {
node.level[0].forward.backward = node
} else {
skiplist.tail = node
}
skiplist.length++
return node
}
/*
* param node: node to delete
* param update: backward node (of target)
*/
func (skiplist *skiplist) removeNode(node *Node, update []*Node) {
for i := int16(0); i < skiplist.level; i++ {
if update[i].level[i].forward == node {
update[i].level[i].span += node.level[i].span - 1
update[i].level[i].forward = node.level[i].forward
} else {
update[i].level[i].span--
}
}
if node.level[0].forward != nil {
node.level[0].forward.backward = node.backward
} else {
skiplist.tail = node.backward
}
for skiplist.level > 1 && skiplist.header.level[skiplist.level-1].forward == nil {
skiplist.level--
}
skiplist.length--
}
/*
* return: has found and removed node
*/
func (skiplist *skiplist) remove(member string, score float64) bool {
/*
* find backward node (of target) or last node of each level
* their forward need to be updated
*/
update := make([]*Node, maxLevel)
node := skiplist.header
for i := skiplist.level - 1; i >= 0; i-- {
for node.level[i].forward != nil &&
(node.level[i].forward.Score < score ||
(node.level[i].forward.Score == score &&
node.level[i].forward.Member < member)) {
node = node.level[i].forward
}
update[i] = node
}
node = node.level[0].forward
if node != nil && score == node.Score && node.Member == member {
skiplist.removeNode(node, update)
// free x
return true
}
return false
}
/*
* return: 1 based rank, 0 means member not found
*/
func (skiplist *skiplist) getRank(member string, score float64) int64 {
var rank int64 = 0
x := skiplist.header
for i := skiplist.level - 1; i >= 0; i-- {
for x.level[i].forward != nil &&
(x.level[i].forward.Score < score ||
(x.level[i].forward.Score == score &&
x.level[i].forward.Member <= member)) {
rank += x.level[i].span
x = x.level[i].forward
}
/* x might be equal to zsl->header, so test if obj is non-NULL */
if x.Member == member {
return rank
}
}
return 0
}
/*
* 1-based rank
*/
func (skiplist *skiplist) getByRank(rank int64) *Node {
var i int64 = 0
n := skiplist.header
// scan from top level
for level := skiplist.level - 1; level >= 0; level-- {
for n.level[level].forward != nil && (i+n.level[level].span) <= rank {
i += n.level[level].span
n = n.level[level].forward
}
if i == rank {
return n
}
}
return nil
}
func (skiplist *skiplist) hasInRange(min *ScoreBorder, max *ScoreBorder) bool {
// min & max = empty
if min.Value > max.Value || (min.Value == max.Value && (min.Exclude || max.Exclude)) {
return false
}
// min > tail
n := skiplist.tail
if n == nil || !min.less(n.Score) {
return false
}
// max < head
n = skiplist.header.level[0].forward
if n == nil || !max.greater(n.Score) {
return false
}
return true
}
func (skiplist *skiplist) getFirstInScoreRange(min *ScoreBorder, max *ScoreBorder) *Node {
if !skiplist.hasInRange(min, max) {
return nil
}
n := skiplist.header
// scan from top level
for level := skiplist.level - 1; level >= 0; level-- {
// if forward is not in range than move forward
for n.level[level].forward != nil && !min.less(n.level[level].forward.Score) {
n = n.level[level].forward
}
}
/* This is an inner range, so the next node cannot be NULL. */
n = n.level[0].forward
if !max.greater(n.Score) {
return nil
}
return n
}
func (skiplist *skiplist) getLastInScoreRange(min *ScoreBorder, max *ScoreBorder) *Node {
if !skiplist.hasInRange(min, max) {
return nil
}
n := skiplist.header
// scan from top level
for level := skiplist.level - 1; level >= 0; level-- {
for n.level[level].forward != nil && max.greater(n.level[level].forward.Score) {
n = n.level[level].forward
}
}
if !min.less(n.Score) {
return nil
}
return n
}
/*
* return removed elements
*/
func (skiplist *skiplist) RemoveRangeByScore(min *ScoreBorder, max *ScoreBorder) (removed []*Element) {
update := make([]*Node, maxLevel)
removed = make([]*Element, 0)
// find backward nodes (of target range) or last node of each level
node := skiplist.header
for i := skiplist.level - 1; i >= 0; i-- {
for node.level[i].forward != nil {
if min.less(node.level[i].forward.Score) { // already in range
break
}
node = node.level[i].forward
}
update[i] = node
}
// node is the first one within range
node = node.level[0].forward
// remove nodes in range
for node != nil {
if !max.greater(node.Score) { // already out of range
break
}
next := node.level[0].forward
removedElement := node.Element
removed = append(removed, &removedElement)
skiplist.removeNode(node, update)
node = next
}
return removed
}
// 1-based rank, including start, exclude stop
func (skiplist *skiplist) RemoveRangeByRank(start int64, stop int64) (removed []*Element) {
var i int64 = 0 // rank of iterator
update := make([]*Node, maxLevel)
removed = make([]*Element, 0)
// scan from top level
node := skiplist.header
for level := skiplist.level - 1; level >= 0; level-- {
for node.level[level].forward != nil && (i+node.level[level].span) < start {
i += node.level[level].span
node = node.level[level].forward
}
update[level] = node
}
i++
node = node.level[0].forward // first node in range
// remove nodes in range
for node != nil && i < stop {
next := node.level[0].forward
removedElement := node.Element
removed = append(removed, &removedElement)
skiplist.removeNode(node, update)
node = next
i++
}
return removed
}

226
datastruct/sortedset/sortedset.go Normal file
View File

@@ -0,0 +1,226 @@
package sortedset
import (
"strconv"
)
type SortedSet struct {
dict map[string]*Element
skiplist *skiplist
}
func Make() *SortedSet {
return &SortedSet{
dict: make(map[string]*Element),
skiplist: makeSkiplist(),
}
}
/*
* return: has inserted new node
*/
func (sortedSet *SortedSet) Add(member string, score float64) bool {
element, ok := sortedSet.dict[member]
sortedSet.dict[member] = &Element{
Member: member,
Score: score,
}
if ok {
if score != element.Score {
sortedSet.skiplist.remove(member, score)
sortedSet.skiplist.insert(member, score)
}
return false
} else {
sortedSet.skiplist.insert(member, score)
return true
}
}
func (sortedSet *SortedSet) Len() int64 {
return int64(len(sortedSet.dict))
}
func (sortedSet *SortedSet) Get(member string) (element *Element, ok bool) {
element, ok = sortedSet.dict[member]
if !ok {
return nil, false
}
return element, true
}
func (sortedSet *SortedSet) Remove(member string) bool {
v, ok := sortedSet.dict[member]
if ok {
sortedSet.skiplist.remove(member, v.Score)
delete(sortedSet.dict, member)
return true
}
return false
}
/**
* get 0-based rank
*/
func (sortedSet *SortedSet) GetRank(member string, desc bool) (rank int64) {
element, ok := sortedSet.dict[member]
if !ok {
return -1
}
r := sortedSet.skiplist.getRank(member, element.Score)
if desc {
r = sortedSet.skiplist.length - r
} else {
r--
}
return r
}
/**
* traverse [start, stop), 0-based rank
*/
func (sortedSet *SortedSet) ForEach(start int64, stop int64, desc bool, consumer func(element *Element) bool) {
size := int64(sortedSet.Len())
if start < 0 || start >= size {
panic("illegal start " + strconv.FormatInt(start, 10))
}
if stop < start || stop > size {
panic("illegal end " + strconv.FormatInt(stop, 10))
}
// find start node
var node *Node
if desc {
node = sortedSet.skiplist.tail
if start > 0 {
node = sortedSet.skiplist.getByRank(int64(size - start))
}
} else {
node = sortedSet.skiplist.header.level[0].forward
if start > 0 {
node = sortedSet.skiplist.getByRank(int64(start + 1))
}
}
sliceSize := int(stop - start)
for i := 0; i < sliceSize; i++ {
if !consumer(&node.Element) {
break
}
if desc {
node = node.backward
} else {
node = node.level[0].forward
}
}
}
/**
* return [start, stop), 0-based rank
* assert start in [0, size), stop in [start, size]
*/
func (sortedSet *SortedSet) Range(start int64, stop int64, desc bool) []*Element {
sliceSize := int(stop - start)
slice := make([]*Element, sliceSize)
i := 0
sortedSet.ForEach(start, stop, desc, func(element *Element) bool {
slice[i] = element
i++
return true
})
return slice
}
func (sortedSet *SortedSet) Count(min *ScoreBorder, max *ScoreBorder) int64 {
var i int64 = 0
// ascending order
sortedSet.ForEach(0, sortedSet.Len(), false, func(element *Element) bool {
gtMin := min.less(element.Score) // greater than min
if !gtMin {
// has not into range, continue foreach
return true
}
ltMax := max.greater(element.Score) // less than max
if !ltMax {
// break through score border, break foreach
return false
}
// gtMin && ltMax
i++
return true
})
return i
}
func (sortedSet *SortedSet) ForEachByScore(min *ScoreBorder, max *ScoreBorder, offset int64, limit int64, desc bool, consumer func(element *Element) bool) {
// find start node
var node *Node
if desc {
node = sortedSet.skiplist.getLastInScoreRange(min, max)
} else {
node = sortedSet.skiplist.getFirstInScoreRange(min, max)
}
for node != nil && offset > 0 {
if desc {
node = node.backward
} else {
node = node.level[0].forward
}
offset--
}
// A negative limit returns all elements from the offset
for i := 0; (i < int(limit) || limit < 0) && node != nil; i++ {
if !consumer(&node.Element) {
break
}
if desc {
node = node.backward
} else {
node = node.level[0].forward
}
if node == nil {
break
}
gtMin := min.less(node.Element.Score) // greater than min
ltMax := max.greater(node.Element.Score)
if !gtMin || !ltMax {
break // break through score border
}
}
}
/*
* param limit: <0 means no limit
*/
func (sortedSet *SortedSet) RangeByScore(min *ScoreBorder, max *ScoreBorder, offset int64, limit int64, desc bool) []*Element {
if limit == 0 || offset < 0 {
return make([]*Element, 0)
}
slice := make([]*Element, 0)
sortedSet.ForEachByScore(min, max, offset, limit, desc, func(element *Element) bool {
slice = append(slice, element)
return true
})
return slice
}
func (sortedSet *SortedSet) RemoveByScore(min *ScoreBorder, max *ScoreBorder) int64 {
removed := sortedSet.skiplist.RemoveRangeByScore(min, max)
for _, element := range removed {
delete(sortedSet.dict, element.Member)
}
return int64(len(removed))
}
/*
* 0-based rank, [start, stop)
*/
func (sortedSet *SortedSet) RemoveByRank(start int64, stop int64) int64 {
removed := sortedSet.skiplist.RemoveRangeByRank(start+1, stop+1)
for _, element := range removed {
delete(sortedSet.dict, element.Member)
}
return int64(len(removed))
}

28
datastruct/utils/utils.go Normal file
View File

@@ -0,0 +1,28 @@
package utils
func Equals(a interface{}, b interface{}) bool {
sliceA, okA := a.([]byte)
sliceB, okB := b.([]byte)
if okA && okB {
return BytesEquals(sliceA, sliceB)
}
return a == b
}
func BytesEquals(a []byte, b []byte) bool {
if (a == nil && b != nil) || (a != nil && b == nil) {
return false
}
if len(a) != len(b) {
return false
}
size := len(a)
for i := 0; i < size; i++ {
av := a[i]
bv := b[i]
if av != bv {
return false
}
}
return true
}