mirror of
https://github.com/HDT3213/godis.git
synced 2025-11-03 11:02:17 +08:00
refactor project structure
This commit is contained in:
hdt3213
280
datastruct/dict/concurrent.go
Normal file
280
datastruct/dict/concurrent.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package dict
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type ConcurrentDict struct {
|
||||
table []*Shard
|
||||
count int32
|
||||
}
|
||||
|
||||
type Shard struct {
|
||||
m map[string]interface{}
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
func computeCapacity(param int) (size int) {
|
||||
if param <= 16 {
|
||||
return 16
|
||||
}
|
||||
n := param - 1
|
||||
n |= n >> 1
|
||||
n |= n >> 2
|
||||
n |= n >> 4
|
||||
n |= n >> 8
|
||||
n |= n >> 16
|
||||
if n < 0 {
|
||||
return math.MaxInt32
|
||||
} else {
|
||||
return int(n + 1)
|
||||
}
|
||||
}
|
||||
|
||||
func MakeConcurrent(shardCount int) *ConcurrentDict {
|
||||
shardCount = computeCapacity(shardCount)
|
||||
table := make([]*Shard, shardCount)
|
||||
for i := 0; i < shardCount; i++ {
|
||||
table[i] = &Shard{
|
||||
m: make(map[string]interface{}),
|
||||
}
|
||||
}
|
||||
d := &ConcurrentDict{
|
||||
count: 0,
|
||||
table: table,
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
const prime32 = uint32(16777619)
|
||||
|
||||
func fnv32(key string) uint32 {
|
||||
hash := uint32(2166136261)
|
||||
for i := 0; i < len(key); i++ {
|
||||
hash *= prime32
|
||||
hash ^= uint32(key[i])
|
||||
}
|
||||
return hash
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) spread(hashCode uint32) uint32 {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
tableSize := uint32(len(dict.table))
|
||||
return (tableSize - 1) & uint32(hashCode)
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) getShard(index uint32) *Shard {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
return dict.table[index]
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) Get(key string) (val interface{}, exists bool) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
hashCode := fnv32(key)
|
||||
index := dict.spread(hashCode)
|
||||
shard := dict.getShard(index)
|
||||
shard.mutex.RLock()
|
||||
defer shard.mutex.RUnlock()
|
||||
val, exists = shard.m[key]
|
||||
return
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) Len() int {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
return int(atomic.LoadInt32(&dict.count))
|
||||
}
|
||||
|
||||
// return the number of new inserted key-value
|
||||
func (dict *ConcurrentDict) Put(key string, val interface{}) (result int) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
hashCode := fnv32(key)
|
||||
index := dict.spread(hashCode)
|
||||
shard := dict.getShard(index)
|
||||
shard.mutex.Lock()
|
||||
defer shard.mutex.Unlock()
|
||||
|
||||
if _, ok := shard.m[key]; ok {
|
||||
shard.m[key] = val
|
||||
return 0
|
||||
} else {
|
||||
shard.m[key] = val
|
||||
dict.addCount()
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
// return the number of updated key-value
|
||||
func (dict *ConcurrentDict) PutIfAbsent(key string, val interface{}) (result int) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
hashCode := fnv32(key)
|
||||
index := dict.spread(hashCode)
|
||||
shard := dict.getShard(index)
|
||||
shard.mutex.Lock()
|
||||
defer shard.mutex.Unlock()
|
||||
|
||||
if _, ok := shard.m[key]; ok {
|
||||
return 0
|
||||
} else {
|
||||
shard.m[key] = val
|
||||
dict.addCount()
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
// return the number of updated key-value
|
||||
func (dict *ConcurrentDict) PutIfExists(key string, val interface{}) (result int) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
hashCode := fnv32(key)
|
||||
index := dict.spread(hashCode)
|
||||
shard := dict.getShard(index)
|
||||
shard.mutex.Lock()
|
||||
defer shard.mutex.Unlock()
|
||||
|
||||
if _, ok := shard.m[key]; ok {
|
||||
shard.m[key] = val
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// return the number of deleted key-value
|
||||
func (dict *ConcurrentDict) Remove(key string) (result int) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
hashCode := fnv32(key)
|
||||
index := dict.spread(hashCode)
|
||||
shard := dict.getShard(index)
|
||||
shard.mutex.Lock()
|
||||
defer shard.mutex.Unlock()
|
||||
|
||||
if _, ok := shard.m[key]; ok {
|
||||
delete(shard.m, key)
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) addCount() int32 {
|
||||
return atomic.AddInt32(&dict.count, 1)
|
||||
}
|
||||
|
||||
/*
|
||||
* may not contains new entry inserted during traversal
|
||||
*/
|
||||
func (dict *ConcurrentDict) ForEach(consumer Consumer) {
|
||||
if dict == nil {
|
||||
panic("dict is nil")
|
||||
}
|
||||
|
||||
for _, shard := range dict.table {
|
||||
shard.mutex.RLock()
|
||||
func() {
|
||||
defer shard.mutex.RUnlock()
|
||||
for key, value := range shard.m {
|
||||
continues := consumer(key, value)
|
||||
if !continues {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) Keys() []string {
|
||||
keys := make([]string, dict.Len())
|
||||
i := 0
|
||||
dict.ForEach(func(key string, val interface{}) bool {
|
||||
if i < len(keys) {
|
||||
keys[i] = key
|
||||
i++
|
||||
} else {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return keys
|
||||
}
|
||||
|
||||
func (shard *Shard) RandomKey() string {
|
||||
if shard == nil {
|
||||
panic("shard is nil")
|
||||
}
|
||||
shard.mutex.RLock()
|
||||
defer shard.mutex.RUnlock()
|
||||
|
||||
for key := range shard.m {
|
||||
return key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) RandomKeys(limit int) []string {
|
||||
size := dict.Len()
|
||||
if limit >= size {
|
||||
return dict.Keys()
|
||||
}
|
||||
shardCount := len(dict.table)
|
||||
|
||||
result := make([]string, limit)
|
||||
for i := 0; i < limit; {
|
||||
shard := dict.getShard(uint32(rand.Intn(shardCount)))
|
||||
if shard == nil {
|
||||
continue
|
||||
}
|
||||
key := shard.RandomKey()
|
||||
if key != "" {
|
||||
result[i] = key
|
||||
i++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (dict *ConcurrentDict) RandomDistinctKeys(limit int) []string {
|
||||
size := dict.Len()
|
||||
if limit >= size {
|
||||
return dict.Keys()
|
||||
}
|
||||
|
||||
shardCount := len(dict.table)
|
||||
result := make(map[string]bool)
|
||||
for len(result) < limit {
|
||||
shardIndex := uint32(rand.Intn(shardCount))
|
||||
shard := dict.getShard(shardIndex)
|
||||
if shard == nil {
|
||||
continue
|
||||
}
|
||||
key := shard.RandomKey()
|
||||
if key != "" {
|
||||
result[key] = true
|
||||
}
|
||||
}
|
||||
arr := make([]string, limit)
|
||||
i := 0
|
||||
for k := range result {
|
||||
arr[i] = k
|
||||
i++
|
||||
}
|
||||
return arr
|
||||
}
|
||||
16
datastruct/dict/dict.go
Normal file
16
datastruct/dict/dict.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package dict
|
||||
|
||||
type Consumer func(key string, val interface{}) bool
|
||||
|
||||
type Dict interface {
|
||||
Get(key string) (val interface{}, exists bool)
|
||||
Len() int
|
||||
Put(key string, val interface{}) (result int)
|
||||
PutIfAbsent(key string, val interface{}) (result int)
|
||||
PutIfExists(key string, val interface{}) (result int)
|
||||
Remove(key string) (result int)
|
||||
ForEach(consumer Consumer)
|
||||
Keys() []string
|
||||
RandomKeys(limit int) []string
|
||||
RandomDistinctKeys(limit int) []string
|
||||
}
|
||||
244
datastruct/dict/dict_test.go
Normal file
244
datastruct/dict/dict_test.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package dict
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPut(t *testing.T) {
|
||||
d := MakeConcurrent(0)
|
||||
count := 100
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func(i int) {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
ret := d.Put(key, i)
|
||||
if ret != 1 { // insert 1
|
||||
t.Error("put test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key: " + key)
|
||||
}
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) + ", key: " + key)
|
||||
}
|
||||
} else {
|
||||
_, ok := d.Get(key)
|
||||
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestPutIfAbsent(t *testing.T) {
|
||||
d := MakeConcurrent(0)
|
||||
count := 100
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func(i int) {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
ret := d.PutIfAbsent(key, i)
|
||||
if ret != 1 { // insert 1
|
||||
t.Error("put test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key: " + key)
|
||||
}
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) +
|
||||
", key: " + key)
|
||||
}
|
||||
} else {
|
||||
_, ok := d.Get(key)
|
||||
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
|
||||
}
|
||||
|
||||
// update
|
||||
ret = d.PutIfAbsent(key, i*10)
|
||||
if ret != 0 { // no update
|
||||
t.Error("put test failed: expected result 0, actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
val, ok = d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal) + ", key: " + key)
|
||||
}
|
||||
} else {
|
||||
t.Error("put test failed: expected true, actual: false, key: " + key)
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestPutIfExists(t *testing.T) {
|
||||
d := MakeConcurrent(0)
|
||||
count := 100
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func(i int) {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
// insert
|
||||
ret := d.PutIfExists(key, i)
|
||||
if ret != 0 { // insert
|
||||
t.Error("put test failed: expected result 0, actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
|
||||
d.Put(key, i)
|
||||
d.PutIfExists(key, 10*i)
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != 10*i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(10*i) + ", actual: " + strconv.Itoa(intVal))
|
||||
}
|
||||
} else {
|
||||
_, ok := d.Get(key)
|
||||
t.Error("put test failed: expected true, actual: false, key: " + key + ", retry: " + strconv.FormatBool(ok))
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestRemove(t *testing.T) {
|
||||
d := MakeConcurrent(0)
|
||||
|
||||
// remove head node
|
||||
for i := 0; i < 100; i++ {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
d.Put(key, i)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
key := "k" + strconv.Itoa(i)
|
||||
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
|
||||
}
|
||||
} else {
|
||||
t.Error("put test failed: expected true, actual: false")
|
||||
}
|
||||
|
||||
ret := d.Remove(key)
|
||||
if ret != 1 {
|
||||
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret) + ", key:" + key)
|
||||
}
|
||||
_, ok = d.Get(key)
|
||||
if ok {
|
||||
t.Error("remove test failed: expected true, actual false")
|
||||
}
|
||||
ret = d.Remove(key)
|
||||
if ret != 0 {
|
||||
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
}
|
||||
|
||||
// remove tail node
|
||||
d = MakeConcurrent(0)
|
||||
for i := 0; i < 100; i++ {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
d.Put(key, i)
|
||||
}
|
||||
for i := 9; i >= 0; i-- {
|
||||
key := "k" + strconv.Itoa(i)
|
||||
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
|
||||
}
|
||||
} else {
|
||||
t.Error("put test failed: expected true, actual: false")
|
||||
}
|
||||
|
||||
ret := d.Remove(key)
|
||||
if ret != 1 {
|
||||
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
_, ok = d.Get(key)
|
||||
if ok {
|
||||
t.Error("remove test failed: expected true, actual false")
|
||||
}
|
||||
ret = d.Remove(key)
|
||||
if ret != 0 {
|
||||
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
}
|
||||
|
||||
// remove middle node
|
||||
d = MakeConcurrent(0)
|
||||
d.Put("head", 0)
|
||||
for i := 0; i < 10; i++ {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
d.Put(key, i)
|
||||
}
|
||||
d.Put("tail", 0)
|
||||
for i := 9; i >= 0; i-- {
|
||||
key := "k" + strconv.Itoa(i)
|
||||
|
||||
val, ok := d.Get(key)
|
||||
if ok {
|
||||
intVal, _ := val.(int)
|
||||
if intVal != i {
|
||||
t.Error("put test failed: expected " + strconv.Itoa(i) + ", actual: " + strconv.Itoa(intVal))
|
||||
}
|
||||
} else {
|
||||
t.Error("put test failed: expected true, actual: false")
|
||||
}
|
||||
|
||||
ret := d.Remove(key)
|
||||
if ret != 1 {
|
||||
t.Error("remove test failed: expected result 1, actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
_, ok = d.Get(key)
|
||||
if ok {
|
||||
t.Error("remove test failed: expected true, actual false")
|
||||
}
|
||||
ret = d.Remove(key)
|
||||
if ret != 0 {
|
||||
t.Error("remove test failed: expected result 0 actual: " + strconv.Itoa(ret))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEach(t *testing.T) {
|
||||
d := MakeConcurrent(0)
|
||||
size := 100
|
||||
for i := 0; i < size; i++ {
|
||||
// insert
|
||||
key := "k" + strconv.Itoa(i)
|
||||
d.Put(key, i)
|
||||
}
|
||||
i := 0
|
||||
d.ForEach(func(key string, value interface{}) bool {
|
||||
intVal, _ := value.(int)
|
||||
expectedKey := "k" + strconv.Itoa(intVal)
|
||||
if key != expectedKey {
|
||||
t.Error("remove test failed: expected " + expectedKey + ", actual: " + key)
|
||||
}
|
||||
i++
|
||||
return true
|
||||
})
|
||||
if i != size {
|
||||
t.Error("remove test failed: expected " + strconv.Itoa(size) + ", actual: " + strconv.Itoa(i))
|
||||
}
|
||||
}
|
||||
108
datastruct/dict/simple.go
Normal file
108
datastruct/dict/simple.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package dict
|
||||
|
||||
type SimpleDict struct {
|
||||
m map[string]interface{}
|
||||
}
|
||||
|
||||
func MakeSimple() *SimpleDict {
|
||||
return &SimpleDict{
|
||||
m: make(map[string]interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) Get(key string) (val interface{}, exists bool) {
|
||||
val, ok := dict.m[key]
|
||||
return val, ok
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) Len() int {
|
||||
if dict.m == nil {
|
||||
panic("m is nil")
|
||||
}
|
||||
return len(dict.m)
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) Put(key string, val interface{}) (result int) {
|
||||
_, existed := dict.m[key]
|
||||
dict.m[key] = val
|
||||
if existed {
|
||||
return 0
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) PutIfAbsent(key string, val interface{}) (result int) {
|
||||
_, existed := dict.m[key]
|
||||
if existed {
|
||||
return 0
|
||||
} else {
|
||||
dict.m[key] = val
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) PutIfExists(key string, val interface{}) (result int) {
|
||||
_, existed := dict.m[key]
|
||||
if existed {
|
||||
dict.m[key] = val
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) Remove(key string) (result int) {
|
||||
_, existed := dict.m[key]
|
||||
delete(dict.m, key)
|
||||
if existed {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) Keys() []string {
|
||||
result := make([]string, len(dict.m))
|
||||
i := 0
|
||||
for k := range dict.m {
|
||||
result[i] = k
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) ForEach(consumer Consumer) {
|
||||
for k, v := range dict.m {
|
||||
if !consumer(k, v) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) RandomKeys(limit int) []string {
|
||||
result := make([]string, limit)
|
||||
for i := 0; i < limit; i++ {
|
||||
for k := range dict.m {
|
||||
result[i] = k
|
||||
break
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (dict *SimpleDict) RandomDistinctKeys(limit int) []string {
|
||||
size := limit
|
||||
if size > len(dict.m) {
|
||||
size = len(dict.m)
|
||||
}
|
||||
result := make([]string, size)
|
||||
i := 0
|
||||
for k := range dict.m {
|
||||
if i == limit {
|
||||
break
|
||||
}
|
||||
result[i] = k
|
||||
i++
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user