apps/redis-go
0
0
Fork 0
mirror of https://github.com/HDT3213/godis.git synced 2025-10-05 00:42:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

raft cluster

Browse Source 
wip: raft does not care about migrating

wip: optimize code

wip: raft election

wip

wip: fix raft leader missing log entries

wip

fix a dead lock

batch set slot route

wip: raft persist

wip

refactor cluster suite

remove relay

rename relay2

refactor: allow customizing client factory

test raft

refactor re-balance

avoid errors caused by inconsistent status on follower nodes during raft commits

test raft election
This commit is contained in:
finley
2023-01-02 21:27:06 +08:00
parent df672d4c92
commit bf7f628810
54 changed files with 3122 additions and 703 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
cluster/cluster.go
View File

@@ -6,100 +6,112 @@ import (
"github.com/hdt3213/godis/config"
database2 "github.com/hdt3213/godis/database"
"github.com/hdt3213/godis/datastruct/dict"
"github.com/hdt3213/godis/datastruct/set"
"github.com/hdt3213/godis/interface/database"
"github.com/hdt3213/godis/interface/redis"
"github.com/hdt3213/godis/lib/consistenthash"
"github.com/hdt3213/godis/lib/idgenerator"
"github.com/hdt3213/godis/lib/logger"
"github.com/hdt3213/godis/lib/pool"
"github.com/hdt3213/godis/lib/utils"
"github.com/hdt3213/godis/redis/client"
"github.com/hdt3213/godis/redis/parser"
"github.com/hdt3213/godis/redis/protocol"
"os"
"path"
"runtime/debug"
"strings"
"sync"
)
type PeerPicker interface {
AddNode(keys ...string)
PickNode(key string) string
}
// Cluster represents a node of godis cluster
// it holds part of data and coordinates other nodes to finish transactions
type Cluster struct {
self string
nodes []string
peerPicker PeerPicker
nodeConnections map[string]*pool.Pool
self string
addr string
db database.DBEngine
transactions *dict.SimpleDict // id -> Transaction
topology topology
slotMu sync.RWMutex
slots map[uint32]*hostSlot
idGenerator *idgenerator.IDGenerator
idGenerator *idgenerator.IDGenerator
// use a variable to allow injecting stub for testing
relayImpl func(cluster *Cluster, node string, c redis.Connection, cmdLine CmdLine) redis.Reply
clientFactory clientFactory
}
type peerClient interface {
Send(args [][]byte) redis.Reply
}
type peerStream interface {
Stream() <-chan *parser.Payload
Close() error
}
type clientFactory interface {
GetPeerClient(peerAddr string) (peerClient, error)
ReturnPeerClient(peerAddr string, peerClient peerClient) error
NewStream(peerAddr string, cmdLine CmdLine) (peerStream, error)
Close() error
}
type relayFunc func(cluster *Cluster, node string, c redis.Connection, cmdLine CmdLine) redis.Reply
const (
replicas = 4
slotStateHost = iota
slotStateImporting
slotStateMovingOut
)
// hostSlot stores status of host which hosted by current node
type hostSlot struct {
state uint32
mu sync.RWMutex
// OldNodeID is the node which is moving out this slot
// only valid during slot is importing
oldNodeID string
// OldNodeID is the node which is importing this slot
// only valid during slot is moving out
newNodeID string
/* importedKeys stores imported keys during migrating progress
* While this slot is migrating, if importedKeys does not have the given key, then current node will import key before execute commands
*
* In a migrating slot, the slot on the old node is immutable, we only delete a key in the new node.
* Therefore, we must distinguish between non-migrated key and deleted key.
* Even if a key has been deleted, it still exists in importedKeys, so we can distinguish between non-migrated and deleted.
*/
importedKeys *set.Set
// keys stores all keys in this slot
// Cluster.makeInsertCallback and Cluster.makeDeleteCallback will keep keys up to time
keys *set.Set
}
// if only one node involved in a transaction, just execute the command don't apply tcc procedure
var allowFastTransaction = true
// MakeCluster creates and starts a node of cluster
func MakeCluster() *Cluster {
cluster := &Cluster{
self: config.Properties.Self,
db: database2.NewStandaloneServer(),
transactions: dict.MakeSimple(),
peerPicker: consistenthash.New(replicas, nil),
nodeConnections: make(map[string]*pool.Pool),
idGenerator: idgenerator.MakeGenerator(config.Properties.Self),
relayImpl: defaultRelayImpl,
self: config.Properties.Self,
addr: config.Properties.AnnounceAddress(),
db: database2.NewStandaloneServer(),
transactions: dict.MakeSimple(),
idGenerator: idgenerator.MakeGenerator(config.Properties.Self),
clientFactory: newDefaultClientFactory(),
}
contains := make(map[string]struct{})
nodes := make([]string, 0, len(config.Properties.Peers)+1)
for _, peer := range config.Properties.Peers {
if _, ok := contains[peer]; ok {
continue
}
contains[peer] = struct{}{}
nodes = append(nodes, peer)
topologyPersistFile := path.Join(config.Properties.Dir, config.Properties.ClusterConfigFile)
cluster.topology = newRaft(cluster, topologyPersistFile)
cluster.db.SetKeyInsertedCallback(cluster.makeInsertCallback())
cluster.db.SetKeyDeletedCallback(cluster.makeDeleteCallback())
cluster.slots = make(map[uint32]*hostSlot)
var err error
if topologyPersistFile != "" && fileExists(topologyPersistFile) {
err = cluster.LoadConfig()
} else if config.Properties.ClusterAsSeed {
err = cluster.startAsSeed(config.Properties.AnnounceAddress())
} else {
err = cluster.Join(config.Properties.ClusterSeed)
}
nodes = append(nodes, config.Properties.Self)
cluster.peerPicker.AddNode(nodes...)
connectionPoolConfig := pool.Config{
MaxIdle: 1,
MaxActive: 16,
if err != nil {
panic(err)
}
for _, p := range config.Properties.Peers {
peer := p
factory := func() (interface{}, error) {
c, err := client.MakeClient(peer)
if err != nil {
return nil, err
}
c.Start()
// all peers of cluster should use the same password
if config.Properties.RequirePass != "" {
c.Send(utils.ToCmdLine("AUTH", config.Properties.RequirePass))
}
return c, nil
}
finalizer := func(x interface{}) {
cli, ok := x.(client.Client)
if !ok {
return
}
cli.Close()
}
cluster.nodeConnections[peer] = pool.New(factory, finalizer, connectionPoolConfig)
}
cluster.nodes = nodes
return cluster
}
@@ -108,14 +120,11 @@ type CmdFunc func(cluster *Cluster, c redis.Connection, cmdLine CmdLine) redis.R
// Close stops current node of cluster
func (cluster *Cluster) Close() {
_ = cluster.topology.Close()
cluster.db.Close()
for _, pool := range cluster.nodeConnections {
pool.Close()
}
cluster.clientFactory.Close()
}
var router = makeRouter()
func isAuthenticated(c redis.Connection) bool {
if config.Properties.RequirePass == "" {
return true
@@ -155,10 +164,7 @@ func (cluster *Cluster) Exec(c redis.Connection, cmdLine [][]byte) (result redis
}
return execMulti(cluster, c, nil)
} else if cmdName == "select" {
if len(cmdLine) != 2 {
return protocol.MakeArgNumErrReply(cmdName)
}
return execSelect(c, cmdLine)
return protocol.MakeErrReply("select not supported in cluster")
}
if c != nil && c.InMultiState() {
return database2.EnqueueCmd(c, cmdLine)
@@ -175,3 +181,38 @@ func (cluster *Cluster) Exec(c redis.Connection, cmdLine [][]byte) (result redis
func (cluster *Cluster) AfterClientClose(c redis.Connection) {
cluster.db.AfterClientClose(c)
}
func (cluster *Cluster) makeInsertCallback() database.KeyEventCallback {
return func(dbIndex int, key string, entity *database.DataEntity) {
slotId := getSlot(key)
cluster.slotMu.RLock()
slot, ok := cluster.slots[slotId]
cluster.slotMu.RUnlock()
// As long as the command is executed, we should update slot.keys regardless of slot.state
if ok {
slot.mu.Lock()
defer slot.mu.Unlock()
slot.keys.Add(key)
}
}
}
func (cluster *Cluster) makeDeleteCallback() database.KeyEventCallback {
return func(dbIndex int, key string, entity *database.DataEntity) {
slotId := getSlot(key)
cluster.slotMu.RLock()
slot, ok := cluster.slots[slotId]
cluster.slotMu.RUnlock()
// As long as the command is executed, we should update slot.keys regardless of slot.state
if ok {
slot.mu.Lock()
defer slot.mu.Unlock()
slot.keys.Remove(key)
}
}
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
return err == nil && !info.IsDir()
}