From d6bbf0315cb8c308b507e072ec31da41159945e6 Mon Sep 17 00:00:00 2001 From: finley Date: Sun, 25 May 2025 21:04:49 +0800 Subject: [PATCH] refactor code structure for gnet --- README.md | 71 ++++++++----------- README_CN.md | 69 ++++++++---------- config/config.go | 2 +- main.go | 37 +++++----- gnet/parser.go => redis/parser/parserv2.go | 4 +- .../parser/parserv2_test.go | 20 +++++- {gnet => redis/server/gnet}/server.go | 12 +++- redis/server/gnet/server_test.go | 45 ++++++++++++ redis/server/{ => std}/pubsub_test.go | 2 +- redis/server/{ => std}/server.go | 20 ++++-- redis/server/{ => std}/server_test.go | 2 +- 11 files changed, 167 insertions(+), 117 deletions(-) rename gnet/parser.go => redis/parser/parserv2.go (97%) rename gnet/parser_test.go => redis/parser/parserv2_test.go (59%) rename {gnet => redis/server/gnet}/server.go (83%) create mode 100644 redis/server/gnet/server_test.go rename redis/server/{ => std}/pubsub_test.go (98%) rename redis/server/{ => std}/server.go (87%) rename redis/server/{ => std}/server_test.go (98%) diff --git a/README.md b/README.md index 9b13cd4..ebd78fa 100644 --- a/README.md +++ b/README.md @@ -16,21 +16,19 @@ middleware using golang. Key Features: - Support string, list, hash, set, sorted set, bitmap -- Multi Database and `SELECT` command +- Concurrent Core for better performance - TTL - Publish/Subscribe - GEO - AOF and AOF Rewrite - RDB read and write -- MULTI Commands Transaction is Atomic and Isolated. If any errors are encountered during execution, godis will rollback - the executed commands -- Replication (experimental) -- Server-side Cluster which is transparent to client. You can connect to any node in the cluster to - access all data in the cluster. +- Multi Database and `SELECT` command +- Transaction is **Atomic** and Isolated. If any errors are encountered during execution, godis will rollback the executed commands +- Replication +- Server-side Cluster which is transparent to client. You can connect to any node in the cluster to access all data in the cluster. - Use the raft algorithm to maintain cluster metadata. (experimental) - `MSET`, `MSETNX`, `DEL`, `Rename`, `RenameNX` command is supported and atomically executed in cluster mode, allow over multi node - `MULTI` Commands Transaction is supported within slot in cluster mode -- Concurrent Core, so you don't have to worry about your commands blocking the server too much. If you could read Chinese, you can find more details in [My Blog](https://www.cnblogs.com/Finley/category/1598973.html). @@ -40,6 +38,9 @@ You can get runnable program in the releases of this repository, which supports ```bash ./godis-darwin +``` + +```bash ./godis-linux ``` @@ -85,46 +86,34 @@ See: [commands.md](https://github.com/HDT3213/godis/blob/master/commands.md) Environment: -Go version：1.17 - -System: macOS Catalina 10.15.7 - -CPU: 2.6GHz 6-Core Intel Core i7 - -Memory: 16 GB 2667 MHz DDR4 +Go version：1.23 +System: MacOS Monterey 12.5 M2 Air Performance report by redis-benchmark: ``` -PING_INLINE: 87260.03 requests per second -PING_BULK: 89206.06 requests per second -SET: 85034.02 requests per second -GET: 87565.68 requests per second -INCR: 91157.70 requests per second -LPUSH: 90334.23 requests per second -RPUSH: 90334.23 requests per second -LPOP: 90334.23 requests per second -RPOP: 90415.91 requests per second -SADD: 90909.09 requests per second -HSET: 84104.29 requests per second -SPOP: 82918.74 requests per second -LPUSH (needed to benchmark LRANGE): 78247.26 requests per second -LRANGE_100 (first 100 elements): 26406.13 requests per second -LRANGE_300 (first 300 elements): 11307.10 requests per second -LRANGE_500 (first 450 elements): 7968.13 requests per second -LRANGE_600 (first 600 elements): 6092.73 requests per second -MSET (10 keys): 65487.89 requests per second +PING_INLINE: 179211.45 requests per second, p50=1.031 msec +PING_MBULK: 173611.12 requests per second, p50=1.071 msec +SET: 158478.61 requests per second, p50=1.535 msec +GET: 156985.86 requests per second, p50=1.127 msec +INCR: 164473.69 requests per second, p50=1.063 msec +LPUSH: 151285.92 requests per second, p50=1.079 msec +RPUSH: 176678.45 requests per second, p50=1.023 msec +LPOP: 177619.89 requests per second, p50=1.039 msec +RPOP: 172413.80 requests per second, p50=1.039 msec +SADD: 159489.64 requests per second, p50=1.047 msec +HSET: 175131.36 requests per second, p50=1.031 msec +SPOP: 170648.45 requests per second, p50=1.031 msec +ZADD: 165289.25 requests per second, p50=1.039 msec +ZPOPMIN: 185528.77 requests per second, p50=0.999 msec +LPUSH (needed to benchmark LRANGE): 172117.05 requests per second, p50=1.055 msec +LRANGE_100 (first 100 elements): 46511.62 requests per second, p50=4.063 msec +LRANGE_300 (first 300 elements): 21217.91 requests per second, p50=9.311 msec +LRANGE_500 (first 500 elements): 13331.56 requests per second, p50=14.407 msec +LRANGE_600 (first 600 elements): 11153.25 requests per second, p50=17.007 msec +MSET (10 keys): 88417.33 requests per second, p50=3.687 msec ``` -## Todo List - -+ [x] `Multi` Command -+ [x] `Watch` Command and CAS support -+ [ ] Stream support -+ [x] RDB file loader -+ [ ] Master-Slave mode -+ [ ] Sentinel - ## Read My Code If you want to read my code in this repository, here is a simple guidance. diff --git a/README_CN.md b/README_CN.md index 64a2053..e93ddf6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -11,18 +11,16 @@ Godis 是一个用 Go 语言实现的 Redis 服务器。本项目旨在为尝试 关键功能: - 支持 string, list, hash, set, sorted set, bitmap 数据结构 +- 并行内核，提供更优秀的性能 - 自动过期功能(TTL) - 发布订阅 - 地理位置 -- AOF 持久化及 AOF 重写 -- 加载和导出 RDB 文件 -- 主从复制 (测试中) -- Multi 命令开启的事务具有`原子性`和`隔离性`. 若在执行过程中遇到错误, godis 会回滚已执行的命令 +- AOF 持久化、RDB 持久化、aof-use-rdb-preamble 混合持久化 +- 主从复制 +- Multi 命令开启的事务具有**原子性**和隔离性. 若在执行过程中遇到错误, godis 会回滚已执行的命令 - 内置集群模式. 集群对客户端是透明的, 您可以像使用单机版 redis 一样使用 godis 集群 - 使用 raft 算法维护集群元数据(测试中) - `MSET`, `MSETNX`, `DEL`, `Rename`, `RenameNX` 命令在集群模式下原子性执行, 允许 key 在集群的不同节点上 - - 在集群模式下支持在同一个 slot 内执行事务 -- 并行引擎, 无需担心您的操作会阻塞整个服务器. 可以在[我的博客](https://www.cnblogs.com/Finley/category/1598973.html)了解更多关于 Godis 的信息。 @@ -33,6 +31,9 @@ Godis 的信息。 ```bash ./godis-darwin +``` + +```bash ./godis-linux ``` @@ -74,46 +75,34 @@ redis-cli -p 6399 环境: -Go version：1.17 - -System: macOS Catalina 10.15.7 - -CPU: 2.6GHz 6-Core Intel Core i7 - -Memory: 16 GB 2667 MHz DDR4 +Go version: 1.23 +System: MacOS Monterey 12.5 M2 Air redis-benchmark 测试结果: ``` -PING_INLINE: 87260.03 requests per second -PING_BULK: 89206.06 requests per second -SET: 85034.02 requests per second -GET: 87565.68 requests per second -INCR: 91157.70 requests per second -LPUSH: 90334.23 requests per second -RPUSH: 90334.23 requests per second -LPOP: 90334.23 requests per second -RPOP: 90415.91 requests per second -SADD: 90909.09 requests per second -HSET: 84104.29 requests per second -SPOP: 82918.74 requests per second -LPUSH (needed to benchmark LRANGE): 78247.26 requests per second -LRANGE_100 (first 100 elements): 26406.13 requests per second -LRANGE_300 (first 300 elements): 11307.10 requests per second -LRANGE_500 (first 450 elements): 7968.13 requests per second -LRANGE_600 (first 600 elements): 6092.73 requests per second -MSET (10 keys): 65487.89 requests per second +PING_INLINE: 179211.45 requests per second, p50=1.031 msec +PING_MBULK: 173611.12 requests per second, p50=1.071 msec +SET: 158478.61 requests per second, p50=1.535 msec +GET: 156985.86 requests per second, p50=1.127 msec +INCR: 164473.69 requests per second, p50=1.063 msec +LPUSH: 151285.92 requests per second, p50=1.079 msec +RPUSH: 176678.45 requests per second, p50=1.023 msec +LPOP: 177619.89 requests per second, p50=1.039 msec +RPOP: 172413.80 requests per second, p50=1.039 msec +SADD: 159489.64 requests per second, p50=1.047 msec +HSET: 175131.36 requests per second, p50=1.031 msec +SPOP: 170648.45 requests per second, p50=1.031 msec +ZADD: 165289.25 requests per second, p50=1.039 msec +ZPOPMIN: 185528.77 requests per second, p50=0.999 msec +LPUSH (needed to benchmark LRANGE): 172117.05 requests per second, p50=1.055 msec +LRANGE_100 (first 100 elements): 46511.62 requests per second, p50=4.063 msec +LRANGE_300 (first 300 elements): 21217.91 requests per second, p50=9.311 msec +LRANGE_500 (first 500 elements): 13331.56 requests per second, p50=14.407 msec +LRANGE_600 (first 600 elements): 11153.25 requests per second, p50=17.007 msec +MSET (10 keys): 88417.33 requests per second, p50=3.687 msec ``` -## 开发计划 - -+ [x] `Multi` 命令 -+ [x] `Watch` 命令和 CAS 支持 -+ [ ] Stream 队列 -+ [ ] 加载 RDB 文件 -+ [ ] 主从模式 -+ [ ] 哨兵 - ## 如何阅读源码 本项目的目录结构: diff --git a/config/config.go b/config/config.go index 6e458ec..c927e05 100644 --- a/config/config.go +++ b/config/config.go @@ -40,7 +40,7 @@ type ServerProperties struct { SlaveAnnouncePort int `cfg:"slave-announce-port"` SlaveAnnounceIP string `cfg:"slave-announce-ip"` ReplTimeout int `cfg:"repl-timeout"` - + UseGnet bool `cfg:"use-gnet"` ClusterEnable bool `cfg:"cluster-enable"` ClusterAsSeed bool `cfg:"cluster-as-seed"` ClusterSeed string `cfg:"cluster-seed"` diff --git a/main.go b/main.go index 6803561..f3e7b7c 100644 --- a/main.go +++ b/main.go @@ -6,15 +6,12 @@ import ( "github.com/hdt3213/godis/cluster" "github.com/hdt3213/godis/config" - database2 "github.com/hdt3213/godis/database" - "github.com/hdt3213/godis/gnet" - "github.com/hdt3213/godis/interface/database" + "github.com/hdt3213/godis/database" + idatabase "github.com/hdt3213/godis/interface/database" "github.com/hdt3213/godis/lib/logger" "github.com/hdt3213/godis/lib/utils" - - // RedisServer "github.com/hdt3213/godis/redis/server" - // "github.com/hdt3213/godis/tcp" - gnetv2 "github.com/panjf2000/gnet/v2" + "github.com/hdt3213/godis/redis/server/gnet" + stdserver "github.com/hdt3213/godis/redis/server/std" ) var banner = ` @@ -58,21 +55,21 @@ func main() { config.SetupConfig(configFilename) } listenAddr := fmt.Sprintf("%s:%d", config.Properties.Bind, config.Properties.Port) - // err := tcp.ListenAndServeWithSignal(&tcp.Config{ - // Address: fmt.Sprintf("%s:%d", config.Properties.Bind, config.Properties.Port), - // }, RedisServer.MakeHandler()) - // if err != nil { - // logger.Error(err) - // } - - var db database.DB - if config.Properties.ClusterEnable { - db = cluster.MakeCluster() + + var err error + if config.Properties.UseGnet { + var db idatabase.DB + if config.Properties.ClusterEnable { + db = cluster.MakeCluster() + } else { + db = database.NewStandaloneServer() + } + server := gnet.NewGnetServer(db) + err = server.Run(listenAddr) } else { - db = database2.NewStandaloneServer() + handler := stdserver.MakeHandler() + err = stdserver.Serve(listenAddr, handler) } - server := gnet.NewGnetServer(db) - err := gnetv2.Run(server, "tcp://" + listenAddr, gnetv2.WithMulticore(true)) if err != nil { logger.Errorf("start server failed: %v", err) } diff --git a/gnet/parser.go b/redis/parser/parserv2.go similarity index 97% rename from gnet/parser.go rename to redis/parser/parserv2.go index 54b4a44..57a57a3 100644 --- a/gnet/parser.go +++ b/redis/parser/parserv2.go @@ -1,4 +1,4 @@ -package gnet +package parser import ( "errors" @@ -7,7 +7,7 @@ import ( "strings" ) -func Parse(r io.Reader) ([][]byte, error) { +func ParseV2(r io.Reader) ([][]byte, error) { // 读取起始字符 '*' buf := make([]byte, 1) _, err := io.ReadFull(r, buf) diff --git a/gnet/parser_test.go b/redis/parser/parserv2_test.go similarity index 59% rename from gnet/parser_test.go rename to redis/parser/parserv2_test.go index 366e6bc..1254981 100644 --- a/gnet/parser_test.go +++ b/redis/parser/parserv2_test.go @@ -1,4 +1,4 @@ -package gnet +package parser import ( "bytes" @@ -18,15 +18,29 @@ func BenchmarkParseSETCommand(b *testing.B) { for i := 0; i < subB.N; i++ { reader := bytes.NewReader(cmd) - _, err := Parse(reader) + _, err := ParseV2(reader) if err != nil { - subB.Fatalf("解析失败: %v", err) + subB.Fatalf("parse failed: %v", err) } } }) } } +func TestParseV2(t *testing.T) { + value := bytes.Repeat([]byte("a"), 100) + data := []byte(fmt.Sprintf("*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$%d\r\n%s\r\n", len(value), value)) + cmdLine, err := ParseV2(bytes.NewBuffer(data)) + if err != nil { + t.Error(err) + return + } + if len(cmdLine) != 3 || string(cmdLine[0]) != "SET" || string(cmdLine[1]) != "key" || string(cmdLine[2]) != string(value) { + t.Error("parse error") + return + } +} + func formatSize(size int) string { units := []string{"B", "KB", "MB"} unitIndex := 0 diff --git a/gnet/server.go b/redis/server/gnet/server.go similarity index 83% rename from gnet/server.go rename to redis/server/gnet/server.go index b048325..8a9e06d 100644 --- a/gnet/server.go +++ b/redis/server/gnet/server.go @@ -1,12 +1,14 @@ package gnet import ( + "context" "sync/atomic" "github.com/hdt3213/godis/interface/database" "github.com/hdt3213/godis/interface/redis" "github.com/hdt3213/godis/lib/logger" "github.com/hdt3213/godis/redis/connection" + "github.com/hdt3213/godis/redis/parser" "github.com/panjf2000/gnet/v2" ) @@ -23,6 +25,10 @@ func NewGnetServer(db database.DB) *GnetServer { } } +func (s *GnetServer) Run(listenAddr string) error { + return gnet.Run(s, "tcp://"+listenAddr, gnet.WithMulticore(true)) +} + func (s *GnetServer) OnBoot(eng gnet.Engine) (action gnet.Action) { s.eng = eng return @@ -47,7 +53,7 @@ func (s *GnetServer) OnClose(c gnet.Conn, err error) (action gnet.Action) { func (s *GnetServer) OnTraffic(c gnet.Conn) (action gnet.Action) { conn := c.Context().(redis.Connection) - cmdLine, err := Parse(c) + cmdLine, err := parser.ParseV2(c) if err != nil { logger.Infof("parse command line failed: %v", err) return gnet.Close @@ -62,3 +68,7 @@ func (s *GnetServer) OnTraffic(c gnet.Conn) (action gnet.Action) { } return gnet.None } + +func (s *GnetServer) Close() { + s.eng.Stop(context.Background()) +} \ No newline at end of file diff --git a/redis/server/gnet/server_test.go b/redis/server/gnet/server_test.go new file mode 100644 index 0000000..0a9b246 --- /dev/null +++ b/redis/server/gnet/server_test.go @@ -0,0 +1,45 @@ +package gnet + +import ( + "bufio" + "net" + "testing" + + "github.com/hdt3213/godis/database" +) + +func TestListenAndServe(t *testing.T) { + var err error + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Error(err) + return + } + addr := listener.Addr().String() + db := database.NewStandaloneServer() + server := NewGnetServer(db) + go server.Run(addr) + + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Error(err) + return + } + _, err = conn.Write([]byte("PING\r\n")) + if err != nil { + t.Error(err) + return + } + bufReader := bufio.NewReader(conn) + line, _, err := bufReader.ReadLine() + if err != nil { + t.Error(err) + return + } + if string(line) != "+PONG" { + t.Error("get wrong response") + return + } + conn.Close() + server.Close() +} diff --git a/redis/server/pubsub_test.go b/redis/server/std/pubsub_test.go similarity index 98% rename from redis/server/pubsub_test.go rename to redis/server/std/pubsub_test.go index bada3f4..e79b3ad 100644 --- a/redis/server/pubsub_test.go +++ b/redis/server/std/pubsub_test.go @@ -1,4 +1,4 @@ -package server +package std import ( "github.com/hdt3213/godis/lib/utils" diff --git a/redis/server/server.go b/redis/server/std/server.go similarity index 87% rename from redis/server/server.go rename to redis/server/std/server.go index abee86d..1641f8f 100644 --- a/redis/server/server.go +++ b/redis/server/std/server.go @@ -1,4 +1,4 @@ -package server +package std /* * A tcp.Handler implements redis protocol @@ -13,14 +13,14 @@ import ( "github.com/hdt3213/godis/cluster" "github.com/hdt3213/godis/config" - database2 "github.com/hdt3213/godis/database" - "github.com/hdt3213/godis/interface/database" - "github.com/hdt3213/godis/interface/redis" + "github.com/hdt3213/godis/database" + idatabase "github.com/hdt3213/godis/interface/database" "github.com/hdt3213/godis/lib/logger" "github.com/hdt3213/godis/lib/sync/atomic" "github.com/hdt3213/godis/redis/connection" "github.com/hdt3213/godis/redis/parser" "github.com/hdt3213/godis/redis/protocol" + "github.com/hdt3213/godis/tcp" ) var ( @@ -30,23 +30,29 @@ var ( // Handler implements tcp.Handler and serves as a redis server type Handler struct { activeConn sync.Map // *client -> placeholder - db database.DB + db idatabase.DB closing atomic.Boolean // refusing new client and new request } // MakeHandler creates a Handler instance func MakeHandler() *Handler { - var db database.DB + var db idatabase.DB if config.Properties.ClusterEnable { db = cluster.MakeCluster() } else { - db = database2.NewStandaloneServer() + db = database.NewStandaloneServer() } return &Handler{ db: db, } } +func Serve(addr string, handler *Handler) error { + return tcp.ListenAndServeWithSignal(&tcp.Config{ + Address: addr, + }, handler) +} + func (h *Handler) closeClient(client *connection.Connection) { _ = client.Close() h.db.AfterClientClose(client) diff --git a/redis/server/server_test.go b/redis/server/std/server_test.go similarity index 98% rename from redis/server/server_test.go rename to redis/server/std/server_test.go index 0c69c95..de7c3aa 100644 --- a/redis/server/server_test.go +++ b/redis/server/std/server_test.go @@ -1,4 +1,4 @@ -package server +package std import ( "bufio"