diff --git a/cmd/eagle/go.mod b/cmd/eagle/go.mod index e2313e7..3e18d32 100644 --- a/cmd/eagle/go.mod +++ b/cmd/eagle/go.mod @@ -1,7 +1,8 @@ module github.com/go-eagle/eagle/cmd/eagle -go 1.22 -toolchain go1.24.1 +go 1.23.0 + +toolchain go1.24.3 require ( github.com/AlecAivazis/survey/v2 v2.2.12 diff --git a/cmd/eagle/internal/cache/add/template.go b/cmd/eagle/internal/cache/add/template.go index cc59116..69ca262 100644 --- a/cmd/eagle/internal/cache/add/template.go +++ b/cmd/eagle/internal/cache/add/template.go @@ -48,10 +48,10 @@ type {{.LcName}}Cache struct { // New{{.Name}}Cache new a cache func New{{.Name}}Cache(rdb *redis.Client) {{.Name}}Cache { - jsonEncoding := encoding.JSONEncoding{} + sonicEncoding := encoding.SonicEncoding{} cachePrefix := "" return &{{.LcName}}Cache{ - cache: cache.NewRedisCache(rdb, cachePrefix, jsonEncoding, func() interface{} { + cache: cache.NewRedisCache(rdb, cachePrefix, sonicEncoding, func() interface{} { return &model.{{.Name}}Model{} }), } diff --git a/cmd/eagle/internal/repo/add/template.go b/cmd/eagle/internal/repo/add/template.go index e59b9bd..6057611 100644 --- a/cmd/eagle/internal/repo/add/template.go +++ b/cmd/eagle/internal/repo/add/template.go @@ -16,6 +16,7 @@ import ( "time" localCache "github.com/go-eagle/eagle/pkg/cache" + cacheBase "github.com/go-eagle/eagle/pkg/cache" "github.com/go-eagle/eagle/pkg/encoding" "github.com/pkg/errors" "github.com/spf13/cast" @@ -57,7 +58,7 @@ func New{{.Name}}Repo(db *dal.DBClient, cache cache.{{.Name}}Cache) {{.Name}}Rep db: db, tracer: otel.Tracer("{{.LcName}}"), cache: cache, - localCache: localCache.NewMemoryCache("local:{{.LcName}}:", encoding.JSONEncoding{}), + localCache: localCache.NewMemoryCache("local:{{.LcName}}:", encoding.SonicEncoding{}), sg: singleflight.Group{}, } } @@ -124,48 +125,49 @@ func (r *{{.LcName}}Repo) Get{{.Name}}(ctx context.Context, id int64) (ret *mode // read redis cache ret, err = r.cache.Get{{.Name}}Cache(ctx, id) - if err != nil { - return nil, err - } - if ret != nil && ret.ID > 0 { - return ret, nil - } - - // get data from db - // 避免缓存击穿(瞬间有大量请求过来) - val, err, _ := r.sg.Do("sg:{{.LcName}}:"+cast.ToString(id), func() (interface{}, error) { - // read db or rpc - data, err := dao.{{.Name}}Model.WithContext(ctx).Where(dao.{{.Name}}Model.ID.Eq(id)).Last() + if errors.Is(err, cacheBase.ErrPlaceholder) { + return nil, gorm.ErrRecordNotFound + } else if errors.Is(err, redis.ErrRedisNotFound) { + // get data from db + // 避免缓存击穿(瞬间有大量请求过来) + val, err, _ := r.sg.Do("sg:{{.LcName}}:"+cast.ToString(id), func() (interface{}, error) { + // read db or rpc + data, err := dao.{{.Name}}Model.WithContext(ctx).Where(dao.{{.Name}}Model.ID.Eq(id)).Last() + if err != nil { + // cache not found and set empty cache to avoid 缓存穿透 + // Note: 如果缓存空数据太多，会大大降低缓存命中率，可以改为使用布隆过滤器 + if errors.Is(err, gorm.ErrRecordNotFound) { + r.cache.SetCacheWithNotFound(ctx, id) + } + return nil, errors.Wrapf(err, "[repo] get {{.Name}} from db error, id: %d", id) + } + + // write cache + if data != nil && data.ID > 0 { + // write redis + err = r.cache.Set{{.Name}}Cache(ctx, id, data, 5*time.Minute) + if err != nil { + return nil, errors.WithMessage(err, "[repo] Get{{.Name}} Set{{.Name}}Cache error") + } + + // write local cache + err = r.localCache.Set(ctx, cast.ToString(id), data, 2*time.Minute) + if err != nil { + return nil, errors.WithMessage(err, "[repo] Get{{.Name}} localCache set error") + } + } + return data, nil + }) if err != nil { - // cache not found and set empty cache to avoid 缓存穿透 - // Note: 如果缓存空数据太多，会大大降低缓存命中率，可以改为使用布隆过滤器 - if errors.Is(err, gorm.ErrRecordNotFound) { - r.cache.SetCacheWithNotFound(ctx, id) - } - return nil, errors.Wrapf(err, "[repo] get {{.Name}} from db error, id: %d", id) + return nil, err } - - // write cache - if data != nil && data.ID > 0 { - // write redis - err = r.cache.Set{{.Name}}Cache(ctx, id, data, 5*time.Minute) - if err != nil { - return nil, errors.WithMessage(err, "[repo] Get{{.Name}} Set{{.Name}}Cache error") - } - - // write local cache - err = r.localCache.Set(ctx, cast.ToString(id), data, 2*time.Minute) - if err != nil { - return nil, errors.WithMessage(err, "[repo] Get{{.Name}} localCache set error") - } - } - return data, nil - }) - if err != nil { + + return val.(*model.{{.Name}}Model), nil + } else if err != nil { return nil, err } + return ret, nil - return val.(*model.{{.Name}}Model), nil {{- else }} // read db ret, err = dao.{{.Name}}Model.WithContext(ctx).Where(dao.{{.Name}}Model.ID.Eq(id)).Last() diff --git a/cmd/protoc-gen-go-gin/gin.go b/cmd/protoc-gen-go-gin/gin.go index 4571466..0f8cb0b 100644 --- a/cmd/protoc-gen-go-gin/gin.go +++ b/cmd/protoc-gen-go-gin/gin.go @@ -18,6 +18,7 @@ const ( metadataPackage = protogen.GoImportPath("google.golang.org/grpc/metadata") eagleAppPackage = protogen.GoImportPath("github.com/go-eagle/eagle/pkg/app") errCodePackage = protogen.GoImportPath("github.com/go-eagle/eagle/pkg/errcode") + utilsPackage = protogen.GoImportPath("github.com/go-eagle/eagle/pkg/utils") deprecationComment = "// Deprecated: Do not use." ) @@ -40,7 +41,7 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated g.P() g.P("// ", contextPackage.Ident("")) g.P("// ", metadataPackage.Ident("")) - g.P("// ", ginPackage.Ident(""), eagleAppPackage.Ident(""), errCodePackage.Ident("")) + g.P("// ", ginPackage.Ident(""), eagleAppPackage.Ident(""), errCodePackage.Ident(""), utilsPackage.Ident("")) g.P() for _, service := range file.Services { diff --git a/cmd/protoc-gen-go-gin/template.go b/cmd/protoc-gen-go-gin/template.go index c557a43..785362e 100644 --- a/cmd/protoc-gen-go-gin/template.go +++ b/cmd/protoc-gen-go-gin/template.go @@ -31,14 +31,19 @@ type {{$.Name}} struct{ {{range .Methods}} func (s *{{$.Name}}) {{ .HandlerName }} (ctx *gin.Context) { var in {{.Request}} + var err error {{if eq .Method "GET" }} - if err := ctx.ShouldBindQuery(&in); err != nil { + if err = ctx.ShouldBindQuery(&in); err != nil { app.Error(ctx, errcode.ErrInvalidParam.WithDetails(err.Error())) return } {{if .HasPathParams }} // make sure the uri include :id - in.Id = ctx.Param("id") + in.Id, err = utils.StringToInt64((ctx.Param("id"))) + if err != nil { + app.Error(ctx, errcode.ErrInvalidParam.WithDetails(err.Error())) + return + } {{end}} {{else if eq .Method "POST" "PUT" "PATCH" "DELETE"}} if err := ctx.ShouldBindJSON(&in); err != nil { @@ -47,7 +52,11 @@ func (s *{{$.Name}}) {{ .HandlerName }} (ctx *gin.Context) { } {{if .HasPathParams }} // make sure the uri include :id - in.Id = ctx.Param("id") + in.Id, err = utils.StringToInt64((ctx.Param("id"))) + if err != nil { + app.Error(ctx, errcode.ErrInvalidParam.WithDetails(err.Error())) + return + } {{end}} {{else}} if err := ctx.ShouldBind(&in); err != nil { diff --git a/examples/trace/main.go b/examples/trace/main.go index ec7f433..340e8d7 100644 --- a/examples/trace/main.go +++ b/examples/trace/main.go @@ -78,7 +78,8 @@ func main() { } // init service - service.Svc = service.New(repository.New(model.GetDB())) + db, _ := model.GetDB() + service.Svc = service.New(repository.New(db)) gin.SetMode(cfg.Mode) diff --git a/internal/cache/cache_client.go b/internal/cache/cache_client.go index 401839b..a8c3f2b 100644 --- a/internal/cache/cache_client.go +++ b/internal/cache/cache_client.go @@ -10,9 +10,9 @@ import ( ) func getCacheClient(ctx context.Context) cache.Cache { - jsonEncoding := encoding.JSONEncoding{} + sonicEncoding := encoding.SonicEncoding{} cachePrefix := "" - client := cache.NewRedisCache(redis.RedisClient, cachePrefix, jsonEncoding, func() interface{} { + client := cache.NewRedisCache(redis.RedisClient, cachePrefix, sonicEncoding, func() interface{} { return &model.UserBaseModel{} }) diff --git a/pkg/encoding/sonic_encoding.go b/pkg/encoding/sonic_encoding.go new file mode 100644 index 0000000..0b4bfd6 --- /dev/null +++ b/pkg/encoding/sonic_encoding.go @@ -0,0 +1,45 @@ +package encoding + +import ( + "github.com/bytedance/sonic" + "github.com/golang/snappy" +) + +type SonicEncoding struct { +} + +func (s SonicEncoding) Marshal(v interface{}) ([]byte, error) { + buf, err := sonic.Marshal(v) + return buf, err +} + +func (s SonicEncoding) Unmarshal(data []byte, value interface{}) error { + err := sonic.Unmarshal(data, value) + if err != nil { + return err + } + return nil +} + +// SonicSnappyEncoding Sonic json格式和snappy压缩 +type SonicSnappyEncoding struct{} + +// Marshal 序列化 +func (s SonicSnappyEncoding) Marshal(v interface{}) (data []byte, err error) { + b, err := sonic.Marshal(v) + if err != nil { + return nil, err + } + d := snappy.Encode(nil, b) + return d, nil +} + +// Unmarshal 反序列化 +func (s SonicSnappyEncoding) Unmarshal(data []byte, value interface{}) error { + b, err := snappy.Decode(nil, data) + if err != nil { + return err + } + + return sonic.Unmarshal(b, value) +} diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 8531225..8bfb2e0 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -6,7 +6,7 @@ import ( "strings" "unsafe" - jsoniter "github.com/json-iterator/go" + "github.com/bytedance/sonic" ) // IsEmpty 是否是空字符串 @@ -97,13 +97,10 @@ func StringToBytes(s string) []byte { return *(*[]byte)(unsafe.Pointer(&h)) } -// 对于序列化和反序列化场景较多的服务可以使用性能更高的json-iterator -// https://github.com/json-iterator/go -var Json = jsoniter.Config{ - EscapeHTML: true, - SortMapKeys: true, - ValidateJsonRawMessage: true, - UseNumber: true, +var Json = sonic.Config{ + EscapeHTML: true, + SortMapKeys: true, + UseNumber: true, }.Froze() func Stringify(obj interface{}) string {