From cc9cf382e78af30b384cd349a6256b11c5191af2 Mon Sep 17 00:00:00 2001 From: Eric-Guo Date: Wed, 3 Sep 2025 09:33:45 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8Fix=20api/v1/users/condition=20end=20p?= =?UTF-8?q?oint=20can=20not=20handle=20filter=20with=20Chinese=20payload?= =?UTF-8?q?=20like:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit { "columns": [ { "name": "chinese_name", "exp": "like", "value": "过%" } ] } --- cmd/sponge/commands/generate/template.go | 4 +- configs/serverNameExample.yml | 2 +- pkg/conf/test.yml | 2 +- pkg/sgorm/README.md | 2 +- pkg/sgorm/mysql/mysql.go | 7 +- pkg/sgorm/query/query_condition.go | 13 +-- pkg/utils/dsn.go | 105 ++++++++++++++++++++++- 7 files changed, 120 insertions(+), 15 deletions(-) diff --git a/cmd/sponge/commands/generate/template.go b/cmd/sponge/commands/generate/template.go index 4a93ea7..dd29bdd 100644 --- a/cmd/sponge/commands/generate/template.go +++ b/cmd/sponge/commands/generate/template.go @@ -488,7 +488,7 @@ database: # mysql settings mysql: # dsn format, :@(:)/?[k=v& ......] - dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4" + dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci" enableLog: true # whether to turn on printing of all logs maxIdleConns: 10 # set the maximum number of connections in the idle connection pool maxOpenConns: 100 # set the maximum number of open database connections @@ -535,7 +535,7 @@ database: # mysql settings mysql: # dsn format, :@(:)/?[k=v& ......] - dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4" + dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci" enableLog: true # whether to turn on printing of all logs maxIdleConns: 10 # set the maximum number of connections in the idle connection pool maxOpenConns: 100 # set the maximum number of open database connections diff --git a/configs/serverNameExample.yml b/configs/serverNameExample.yml index c164227..23b4295 100644 --- a/configs/serverNameExample.yml +++ b/configs/serverNameExample.yml @@ -86,7 +86,7 @@ database: # mysql settings mysql: # dsn format, :@(:)/?[k=v& ......] - dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4" + dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci" enableLog: true # whether to turn on printing of all logs maxIdleConns: 10 # set the maximum number of connections in the idle connection pool maxOpenConns: 100 # set the maximum number of open database connections diff --git a/pkg/conf/test.yml b/pkg/conf/test.yml index b462371..1443e01 100644 --- a/pkg/conf/test.yml +++ b/pkg/conf/test.yml @@ -10,7 +10,7 @@ database: # mysql settings mysql: # dsn format, :@(127.0.0.1:3306)/?[k=v& ......] - dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4" + dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci" # redis settings redis: diff --git a/pkg/sgorm/README.md b/pkg/sgorm/README.md index 84ef616..9ecc1ee 100644 --- a/pkg/sgorm/README.md +++ b/pkg/sgorm/README.md @@ -13,7 +13,7 @@ Support `mysql`, `postgresql`, `sqlite`. ```go import "github.com/go-dev-frame/sponge/pkg/sgorm/mysql" - var dsn = "root:123456@(127.0.0.1:3306)/test?charset=utf8mb4&parseTime=True&loc=Local" + var dsn = "root:123456@(127.0.0.1:3306)/test?charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&loc=Local" // case 1: connect to the database using the default settings db, err := mysql.Init(dsn) diff --git a/pkg/sgorm/mysql/mysql.go b/pkg/sgorm/mysql/mysql.go index 9101a6a..f8f808c 100644 --- a/pkg/sgorm/mysql/mysql.go +++ b/pkg/sgorm/mysql/mysql.go @@ -16,6 +16,7 @@ import ( "github.com/go-dev-frame/sponge/pkg/sgorm/dbclose" "github.com/go-dev-frame/sponge/pkg/sgorm/glog" + "github.com/go-dev-frame/sponge/pkg/utils" ) // Init mysql @@ -35,7 +36,7 @@ func Init(dsn string, opts ...Option) (*gorm.DB, error) { if err != nil { return nil, err } - db.Set("gorm:table_options", "CHARSET=utf8mb4") // automatic appending of table suffixes when creating tables + db.Set("gorm:table_options", "CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci") // automatic appending of table suffixes when creating tables // register trace plugin if o.enableTrace { @@ -108,14 +109,14 @@ func rwSeparationPlugin(o *options) gorm.Plugin { slaves := []gorm.Dialector{} for _, dsn := range o.slavesDsn { slaves = append(slaves, mysqlDriver.New(mysqlDriver.Config{ - DSN: dsn, + DSN: utils.AdaptiveMysqlDsn(dsn), })) } masters := []gorm.Dialector{} for _, dsn := range o.mastersDsn { masters = append(masters, mysqlDriver.New(mysqlDriver.Config{ - DSN: dsn, + DSN: utils.AdaptiveMysqlDsn(dsn), })) } diff --git a/pkg/sgorm/query/query_condition.go b/pkg/sgorm/query/query_condition.go index b382975..317bd58 100644 --- a/pkg/sgorm/query/query_condition.go +++ b/pkg/sgorm/query/query_condition.go @@ -145,12 +145,13 @@ func (c *Column) checkExp() (string, error) { if !ok1 { return symbol, fmt.Errorf("invalid value type '%s'", c.Value) } - l := len(val) - if l > 2 { - val2 := val[1 : l-1] - val2 = strings.ReplaceAll(val2, "%", "\\%") - val2 = strings.ReplaceAll(val2, "_", "\\_") - val = string(val[0]) + val2 + string(val[l-1]) + // Use rune-safe slicing to preserve multi-byte characters + r := []rune(val) + if len(r) > 2 { + middle := string(r[1 : len(r)-1]) + middle = strings.ReplaceAll(middle, "%", "\\%") + middle = strings.ReplaceAll(middle, "_", "\\_") + val = string(r[0]) + middle + string(r[len(r)-1]) } if strings.HasPrefix(val, "%") || strings.HasPrefix(val, "_") || diff --git a/pkg/utils/dsn.go b/pkg/utils/dsn.go index 031b0a1..efe0e3f 100644 --- a/pkg/utils/dsn.go +++ b/pkg/utils/dsn.go @@ -8,7 +8,110 @@ import ( // AdaptiveMysqlDsn adaptation of various mysql format dsn address func AdaptiveMysqlDsn(dsn string) string { - return strings.ReplaceAll(dsn, "mysql://", "") + // remove optional scheme prefix + dsn = strings.ReplaceAll(dsn, "mysql://", "") + + // ensure a valid network/address section for go-sql-driver/mysql + // Expected forms: + // user:pass@tcp(127.0.0.1:3306)/db + // user:pass@unix(/path/mysql.sock)/db + // If it's like '@(127.0.0.1:3306)' → add 'tcp' + // If it's like '@127.0.0.1:3306' → wrap to '@tcp(127.0.0.1:3306)' + at := strings.Index(dsn, "@") + if at != -1 { + afterAt := dsn[at+1:] + slashIdx := strings.Index(afterAt, "/") + if slashIdx != -1 { + addrPart := afterAt[:slashIdx] + // If empty addrPart, nothing to fix + if addrPart != "" { + if strings.HasPrefix(addrPart, "(") { + // missing protocol + dsn = strings.Replace(dsn, "@(", "@tcp(", 1) + } else if !(strings.HasPrefix(addrPart, "tcp(") || strings.HasPrefix(addrPart, "unix(")) { + // no parentheses and no protocol → wrap with tcp() + dsn = strings.Replace(dsn, "@"+addrPart, "@tcp("+addrPart+")", 1) + } + } + } + } + + // ensure the connection prefers utf8mb4 to avoid collation mismatch + // issues with MySQL 8 (e.g. mixing utf8mb3_general_ci and utf8mb4_0900_ai_ci). + qIdx := strings.Index(dsn, "?") + if qIdx == -1 { + // no query string → add charset parameter + return dsn + "?charset=utf8mb4" + } + + prefix := dsn[:qIdx] + queryStr := dsn[qIdx+1:] + parts := strings.Split(queryStr, "&") + + hasCharset := false + hasCollation := false + for i, p := range parts { + if strings.HasPrefix(p, "charset=") { + hasCharset = true + val := strings.TrimPrefix(p, "charset=") + // split by comma and de-duplicate while ensuring utf8mb4 comes first if present/added + charsets := []string{} + for _, cs := range strings.Split(val, ",") { + cs = strings.TrimSpace(cs) + if cs == "" { + continue + } + // skip duplicates + dup := false + for _, existing := range charsets { + if strings.EqualFold(existing, cs) { + dup = true + break + } + } + if !dup { + charsets = append(charsets, cs) + } + } + + // ensure utf8mb4 is present and at the first position + containsUtf8mb4 := false + for _, cs := range charsets { + if strings.EqualFold(cs, "utf8mb4") { + containsUtf8mb4 = true + break + } + } + if !containsUtf8mb4 { + charsets = append([]string{"utf8mb4"}, charsets...) + } else if len(charsets) > 0 && !strings.EqualFold(charsets[0], "utf8mb4") { + // move utf8mb4 to front + newOrder := []string{"utf8mb4"} + for _, cs := range charsets { + if !strings.EqualFold(cs, "utf8mb4") { + newOrder = append(newOrder, cs) + } + } + charsets = newOrder + } + + parts[i] = "charset=" + strings.Join(charsets, ",") + break + } + if strings.HasPrefix(p, "collation=") { + hasCollation = true + } + } + + if !hasCharset { + parts = append(parts, "charset=utf8mb4") + } + if !hasCollation { + // default to a broadly compatible utf8mb4 collation + parts = append(parts, "collation=utf8mb4_general_ci") + } + + return prefix + "?" + strings.Join(parts, "&") } // AdaptivePostgresqlDsn convert postgres dsn to kv string