Fix api/v1/users/condition end point can not handle filter with Chinese payload like:

{
  "columns": [
    {
      "name": "chinese_name",
      "exp": "like",
      "value": "过%"
    }
  ]
}
This commit is contained in:
Eric-Guo
2025-09-03 09:33:45 +08:00
parent b9db5f180e
commit cc9cf382e7
7 changed files with 120 additions and 15 deletions

View File

@@ -488,7 +488,7 @@ database:
# mysql settings
mysql:
# dsn format, <username>:<password>@(<hostname>:<port>)/<db>?[k=v& ......]
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4"
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci"
enableLog: true # whether to turn on printing of all logs
maxIdleConns: 10 # set the maximum number of connections in the idle connection pool
maxOpenConns: 100 # set the maximum number of open database connections
@@ -535,7 +535,7 @@ database:
# mysql settings
mysql:
# dsn format, <username>:<password>@(<hostname>:<port>)/<db>?[k=v& ......]
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4"
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci"
enableLog: true # whether to turn on printing of all logs
maxIdleConns: 10 # set the maximum number of connections in the idle connection pool
maxOpenConns: 100 # set the maximum number of open database connections

View File

@@ -86,7 +86,7 @@ database:
# mysql settings
mysql:
# dsn format, <username>:<password>@(<hostname>:<port>)/<db>?[k=v& ......]
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4"
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci"
enableLog: true # whether to turn on printing of all logs
maxIdleConns: 10 # set the maximum number of connections in the idle connection pool
maxOpenConns: 100 # set the maximum number of open database connections

View File

@@ -10,7 +10,7 @@ database:
# mysql settings
mysql:
# dsn format, <user>:<pass>@(127.0.0.1:3306)/<db>?[k=v& ......]
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8,utf8mb4"
dsn: "root:123456@(192.168.3.37:3306)/account?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_general_ci"
# redis settings
redis:

View File

@@ -13,7 +13,7 @@ Support `mysql`, `postgresql`, `sqlite`.
```go
import "github.com/go-dev-frame/sponge/pkg/sgorm/mysql"
var dsn = "root:123456@(127.0.0.1:3306)/test?charset=utf8mb4&parseTime=True&loc=Local"
var dsn = "root:123456@(127.0.0.1:3306)/test?charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&loc=Local"
// case 1: connect to the database using the default settings
db, err := mysql.Init(dsn)

View File

@@ -16,6 +16,7 @@ import (
"github.com/go-dev-frame/sponge/pkg/sgorm/dbclose"
"github.com/go-dev-frame/sponge/pkg/sgorm/glog"
"github.com/go-dev-frame/sponge/pkg/utils"
)
// Init mysql
@@ -35,7 +36,7 @@ func Init(dsn string, opts ...Option) (*gorm.DB, error) {
if err != nil {
return nil, err
}
db.Set("gorm:table_options", "CHARSET=utf8mb4") // automatic appending of table suffixes when creating tables
db.Set("gorm:table_options", "CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci") // automatic appending of table suffixes when creating tables
// register trace plugin
if o.enableTrace {
@@ -108,14 +109,14 @@ func rwSeparationPlugin(o *options) gorm.Plugin {
slaves := []gorm.Dialector{}
for _, dsn := range o.slavesDsn {
slaves = append(slaves, mysqlDriver.New(mysqlDriver.Config{
DSN: dsn,
DSN: utils.AdaptiveMysqlDsn(dsn),
}))
}
masters := []gorm.Dialector{}
for _, dsn := range o.mastersDsn {
masters = append(masters, mysqlDriver.New(mysqlDriver.Config{
DSN: dsn,
DSN: utils.AdaptiveMysqlDsn(dsn),
}))
}

View File

@@ -145,12 +145,13 @@ func (c *Column) checkExp() (string, error) {
if !ok1 {
return symbol, fmt.Errorf("invalid value type '%s'", c.Value)
}
l := len(val)
if l > 2 {
val2 := val[1 : l-1]
val2 = strings.ReplaceAll(val2, "%", "\\%")
val2 = strings.ReplaceAll(val2, "_", "\\_")
val = string(val[0]) + val2 + string(val[l-1])
// Use rune-safe slicing to preserve multi-byte characters
r := []rune(val)
if len(r) > 2 {
middle := string(r[1 : len(r)-1])
middle = strings.ReplaceAll(middle, "%", "\\%")
middle = strings.ReplaceAll(middle, "_", "\\_")
val = string(r[0]) + middle + string(r[len(r)-1])
}
if strings.HasPrefix(val, "%") ||
strings.HasPrefix(val, "_") ||

View File

@@ -8,7 +8,110 @@ import (
// AdaptiveMysqlDsn adaptation of various mysql format dsn address
func AdaptiveMysqlDsn(dsn string) string {
return strings.ReplaceAll(dsn, "mysql://", "")
// remove optional scheme prefix
dsn = strings.ReplaceAll(dsn, "mysql://", "")
// ensure a valid network/address section for go-sql-driver/mysql
// Expected forms:
// user:pass@tcp(127.0.0.1:3306)/db
// user:pass@unix(/path/mysql.sock)/db
// If it's like '@(127.0.0.1:3306)' → add 'tcp'
// If it's like '@127.0.0.1:3306' → wrap to '@tcp(127.0.0.1:3306)'
at := strings.Index(dsn, "@")
if at != -1 {
afterAt := dsn[at+1:]
slashIdx := strings.Index(afterAt, "/")
if slashIdx != -1 {
addrPart := afterAt[:slashIdx]
// If empty addrPart, nothing to fix
if addrPart != "" {
if strings.HasPrefix(addrPart, "(") {
// missing protocol
dsn = strings.Replace(dsn, "@(", "@tcp(", 1)
} else if !(strings.HasPrefix(addrPart, "tcp(") || strings.HasPrefix(addrPart, "unix(")) {
// no parentheses and no protocol → wrap with tcp()
dsn = strings.Replace(dsn, "@"+addrPart, "@tcp("+addrPart+")", 1)
}
}
}
}
// ensure the connection prefers utf8mb4 to avoid collation mismatch
// issues with MySQL 8 (e.g. mixing utf8mb3_general_ci and utf8mb4_0900_ai_ci).
qIdx := strings.Index(dsn, "?")
if qIdx == -1 {
// no query string → add charset parameter
return dsn + "?charset=utf8mb4"
}
prefix := dsn[:qIdx]
queryStr := dsn[qIdx+1:]
parts := strings.Split(queryStr, "&")
hasCharset := false
hasCollation := false
for i, p := range parts {
if strings.HasPrefix(p, "charset=") {
hasCharset = true
val := strings.TrimPrefix(p, "charset=")
// split by comma and de-duplicate while ensuring utf8mb4 comes first if present/added
charsets := []string{}
for _, cs := range strings.Split(val, ",") {
cs = strings.TrimSpace(cs)
if cs == "" {
continue
}
// skip duplicates
dup := false
for _, existing := range charsets {
if strings.EqualFold(existing, cs) {
dup = true
break
}
}
if !dup {
charsets = append(charsets, cs)
}
}
// ensure utf8mb4 is present and at the first position
containsUtf8mb4 := false
for _, cs := range charsets {
if strings.EqualFold(cs, "utf8mb4") {
containsUtf8mb4 = true
break
}
}
if !containsUtf8mb4 {
charsets = append([]string{"utf8mb4"}, charsets...)
} else if len(charsets) > 0 && !strings.EqualFold(charsets[0], "utf8mb4") {
// move utf8mb4 to front
newOrder := []string{"utf8mb4"}
for _, cs := range charsets {
if !strings.EqualFold(cs, "utf8mb4") {
newOrder = append(newOrder, cs)
}
}
charsets = newOrder
}
parts[i] = "charset=" + strings.Join(charsets, ",")
break
}
if strings.HasPrefix(p, "collation=") {
hasCollation = true
}
}
if !hasCharset {
parts = append(parts, "charset=utf8mb4")
}
if !hasCollation {
// default to a broadly compatible utf8mb4 collation
parts = append(parts, "collation=utf8mb4_general_ci")
}
return prefix + "?" + strings.Join(parts, "&")
}
// AdaptivePostgresqlDsn convert postgres dsn to kv string