package/storage
1
0
Fork 0
mirror of https://github.com/gofiber/storage.git synced 2025-10-05 08:37:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🐛 Fix bug preventing non-utf8 characters being saved in some databases (#87)

Browse Source
This commit is contained in:
hi019
2021-05-05 21:17:52 -04:00
committed by GitHub
parent aedc03b9c3
commit 6a0f72c350
8 changed files with 113 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
MIGRATE.md Normal file
View File

@@ -0,0 +1,21 @@
This document contains instructions for migrating to various storage versions.
### 0.1 -> 0.2
v0.2 fixes [a bug](https://github.com/gofiber/fiber/issues/1258) in MYSQL, Postgres and Arangodb in which
inserting non-UTF8 characters would trigger a panic due to the values being saved in a TEXT column instead of a
BYTEA/BLOB column. Migration instructions (note you may need to adjust the table names if you have supplied a custom
config to the storage):
**Postgres**
```sql
ALTER TABLE fiber_storage
ALTER COLUMN v TYPE BYTEA USING v::bytea;
```
**MYSQL**
```sql
ALTER TABLE fiber_storage MODIFY COLUMN v BLOB;
```
**Arangodb**
No migration other then updating the library is necessary.

11
arangodb/arangodb_test.go
View File

@@ -118,6 +118,17 @@ func Test_ARANGODB_Reset(t *testing.T) {
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_ARANGODB_Non_UTF8(t *testing.T) {
val := []byte("0xF5")
err := testStore.Set("0xF6", val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("0xF6")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_ARANGODB_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}

26
mysql/mysql.go
View File

@@ -3,10 +3,10 @@ package mysql
import (
"database/sql"
"fmt"
"strings"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/gofiber/utils"
)
// Storage interface that is implemented by storage providers
@@ -23,15 +23,19 @@ type Storage struct {
}
var (
checkSchemaMsg = "The `v` row has an incorrect data type. " +
"It should be BLOB but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
dropQuery = "DROP TABLE IF EXISTS %s;"
initQuery = []string{
`CREATE TABLE IF NOT EXISTS %s (
k VARCHAR(64) NOT NULL DEFAULT '',
v TEXT NOT NULL,
v BLOB NOT NULL,
e BIGINT NOT NULL DEFAULT '0',
PRIMARY KEY (k)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`,
}
checkSchemaQuery = `SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS
WHERE table_name = '%s' AND COLUMN_NAME = 'v';`
)
// New creates a new storage
@@ -85,6 +89,8 @@ func New(config ...Config) *Storage {
sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= ? AND e != 0", cfg.Table),
}
store.checkSchema(cfg.Table)
// Start garbage collector
go store.gcTicker()
@@ -133,8 +139,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
if exp != 0 {
expSeconds = time.Now().Add(exp).Unix()
}
valStr := utils.UnsafeString(val)
_, err := s.db.Exec(s.sqlInsert, key, valStr, expSeconds, valStr, expSeconds)
_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds, val, expSeconds)
return err
}
@@ -178,3 +183,16 @@ func (s *Storage) gcTicker() {
func (s *Storage) gc(t time.Time) {
_, _ = s.db.Exec(s.sqlGC, t.Unix())
}
func (s *Storage) checkSchema(tableName string) {
var data []byte
row := s.db.QueryRow(fmt.Sprintf(checkSchemaQuery, tableName))
if err := row.Scan(&data); err != nil {
panic(err)
}
if strings.ToLower(string(data)) != "blob" {
fmt.Printf(checkSchemaMsg, string(data))
}
}

11
mysql/mysql_test.go
View File

@@ -148,6 +148,17 @@ func Test_MYSQL_GC(t *testing.T) {
}
func Test_MYSQL_Non_UTF8(t *testing.T) {
val := []byte("0xF5")
err := testStore.Set("0xF6", val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("0xF6")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_MYSQL_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}

27
postgres/postgres.go
View File

@@ -5,9 +5,9 @@ import (
"errors"
"fmt"
"net/url"
"strings"
"time"
"github.com/gofiber/utils"
_ "github.com/lib/pq"
)
@@ -25,15 +25,19 @@ type Storage struct {
}
var (
checkSchemaMsg = "The `v` row has an incorrect data type. " +
"It should be BYTEA but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
dropQuery = `DROP TABLE IF EXISTS %s;`
initQuery = []string{
`CREATE TABLE IF NOT EXISTS %s (
k VARCHAR(64) PRIMARY KEY NOT NULL DEFAULT '',
v TEXT NOT NULL,
v BYTEA NOT NULL,
e BIGINT NOT NULL DEFAULT '0'
);`,
`CREATE INDEX IF NOT EXISTS e ON %s (e);`,
}
checkSchemaQuery = `SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS
WHERE table_name = '%s' AND COLUMN_NAME = 'v';`
)
// New creates a new storage
@@ -103,6 +107,8 @@ func New(config ...Config) *Storage {
sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= $1 AND e != 0", cfg.Table),
}
store.checkSchema(cfg.Table)
// Start garbage collector
go store.gcTicker()
@@ -137,7 +143,6 @@ func (s *Storage) Get(key string) ([]byte, error) {
return data, nil
}
// Set key with value
// Set key with value
func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
// Ain't Nobody Got Time For That
@@ -148,8 +153,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
if exp != 0 {
expSeconds = time.Now().Add(exp).Unix()
}
valStr := utils.UnsafeString(val)
_, err := s.db.Exec(s.sqlInsert, key, valStr, expSeconds, valStr, expSeconds)
_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds, val, expSeconds)
return err
}
@@ -193,3 +197,16 @@ func (s *Storage) gcTicker() {
func (s *Storage) gc(t time.Time) {
_, _ = s.db.Exec(s.sqlGC, t.Unix())
}
func (s *Storage) checkSchema(tableName string) {
var data []byte
row := s.db.QueryRow(fmt.Sprintf(checkSchemaQuery, tableName))
if err := row.Scan(&data); err != nil {
panic(err)
}
if strings.ToLower(string(data)) != "bytea" {
fmt.Printf(checkSchemaMsg, string(data))
}
}

11
postgres/postgres_test.go
View File

@@ -148,6 +148,17 @@ func Test_Postgres_GC(t *testing.T) {
}
func Test_Postgres_Non_UTF8(t *testing.T) {
val := []byte("0xF5")
err := testStore.Set("0xF6", val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("0xF6")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_Postgres_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}

9
sqlite3/sqlite3.go
View File

@@ -5,8 +5,6 @@ import (
"fmt"
"time"
"github.com/gofiber/utils"
_ "github.com/mattn/go-sqlite3"
)
@@ -24,11 +22,13 @@ type Storage struct {
}
var (
checkSchemaMsg = "The `v` row has an incorrect data type. " +
"It should be BLOB but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
dropQuery = `DROP TABLE IF EXISTS %s;`
initQuery = []string{
`CREATE TABLE IF NOT EXISTS %s (
k VARCHAR(64) PRIMARY KEY NOT NULL DEFAULT '',
v TEXT NOT NULL,
v BLOB NOT NULL,
e BIGINT NOT NULL DEFAULT '0'
);`,
`CREATE INDEX IF NOT EXISTS e ON %s (e);`,
@@ -115,7 +115,6 @@ func (s *Storage) Get(key string) ([]byte, error) {
return data, nil
}
// Set key with value
// Set key with value
func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
// Ain't Nobody Got Time For That
@@ -126,7 +125,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
if exp != 0 {
expSeconds = time.Now().Add(exp).Unix()
}
_, err := s.db.Exec(s.sqlInsert, key, utils.UnsafeString(val), expSeconds)
_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds)
return err
}

11
sqlite3/sqlite3_test.go
View File

@@ -145,6 +145,17 @@ func Test_SQLite3_GC(t *testing.T) {
}
func Test_SQLite3_Non_UTF8(t *testing.T) {
val := []byte("0xF5")
err := testStore.Set("0xF6", val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("0xF6")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_SQLite3_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}