🐛 Fix bug preventing non-utf8 characters being saved in some databases (#87)

MIGRATE.md

@@ -0,0 +1,21 @@
+This document contains instructions for migrating to various storage versions.
+
+### 0.1 -> 0.2
+v0.2 fixes [a bug](https://github.com/gofiber/fiber/issues/1258) in MYSQL, Postgres and Arangodb in which 
+inserting non-UTF8 characters would trigger a panic due to the values being saved in a TEXT column instead of a 
+BYTEA/BLOB column. Migration instructions (note you may need to adjust the table names if you have supplied a custom
+config to the storage):
+
+**Postgres**
+```sql
+ALTER TABLE fiber_storage
+ALTER COLUMN v TYPE BYTEA USING v::bytea;
+```
+
+**MYSQL**
+```sql
+ALTER TABLE fiber_storage MODIFY COLUMN v BLOB;
+```
+
+**Arangodb**
+No migration other then updating the library is necessary.

arangodb/arangodb_test.go

@@ -118,6 +118,17 @@ func Test_ARANGODB_Reset(t *testing.T) {
 	utils.AssertEqual(t, true, len(result) == 0)
 }
 
+func Test_ARANGODB_Non_UTF8(t *testing.T) {
+	val := []byte("0xF5")
+
+	err := testStore.Set("0xF6", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get("0xF6")
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, val, result)
+}
+
 func Test_ARANGODB_Close(t *testing.T) {
 	utils.AssertEqual(t, nil, testStore.Close())
 }

mysql/mysql.go

@@ -3,10 +3,10 @@ package mysql
 import (
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
 	_ "github.com/go-sql-driver/mysql"
-	"github.com/gofiber/utils"
 )
 
 // Storage interface that is implemented by storage providers
@@ -23,15 +23,19 @@ type Storage struct {
 }
 
 var (
+	checkSchemaMsg = "The `v` row has an incorrect data type. " +
+		"It should be BLOB but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
 	dropQuery = "DROP TABLE IF EXISTS %s;"
 	initQuery = []string{
 		`CREATE TABLE IF NOT EXISTS %s ( 
 			k  VARCHAR(64) NOT NULL DEFAULT '', 
-			v  TEXT NOT NULL, 
+			v  BLOB NOT NULL, 
 			e  BIGINT NOT NULL DEFAULT '0', 
 			PRIMARY KEY (k)
 		) ENGINE=InnoDB DEFAULT CHARSET=utf8;`,
 	}
+	checkSchemaQuery = `SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS
+		WHERE table_name = '%s' AND COLUMN_NAME = 'v';`
 )
 
 // New creates a new storage
@@ -85,6 +89,8 @@ func New(config ...Config) *Storage {
 		sqlGC:      fmt.Sprintf("DELETE FROM %s WHERE e <= ? AND e != 0", cfg.Table),
 	}
 
+	store.checkSchema(cfg.Table)
+
 	// Start garbage collector
 	go store.gcTicker()
 
@@ -133,8 +139,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
 	if exp != 0 {
 		expSeconds = time.Now().Add(exp).Unix()
 	}
-	valStr := utils.UnsafeString(val)
+	_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds, val, expSeconds)
-	_, err := s.db.Exec(s.sqlInsert, key, valStr, expSeconds, valStr, expSeconds)
 	return err
 }
 
@@ -178,3 +183,16 @@ func (s *Storage) gcTicker() {
 func (s *Storage) gc(t time.Time) {
 	_, _ = s.db.Exec(s.sqlGC, t.Unix())
 }
+
+func (s *Storage) checkSchema(tableName string) {
+	var data []byte
+
+	row := s.db.QueryRow(fmt.Sprintf(checkSchemaQuery, tableName))
+	if err := row.Scan(&data); err != nil {
+		panic(err)
+	}
+
+	if strings.ToLower(string(data)) != "blob" {
+		fmt.Printf(checkSchemaMsg, string(data))
+	}
+}

mysql/mysql_test.go

@@ -148,6 +148,17 @@ func Test_MYSQL_GC(t *testing.T) {
 
 }
 
+func Test_MYSQL_Non_UTF8(t *testing.T) {
+	val := []byte("0xF5")
+
+	err := testStore.Set("0xF6", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get("0xF6")
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, val, result)
+}
+
 func Test_MYSQL_Close(t *testing.T) {
 	utils.AssertEqual(t, nil, testStore.Close())
 }

postgres/postgres.go

@@ -5,9 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 
-	"github.com/gofiber/utils"
 	_ "github.com/lib/pq"
 )
 
@@ -25,15 +25,19 @@ type Storage struct {
 }
 
 var (
+	checkSchemaMsg = "The `v` row has an incorrect data type. " +
+		"It should be BYTEA but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
 	dropQuery = `DROP TABLE IF EXISTS %s;`
 	initQuery = []string{
 		`CREATE TABLE IF NOT EXISTS %s (
 			k  VARCHAR(64) PRIMARY KEY NOT NULL DEFAULT '',
-			v  TEXT NOT NULL,
+			v  BYTEA NOT NULL,
 			e  BIGINT NOT NULL DEFAULT '0'
 		);`,
 		`CREATE INDEX IF NOT EXISTS e ON %s (e);`,
 	}
+	checkSchemaQuery = `SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS
+		WHERE table_name = '%s' AND COLUMN_NAME = 'v';`
 )
 
 // New creates a new storage
@@ -103,6 +107,8 @@ func New(config ...Config) *Storage {
 		sqlGC:      fmt.Sprintf("DELETE FROM %s WHERE e <= $1 AND e != 0", cfg.Table),
 	}
 
+	store.checkSchema(cfg.Table)
+
 	// Start garbage collector
 	go store.gcTicker()
 
@@ -137,7 +143,6 @@ func (s *Storage) Get(key string) ([]byte, error) {
 	return data, nil
 }
 
-// Set key with value
 // Set key with value
 func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
 	// Ain't Nobody Got Time For That
@@ -148,8 +153,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
 	if exp != 0 {
 		expSeconds = time.Now().Add(exp).Unix()
 	}
-	valStr := utils.UnsafeString(val)
+	_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds, val, expSeconds)
-	_, err := s.db.Exec(s.sqlInsert, key, valStr, expSeconds, valStr, expSeconds)
 	return err
 }
 
@@ -193,3 +197,16 @@ func (s *Storage) gcTicker() {
 func (s *Storage) gc(t time.Time) {
 	_, _ = s.db.Exec(s.sqlGC, t.Unix())
 }
+
+func (s *Storage) checkSchema(tableName string) {
+	var data []byte
+
+	row := s.db.QueryRow(fmt.Sprintf(checkSchemaQuery, tableName))
+	if err := row.Scan(&data); err != nil {
+		panic(err)
+	}
+
+	if strings.ToLower(string(data)) != "bytea" {
+		fmt.Printf(checkSchemaMsg, string(data))
+	}
+}

postgres/postgres_test.go

@@ -148,6 +148,17 @@ func Test_Postgres_GC(t *testing.T) {
 
 }
 
+func Test_Postgres_Non_UTF8(t *testing.T) {
+	val := []byte("0xF5")
+
+	err := testStore.Set("0xF6", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get("0xF6")
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, val, result)
+}
+
 func Test_Postgres_Close(t *testing.T) {
 	utils.AssertEqual(t, nil, testStore.Close())
 }

sqlite3/sqlite3.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/gofiber/utils"
-
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -24,11 +22,13 @@ type Storage struct {
 }
 
 var (
+	checkSchemaMsg = "The `v` row has an incorrect data type. " +
+		"It should be BLOB but is instead %s. This will cause encoding-related panics if the DB is not migrated (see https://github.com/gofiber/storage/blob/main/MIGRATE.md)."
 	dropQuery = `DROP TABLE IF EXISTS %s;`
 	initQuery = []string{
 		`CREATE TABLE IF NOT EXISTS %s (
 			k  VARCHAR(64) PRIMARY KEY NOT NULL DEFAULT '',
-			v TEXT NOT NULL,
+			v  BLOB NOT NULL,
 			e  BIGINT NOT NULL DEFAULT '0'
 		);`,
 		`CREATE INDEX IF NOT EXISTS e ON %s (e);`,
@@ -115,7 +115,6 @@ func (s *Storage) Get(key string) ([]byte, error) {
 	return data, nil
 }
 
-// Set key with value
 // Set key with value
 func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
 	// Ain't Nobody Got Time For That
@@ -126,7 +125,7 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
 	if exp != 0 {
 		expSeconds = time.Now().Add(exp).Unix()
 	}
-	_, err := s.db.Exec(s.sqlInsert, key, utils.UnsafeString(val), expSeconds)
+	_, err := s.db.Exec(s.sqlInsert, key, val, expSeconds)
 	return err
 }
 

sqlite3/sqlite3_test.go

@@ -145,6 +145,17 @@ func Test_SQLite3_GC(t *testing.T) {
 
 }
 
+func Test_SQLite3_Non_UTF8(t *testing.T) {
+	val := []byte("0xF5")
+
+	err := testStore.Set("0xF6", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get("0xF6")
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, val, result)
+}
+
 func Test_SQLite3_Close(t *testing.T) {
 	utils.AssertEqual(t, nil, testStore.Close())
 }