From a66230ef170f64d6b72bbdecc1a4c27ce3a3504a Mon Sep 17 00:00:00 2001 From: hi019 <65871571+hi019@users.noreply.github.com> Date: Thu, 18 Mar 2021 17:09:06 -0400 Subject: [PATCH] Fix GC deleting values without expiry (#54) * Test MYSQL Fix * Test * Add postgres * Fix mysql * Fix sqlite --- mysql/mysql.go | 15 ++++++++++----- mysql/mysql_test.go | 26 ++++++++++++++++++++++++++ postgres/postgres.go | 15 ++++++++++----- postgres/postgres_test.go | 26 ++++++++++++++++++++++++++ sqlite3/sqlite3.go | 15 ++++++++++----- sqlite3/sqlite3_test.go | 26 ++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 15 deletions(-) diff --git a/mysql/mysql.go b/mysql/mysql.go index 8394fc93..d1052cc3 100644 --- a/mysql/mysql.go +++ b/mysql/mysql.go @@ -82,11 +82,11 @@ func New(config ...Config) *Storage { sqlInsert: fmt.Sprintf("INSERT INTO %s (k, v, e) VALUES (?,?,?) ON DUPLICATE KEY UPDATE v = ?, e = ?", cfg.Table), sqlDelete: fmt.Sprintf("DELETE FROM %s WHERE k=?", cfg.Table), sqlReset: fmt.Sprintf("DELETE FROM %s;", cfg.Table), - sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= ?", cfg.Table), + sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= ? AND e != 0", cfg.Table), } // Start garbage collector - go store.gc() + go store.gcTicker() return store } @@ -160,8 +160,8 @@ func (s *Storage) Close() error { return s.db.Close() } -// Garbage collector to delete expired keys -func (s *Storage) gc() { +// gcTicker starts the gc ticker +func (s *Storage) gcTicker() { ticker := time.NewTicker(s.gcInterval) defer ticker.Stop() for { @@ -169,7 +169,12 @@ func (s *Storage) gc() { case <-s.done: return case t := <-ticker.C: - _, _ = s.db.Exec(s.sqlGC, t.Unix()) + s.gc(t) } } } + +// gc deletes all expired entries +func (s *Storage) gc(t time.Time) { + _, _ = s.db.Exec(s.sqlGC, t.Unix()) +} diff --git a/mysql/mysql_test.go b/mysql/mysql_test.go index 1b86b938..86b7fa39 100644 --- a/mysql/mysql_test.go +++ b/mysql/mysql_test.go @@ -1,6 +1,7 @@ package mysql import ( + "database/sql" "os" "testing" "time" @@ -122,6 +123,31 @@ func Test_MYSQL_Reset(t *testing.T) { utils.AssertEqual(t, true, len(result) == 0) } +func Test_MYSQL_GC(t *testing.T) { + var ( + testVal = []byte("doe") + ) + + // This key should expire + err := testStore.Set("john", testVal, time.Nanosecond) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + row := testStore.db.QueryRow(testStore.sqlSelect, "john") + err = row.Scan(nil, nil) + utils.AssertEqual(t, sql.ErrNoRows, err) + + // This key should not expire + err = testStore.Set("john", testVal, 0) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + val, err := testStore.Get("john") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, testVal, val) + +} + func Test_MYSQL_Close(t *testing.T) { utils.AssertEqual(t, nil, testStore.Close()) } diff --git a/postgres/postgres.go b/postgres/postgres.go index 96aa81d8..87fa0aeb 100644 --- a/postgres/postgres.go +++ b/postgres/postgres.go @@ -100,11 +100,11 @@ func New(config ...Config) *Storage { sqlInsert: fmt.Sprintf("INSERT INTO %s (k, v, e) VALUES ($1, $2, $3) ON CONFLICT (k) DO UPDATE SET v = $4, e = $5", cfg.Table), sqlDelete: fmt.Sprintf("DELETE FROM %s WHERE k=$1", cfg.Table), sqlReset: fmt.Sprintf("DELETE FROM %s;", cfg.Table), - sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= $1", cfg.Table), + sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= $1 AND e != 0", cfg.Table), } // Start garbage collector - go store.gc() + go store.gcTicker() return store } @@ -175,8 +175,8 @@ func (s *Storage) Close() error { return s.db.Close() } -// GC deletes all expired entries -func (s *Storage) gc() { +// gcTicker starts the gc ticker +func (s *Storage) gcTicker() { ticker := time.NewTicker(s.gcInterval) defer ticker.Stop() for { @@ -184,7 +184,12 @@ func (s *Storage) gc() { case <-s.done: return case t := <-ticker.C: - _, _ = s.db.Exec(s.sqlGC, t.Unix()) + s.gc(t) } } } + +// gc deletes all expired entries +func (s *Storage) gc(t time.Time) { + _, _ = s.db.Exec(s.sqlGC, t.Unix()) +} diff --git a/postgres/postgres_test.go b/postgres/postgres_test.go index 566d2f0f..1f622e1c 100644 --- a/postgres/postgres_test.go +++ b/postgres/postgres_test.go @@ -1,6 +1,7 @@ package postgres import ( + "database/sql" "os" "testing" "time" @@ -122,6 +123,31 @@ func Test_Postgres_Reset(t *testing.T) { utils.AssertEqual(t, true, len(result) == 0) } +func Test_Postgres_GC(t *testing.T) { + var ( + testVal = []byte("doe") + ) + + // This key should expire + err := testStore.Set("john", testVal, time.Nanosecond) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + row := testStore.db.QueryRow(testStore.sqlSelect, "john") + err = row.Scan(nil, nil) + utils.AssertEqual(t, sql.ErrNoRows, err) + + // This key should not expire + err = testStore.Set("john", testVal, 0) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + val, err := testStore.Get("john") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, testVal, val) + +} + func Test_Postgres_Close(t *testing.T) { utils.AssertEqual(t, nil, testStore.Close()) } diff --git a/sqlite3/sqlite3.go b/sqlite3/sqlite3.go index 14ee9814..4730db8e 100644 --- a/sqlite3/sqlite3.go +++ b/sqlite3/sqlite3.go @@ -81,11 +81,11 @@ func New(config ...Config) *Storage { sqlInsert: fmt.Sprintf("INSERT OR REPLACE INTO %s (k, v, e) VALUES (?,?,?)", cfg.Table), sqlDelete: fmt.Sprintf("DELETE FROM %s WHERE k=?", cfg.Table), sqlReset: fmt.Sprintf("DELETE FROM %s;", cfg.Table), - sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= ?", cfg.Table), + sqlGC: fmt.Sprintf("DELETE FROM %s WHERE e <= ? AND e != 0", cfg.Table), } // Start garbage collector - go store.gc() + go store.gcTicker() return store } @@ -152,8 +152,8 @@ func (s *Storage) Close() error { return s.db.Close() } -// GC deletes all expired entries -func (s *Storage) gc() { +// gcTicker starts the gc ticker +func (s *Storage) gcTicker() { ticker := time.NewTicker(s.gcInterval) defer ticker.Stop() for { @@ -161,7 +161,12 @@ func (s *Storage) gc() { case <-s.done: return case t := <-ticker.C: - _, _ = s.db.Exec(s.sqlGC, t.Unix()) + s.gc(t) } } } + +// gc deletes all expired entries +func (s *Storage) gc(t time.Time) { + _, _ = s.db.Exec(s.sqlGC, t.Unix()) +} diff --git a/sqlite3/sqlite3_test.go b/sqlite3/sqlite3_test.go index 4a93b8f5..9b9fe3f4 100644 --- a/sqlite3/sqlite3_test.go +++ b/sqlite3/sqlite3_test.go @@ -1,6 +1,7 @@ package sqlite3 import ( + "database/sql" "testing" "time" @@ -119,6 +120,31 @@ func Test_SQLite3_Reset(t *testing.T) { utils.AssertEqual(t, true, len(result) == 0) } +func Test_SQLite3_GC(t *testing.T) { + var ( + testVal = []byte("doe") + ) + + // This key should expire + err := testStore.Set("john", testVal, time.Nanosecond) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + row := testStore.db.QueryRow(testStore.sqlSelect, "john") + err = row.Scan(nil, nil) + utils.AssertEqual(t, sql.ErrNoRows, err) + + // This key should not expire + err = testStore.Set("john", testVal, 0) + utils.AssertEqual(t, nil, err) + + testStore.gc(time.Now()) + val, err := testStore.Get("john") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, testVal, val) + +} + func Test_SQLite3_Close(t *testing.T) { utils.AssertEqual(t, nil, testStore.Close()) }