diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 975c9586..3dcc2585 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -140,3 +140,9 @@ updates: - "๐Ÿค– Dependencies" schedule: interval: "daily" + - package-ecosystem: "gomod" + directory: "/scylladb/" # Location of package manifests + labels: + - "๐Ÿค– Dependencies" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/release-drafter-scylladb.yml b/.github/release-drafter-scylladb.yml new file mode 100644 index 00000000..27204e68 --- /dev/null +++ b/.github/release-drafter-scylladb.yml @@ -0,0 +1,41 @@ +name-template: 'ScyllaDb - v$RESOLVED_VERSION' +tag-template: 'scylladb/v$RESOLVED_VERSION' +tag-prefix: scylladb/v +include-paths: + - scylladb +categories: + - title: '๐Ÿš€ New' + labels: + - 'โœ๏ธ Feature' + - title: '๐Ÿงน Updates' + labels: + - '๐Ÿงน Updates' + - '๐Ÿค– Dependencies' + - title: '๐Ÿ› Fixes' + labels: + - 'โ˜ข๏ธ Bug' + - title: '๐Ÿ“š Documentation' + labels: + - '๐Ÿ“’ Documentation' +change-template: '- $TITLE (#$NUMBER)' +change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. +version-resolver: + major: + labels: + - 'major' + minor: + labels: + - 'minor' + - 'โœ๏ธ Feature' + patch: + labels: + - 'patch' + - '๐Ÿ“’ Documentation' + - 'โ˜ข๏ธ Bug' + - '๐Ÿค– Dependencies' + - '๐Ÿงน Updates' + default: patch +template: | + $CHANGES + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...scylladb/v$RESOLVED_VERSION + Thank you $CONTRIBUTORS for making this update possible. \ No newline at end of file diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml index 2b30875b..0ff5f1f5 100644 --- a/.github/workflows/gosec.yml +++ b/.github/workflows/gosec.yml @@ -124,3 +124,7 @@ jobs: working-directory: ./rueidis run: gosec ./... # ----- + - name: Run Gosec (scylladb) + working-directory: ./scylladb + run: gosec ./... + # ----- diff --git a/.github/workflows/release-drafter-scylladb.yml b/.github/workflows/release-drafter-scylladb.yml new file mode 100644 index 00000000..3fdb5fba --- /dev/null +++ b/.github/workflows/release-drafter-scylladb.yml @@ -0,0 +1,19 @@ +name: Release Drafter ScyllaDb +on: + push: + # branches to consider in the event; optional, defaults to all + branches: + - master + - main + paths: + - 'scylladb/**' +jobs: + draft_release_scylladb: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: release-drafter/release-drafter@v5 + with: + config-name: release-drafter-scylladb.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test-scylladb.yml b/.github/workflows/test-scylladb.yml new file mode 100644 index 00000000..c76def0d --- /dev/null +++ b/.github/workflows/test-scylladb.yml @@ -0,0 +1,43 @@ +on: + push: + branches: + - master + - main + paths: + - 'scylladb/**' + pull_request: + paths: + - 'scylladb/**' + +name: "Tests ScyllaDb" + +jobs: + Tests: + runs-on: ubuntu-latest + + strategy: + matrix: + go-version: + - 1.19.x + - 1.20.x + + steps: + - name: Fetch Repository + uses: actions/checkout@v3 + + - name: Run ScyllaDb + run: | + docker run --name scylladb -p 9042:9042 -d scylladb/scylla:latest --listen-address 0.0.0.0 + sleep 30 # Wait for ScyllaDb to initialize + + - name: Create Default Keyspace + run: | + docker exec -it scylladb cqlsh -e "CREATE KEYSPACE IF NOT EXISTS scylla_db WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};" + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '${{ matrix.go-version }}' + + - name: Run Test + run: cd ./scylladb && go test ./... -v -race diff --git a/scylladb/README.md b/scylladb/README.md new file mode 100644 index 00000000..357a210d --- /dev/null +++ b/scylladb/README.md @@ -0,0 +1,111 @@ +# ScyllaDB + +A ScyllaDB storage driver using [gocql/gocql]("https://github.com/gocql/gocql"). + +### Table of Contents +- [Signatures](#signatures) +- [Installation](#installation) +- [Examples](#examples) +- [Config](#config) +- [Default Config](#default-config) + +### Signatures +```go +func New(config ...Config) Storage +func (s *Storage) Get(key string) ([]byte, error) +func (s *Storage) Set(key string, val []byte, exp time.Duration) error +func (s *Storage) Delete(key string) error +func (s *Storage) Reset() error +func (s *Storage) Close() error +func (s *Storage) Conn() *gocql.Session +``` +### Installation +ScyllaDB is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet: +```bash +go mod init github.com// +``` +And then install the scylladb implementation: +```bash +go get github.com/gofiber/storage/scylladb +``` + +### Examples +Import the storage package. +```go +import "github.com/gofiber/storage/scylladb" +``` + +You can use the following possibilities to create a storage: +```go +// Initialize default config +store := scylladb.New() + +// Initialize custom config +store := scylladb.New(scylladb.Config{ + Host: "127.0.0.1", + Port: 9042, + Database: "fiber", + Collection: "fiber_storage", + Reset: false, +}) + + +``` + +### Config +```go +type Config struct { + // Host name where the DB is hosted + // + // Optional. Default is "127.0.0.1" + Host string + + // Port where the DB is listening on + // + // Optional. Default is 9042 + Port int + + // Server username + // + // Optional. Default is "" + Username string + + // Server password + // + // Optional. Default is "" + Password string + + // Keyspace name + // + // Optional. Default is "scylladb_db" + Keyspace string + + // Number of replication + // + // Optional. Default 1 + ReplicationFactor int + + // Database to be operated on in the cluster. + // + // Optional. Default is "". + Table string + + // Reset clears any existing keys in existing Table + // + // Optional. Default is false + Reset bool +} +``` + +### Default Config +```go +var ConfigDefault = Config{ + Host: "127.0.0.1", + Port: 9042, + Username: "", + Password: "", + Table: "scylladb_table", + Keyspace: "scylladb_db", + ReplicationFactor: 1, +} +``` diff --git a/scylladb/config.go b/scylladb/config.go new file mode 100644 index 00000000..cb3b5d35 --- /dev/null +++ b/scylladb/config.go @@ -0,0 +1,82 @@ +package scylladb + +type Config struct { + // Host name where the DB is hosted + // + // Optional. Default is "127.0.0.1" + Hosts []string + + // Server username + // + // Optional. Default is "" + Username string + + // Server password + // + // Optional. Default is "" + Password string + + // Name of the keyspace + // + // Optional. Default is "scylla_db" + Keyspace string + + // Level of the consistency + // + // Optional. Default is "LOCAL_ONE" + Consistency string + + // Number of replication factor + // + // Optional. Default 1 + ReplicationFactor int + + // Database to be operated on in the cluster. + // + // Optional. Default is "scylla_table". + Table string + + // Reset clears any existing keys in existing Table + // + // Optional. Default is false + Reset bool +} + +var ConfigDefault = Config{ + Hosts: []string{"172.19.0.10"}, + Username: "", + Password: "", + Table: "scylla_table", + Keyspace: "scylla_db", + Consistency: "LOCAL_ONE", + ReplicationFactor: 1, +} + +func configDefault(config ...Config) Config { + // Return default config if nothing provided + if len(config) < 1 { + return ConfigDefault + } + + // Override default config + cfg := config[0] + + // Set default values + if cfg.Hosts == nil { + cfg.Hosts = ConfigDefault.Hosts + } + if cfg.Keyspace == "" { + cfg.Keyspace = ConfigDefault.Keyspace + } + if cfg.Table == "" { + cfg.Table = ConfigDefault.Table + } + if cfg.Consistency == "" { + cfg.Consistency = ConfigDefault.Consistency + } + if cfg.ReplicationFactor <= 0 { + cfg.ReplicationFactor = ConfigDefault.ReplicationFactor + } + + return cfg +} diff --git a/scylladb/go.mod b/scylladb/go.mod new file mode 100644 index 00000000..a4270c96 --- /dev/null +++ b/scylladb/go.mod @@ -0,0 +1,16 @@ +module github.com/gofiber/storage/scylladb + +go 1.19 + +require ( + github.com/gocql/gocql v1.6.0 + github.com/gofiber/utils v1.1.0 +) + +require ( + github.com/golang/snappy v0.0.4 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + gopkg.in/inf.v0 v0.9.1 // indirect +) + +replace github.com/gocql/gocql => github.com/scylladb/gocql v1.11.1 diff --git a/scylladb/go.sum b/scylladb/go.sum new file mode 100644 index 00000000..a15a8785 --- /dev/null +++ b/scylladb/go.sum @@ -0,0 +1,41 @@ +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= +github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/scylladb/gocql v1.11.1 h1:AlIPHHZf2l0Cbj8wGjfELspaGfnd4meGj9sPQnr5dn8= +github.com/scylladb/gocql v1.11.1/go.mod h1:ZLEJ0EVE5JhmtxIW2stgHq/v1P4fWap0qyyXSKyV8K0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/scylladb/scylladb.go b/scylladb/scylladb.go new file mode 100644 index 00000000..57b6f627 --- /dev/null +++ b/scylladb/scylladb.go @@ -0,0 +1,140 @@ +package scylladb + +import ( + "context" + "errors" + "fmt" + "github.com/gocql/gocql" + "time" +) + +type Storage struct { + Cluster *gocql.ClusterConfig + Session *gocql.Session + + cqlSelect string + cqlInsert string + cqlDelete string + cqlReset string + cqlGC string +} + +var ( + createKeyspaceQuery = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};" + + dropQuery = "DROP TABLE IF EXISTS %s;" + + initQuery = []string{ + `CREATE TABLE IF NOT EXISTS %s ( + k text PRIMARY KEY, + v blob, + e bigint + );`, + } +) + +func New(config ...Config) *Storage { + cfg := configDefault(config...) + + cluster := gocql.NewCluster(cfg.Hosts...) + cluster.Consistency = gocql.ParseConsistency(cfg.Consistency) + cluster.Keyspace = cfg.Keyspace + + session, err := cluster.CreateSession() + if err != nil { + panic(err) + } + + // Primitive ping method + if err := session.Query("SELECT release_version FROM system.local").Exec(); err != nil { + session.Close() + panic(err) + } + + // Create keyspace if it does not exist + if err := session.Query(fmt.Sprintf(createKeyspaceQuery, cfg.Keyspace)).Exec(); err != nil { + session.Close() + panic(err) + } + + // Drop table if reset set + ctx := context.Background() + if cfg.Reset { + if err := session.Query(dropQuery, cfg.Table).WithContext(ctx).Exec(); err != nil { + session.Close() + panic(err) + } + } + + // Init database queries + ctx = context.Background() + for _, query := range initQuery { + + if err := session.Query(fmt.Sprintf(query, cfg.Table)).WithContext(ctx).Exec(); err != nil { + session.Close() + panic(err) + } + } + + storage := &Storage{ + Cluster: cluster, + Session: session, + + cqlSelect: fmt.Sprintf(`SELECT v, e FROM %s WHERE k=?`, cfg.Table), + cqlInsert: fmt.Sprintf(`INSERT INTO %s (k, v, e) VALUES (?, ?, ?)`, cfg.Table), + cqlDelete: fmt.Sprintf(`DELETE FROM %s WHERE k=?`, cfg.Table), + cqlReset: fmt.Sprintf(`TRUNCATE %s`, cfg.Table), + cqlGC: fmt.Sprintf(`DELETE FROM %s WHERE e <= ? AND e != 0`, cfg.Table), + } + + return storage +} + +// Get value by key +func (s *Storage) Get(key string) ([]byte, error) { + ctx := context.Background() + var ( + data []byte + exp int64 = 0 + ) + if err := s.Session.Query(s.cqlSelect, key).WithContext(ctx).Scan(&data, &exp); err != nil { + if errors.Is(err, gocql.ErrNotFound) { + return nil, nil + } + return nil, err + } + + return data, nil +} + +// Set sets a value in the storage for the provided key +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { + ctx := context.Background() + + return s.Session.Query(s.cqlInsert, key, val, int64(exp.Seconds())).WithContext(ctx).Exec() +} + +// Delete deletes a value from the storage based on the provided key +func (s *Storage) Delete(key string) error { + ctx := context.Background() + + return s.Session.Query(s.cqlDelete, key).WithContext(ctx).Exec() +} + +// Reset resets the storage +func (s *Storage) Reset() error { + ctx := context.Background() + + return s.Session.Query(s.cqlReset).WithContext(ctx).Exec() +} + +// Close closes the connection to the storage +func (s *Storage) Close() error { + s.Session.Close() + return nil +} + +// Conn returns session +func (s *Storage) Conn() *gocql.Session { + return s.Session +} diff --git a/scylladb/scylladb_test.go b/scylladb/scylladb_test.go new file mode 100644 index 00000000..dde3755f --- /dev/null +++ b/scylladb/scylladb_test.go @@ -0,0 +1,114 @@ +package scylladb + +import ( + "github.com/gofiber/utils" + "testing" + "time" +) + +var testStore = New(Config{}) + +func Test_Scylla_Set(t *testing.T) { + // Create a new instance of the Storage + var ( + key = "john" + value = []byte("doe") + ) + err := testStore.Set(key, value, time.Minute) + utils.AssertEqual(t, nil, err, "Failed to set the value") +} + +func Test_Scylla_Set_Override(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) +} + +func Test_Scylla_Get(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + result, err := testStore.Get(key) + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, val, result) +} + +func Test_Scylla_Set_Expiration(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + exp = 1 * time.Second + ) + + err := testStore.Set(key, val, exp) + utils.AssertEqual(t, nil, err) + + time.Sleep(1100 * time.Millisecond) +} + +func Test_Scylla_Get_NotExist(t *testing.T) { + + result, err := testStore.Get("not-exist") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, true, len(result) == 0) +} + +func Test_Scylla_Delete(t *testing.T) { + var ( + key = "john" + val = []byte("doe") + ) + + err := testStore.Set(key, val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Delete(key) + utils.AssertEqual(t, nil, err) + + result, err := testStore.Get(key) + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, true, len(result) == 0) +} + +func Test_Scylla_Reset(t *testing.T) { + var ( + val = []byte("doe") + ) + + err := testStore.Set("john1", val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Set("john2", val, 0) + utils.AssertEqual(t, nil, err) + + err = testStore.Reset() + utils.AssertEqual(t, nil, err) + + result, err := testStore.Get("john1") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, true, len(result) == 0) + + result, err = testStore.Get("john2") + utils.AssertEqual(t, nil, err) + utils.AssertEqual(t, true, len(result) == 0) +} + +func Test_Scylla_Close(t *testing.T) { + utils.AssertEqual(t, nil, testStore.Close()) +} + +func Test_Scylla_Conn(t *testing.T) { + utils.AssertEqual(t, true, testStore.Conn() != nil) +}