New Storage Driver: ScyllaDb

This commit is contained in:
chris.grundling
2023-10-24 21:16:48 +02:00
parent 4b59c5aa07
commit 2f5ae09380
11 changed files with 617 additions and 0 deletions

View File

@@ -140,3 +140,9 @@ updates:
- "🤖 Dependencies"
schedule:
interval: "daily"
- package-ecosystem: "gomod"
directory: "/scylladb/" # Location of package manifests
labels:
- "🤖 Dependencies"
schedule:
interval: "daily"

41
.github/release-drafter-scylladb.yml vendored Normal file
View File

@@ -0,0 +1,41 @@
name-template: 'ScyllaDb - v$RESOLVED_VERSION'
tag-template: 'scylladb/v$RESOLVED_VERSION'
tag-prefix: scylladb/v
include-paths:
- scylladb
categories:
- title: '🚀 New'
labels:
- '✏️ Feature'
- title: '🧹 Updates'
labels:
- '🧹 Updates'
- '🤖 Dependencies'
- title: '🐛 Fixes'
labels:
- '☢️ Bug'
- title: '📚 Documentation'
labels:
- '📒 Documentation'
change-template: '- $TITLE (#$NUMBER)'
change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
version-resolver:
major:
labels:
- 'major'
minor:
labels:
- 'minor'
- '✏️ Feature'
patch:
labels:
- 'patch'
- '📒 Documentation'
- '☢️ Bug'
- '🤖 Dependencies'
- '🧹 Updates'
default: patch
template: |
$CHANGES
**Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...scylladb/v$RESOLVED_VERSION
Thank you $CONTRIBUTORS for making this update possible.

View File

@@ -124,3 +124,7 @@ jobs:
working-directory: ./rueidis
run: gosec ./...
# -----
- name: Run Gosec (scylladb)
working-directory: ./scylladb
run: gosec ./...
# -----

View File

@@ -0,0 +1,19 @@
name: Release Drafter ScyllaDb
on:
push:
# branches to consider in the event; optional, defaults to all
branches:
- master
- main
paths:
- 'scylladb/**'
jobs:
draft_release_scylladb:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: release-drafter/release-drafter@v5
with:
config-name: release-drafter-scylladb.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

43
.github/workflows/test-scylladb.yml vendored Normal file
View File

@@ -0,0 +1,43 @@
on:
push:
branches:
- master
- main
paths:
- 'scylladb/**'
pull_request:
paths:
- 'scylladb/**'
name: "Tests ScyllaDb"
jobs:
Tests:
runs-on: ubuntu-latest
strategy:
matrix:
go-version:
- 1.19.x
- 1.20.x
steps:
- name: Fetch Repository
uses: actions/checkout@v3
- name: Run ScyllaDb
run: |
docker run --name scylladb -p 9042:9042 -d scylladb/scylla:latest --listen-address 0.0.0.0
sleep 30 # Wait for ScyllaDb to initialize
- name: Create Default Keyspace
run: |
docker exec -it scylladb cqlsh -e "CREATE KEYSPACE IF NOT EXISTS scylla_db WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};"
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '${{ matrix.go-version }}'
- name: Run Test
run: cd ./scylladb && go test ./... -v -race

111
scylladb/README.md Normal file
View File

@@ -0,0 +1,111 @@
# ScyllaDB
A ScyllaDB storage driver using [gocql/gocql]("https://github.com/gocql/gocql").
### Table of Contents
- [Signatures](#signatures)
- [Installation](#installation)
- [Examples](#examples)
- [Config](#config)
- [Default Config](#default-config)
### Signatures
```go
func New(config ...Config) Storage
func (s *Storage) Get(key string) ([]byte, error)
func (s *Storage) Set(key string, val []byte, exp time.Duration) error
func (s *Storage) Delete(key string) error
func (s *Storage) Reset() error
func (s *Storage) Close() error
func (s *Storage) Conn() *gocql.Session
```
### Installation
ScyllaDB is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet:
```bash
go mod init github.com/<user>/<repo>
```
And then install the scylladb implementation:
```bash
go get github.com/gofiber/storage/scylladb
```
### Examples
Import the storage package.
```go
import "github.com/gofiber/storage/scylladb"
```
You can use the following possibilities to create a storage:
```go
// Initialize default config
store := scylladb.New()
// Initialize custom config
store := scylladb.New(scylladb.Config{
Host: "127.0.0.1",
Port: 9042,
Database: "fiber",
Collection: "fiber_storage",
Reset: false,
})
```
### Config
```go
type Config struct {
// Host name where the DB is hosted
//
// Optional. Default is "127.0.0.1"
Host string
// Port where the DB is listening on
//
// Optional. Default is 9042
Port int
// Server username
//
// Optional. Default is ""
Username string
// Server password
//
// Optional. Default is ""
Password string
// Keyspace name
//
// Optional. Default is "scylladb_db"
Keyspace string
// Number of replication
//
// Optional. Default 1
ReplicationFactor int
// Database to be operated on in the cluster.
//
// Optional. Default is "".
Table string
// Reset clears any existing keys in existing Table
//
// Optional. Default is false
Reset bool
}
```
### Default Config
```go
var ConfigDefault = Config{
Host: "127.0.0.1",
Port: 9042,
Username: "",
Password: "",
Table: "scylladb_table",
Keyspace: "scylladb_db",
ReplicationFactor: 1,
}
```

82
scylladb/config.go Normal file
View File

@@ -0,0 +1,82 @@
package scylladb
type Config struct {
// Host name where the DB is hosted
//
// Optional. Default is "127.0.0.1"
Hosts []string
// Server username
//
// Optional. Default is ""
Username string
// Server password
//
// Optional. Default is ""
Password string
// Name of the keyspace
//
// Optional. Default is "scylla_db"
Keyspace string
// Level of the consistency
//
// Optional. Default is "LOCAL_ONE"
Consistency string
// Number of replication factor
//
// Optional. Default 1
ReplicationFactor int
// Database to be operated on in the cluster.
//
// Optional. Default is "scylla_table".
Table string
// Reset clears any existing keys in existing Table
//
// Optional. Default is false
Reset bool
}
var ConfigDefault = Config{
Hosts: []string{"172.19.0.10"},
Username: "",
Password: "",
Table: "scylla_table",
Keyspace: "scylla_db",
Consistency: "LOCAL_ONE",
ReplicationFactor: 1,
}
func configDefault(config ...Config) Config {
// Return default config if nothing provided
if len(config) < 1 {
return ConfigDefault
}
// Override default config
cfg := config[0]
// Set default values
if cfg.Hosts == nil {
cfg.Hosts = ConfigDefault.Hosts
}
if cfg.Keyspace == "" {
cfg.Keyspace = ConfigDefault.Keyspace
}
if cfg.Table == "" {
cfg.Table = ConfigDefault.Table
}
if cfg.Consistency == "" {
cfg.Consistency = ConfigDefault.Consistency
}
if cfg.ReplicationFactor <= 0 {
cfg.ReplicationFactor = ConfigDefault.ReplicationFactor
}
return cfg
}

16
scylladb/go.mod Normal file
View File

@@ -0,0 +1,16 @@
module github.com/gofiber/storage/scylladb
go 1.19
require (
github.com/gocql/gocql v1.6.0
github.com/gofiber/utils v1.1.0
)
require (
github.com/golang/snappy v0.0.4 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
)
replace github.com/gocql/gocql => github.com/scylladb/gocql v1.11.1

41
scylladb/go.sum Normal file
View File

@@ -0,0 +1,41 @@
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM=
github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0=
github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/scylladb/gocql v1.11.1 h1:AlIPHHZf2l0Cbj8wGjfELspaGfnd4meGj9sPQnr5dn8=
github.com/scylladb/gocql v1.11.1/go.mod h1:ZLEJ0EVE5JhmtxIW2stgHq/v1P4fWap0qyyXSKyV8K0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8=
golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

140
scylladb/scylladb.go Normal file
View File

@@ -0,0 +1,140 @@
package scylladb
import (
"context"
"errors"
"fmt"
"github.com/gocql/gocql"
"time"
)
type Storage struct {
Cluster *gocql.ClusterConfig
Session *gocql.Session
cqlSelect string
cqlInsert string
cqlDelete string
cqlReset string
cqlGC string
}
var (
createKeyspaceQuery = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"
dropQuery = "DROP TABLE IF EXISTS %s;"
initQuery = []string{
`CREATE TABLE IF NOT EXISTS %s (
k text PRIMARY KEY,
v blob,
e bigint
);`,
}
)
func New(config ...Config) *Storage {
cfg := configDefault(config...)
cluster := gocql.NewCluster(cfg.Hosts...)
cluster.Consistency = gocql.ParseConsistency(cfg.Consistency)
cluster.Keyspace = cfg.Keyspace
session, err := cluster.CreateSession()
if err != nil {
panic(err)
}
// Primitive ping method
if err := session.Query("SELECT release_version FROM system.local").Exec(); err != nil {
session.Close()
panic(err)
}
// Create keyspace if it does not exist
if err := session.Query(fmt.Sprintf(createKeyspaceQuery, cfg.Keyspace)).Exec(); err != nil {
session.Close()
panic(err)
}
// Drop table if reset set
ctx := context.Background()
if cfg.Reset {
if err := session.Query(dropQuery, cfg.Table).WithContext(ctx).Exec(); err != nil {
session.Close()
panic(err)
}
}
// Init database queries
ctx = context.Background()
for _, query := range initQuery {
if err := session.Query(fmt.Sprintf(query, cfg.Table)).WithContext(ctx).Exec(); err != nil {
session.Close()
panic(err)
}
}
storage := &Storage{
Cluster: cluster,
Session: session,
cqlSelect: fmt.Sprintf(`SELECT v, e FROM %s WHERE k=?`, cfg.Table),
cqlInsert: fmt.Sprintf(`INSERT INTO %s (k, v, e) VALUES (?, ?, ?)`, cfg.Table),
cqlDelete: fmt.Sprintf(`DELETE FROM %s WHERE k=?`, cfg.Table),
cqlReset: fmt.Sprintf(`TRUNCATE %s`, cfg.Table),
cqlGC: fmt.Sprintf(`DELETE FROM %s WHERE e <= ? AND e != 0`, cfg.Table),
}
return storage
}
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
ctx := context.Background()
var (
data []byte
exp int64 = 0
)
if err := s.Session.Query(s.cqlSelect, key).WithContext(ctx).Scan(&data, &exp); err != nil {
if errors.Is(err, gocql.ErrNotFound) {
return nil, nil
}
return nil, err
}
return data, nil
}
// Set sets a value in the storage for the provided key
func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
ctx := context.Background()
return s.Session.Query(s.cqlInsert, key, val, int64(exp.Seconds())).WithContext(ctx).Exec()
}
// Delete deletes a value from the storage based on the provided key
func (s *Storage) Delete(key string) error {
ctx := context.Background()
return s.Session.Query(s.cqlDelete, key).WithContext(ctx).Exec()
}
// Reset resets the storage
func (s *Storage) Reset() error {
ctx := context.Background()
return s.Session.Query(s.cqlReset).WithContext(ctx).Exec()
}
// Close closes the connection to the storage
func (s *Storage) Close() error {
s.Session.Close()
return nil
}
// Conn returns session
func (s *Storage) Conn() *gocql.Session {
return s.Session
}

114
scylladb/scylladb_test.go Normal file
View File

@@ -0,0 +1,114 @@
package scylladb
import (
"github.com/gofiber/utils"
"testing"
"time"
)
var testStore = New(Config{})
func Test_Scylla_Set(t *testing.T) {
// Create a new instance of the Storage
var (
key = "john"
value = []byte("doe")
)
err := testStore.Set(key, value, time.Minute)
utils.AssertEqual(t, nil, err, "Failed to set the value")
}
func Test_Scylla_Set_Override(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
}
func Test_Scylla_Get(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get(key)
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_Scylla_Set_Expiration(t *testing.T) {
var (
key = "john"
val = []byte("doe")
exp = 1 * time.Second
)
err := testStore.Set(key, val, exp)
utils.AssertEqual(t, nil, err)
time.Sleep(1100 * time.Millisecond)
}
func Test_Scylla_Get_NotExist(t *testing.T) {
result, err := testStore.Get("not-exist")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_Scylla_Delete(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Delete(key)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get(key)
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_Scylla_Reset(t *testing.T) {
var (
val = []byte("doe")
)
err := testStore.Set("john1", val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Set("john2", val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Reset()
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("john1")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
result, err = testStore.Get("john2")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_Scylla_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}
func Test_Scylla_Conn(t *testing.T) {
utils.AssertEqual(t, true, testStore.Conn() != nil)
}