diff --git a/.github/README.md b/.github/README.md index b57714a0..bed95302 100644 --- a/.github/README.md +++ b/.github/README.md @@ -1,17 +1 @@ -# đŸ“Ļ Storage drivers for [Fiber](https://github.com/gofiber/fiber) - -Premade storage drivers that implement [`fiber.Storage`](https://github.com/gofiber/fiber/blob/ba08653c92f86bc69956b23714f919b705d9381e/app.go#L39-L50), to be used with various Fiber middlewares. - -## 📑 Contents - -* [In-memory](/memory) -* [Memcached](/memcached) -* [MongoDB](/mongodb) -* [MySQL](/mysql) -* [Postgres](/postgres) -* [Redis](/redis) -* [SQLite3](/sqlite3) - -## 🤔 Something missing? - -If you've got a custom storage driver you made that's not listed here, why not submit a [PR](https://github.com/gofiber/storage/pulls) to add it? \ No newline at end of file +# redis diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..665e4dea --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,28 @@ +on: [push] +name: Benchmark +jobs: + Compare: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: 1.15.x + - name: Fetch Repository + uses: actions/checkout@v2 + - name: Run Benchmark + run: set -o pipefail; go test ./... -benchmem -run=^$ -bench . | tee output.txt + - name: Get Previous Benchmark Results + uses: actions/cache@v1 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + - name: Save Benchmark Results + uses: rhysd/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: output.txt + github-token: ${{ secrets.BENCHMARK_TOKEN }} + fail-on-alert: true + comment-on-alert: true + auto-push: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..f92c6776 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,54 @@ +name: "CodeQL" + +on: + push: + branches: [master, ] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + schedule: + - cron: '0 3 * * 6' + +jobs: + analyse: + name: Analyse + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + with: + languages: go + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 00000000..f9bca034 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,12 @@ +on: [push, pull_request] +name: Linter +jobs: + Golint: + runs-on: ubuntu-latest + steps: + - name: Fetch Repository + uses: actions/checkout@v2 + - name: Run Golint + uses: reviewdog/action-golangci-lint@v1 + with: + golangci_lint_flags: "--tests=false" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..4a2d0313 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,12 @@ +on: [push, pull_request] +name: Security +jobs: + Gosec: + runs-on: ubuntu-latest + steps: + - name: Fetch Repository + uses: actions/checkout@v2 + - name: Run Gosec + uses: securego/gosec@master + with: + args: -exclude-dir=internal/*/ ./... \ No newline at end of file diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml new file mode 100644 index 00000000..6fb513de --- /dev/null +++ b/.github/workflows/snyk.yml @@ -0,0 +1,11 @@ +on: [push, pull_request_target] +name: Snyk security +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/golang@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..88149107 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +on: [push, pull_request] +name: Test +jobs: + Build: + strategy: + matrix: + go-version: [1.14.x, 1.15.x] + platform: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.platform }} + steps: + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Fetch Repository + uses: actions/checkout@v2 + - name: Run Test + run: go test ./... -v -race \ No newline at end of file diff --git a/.gitignore b/.gitignore deleted file mode 100644 index a527526f..00000000 --- a/.gitignore +++ /dev/null @@ -1,184 +0,0 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/windows,jetbrains+all,go,linux,macos -# Edit at https://www.toptal.com/developers/gitignore?templates=windows,jetbrains+all,go,linux,macos - -### Go ### -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -### Go Patch ### -/vendor/ -/Godeps/ - -### JetBrains+all ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### JetBrains+all Patch ### -# Ignores the whole .idea folder and all .iml files -# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 - -.idea/ - -# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 - -*.iml -modules.xml -.idea/misc.xml -*.ipr - -# Sonarlint plugin -.idea/sonarlint - -### Linux ### -*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - -# KDE directory preferences -.directory - -# Linux trash folder which might appear on any partition or disk -.Trash-* - -# .nfs files are created when an open file is removed but is still being accessed -.nfs* - -### macOS ### -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Windows ### -# Windows thumbnail cache files -Thumbs.db -Thumbs.db:encryptable -ehthumbs.db -ehthumbs_vista.db - -# Dump file -*.stackdump - -# Folder config file -[Dd]esktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msix -*.msm -*.msp - -# Windows shortcuts -*.lnk - -# End of https://www.toptal.com/developers/gitignore/api/windows,jetbrains+all,go,linux,macos diff --git a/memcached/memcached.go b/memcached/memcached.go index b2653f13..e03fa908 100644 --- a/memcached/memcached.go +++ b/memcached/memcached.go @@ -16,26 +16,26 @@ type Config struct { var ConfigDefault = Config{} // New creates a new storage -func New(config ...Config) Storage { - return Storage{} +func New(config ...Config) *Storage { + return &Storage{} } // Get value by key -func (store *Storage) Get(key string) ([]byte, error) { +func (s *Storage) Get(key string) ([]byte, error) { return []byte{}, nil } // Set key with value -func (store *Storage) Set(key string, val []byte, exp time.Duration) error { +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { return nil } // Delete key by key -func (store *Storage) Delete(key string) error { +func (s *Storage) Delete(key string) error { return nil } // Clear all keys -func (store *Storage) Clear() error { +func (s *Storage) Clear() error { return nil } diff --git a/mongodb/config.go b/mongodb/config.go new file mode 100644 index 00000000..67359d43 --- /dev/null +++ b/mongodb/config.go @@ -0,0 +1,72 @@ +package mongodb + +import ( + "crypto/tls" + "time" + + "go.mongodb.org/mongo-driver/bson/bsoncodec" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/readpref" + "go.mongodb.org/mongo-driver/mongo/writeconcern" +) + +// Config defines the config for storage. +type Config struct { + // Custom options + Addr string + Database string + Collection string + + // https://pkg.go.dev/go.mongodb.org/mongo-driver@v1.4.2/mongo/options#ClientOptions + AppName string + Auth options.Credential + AutoEncryptionOptions *options.AutoEncryptionOptions + ConnectTimeout time.Duration + Compressors []string + Dialer options.ContextDialer + Direct bool + DisableOCSPEndpointCheck bool + HeartbeatInterval time.Duration + Hosts []string + LocalThreshold time.Duration + MaxConnIdleTime time.Duration + MaxPoolSize uint64 + MinPoolSize uint64 + PoolMonitor *event.PoolMonitor + Monitor *event.CommandMonitor + ReadConcern *readconcern.ReadConcern + ReadPreference *readpref.ReadPref + Registry *bsoncodec.Registry + ReplicaSet string + RetryReads bool + RetryWrites bool + ServerSelectionTimeout time.Duration + SocketTimeout time.Duration + TLSConfig *tls.Config + WriteConcern *writeconcern.WriteConcern + ZlibLevel int + ZstdLevel int +} + +// ConfigDefault is the default config +var ConfigDefault = Config{ + Addr: "127.0.0.1:27017", + Database: "_database", + Collection: "_storage", +} + +// Helper function to set default values +func configDefault(cfg Config) Config { + if cfg.Addr == "" { + cfg.Addr = ConfigDefault.Addr + } + if cfg.Database == "" { + cfg.Database = ConfigDefault.Database + } + if cfg.Collection == "" { + cfg.Collection = ConfigDefault.Collection + } + return cfg +} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index bd3ad807..74bc9698 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -2,16 +2,17 @@ package mongodb import ( "context" + "time" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" ) // Storage interface that is implemented by storage providers type Storage struct { - col *mongo.Collection + db *mongo.Collection } type MongoStorage struct { @@ -22,7 +23,55 @@ type MongoStorage struct { } // New creates a new MongoDB storage -func New(col *mongo.Collection) *Storage { +func New(config ...Config) *Storage { + // Set default config + cfg := ConfigDefault + + // Override config if provided + if len(config) > 0 { + cfg = configDefault(config[0]) + } + + // Set mongo options + opt := options.Client() + opt.SetAppName(cfg.AppName) + opt.SetAuth(cfg.Auth) + opt.SetAutoEncryptionOptions(cfg.AutoEncryptionOptions) + opt.SetConnectTimeout(cfg.ConnectTimeout) + opt.SetCompressors(cfg.Compressors) + opt.SetDialer(cfg.Dialer) + opt.SetDirect(cfg.Direct) + opt.SetDisableOCSPEndpointCheck(cfg.DisableOCSPEndpointCheck) + opt.SetHeartbeatInterval(cfg.HeartbeatInterval) + opt.SetHosts(cfg.Hosts) + opt.SetLocalThreshold(cfg.LocalThreshold) + opt.SetMaxConnIdleTime(cfg.MaxConnIdleTime) + opt.SetMaxPoolSize(cfg.MaxPoolSize) + opt.SetMinPoolSize(cfg.MinPoolSize) + opt.SetPoolMonitor(cfg.PoolMonitor) + opt.SetMonitor(cfg.Monitor) + opt.SetReadConcern(cfg.ReadConcern) + opt.SetReadPreference(cfg.ReadPreference) + opt.SetRegistry(cfg.Registry) + opt.SetReplicaSet(cfg.ReplicaSet) + opt.SetRetryReads(cfg.RetryReads) + opt.SetRetryWrites(cfg.RetryWrites) + opt.SetServerSelectionTimeout(cfg.ServerSelectionTimeout) + opt.SetSocketTimeout(cfg.SocketTimeout) + opt.SetTLSConfig(cfg.TLSConfig) + opt.SetWriteConcern(cfg.WriteConcern) + opt.SetZlibLevel(cfg.ZlibLevel) + opt.SetZstdLevel(cfg.ZstdLevel) + + // Create mongo client + client, err := mongo.NewClient(opt.ApplyURI("mongodb://" + cfg.Addr)) + if err != nil { + panic(err) + } + + // Get collection from database + db := client.Database(cfg.Database).Collection(cfg.Collection) + // expired data may exist for some time beyond the 60 second period between runs of the background task. // more on https://docs.mongodb.com/manual/core/index-ttl/ indexModel := mongo.IndexModel{ @@ -33,18 +82,18 @@ func New(col *mongo.Collection) *Storage { Options: options.Index().SetExpireAfterSeconds(0), } - if _, err := col.Indexes().CreateOne(context.TODO(), indexModel); err != nil { + if _, err := db.Indexes().CreateOne(context.TODO(), indexModel); err != nil { panic(err) } return &Storage{ - col: col, + db: db, } } // Get value by key func (s *Storage) Get(key string) ([]byte, error) { - res := s.col.FindOne(context.TODO(), bson.M{"key": key}) + res := s.db.FindOne(context.TODO(), bson.M{"key": key}) result := MongoStorage{} if err := res.Err(); err != nil { @@ -68,17 +117,17 @@ func (s *Storage) Set(key string, val []byte, exp time.Duration) error { if exp != 0 { replace.Exp = time.Now().Add(exp).UTC() } - _, err := s.col.ReplaceOne(context.TODO(), filter, replace, options.Replace().SetUpsert(true)) + _, err := s.db.ReplaceOne(context.TODO(), filter, replace, options.Replace().SetUpsert(true)) return err } // Delete document by key func (s *Storage) Delete(key string) error { - _, err := s.col.DeleteOne(context.TODO(), bson.M{"key": key}) + _, err := s.db.DeleteOne(context.TODO(), bson.M{"key": key}) return err } // Clear all keys by drop collection func (s *Storage) Clear() error { - return s.col.Drop(context.TODO()) + return s.db.Drop(context.TODO()) } diff --git a/mysql/mysql.go b/mysql/mysql.go index 93335f38..64444112 100644 --- a/mysql/mysql.go +++ b/mysql/mysql.go @@ -16,26 +16,26 @@ type Config struct { var ConfigDefault = Config{} // New creates a new storage -func New(config ...Config) Storage { - return Storage{} +func New(config ...Config) *Storage { + return &Storage{} } // Get value by key -func (store *Storage) Get(key string) ([]byte, error) { +func (s *Storage) Get(key string) ([]byte, error) { return []byte{}, nil } // Set key with value -func (store *Storage) Set(key string, val []byte, exp time.Duration) error { +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { return nil } // Delete key by key -func (store *Storage) Delete(key string) error { +func (s *Storage) Delete(key string) error { return nil } // Clear all keys -func (store *Storage) Clear() error { +func (s *Storage) Clear() error { return nil } diff --git a/postgres/postgres.go b/postgres/postgres.go index 1d753a64..c5a4ab48 100644 --- a/postgres/postgres.go +++ b/postgres/postgres.go @@ -16,26 +16,26 @@ type Config struct { var ConfigDefault = Config{} // New creates a new storage -func New(config ...Config) Storage { - return Storage{} +func New(config ...Config) *Storage { + return &Storage{} } // Get value by key -func (store *Storage) Get(key string) ([]byte, error) { +func (s *Storage) Get(key string) ([]byte, error) { return []byte{}, nil } // Set key with value -func (store *Storage) Set(key string, val []byte, exp time.Duration) error { +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { return nil } // Delete key by key -func (store *Storage) Delete(key string) error { +func (s *Storage) Delete(key string) error { return nil } // Clear all keys -func (store *Storage) Clear() error { +func (s *Storage) Clear() error { return nil } diff --git a/redis/README.md b/redis/README.md deleted file mode 100644 index bed95302..00000000 --- a/redis/README.md +++ /dev/null @@ -1 +0,0 @@ -# redis diff --git a/redis/config.go b/redis/config.go index 87a6addc..37a54d12 100644 --- a/redis/config.go +++ b/redis/config.go @@ -11,6 +11,10 @@ import ( // Config defines the config for redis storage. type Config struct { + // Custom options + + // https://pkg.go.dev/github.com/go-redis/redis/v8#Options + // The network type, either tcp or unix. // Default is tcp. Network string @@ -40,9 +44,11 @@ type Config struct { // Maximum number of retries before giving up. // Default is 3 retries. MaxRetries int + // Minimum backoff between each retry. // Default is 8 milliseconds; -1 disables backoff. MinRetryBackoff time.Duration + // Maximum backoff between each retry. // Default is 512 milliseconds; -1 disables backoff. MaxRetryBackoff time.Duration @@ -50,10 +56,12 @@ type Config struct { // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration + // Timeout for socket reads. If reached, commands will fail // with a timeout instead of blocking. Use value -1 for no timeout and 0 for default. // Default is 3 seconds. ReadTimeout time.Duration + // Timeout for socket writes. If reached, commands will fail // with a timeout instead of blocking. // Default is ReadTimeout. @@ -62,20 +70,25 @@ type Config struct { // Maximum number of socket connections. // Default is 10 connections per every CPU as reported by runtime.NumCPU. PoolSize int + // Minimum number of idle connections which is useful when establishing // new connection is slow. MinIdleConns int + // Connection age at which client retires (closes) the connection. // Default is to not close aged connections. MaxConnAge time.Duration + // Amount of time client waits for connection if all connections // are busy before returning an error. // Default is ReadTimeout + 1 second. PoolTimeout time.Duration + // Amount of time after which client closes idle connections. // Should be less than server's timeout. // Default is 5 minutes. -1 disables idle timeout check. IdleTimeout time.Duration + // Frequency of idle checks made by idle connections reaper. // Default is 1 minute. -1 disables idle connections reaper, // but idle connections are still discarded by the client diff --git a/redis/redis.go b/redis/redis.go index a881ee79..507ce461 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -13,7 +13,7 @@ type Storage struct { } // New creates a new redis storage -func New(config ...Config) Storage { +func New(config ...Config) *Storage { // Set default config cfg := ConfigDefault @@ -52,14 +52,14 @@ func New(config ...Config) Storage { panic(err) } // Create new store - return Storage{ + return &Storage{ db: db, } } // Get value by key -func (store *Storage) Get(key string) ([]byte, error) { - val, err := store.db.Get(context.Background(), key).Bytes() +func (s *Storage) Get(key string) ([]byte, error) { + val, err := s.db.Get(context.Background(), key).Bytes() if err != nil { if err != redis.Nil { return nil, err @@ -70,16 +70,16 @@ func (store *Storage) Get(key string) ([]byte, error) { } // Set key with value -func (store *Storage) Set(key string, val []byte, exp time.Duration) error { - return store.db.Set(context.Background(), key, val, exp).Err() +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { + return s.db.Set(context.Background(), key, val, exp).Err() } // Delete key by key -func (store *Storage) Delete(key string) error { - return store.db.Del(context.Background(), key).Err() +func (s *Storage) Delete(key string) error { + return s.db.Del(context.Background(), key).Err() } // Clear all keys -func (store *Storage) Clear() error { - return store.db.FlushDB(context.Background()).Err() +func (s *Storage) Clear() error { + return s.db.FlushDB(context.Background()).Err() } diff --git a/sqlite3/sqlite3.go b/sqlite3/sqlite3.go index f35e117b..c4cd2dfd 100644 --- a/sqlite3/sqlite3.go +++ b/sqlite3/sqlite3.go @@ -16,26 +16,26 @@ type Config struct { var ConfigDefault = Config{} // New creates a new storage -func New(config ...Config) Storage { - return Storage{} +func New(config ...Config) *Storage { + return &Storage{} } // Get value by key -func (store *Storage) Get(key string) ([]byte, error) { +func (s *Storage) Get(key string) ([]byte, error) { return []byte{}, nil } // Set key with value -func (store *Storage) Set(key string, val []byte, exp time.Duration) error { +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { return nil } // Delete key by key -func (store *Storage) Delete(key string) error { +func (s *Storage) Delete(key string) error { return nil } // Clear all keys -func (store *Storage) Clear() error { +func (s *Storage) Clear() error { return nil }