Compare commits

...

6 Commits

Author SHA1 Message Date
Joey
52621660fe Merge pull request #40 from Fenny/main
🩹 fix duration check
2020-11-14 03:14:19 +01:00
Fenny
deb10fb2a7 🩹 fix duration check 2020-11-13 18:35:23 +01:00
Joey
366ce02b88 Merge pull request #39 from Fenny/main
📦 prepare dynamodb
2020-11-12 20:14:40 +01:00
Fenny
c57c524007 📦 add dynamodb 2020-11-12 20:14:21 +01:00
Fenny
e0f4b3ef34 🚀 add dynamodb 2020-11-12 03:34:37 +01:00
Fenny
c0fabf16eb 📦 go 1.14 + 2020-11-12 03:34:26 +01:00
27 changed files with 473 additions and 13 deletions

View File

@@ -53,6 +53,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
var data []byte
err := s.db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))

View File

@@ -41,7 +41,7 @@ func configDefault(config ...Config) Config {
if cfg.Database == "" {
cfg.Database = ConfigDefault.Database
}
if int(cfg.GCInterval) == 0 {
if int(cfg.GCInterval.Seconds()) <= 0 {
cfg.GCInterval = ConfigDefault.GCInterval
}
return cfg

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/badger
go 1.15
go 1.14
require (
github.com/dgraph-io/badger v1.6.2

88
dynamodb/README.md Normal file
View File

@@ -0,0 +1,88 @@
# ⚠ DynamoDB is still in development, do not use in production!
....
### Table of Contents
- [Signatures](#signatures)
- [Installation](#installation)
- [Examples](#examples)
- [Config](#config)
- [Default Config](#default-config)
### Signatures
```go
func New(config Config) Storage
var ErrNotExist = errors.New("key does not exist")
func (s *Storage) Get(key string) ([]byte, error)
func (s *Storage) Set(key string, val []byte, exp time.Duration) error
func (s *Storage) Delete(key string) error
func (s *Storage) Reset() error
func (s *Storage) Close() error
```
### Installation
DynamoDB is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet:
```bash
go mod init github.com/<user>/<repo>
```
And then install the dynamodb implementation:
```bash
go get github.com/gofiber/storage/dynamodb
```
### Examples
Import the storage package.
```go
import "github.com/gofiber/storage/dynamodb"
```
You can use the following possibilities to create a storage:
```go
// Initialize dynamodb
store := dynamodb.New(dynamodb.Config{
})
```
### Config
```go
type Config struct {
// Region of the DynamoDB service you want to use.
// Valid values: https://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region.
// E.g. "us-west-2".
// Optional (read from shared config file or environment variable if not set).
// Environment variable: "AWS_REGION".
Region string
// Name of the DynamoDB table.
// Optional ("fiber_storage" by default).
Table string
// AWS access key ID (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_ACCESS_KEY_ID".
AWSaccessKeyID string
// AWS secret access key (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_SECRET_ACCESS_KEY".
AWSsecretAccessKey string
// CustomEndpoint allows you to set a custom DynamoDB service endpoint.
// This is especially useful if you're running a "DynamoDB local" Docker container for local testing.
// Typical value for the Docker container: "http://localhost:8000".
// See https://hub.docker.com/r/amazon/dynamodb-local/.
// Optional ("" by default)
CustomEndpoint string
}
```
### Default Config
```go
var ConfigDefault = Config{
Table: "fiber_storage",
}
```

82
dynamodb/config.go Normal file
View File

@@ -0,0 +1,82 @@
package dynamodb
import "github.com/aws/aws-sdk-go/aws"
// Config defines the config for storage.
type Config struct {
// Region of the DynamoDB service you want to use.
// Valid values: https://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region.
// E.g. "us-west-2".
// Optional (read from shared config file or environment variable if not set).
// Environment variable: "AWS_REGION".
Region string
// Name of the DynamoDB table.
// Optional ("fiber_storage" by default).
Table string
// AWS access key ID (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_ACCESS_KEY_ID".
AWSaccessKeyID string
// AWS secret access key (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_SECRET_ACCESS_KEY".
AWSsecretAccessKey string
// CustomEndpoint allows you to set a custom DynamoDB service endpoint.
// This is especially useful if you're running a "DynamoDB local" Docker container for local testing.
// Typical value for the Docker container: "http://localhost:8000".
// See https://hub.docker.com/r/amazon/dynamodb-local/.
// Optional ("" by default)
CustomEndpoint string
// ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv.
// Optional (5 by default, which is the same default value as when creating a table in the web console)
// 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
readCapacityUnits int64
// ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv.
// Optional (5 by default, which is the same default value as when creating a table in the web console)
// 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
writeCapacityUnits int64
// If the table doesn't exist yet, gokv creates it.
// If WaitForTableCreation is true, gokv will block until the table is created, with a timeout of 15 seconds.
// If the table still doesn't exist after 15 seconds, an error is returned.
// If WaitForTableCreation is false, gokv returns the client immediately.
// In the latter case you need to make sure that you don't read from or write to the table before it's created,
// because otherwise you will get ResourceNotFoundException errors.
// Optional (true by default).
waitForTableCreation *bool
}
// ConfigDefault is the default config
var ConfigDefault = Config{
Table: "fiber_storage",
readCapacityUnits: 5,
writeCapacityUnits: 5,
waitForTableCreation: aws.Bool(true),
}
// configDefault is a helper function to set default values
func configDefault(config ...Config) Config {
// Return default config if nothing provided
if len(config) < 1 {
return ConfigDefault
}
// Override default config
cfg := config[0]
// Set default values
if cfg.Table == "" {
cfg.Table = ConfigDefault.Table
}
return cfg
}

238
dynamodb/dynamodb.go Normal file
View File

@@ -0,0 +1,238 @@
package dynamodb
import (
"context"
"errors"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
awsdynamodb "github.com/aws/aws-sdk-go/service/dynamodb"
)
// Storage interface that is implemented by storage providers
type Storage struct {
db *awsdynamodb.DynamoDB
table string
}
// Common storage errors
var ErrNotExist = errors.New("key does not exist")
// New creates a new storage
func New(config Config) *Storage {
// Set default config
cfg := configDefault(config)
// Create db
var creds *credentials.Credentials
if (cfg.AWSaccessKeyID != "" && cfg.AWSsecretAccessKey == "") || (cfg.AWSaccessKeyID == "" && cfg.AWSsecretAccessKey != "") {
panic("[DynamoDB] You need to set BOTH AWSaccessKeyID AND AWSsecretAccessKey")
} else if cfg.AWSaccessKeyID != "" {
// Due to the previous check we can be sure that in this case AWSsecretAccessKey is not empty as well.
creds = credentials.NewStaticCredentials(cfg.AWSaccessKeyID, cfg.AWSsecretAccessKey, "")
}
// Set database options
opt := aws.NewConfig()
if cfg.Region != "" {
opt = opt.WithRegion(cfg.Region)
}
if creds != nil {
opt = opt.WithCredentials(creds)
}
if cfg.CustomEndpoint != "" {
opt = opt.WithEndpoint(cfg.CustomEndpoint)
}
sessionOpt := session.Options{
SharedConfigState: session.SharedConfigEnable,
}
// ...but allow overwrite of region and credentials if they are set in the options.
sessionOpt.Config.MergeIn(opt)
session, err := session.NewSessionWithOptions(sessionOpt)
if err != nil {
panic(err)
}
svc := awsdynamodb.New(session)
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
describeTableInput := awsdynamodb.DescribeTableInput{
TableName: &cfg.Table,
}
_, err = svc.DescribeTableWithContext(timeoutCtx, &describeTableInput)
if err != nil {
awsErr, ok := err.(awserr.Error)
if !ok {
panic(err)
} else if awsErr.Code() == awsdynamodb.ErrCodeResourceNotFoundException {
err = createTable(cfg.Table, cfg.readCapacityUnits, cfg.writeCapacityUnits, *cfg.waitForTableCreation, describeTableInput, svc)
if err != nil {
panic(err)
}
} else {
panic(err)
}
}
// Create storage
store := &Storage{
db: svc,
table: cfg.Table,
}
// Start garbage collector
//go store.gc()
return store
}
// "k" is used as table column name for the key.
var keyAttrName = "k"
// "v" is used as table column name for the value.
var valAttrName = "v"
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
k := make(map[string]*awsdynamodb.AttributeValue)
k[keyAttrName] = &awsdynamodb.AttributeValue{
S: &key,
}
getItemInput := awsdynamodb.GetItemInput{
TableName: &s.table,
Key: k,
}
getItemOutput, err := s.db.GetItem(&getItemInput)
if err != nil {
return nil, err
} else if getItemOutput.Item == nil {
return nil, ErrNotExist
}
attributeVal := getItemOutput.Item[valAttrName]
if attributeVal == nil {
return nil, ErrNotExist
}
return attributeVal.B, nil
}
// Set key with value
// Set key with value
func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
// Ain't Nobody Got Time For That
if len(key) <= 0 || len(val) <= 0 {
return nil
}
item := make(map[string]*awsdynamodb.AttributeValue)
item[keyAttrName] = &awsdynamodb.AttributeValue{
S: &key,
}
item[valAttrName] = &awsdynamodb.AttributeValue{
B: val,
}
putItemInput := awsdynamodb.PutItemInput{
TableName: &s.table,
Item: item,
}
_, err := s.db.PutItem(&putItemInput)
return err
}
// Delete entry by key
func (s *Storage) Delete(key string) error {
// Ain't Nobody Got Time For That
if len(key) <= 0 {
return nil
}
k := make(map[string]*awsdynamodb.AttributeValue)
k[keyAttrName] = &awsdynamodb.AttributeValue{
S: &key,
}
deleteItemInput := awsdynamodb.DeleteItemInput{
TableName: &s.table,
Key: k,
}
_, err := s.db.DeleteItem(&deleteItemInput)
return err
}
// Reset all entries, including unexpired
func (s *Storage) Reset() error {
deleteTableInput := awsdynamodb.DeleteTableInput{
TableName: &s.table,
}
_, err := s.db.DeleteTable(&deleteTableInput)
return err
}
// Close the database
func (s *Storage) Close() error {
// In the DynamoDB implementation this doesn't have any effect.
return nil
}
// GC deletes all expired entries
// func (s *Storage) gc() {
// ticker := time.NewTicker(s.gcInterval)
// defer ticker.Stop()
// for {
// select {
// case <-s.done:
// return
// case t := <-ticker.C:
// _, _ = s.db.Exec(s.sqlGC, t.Unix())
// }
// }
// }
func createTable(tableName string, readCapacityUnits, writeCapacityUnits int64, waitForTableCreation bool, describeTableInput awsdynamodb.DescribeTableInput, svc *awsdynamodb.DynamoDB) error {
keyAttrType := "S" // For "string"
keyType := "HASH" // As opposed to "RANGE"
createTableInput := awsdynamodb.CreateTableInput{
TableName: &tableName,
AttributeDefinitions: []*awsdynamodb.AttributeDefinition{{
AttributeName: &keyAttrName,
AttributeType: &keyAttrType,
}},
KeySchema: []*awsdynamodb.KeySchemaElement{{
AttributeName: &keyAttrName,
KeyType: &keyType,
}},
ProvisionedThroughput: &awsdynamodb.ProvisionedThroughput{
ReadCapacityUnits: &readCapacityUnits,
WriteCapacityUnits: &writeCapacityUnits,
},
}
_, err := svc.CreateTable(&createTableInput)
if err != nil {
return err
}
// If configured (true by default), block until the table is created.
// Typical table creation duration is 10 seconds.
if waitForTableCreation {
for try := 1; try < 16; try++ {
describeTableOutput, err := svc.DescribeTable(&describeTableInput)
if err != nil || *describeTableOutput.Table.TableStatus == "CREATING" {
time.Sleep(1 * time.Second)
} else {
break
}
}
// Last try (16th) after 15 seconds of waiting.
// Now handle error as such.
describeTableOutput, err := svc.DescribeTable(&describeTableInput)
if err != nil {
return errors.New("The DynamoDB table couldn't be created")
}
if *describeTableOutput.Table.TableStatus == "CREATING" {
return errors.New("The DynamoDB table took too long to be created")
}
}
return nil
}

View File

@@ -0,0 +1 @@
package dynamodb

5
dynamodb/go.mod Normal file
View File

@@ -0,0 +1,5 @@
module github.com/gofiber/storage/dynamodb
go 1.14
require github.com/aws/aws-sdk-go v1.35.26

22
dynamodb/go.sum Normal file
View File

@@ -0,0 +1,22 @@
github.com/aws/aws-sdk-go v1.35.26 h1:MawRvDpAp/Ai859dPC1xo1fdU/BIkijoHj0DwXLXXkI=
github.com/aws/aws-sdk-go v1.35.26/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/memcache
go 1.15
go 1.14
require (
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b

View File

@@ -60,6 +60,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
item, err := s.db.Get(key)
if err == mc.ErrCacheMiss {
return nil, ErrNotExist

View File

@@ -26,7 +26,7 @@ func configDefault(config ...Config) Config {
cfg := config[0]
// Set default values
if int(cfg.GCInterval) == 0 {
if int(cfg.GCInterval.Seconds()) <= 0 {
cfg.GCInterval = ConfigDefault.GCInterval
}
return cfg

View File

@@ -1,5 +1,5 @@
module github.com/gofiber/storage/memory
go 1.15
go 1.14
require github.com/gofiber/utils v0.1.2

View File

@@ -42,6 +42,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
s.mux.RLock()
v, ok := s.db[key]
s.mux.RUnlock()

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/mongodb
go 1.15
go 1.14
require (
github.com/gofiber/utils v0.1.2

View File

@@ -111,6 +111,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
res := s.col.FindOne(context.Background(), bson.M{"key": key})
item := s.acquireItem()

View File

@@ -96,7 +96,7 @@ func configDefault(config ...Config) Config {
if cfg.Table == "" {
cfg.Table = ConfigDefault.Table
}
if int(cfg.GCInterval) == 0 {
if int(cfg.GCInterval.Seconds()) <= 0 {
cfg.GCInterval = ConfigDefault.GCInterval
}
return cfg

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/mysql
go 1.15
go 1.14
require (
github.com/go-sql-driver/mysql v1.5.0

View File

@@ -99,6 +99,9 @@ var noRows = "sql: no rows in result set"
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
row := s.db.QueryRow(s.sqlSelect, key)
// Add db response to data

View File

@@ -119,7 +119,7 @@ func configDefault(config ...Config) Config {
if cfg.Table == "" {
cfg.Table = ConfigDefault.Table
}
if int(cfg.GCInterval) == 0 {
if int(cfg.GCInterval.Seconds()) <= 0 {
cfg.GCInterval = ConfigDefault.GCInterval
}
return cfg

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/postgres
go 1.15
go 1.14
require (
github.com/gofiber/utils v0.1.2

View File

@@ -116,6 +116,9 @@ var noRows = errors.New("sql: no rows in result set")
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
row := s.db.QueryRow(s.sqlSelect, key)
// Add db response to data
var (

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/redis
go 1.15
go 1.14
require (
github.com/go-redis/redis/v8 v8.3.3

View File

@@ -50,6 +50,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
val, err := s.db.Get(context.Background(), key).Bytes()
if err == redis.Nil {
return nil, ErrNotExist

View File

@@ -64,7 +64,7 @@ func configDefault(config ...Config) Config {
if cfg.Table == "" {
cfg.Table = ConfigDefault.Table
}
if int(cfg.GCInterval) == 0 {
if int(cfg.GCInterval.Seconds()) <= 0 {
cfg.GCInterval = ConfigDefault.GCInterval
}
return cfg

View File

@@ -1,6 +1,6 @@
module github.com/gofiber/storage/sqlite3
go 1.15
go 1.14
require (
github.com/gofiber/utils v0.1.2

View File

@@ -97,6 +97,9 @@ func New(config ...Config) *Storage {
// Get value by key
func (s *Storage) Get(key string) ([]byte, error) {
if len(key) <= 0 {
return nil, ErrNotExist
}
row := s.db.QueryRow(s.sqlSelect, key)
// Add db response to data
var (