diff --git a/badger/badger.go b/badger/badger.go index 38a32473..d5be5b55 100644 --- a/badger/badger.go +++ b/badger/badger.go @@ -53,6 +53,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } var data []byte err := s.db.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(key)) diff --git a/dynamodb/README.md b/dynamodb/README.md index 290f6a11..29c63527 100644 --- a/dynamodb/README.md +++ b/dynamodb/README.md @@ -1,4 +1,4 @@ -# DynamoDB +# ⚠ DynamoDB is still in development, do not use in production! .... @@ -12,7 +12,7 @@ ### Signatures ```go -func New(config ...Config) Storage +func New(config Config) Storage var ErrNotExist = errors.New("key does not exist") @@ -41,10 +41,7 @@ import "github.com/gofiber/storage/dynamodb" You can use the following possibilities to create a storage: ```go -// Initialize default config -store := dynamodb.New() - -// Initialize custom config +// Initialize dynamodb store := dynamodb.New(dynamodb.Config{ }) @@ -53,13 +50,39 @@ store := dynamodb.New(dynamodb.Config{ ### Config ```go type Config struct { + // Region of the DynamoDB service you want to use. + // Valid values: https://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region. + // E.g. "us-west-2". + // Optional (read from shared config file or environment variable if not set). + // Environment variable: "AWS_REGION". + Region string + // Name of the DynamoDB table. + // Optional ("fiber_storage" by default). + Table string + + // AWS access key ID (part of the credentials). + // Optional (read from shared credentials file or environment variable if not set). + // Environment variable: "AWS_ACCESS_KEY_ID". + AWSaccessKeyID string + + // AWS secret access key (part of the credentials). + // Optional (read from shared credentials file or environment variable if not set). + // Environment variable: "AWS_SECRET_ACCESS_KEY". + AWSsecretAccessKey string + + // CustomEndpoint allows you to set a custom DynamoDB service endpoint. + // This is especially useful if you're running a "DynamoDB local" Docker container for local testing. + // Typical value for the Docker container: "http://localhost:8000". + // See https://hub.docker.com/r/amazon/dynamodb-local/. + // Optional ("" by default) + CustomEndpoint string } ``` ### Default Config ```go var ConfigDefault = Config{ - + Table: "fiber_storage", } ``` diff --git a/dynamodb/config.go b/dynamodb/config.go index f170aea6..e28f656a 100644 --- a/dynamodb/config.go +++ b/dynamodb/config.go @@ -1,11 +1,68 @@ package dynamodb +import "github.com/aws/aws-sdk-go/aws" + // Config defines the config for storage. type Config struct { + // Region of the DynamoDB service you want to use. + // Valid values: https://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region. + // E.g. "us-west-2". + // Optional (read from shared config file or environment variable if not set). + // Environment variable: "AWS_REGION". + Region string + + // Name of the DynamoDB table. + // Optional ("fiber_storage" by default). + Table string + + // AWS access key ID (part of the credentials). + // Optional (read from shared credentials file or environment variable if not set). + // Environment variable: "AWS_ACCESS_KEY_ID". + AWSaccessKeyID string + + // AWS secret access key (part of the credentials). + // Optional (read from shared credentials file or environment variable if not set). + // Environment variable: "AWS_SECRET_ACCESS_KEY". + AWSsecretAccessKey string + + // CustomEndpoint allows you to set a custom DynamoDB service endpoint. + // This is especially useful if you're running a "DynamoDB local" Docker container for local testing. + // Typical value for the Docker container: "http://localhost:8000". + // See https://hub.docker.com/r/amazon/dynamodb-local/. + // Optional ("" by default) + CustomEndpoint string + + // ReadCapacityUnits of the table. + // Only required when the table doesn't exist yet and is created by gokv. + // Optional (5 by default, which is the same default value as when creating a table in the web console) + // 25 RCUs are included in the free tier (across all tables). + // For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput. + // For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput. + readCapacityUnits int64 + // ReadCapacityUnits of the table. + // Only required when the table doesn't exist yet and is created by gokv. + // Optional (5 by default, which is the same default value as when creating a table in the web console) + // 25 RCUs are included in the free tier (across all tables). + // For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput. + // For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput. + writeCapacityUnits int64 + // If the table doesn't exist yet, gokv creates it. + // If WaitForTableCreation is true, gokv will block until the table is created, with a timeout of 15 seconds. + // If the table still doesn't exist after 15 seconds, an error is returned. + // If WaitForTableCreation is false, gokv returns the client immediately. + // In the latter case you need to make sure that you don't read from or write to the table before it's created, + // because otherwise you will get ResourceNotFoundException errors. + // Optional (true by default). + waitForTableCreation *bool } // ConfigDefault is the default config -var ConfigDefault = Config{} +var ConfigDefault = Config{ + Table: "fiber_storage", + readCapacityUnits: 5, + writeCapacityUnits: 5, + waitForTableCreation: aws.Bool(true), +} // configDefault is a helper function to set default values func configDefault(config ...Config) Config { @@ -18,6 +75,8 @@ func configDefault(config ...Config) Config { cfg := config[0] // Set default values - + if cfg.Table == "" { + cfg.Table = ConfigDefault.Table + } return cfg } diff --git a/dynamodb/dynamodb.go b/dynamodb/dynamodb.go index eeacc196..b5b72628 100644 --- a/dynamodb/dynamodb.go +++ b/dynamodb/dynamodb.go @@ -1 +1,238 @@ package dynamodb + +import ( + "context" + "errors" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + awsdynamodb "github.com/aws/aws-sdk-go/service/dynamodb" +) + +// Storage interface that is implemented by storage providers +type Storage struct { + db *awsdynamodb.DynamoDB + table string +} + +// Common storage errors +var ErrNotExist = errors.New("key does not exist") + +// New creates a new storage +func New(config Config) *Storage { + // Set default config + cfg := configDefault(config) + + // Create db + var creds *credentials.Credentials + if (cfg.AWSaccessKeyID != "" && cfg.AWSsecretAccessKey == "") || (cfg.AWSaccessKeyID == "" && cfg.AWSsecretAccessKey != "") { + panic("[DynamoDB] You need to set BOTH AWSaccessKeyID AND AWSsecretAccessKey") + } else if cfg.AWSaccessKeyID != "" { + // Due to the previous check we can be sure that in this case AWSsecretAccessKey is not empty as well. + creds = credentials.NewStaticCredentials(cfg.AWSaccessKeyID, cfg.AWSsecretAccessKey, "") + } + + // Set database options + opt := aws.NewConfig() + if cfg.Region != "" { + opt = opt.WithRegion(cfg.Region) + } + if creds != nil { + opt = opt.WithCredentials(creds) + } + if cfg.CustomEndpoint != "" { + opt = opt.WithEndpoint(cfg.CustomEndpoint) + } + + sessionOpt := session.Options{ + SharedConfigState: session.SharedConfigEnable, + } + + // ...but allow overwrite of region and credentials if they are set in the options. + sessionOpt.Config.MergeIn(opt) + session, err := session.NewSessionWithOptions(sessionOpt) + if err != nil { + panic(err) + } + svc := awsdynamodb.New(session) + + timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + describeTableInput := awsdynamodb.DescribeTableInput{ + TableName: &cfg.Table, + } + + _, err = svc.DescribeTableWithContext(timeoutCtx, &describeTableInput) + if err != nil { + awsErr, ok := err.(awserr.Error) + if !ok { + panic(err) + } else if awsErr.Code() == awsdynamodb.ErrCodeResourceNotFoundException { + err = createTable(cfg.Table, cfg.readCapacityUnits, cfg.writeCapacityUnits, *cfg.waitForTableCreation, describeTableInput, svc) + if err != nil { + panic(err) + } + } else { + panic(err) + } + } + + // Create storage + store := &Storage{ + db: svc, + table: cfg.Table, + } + + // Start garbage collector + //go store.gc() + + return store +} + +// "k" is used as table column name for the key. +var keyAttrName = "k" + +// "v" is used as table column name for the value. +var valAttrName = "v" + +// Get value by key +func (s *Storage) Get(key string) ([]byte, error) { + k := make(map[string]*awsdynamodb.AttributeValue) + k[keyAttrName] = &awsdynamodb.AttributeValue{ + S: &key, + } + getItemInput := awsdynamodb.GetItemInput{ + TableName: &s.table, + Key: k, + } + getItemOutput, err := s.db.GetItem(&getItemInput) + if err != nil { + return nil, err + } else if getItemOutput.Item == nil { + return nil, ErrNotExist + } + attributeVal := getItemOutput.Item[valAttrName] + if attributeVal == nil { + return nil, ErrNotExist + } + return attributeVal.B, nil +} + +// Set key with value +// Set key with value +func (s *Storage) Set(key string, val []byte, exp time.Duration) error { + // Ain't Nobody Got Time For That + if len(key) <= 0 || len(val) <= 0 { + return nil + } + item := make(map[string]*awsdynamodb.AttributeValue) + item[keyAttrName] = &awsdynamodb.AttributeValue{ + S: &key, + } + item[valAttrName] = &awsdynamodb.AttributeValue{ + B: val, + } + putItemInput := awsdynamodb.PutItemInput{ + TableName: &s.table, + Item: item, + } + _, err := s.db.PutItem(&putItemInput) + return err +} + +// Delete entry by key +func (s *Storage) Delete(key string) error { + // Ain't Nobody Got Time For That + if len(key) <= 0 { + return nil + } + k := make(map[string]*awsdynamodb.AttributeValue) + k[keyAttrName] = &awsdynamodb.AttributeValue{ + S: &key, + } + deleteItemInput := awsdynamodb.DeleteItemInput{ + TableName: &s.table, + Key: k, + } + _, err := s.db.DeleteItem(&deleteItemInput) + return err +} + +// Reset all entries, including unexpired +func (s *Storage) Reset() error { + deleteTableInput := awsdynamodb.DeleteTableInput{ + TableName: &s.table, + } + _, err := s.db.DeleteTable(&deleteTableInput) + return err +} + +// Close the database +func (s *Storage) Close() error { + // In the DynamoDB implementation this doesn't have any effect. + return nil +} + +// GC deletes all expired entries +// func (s *Storage) gc() { +// ticker := time.NewTicker(s.gcInterval) +// defer ticker.Stop() +// for { +// select { +// case <-s.done: +// return +// case t := <-ticker.C: +// _, _ = s.db.Exec(s.sqlGC, t.Unix()) +// } +// } +// } + +func createTable(tableName string, readCapacityUnits, writeCapacityUnits int64, waitForTableCreation bool, describeTableInput awsdynamodb.DescribeTableInput, svc *awsdynamodb.DynamoDB) error { + keyAttrType := "S" // For "string" + keyType := "HASH" // As opposed to "RANGE" + createTableInput := awsdynamodb.CreateTableInput{ + TableName: &tableName, + AttributeDefinitions: []*awsdynamodb.AttributeDefinition{{ + AttributeName: &keyAttrName, + AttributeType: &keyAttrType, + }}, + KeySchema: []*awsdynamodb.KeySchemaElement{{ + AttributeName: &keyAttrName, + KeyType: &keyType, + }}, + ProvisionedThroughput: &awsdynamodb.ProvisionedThroughput{ + ReadCapacityUnits: &readCapacityUnits, + WriteCapacityUnits: &writeCapacityUnits, + }, + } + _, err := svc.CreateTable(&createTableInput) + if err != nil { + return err + } + // If configured (true by default), block until the table is created. + // Typical table creation duration is 10 seconds. + if waitForTableCreation { + for try := 1; try < 16; try++ { + describeTableOutput, err := svc.DescribeTable(&describeTableInput) + if err != nil || *describeTableOutput.Table.TableStatus == "CREATING" { + time.Sleep(1 * time.Second) + } else { + break + } + } + // Last try (16th) after 15 seconds of waiting. + // Now handle error as such. + describeTableOutput, err := svc.DescribeTable(&describeTableInput) + if err != nil { + return errors.New("The DynamoDB table couldn't be created") + } + if *describeTableOutput.Table.TableStatus == "CREATING" { + return errors.New("The DynamoDB table took too long to be created") + } + } + + return nil +} diff --git a/dynamodb/go.mod b/dynamodb/go.mod index d80c2c89..21bedf1d 100644 --- a/dynamodb/go.mod +++ b/dynamodb/go.mod @@ -1,3 +1,5 @@ module github.com/gofiber/storage/dynamodb go 1.14 + +require github.com/aws/aws-sdk-go v1.35.26 diff --git a/dynamodb/go.sum b/dynamodb/go.sum new file mode 100644 index 00000000..e4788866 --- /dev/null +++ b/dynamodb/go.sum @@ -0,0 +1,22 @@ +github.com/aws/aws-sdk-go v1.35.26 h1:MawRvDpAp/Ai859dPC1xo1fdU/BIkijoHj0DwXLXXkI= +github.com/aws/aws-sdk-go v1.35.26/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/memcache/memcache.go b/memcache/memcache.go index 57977f80..186f225c 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -60,6 +60,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } item, err := s.db.Get(key) if err == mc.ErrCacheMiss { return nil, ErrNotExist diff --git a/memory/memory.go b/memory/memory.go index 0b62457a..8490a528 100644 --- a/memory/memory.go +++ b/memory/memory.go @@ -42,6 +42,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } s.mux.RLock() v, ok := s.db[key] s.mux.RUnlock() diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index 60345b24..b9b5ba70 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -111,6 +111,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } res := s.col.FindOne(context.Background(), bson.M{"key": key}) item := s.acquireItem() diff --git a/mysql/mysql.go b/mysql/mysql.go index 44526bdd..071d0d01 100644 --- a/mysql/mysql.go +++ b/mysql/mysql.go @@ -99,6 +99,9 @@ var noRows = "sql: no rows in result set" // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } row := s.db.QueryRow(s.sqlSelect, key) // Add db response to data diff --git a/postgres/postgres.go b/postgres/postgres.go index dfeb5274..25ff3c3e 100644 --- a/postgres/postgres.go +++ b/postgres/postgres.go @@ -116,6 +116,9 @@ var noRows = errors.New("sql: no rows in result set") // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } row := s.db.QueryRow(s.sqlSelect, key) // Add db response to data var ( diff --git a/redis/redis.go b/redis/redis.go index 0b51a86c..f3e80436 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -50,6 +50,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } val, err := s.db.Get(context.Background(), key).Bytes() if err == redis.Nil { return nil, ErrNotExist diff --git a/sqlite3/sqlite3.go b/sqlite3/sqlite3.go index 7c4003a7..6e1b9497 100644 --- a/sqlite3/sqlite3.go +++ b/sqlite3/sqlite3.go @@ -97,6 +97,9 @@ func New(config ...Config) *Storage { // Get value by key func (s *Storage) Get(key string) ([]byte, error) { + if len(key) <= 0 { + return nil, ErrNotExist + } row := s.db.QueryRow(s.sqlSelect, key) // Add db response to data var (