package/storage
1
0
Fork 0
mirror of https://github.com/gofiber/storage.git synced 2025-11-02 12:34:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🔥 Make DynamoDB Production-Ready [🎌 breaking change] (#323)

Browse Source 
* Update CI/CD tests.

* Update aws-sdk-go to v2, add unit tests, remove warning test, make config better.

* add action

Co-authored-by: wernerr <rene.werner@verivox.com>
This commit is contained in:
M. Efe Çetin
2022-02-10 17:44:59 +03:00
committed by GitHub
parent f9eaa6ae4d
commit 315f14ce58
7 changed files with 450 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/test-dynamodb.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
'on':
- push
- pull_request
name: DynamoDB
jobs:
Tests:
runs-on: ubuntu-latest
services:
mongo:
image: 'amazon/dynamodb-local:latest'
ports:
- '8000:8000'
strategy:
matrix:
go-version:
- 1.14.x
- 1.15.x
- 1.16.x
- 1.17.x
platform:
- ubuntu-latest
- windows-latest
steps:
- name: Install Go
uses: actions/setup-go@v1
with:
go-version: '${{ matrix.go-version }}'
- name: Fetch Repository
uses: actions/checkout@v2
- name: Run Test
run: cd ./dynamodb && go test ./... -v -race

71
dynamodb/README.md
View File

@@ -1,4 +1,7 @@
# ⚠ DynamoDB is still in development, do not use in production! # DynamoDB
A DynamoDB storage driver using [aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2).
**Note:** If config fields of credentials not given, credentials are using from the environment variables, ~/.aws/credentials, or EC2 instance role. If config fields of credentials given, credentials are using from config. Look at: [specifying credentials](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials)
.... ....
@@ -60,28 +63,70 @@ type Config struct {
// Optional ("fiber_storage" by default). // Optional ("fiber_storage" by default).
Table string Table string
// AWS access key ID (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_ACCESS_KEY_ID".
AWSaccessKeyID string
// AWS secret access key (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_SECRET_ACCESS_KEY".
AWSsecretAccessKey string
// CustomEndpoint allows you to set a custom DynamoDB service endpoint. // CustomEndpoint allows you to set a custom DynamoDB service endpoint.
// This is especially useful if you're running a "DynamoDB local" Docker container for local testing. // This is especially useful if you're running a "DynamoDB local" Docker container for local testing.
// Typical value for the Docker container: "http://localhost:8000". // Typical value for the Docker container: "http://localhost:8000".
// See https://hub.docker.com/r/amazon/dynamodb-local/. // See https://hub.docker.com/r/amazon/dynamodb-local/.
// Optional ("" by default) // Optional ("" by default)
CustomEndpoint string Endpoint string
// Credentials overrides AWS access key and AWS secret access key. Not recommended.
//
// Optional. Default is Credentials{}
Credentials Credentials
// The maximum number of times requests that encounter retryable failures should be attempted.
//
// Optional. Default is 3
MaxAttempts int
// Reset clears any existing keys in existing Bucket
//
// Optional. Default is false
Reset bool
// ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv.
// Optional (5 by default, which is the same default value as when creating a table in the web console)
// 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
ReadCapacityUnits int64
// ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv.
// Optional (5 by default, which is the same default value as when creating a table in the web console)
// 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
WriteCapacityUnits int64
// If the table doesn't exist yet, gokv creates it.
// If WaitForTableCreation is true, gokv will block until the table is created, with a timeout of 15 seconds.
// If the table still doesn't exist after 15 seconds, an error is returned.
// If WaitForTableCreation is false, gokv returns the client immediately.
// In the latter case you need to make sure that you don't read from or write to the table before it's created,
// because otherwise you will get ResourceNotFoundException errors.
// Optional (true by default).
WaitForTableCreation *bool
} }
type Credentials struct {
AccessKey string
SecretAccessKey string
}
``` ```
### Default Config ### Default Config
```go ```go
var ConfigDefault = Config{ var ConfigDefault = Config{
Table: "fiber_storage", Table: "fiber_storage",
Credentials: Credentials{},
MaxAttempts: 3,
Reset: false,
ReadCapacityUnits: 5,
WriteCapacityUnits: 5,
WaitForTableCreation: aws.Bool(true),
} }
``` ```

64
dynamodb/config.go
View File

@@ -1,6 +1,6 @@
package dynamodb package dynamodb
import "github.com/aws/aws-sdk-go/aws" import "github.com/aws/aws-sdk-go-v2/aws"
// Config defines the config for storage. // Config defines the config for storage.
type Config struct { type Config struct {
@@ -15,22 +15,27 @@ type Config struct {
// Optional ("fiber_storage" by default). // Optional ("fiber_storage" by default).
Table string Table string
// AWS access key ID (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_ACCESS_KEY_ID".
AWSaccessKeyID string
// AWS secret access key (part of the credentials).
// Optional (read from shared credentials file or environment variable if not set).
// Environment variable: "AWS_SECRET_ACCESS_KEY".
AWSsecretAccessKey string
// CustomEndpoint allows you to set a custom DynamoDB service endpoint. // CustomEndpoint allows you to set a custom DynamoDB service endpoint.
// This is especially useful if you're running a "DynamoDB local" Docker container for local testing. // This is especially useful if you're running a "DynamoDB local" Docker container for local testing.
// Typical value for the Docker container: "http://localhost:8000". // Typical value for the Docker container: "http://localhost:8000".
// See https://hub.docker.com/r/amazon/dynamodb-local/. // See https://hub.docker.com/r/amazon/dynamodb-local/.
// Optional ("" by default) // Optional ("" by default)
CustomEndpoint string Endpoint string
// Credentials overrides AWS access key and AWS secret access key. Not recommended.
//
// Optional. Default is Credentials{}
Credentials Credentials
// The maximum number of times requests that encounter retryable failures should be attempted.
//
// Optional. Default is 3
MaxAttempts int
// Reset clears any existing keys in existing Bucket
//
// Optional. Default is false
Reset bool
// ReadCapacityUnits of the table. // ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv. // Only required when the table doesn't exist yet and is created by gokv.
@@ -38,14 +43,16 @@ type Config struct {
// 25 RCUs are included in the free tier (across all tables). // 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput. // For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput. // For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
readCapacityUnits int64 ReadCapacityUnits int64
// ReadCapacityUnits of the table. // ReadCapacityUnits of the table.
// Only required when the table doesn't exist yet and is created by gokv. // Only required when the table doesn't exist yet and is created by gokv.
// Optional (5 by default, which is the same default value as when creating a table in the web console) // Optional (5 by default, which is the same default value as when creating a table in the web console)
// 25 RCUs are included in the free tier (across all tables). // 25 RCUs are included in the free tier (across all tables).
// For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput. // For example calculations, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/HowItWorks.ProvisionedThroughput.
// For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput. // For limits, see https://github.com/awsdocs/amazon-dynamodb-developer-guide/blob/c420420a59040c5b3dd44a6e59f7c9e55fc922ef/doc_source/Limits.md#capacity-units-and-provisioned-throughput.md#provisioned-throughput.
writeCapacityUnits int64 WriteCapacityUnits int64
// If the table doesn't exist yet, gokv creates it. // If the table doesn't exist yet, gokv creates it.
// If WaitForTableCreation is true, gokv will block until the table is created, with a timeout of 15 seconds. // If WaitForTableCreation is true, gokv will block until the table is created, with a timeout of 15 seconds.
// If the table still doesn't exist after 15 seconds, an error is returned. // If the table still doesn't exist after 15 seconds, an error is returned.
@@ -53,15 +60,23 @@ type Config struct {
// In the latter case you need to make sure that you don't read from or write to the table before it's created, // In the latter case you need to make sure that you don't read from or write to the table before it's created,
// because otherwise you will get ResourceNotFoundException errors. // because otherwise you will get ResourceNotFoundException errors.
// Optional (true by default). // Optional (true by default).
waitForTableCreation *bool WaitForTableCreation *bool
}
type Credentials struct {
AccessKey string
SecretAccessKey string
} }
// ConfigDefault is the default config // ConfigDefault is the default config
var ConfigDefault = Config{ var ConfigDefault = Config{
Table: "fiber_storage", Table: "fiber_storage",
readCapacityUnits: 5, Credentials: Credentials{},
writeCapacityUnits: 5, MaxAttempts: 3,
waitForTableCreation: aws.Bool(true), Reset: false,
ReadCapacityUnits: 5,
WriteCapacityUnits: 5,
WaitForTableCreation: aws.Bool(true),
} }
// configDefault is a helper function to set default values // configDefault is a helper function to set default values
@@ -78,5 +93,18 @@ func configDefault(config ...Config) Config {
if cfg.Table == "" { if cfg.Table == "" {
cfg.Table = ConfigDefault.Table cfg.Table = ConfigDefault.Table
} }
if cfg.MaxAttempts == 0 {
cfg.MaxAttempts = ConfigDefault.MaxAttempts
}
if cfg.ReadCapacityUnits == 0 {
cfg.ReadCapacityUnits = ConfigDefault.ReadCapacityUnits
}
if cfg.WriteCapacityUnits == 0 {
cfg.WriteCapacityUnits = ConfigDefault.WriteCapacityUnits
}
if cfg.WaitForTableCreation == nil {
cfg.WaitForTableCreation = ConfigDefault.WaitForTableCreation
}
return cfg return cfg
} }

291
dynamodb/dynamodb.go
View File

@@ -3,90 +3,23 @@ package dynamodb
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go/aws/credentials" awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go-v2/credentials"
awsdynamodb "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
awsdynamodb "github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
) )
// Storage interface that is implemented by storage providers // Storage interface that is implemented by storage providers
type Storage struct { type Storage struct {
db *awsdynamodb.DynamoDB db *awsdynamodb.Client
table string table string
} requestTimeout time.Duration
// New creates a new storage
func New(config Config) *Storage {
// Set default config
cfg := configDefault(config)
// Create db
var creds *credentials.Credentials
if (cfg.AWSaccessKeyID != "" && cfg.AWSsecretAccessKey == "") || (cfg.AWSaccessKeyID == "" && cfg.AWSsecretAccessKey != "") {
panic("[DynamoDB] You need to set BOTH AWSaccessKeyID AND AWSsecretAccessKey")
} else if cfg.AWSaccessKeyID != "" {
// Due to the previous check we can be sure that in this case AWSsecretAccessKey is not empty as well.
creds = credentials.NewStaticCredentials(cfg.AWSaccessKeyID, cfg.AWSsecretAccessKey, "")
}
// Set database options
opt := aws.NewConfig()
if cfg.Region != "" {
opt = opt.WithRegion(cfg.Region)
}
if creds != nil {
opt = opt.WithCredentials(creds)
}
if cfg.CustomEndpoint != "" {
opt = opt.WithEndpoint(cfg.CustomEndpoint)
}
sessionOpt := session.Options{
SharedConfigState: session.SharedConfigEnable,
}
// ...but allow overwrite of region and credentials if they are set in the options.
sessionOpt.Config.MergeIn(opt)
session, err := session.NewSessionWithOptions(sessionOpt)
if err != nil {
panic(err)
}
svc := awsdynamodb.New(session)
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
describeTableInput := awsdynamodb.DescribeTableInput{
TableName: &cfg.Table,
}
_, err = svc.DescribeTableWithContext(timeoutCtx, &describeTableInput)
if err != nil {
awsErr, ok := err.(awserr.Error)
if !ok {
panic(err)
} else if awsErr.Code() == awsdynamodb.ErrCodeResourceNotFoundException {
err = createTable(cfg.Table, cfg.readCapacityUnits, cfg.writeCapacityUnits, *cfg.waitForTableCreation, describeTableInput, svc)
if err != nil {
panic(err)
}
} else {
panic(err)
}
}
// Create storage
store := &Storage{
db: svc,
table: cfg.Table,
}
// Start garbage collector
//go store.gc()
return store
} }
// "k" is used as table column name for the key. // "k" is used as table column name for the key.
@@ -95,126 +28,182 @@ var keyAttrName = "k"
// "v" is used as table column name for the value. // "v" is used as table column name for the value.
var valAttrName = "v" var valAttrName = "v"
type table struct {
K string
V []byte
}
// New creates a new storage
func New(config Config) *Storage {
// Set default config
cfg := configDefault(config)
awscfg, err := returnAWSConfig(cfg)
if err != nil {
panic(fmt.Sprintf("unable to load SDK config, %v", err))
}
// Create db
sess := awsdynamodb.NewFromConfig(awscfg)
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
describeTableInput := awsdynamodb.DescribeTableInput{
TableName: &cfg.Table,
}
// Create storage
store := &Storage{
db: sess,
table: cfg.Table,
}
// Create table
_, err = sess.DescribeTable(timeoutCtx, &describeTableInput)
if err != nil {
var rnfe *types.ResourceNotFoundException
if errors.As(err, &rnfe) {
err := store.createTable(cfg, describeTableInput)
if err != nil {
panic(err)
}
} else {
panic(err)
}
}
return store
}
// Get value by key // Get value by key
func (s *Storage) Get(key string) ([]byte, error) { func (s *Storage) Get(key string) ([]byte, error) {
k := make(map[string]*awsdynamodb.AttributeValue) ctx, cancel := s.requestContext()
k[keyAttrName] = &awsdynamodb.AttributeValue{ defer cancel()
S: &key,
k := make(map[string]types.AttributeValue)
k[keyAttrName] = &types.AttributeValueMemberS{
Value: key,
} }
getItemInput := awsdynamodb.GetItemInput{ getItemInput := awsdynamodb.GetItemInput{
TableName: &s.table, TableName: &s.table,
Key: k, Key: k,
} }
getItemOutput, err := s.db.GetItem(&getItemInput) getItemOutput, err := s.db.GetItem(ctx, &getItemInput)
if err != nil { if err != nil {
var rnfe *types.ResourceNotFoundException
if errors.As(err, &rnfe) {
return nil, nil
}
return nil, err return nil, err
} else if getItemOutput.Item == nil { } else if getItemOutput.Item == nil {
return nil, nil return nil, nil
} }
attributeVal := getItemOutput.Item[valAttrName]
if attributeVal == nil { item := &table{}
return nil, nil err = attributevalue.UnmarshalMap(getItemOutput.Item, &item)
}
return attributeVal.B, nil return item.V, err
} }
// Set key with value
// Set key with value // Set key with value
func (s *Storage) Set(key string, val []byte, exp time.Duration) error { func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
ctx, cancel := s.requestContext()
defer cancel()
// Ain't Nobody Got Time For That // Ain't Nobody Got Time For That
if len(key) <= 0 || len(val) <= 0 { if len(key) <= 0 || len(val) <= 0 {
return nil return nil
} }
item := make(map[string]*awsdynamodb.AttributeValue)
item[keyAttrName] = &awsdynamodb.AttributeValue{ item := make(map[string]types.AttributeValue)
S: &key, item[keyAttrName] = &types.AttributeValueMemberS{
Value: key,
} }
item[valAttrName] = &awsdynamodb.AttributeValue{ item[valAttrName] = &types.AttributeValueMemberB{
B: val, Value: val,
} }
putItemInput := awsdynamodb.PutItemInput{ putItemInput := awsdynamodb.PutItemInput{
TableName: &s.table, TableName: &s.table,
Item: item, Item: item,
} }
_, err := s.db.PutItem(&putItemInput)
_, err := s.db.PutItem(ctx, &putItemInput)
return err return err
} }
// Delete entry by key // Delete entry by key
func (s *Storage) Delete(key string) error { func (s *Storage) Delete(key string) error {
ctx, cancel := s.requestContext()
defer cancel()
// Ain't Nobody Got Time For That // Ain't Nobody Got Time For That
if len(key) <= 0 { if len(key) <= 0 {
return nil return nil
} }
k := make(map[string]*awsdynamodb.AttributeValue)
k[keyAttrName] = &awsdynamodb.AttributeValue{ k := make(map[string]types.AttributeValue)
S: &key, k[keyAttrName] = &types.AttributeValueMemberS{
Value: key,
} }
deleteItemInput := awsdynamodb.DeleteItemInput{ deleteItemInput := awsdynamodb.DeleteItemInput{
TableName: &s.table, TableName: &s.table,
Key: k, Key: k,
} }
_, err := s.db.DeleteItem(&deleteItemInput)
_, err := s.db.DeleteItem(ctx, &deleteItemInput)
return err return err
} }
// Reset all entries, including unexpired // Reset all entries, including unexpired
func (s *Storage) Reset() error { func (s *Storage) Reset() error {
ctx, cancel := s.requestContext()
defer cancel()
deleteTableInput := awsdynamodb.DeleteTableInput{ deleteTableInput := awsdynamodb.DeleteTableInput{
TableName: &s.table, TableName: &s.table,
} }
_, err := s.db.DeleteTable(&deleteTableInput) _, err := s.db.DeleteTable(ctx, &deleteTableInput)
return err return err
} }
// Close the database // Close the database
func (s *Storage) Close() error { func (s *Storage) Close() error {
// In the DynamoDB implementation this doesn't have any effect.
return nil return nil
} }
// GC deletes all expired entries func (s *Storage) createTable(cfg Config, describeTableInput awsdynamodb.DescribeTableInput) error {
// func (s *Storage) gc() { ctx, cancel := s.requestContext()
// ticker := time.NewTicker(s.gcInterval) defer cancel()
// defer ticker.Stop()
// for {
// select {
// case <-s.done:
// return
// case t := <-ticker.C:
// _, _ = s.db.Exec(s.sqlGC, t.Unix())
// }
// }
// }
func createTable(tableName string, readCapacityUnits, writeCapacityUnits int64, waitForTableCreation bool, describeTableInput awsdynamodb.DescribeTableInput, svc *awsdynamodb.DynamoDB) error {
keyAttrType := "S" // For "string" keyAttrType := "S" // For "string"
keyType := "HASH" // As opposed to "RANGE" keyType := "HASH" // As opposed to "RANGE"
createTableInput := awsdynamodb.CreateTableInput{ createTableInput := awsdynamodb.CreateTableInput{
TableName: &tableName, TableName: &s.table,
AttributeDefinitions: []*awsdynamodb.AttributeDefinition{{ AttributeDefinitions: []types.AttributeDefinition{{
AttributeName: &keyAttrName, AttributeName: &keyAttrName,
AttributeType: &keyAttrType, AttributeType: types.ScalarAttributeType(keyAttrType),
}}, }},
KeySchema: []*awsdynamodb.KeySchemaElement{{ KeySchema: []types.KeySchemaElement{{
AttributeName: &keyAttrName, AttributeName: &keyAttrName,
KeyType: &keyType, KeyType: types.KeyType(keyType),
}}, }},
ProvisionedThroughput: &awsdynamodb.ProvisionedThroughput{ ProvisionedThroughput: &types.ProvisionedThroughput{
ReadCapacityUnits: &readCapacityUnits, ReadCapacityUnits: &cfg.ReadCapacityUnits,
WriteCapacityUnits: &writeCapacityUnits, WriteCapacityUnits: &cfg.WriteCapacityUnits,
}, },
} }
_, err := svc.CreateTable(&createTableInput) _, err := s.db.CreateTable(ctx, &createTableInput)
if err != nil { if err != nil {
return err return err
} }
// If configured (true by default), block until the table is created. // If configured (true by default), block until the table is created.
// Typical table creation duration is 10 seconds. // Typical table creation duration is 10 seconds.
if waitForTableCreation { if *cfg.WaitForTableCreation {
for try := 1; try < 16; try++ { for try := 1; try < 16; try++ {
describeTableOutput, err := svc.DescribeTable(&describeTableInput) describeTableOutput, err := s.db.DescribeTable(ctx, &describeTableInput)
if err != nil || *describeTableOutput.Table.TableStatus == "CREATING" { if err != nil || describeTableOutput.Table.TableStatus == "CREATING" {
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} else { } else {
break break
@@ -222,14 +211,56 @@ func createTable(tableName string, readCapacityUnits, writeCapacityUnits int64,
} }
// Last try (16th) after 15 seconds of waiting. // Last try (16th) after 15 seconds of waiting.
// Now handle error as such. // Now handle error as such.
describeTableOutput, err := svc.DescribeTable(&describeTableInput) describeTableOutput, err := s.db.DescribeTable(ctx, &describeTableInput)
if err != nil { if err != nil {
return errors.New("The DynamoDB table couldn't be created") return errors.New("dynamodb: the table couldn't be created")
} }
if *describeTableOutput.Table.TableStatus == "CREATING" { if describeTableOutput.Table.TableStatus == "CREATING" {
return errors.New("The DynamoDB table took too long to be created") return errors.New("dynamodb: the table took too long to be created")
} }
} }
return nil return nil
} }
// Context for making requests will timeout if a non-zero timeout is configured
func (s *Storage) requestContext() (context.Context, context.CancelFunc) {
if s.requestTimeout > 0 {
return context.WithTimeout(context.Background(), s.requestTimeout)
}
return context.Background(), func() {}
}
func returnAWSConfig(cfg Config) (aws.Config, error) {
endpoint := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if cfg.Endpoint != "" {
return aws.Endpoint{
PartitionID: "aws",
URL: cfg.Endpoint,
SigningRegion: cfg.Region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
if cfg.Credentials != (Credentials{}) {
credentials := credentials.NewStaticCredentialsProvider(cfg.Credentials.AccessKey, cfg.Credentials.SecretAccessKey, "")
return awsconfig.LoadDefaultConfig(context.TODO(),
awsconfig.WithRegion(cfg.Region),
awsconfig.WithEndpointResolverWithOptions(endpoint),
awsconfig.WithCredentialsProvider(credentials),
awsconfig.WithRetryer(func() aws.Retryer {
return retry.AddWithMaxAttempts(retry.NewStandard(), cfg.MaxAttempts)
}),
)
}
return awsconfig.LoadDefaultConfig(context.TODO(),
awsconfig.WithRegion(cfg.Region),
awsconfig.WithEndpointResolverWithOptions(endpoint),
awsconfig.WithRetryer(func() aws.Retryer {
return retry.AddWithMaxAttempts(retry.NewStandard(), cfg.MaxAttempts)
}),
)
}

106
dynamodb/dynamodb_test.go
View File

@@ -1 +1,107 @@
package dynamodb package dynamodb
import (
"testing"
"github.com/gofiber/utils"
)
var testStore = New(
Config{
Table: "fiber_storage",
Endpoint: "http://localhost:8000/",
Region: "us-east-1",
Credentials: Credentials{
AccessKey: "dummy",
SecretAccessKey: "dummy",
},
},
)
func Test_DynamoDB_Set(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
}
func Test_DynamoDB_Set_Override(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
}
func Test_DynamoDB_Get(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get(key)
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, val, result)
}
func Test_DynamoDB_Get_NotExist(t *testing.T) {
result, err := testStore.Get("notexist")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_DynamoDB_Delete(t *testing.T) {
var (
key = "john"
val = []byte("doe")
)
err := testStore.Set(key, val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Delete(key)
utils.AssertEqual(t, nil, err)
result, err := testStore.Get(key)
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_DynamoDB_Reset(t *testing.T) {
var (
val = []byte("doe")
)
err := testStore.Set("john1", val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Set("john2", val, 0)
utils.AssertEqual(t, nil, err)
err = testStore.Reset()
utils.AssertEqual(t, nil, err)
result, err := testStore.Get("john1")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
result, err = testStore.Get("john2")
utils.AssertEqual(t, nil, err)
utils.AssertEqual(t, true, len(result) == 0)
}
func Test_DynamoDB_Close(t *testing.T) {
utils.AssertEqual(t, nil, testStore.Close())
}

9
dynamodb/go.mod
View File

@@ -2,4 +2,11 @@ module github.com/gofiber/storage/dynamodb
go 1.14 go 1.14
require github.com/aws/aws-sdk-go v1.42.50 require (
github.com/aws/aws-sdk-go-v2 v1.13.0
github.com/aws/aws-sdk-go-v2/config v1.13.1
github.com/aws/aws-sdk-go-v2/credentials v1.8.0
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.6.0
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.13.0
github.com/gofiber/utils v0.1.2
)

52
dynamodb/go.sum
View File

@@ -1,24 +1,52 @@
github.com/aws/aws-sdk-go v1.42.50 h1:FA5pbpkLz2fdnMt+AWyHnNaIA269rqr/sYAe3WKCYN4= github.com/aws/aws-sdk-go-v2 v1.13.0 h1:1XIXAfxsEmbhbj5ry3D3vX+6ZcUYvIqSm4CWWEuGZCA=
github.com/aws/aws-sdk-go v1.42.50/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go-v2 v1.13.0/go.mod h1:L6+ZpqHaLbAaxsqV0L4cvxZY7QupWJB4fhkf8LXvC7w=
github.com/aws/aws-sdk-go-v2/config v1.13.1 h1:yLv8bfNoT4r+UvUKQKqRtdnvuWGMK5a82l4ru9Jvnuo=
github.com/aws/aws-sdk-go-v2/config v1.13.1/go.mod h1:Ba5Z4yL/UGbjQUzsiaN378YobhFo0MLfueXGiOsYtEs=
github.com/aws/aws-sdk-go-v2/credentials v1.8.0 h1:8Ow0WcyDesGNL0No11jcgb1JAtE+WtubqXjgxau+S0o=
github.com/aws/aws-sdk-go-v2/credentials v1.8.0/go.mod h1:gnMo58Vwx3Mu7hj1wpcG8DI0s57c9o42UQ6wgTQT5to=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.6.0 h1:qS/1WpMN7RyJD+qQsS+pwtGxxaRJa3qbf6EP7jZwLIg=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.6.0/go.mod h1:LchVYRkk9AQyRgDXWAlJ01H5C1XcODuPK9/RyeCcIYk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 h1:NITDuUZO34mqtOwFWZiXo7yAHj7kf+XPE+EiKuCBNUI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0/go.mod h1:I6/fHT/fH460v09eg2gVrd8B/IqskhNdpcLH0WNO3QI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 h1:CRiQJ4E2RhfDdqbie1ZYDo8QtIo75Mk7oTdJSfwJTMQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4/go.mod h1:XHgQ7Hz2WY2GAn//UXHofLfPXWh+s62MbMOijrg12Lw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 h1:3ADoioDMOtF4uiK59vCpplpCwugEU+v4ZFD29jDL3RQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0/go.mod h1:BsCSJHx5DnDXIrOcqB8KN1/B+hXLG/bi4Y6Vjcx/x9E=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 h1:ixotxbfTCFpqbuwFv/RcZwyzhkxPSYDYEMcj4niB5Uk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5/go.mod h1:R3sWUqPcfXSiF/LSFJhjyJmpg9uV6yP2yv3YZZjldVI=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.13.0 h1:Xlmdkxi8WcIwX5Cy9BS+scWcmvARw8pg0bi7kaeERUY=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.13.0/go.mod h1:eNvoR4P1XQN7xElmYA8cWeFENLY3pfsj/5nFRItzXnA=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.11.0 h1:QN/wfWh/FJud6IKobe7QUMw1J0NfdZVtqvndyFgofCg=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.11.0/go.mod h1:tS6jI0oPA0cVqUdZJe0qea1u7YnCejeTi4o6rAk9VO0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.7.0 h1:F1diQIOkNn8jcez4173r+PLPdkWK7chy74r3fKpDrLI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.7.0/go.mod h1:8ctElVINyp+SjhoZZceUAZw78glZH6R8ox5MVNu5j2s=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.5.0 h1:tzVhIPr/psp8Gb2Blst9mq6HklkhAGPqv2eaiSq6yoU=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.5.0/go.mod h1:u0rI/Mm45zCJe86J5kvPfG7pYzkVZzNjEkoTVbfOYE8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 h1:4QAOB3KrvI1ApJK14sliGr3Ie2pjyvNypn/lfzDHfUw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0/go.mod h1:K/qPe6AP2TGYv4l6n7c88zh9jWBDf6nHhvg1fx/EWfU=
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 h1:1qLJeQGBmNQW3mBNzK2CFmrQNmoXWrscPqsrAaU1aTA=
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0/go.mod h1:vCV4glupK3tR7pw7ks7Y4jYRL86VvxS+g5qk04YeWrU=
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 h1:ksiDXhvNYg0D2/UFkLejsaz3LqpW5yjNQ8Nx9Sn2c0E=
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0/go.mod h1:u0xMJKDvvfocRjiozsoZglVNXRG19043xzp3r2ivLIk=
github.com/aws/smithy-go v1.10.0 h1:gsoZQMNHnX+PaghNw4ynPsyGP7aUCqx5sY2dlPQsZ0w=
github.com/aws/smithy-go v1.10.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/utils v0.1.2 h1:1SH2YEz4RlNS0tJlMJ0bGwO0JkqPqvq6TbHK9tXZKtk=
github.com/gofiber/utils v0.1.2/go.mod h1:pacRFtghAE3UoknMOUiXh2Io/nLWSUHtQCi/3QASsOc=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=