Added azure blob storage

Signed-off-by: kosar <bogdan.kosarevskyi@gmail.com>
2025-10-06 00:57:38 +08:00 · 2022-10-27 23:03:10 +03:00
parent dca8f183e4
commit caca820b8c
9 changed files with 703 additions and 160 deletions
--- a/.github/workflows/security.yml
+++ b/.github/workflows/security.yml
@@ -16,7 +16,7 @@ jobs:
                    go install github.com/securego/gosec/v2/cmd/gosec@latest
            -   name: Run Gosec (root)
                working-directory: .
-                run: "`go env GOPATH`/bin/gosec -exclude-dir=internal -exclude-dir=arangodb -exclude-dir=badger -exclude-dir=dynamodb -exclude-dir=memcache -exclude-dir=memory -exclude-dir=mongodb -exclude-dir=mysql -exclude-dir=postgres -exclude-dir=redis -exclude-dir=ristretto -exclude-dir=sqlite3 -exclude-dir=s3 -exclude-dir=bbolt ./..."
+                run: "`go env GOPATH`/bin/gosec -exclude-dir=internal -exclude-dir=arangodb -exclude-dir=badger -exclude-dir=dynamodb -exclude-dir=memcache -exclude-dir=memory -exclude-dir=mongodb -exclude-dir=mysql -exclude-dir=postgres -exclude-dir=redis -exclude-dir=ristretto -exclude-dir=sqlite3 -exclude-dir=s3 -exclude-dir=bbolt -exclude-dir=azureblob ./..."
                # -----
            -   name: Run Gosec (arangodb)
                working-directory: ./arangodb
@@ -70,3 +70,7 @@ jobs:
                working-directory: ./bbolt
                run: "`go env GOPATH`/bin/gosec -exclude-dir=internal ./..."
                # -----
+            -   name: Run Gosec (azureblob)
+                working-directory: ./azureblob
+                run: "`go env GOPATH`/bin/gosec -exclude-dir=internal ./..."
+                # -----
--- a/.github/workflows/test-azureblob.yml
+++ b/.github/workflows/test-azureblob.yml
@@ -0,0 +1,46 @@
+on:
+    push:
+        branches:
+            - master
+            - main
+    pull_request:
+name: "Tests Azure Blob"
+jobs:
+    Tests:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                go-version:
+                    - 1.18.x
+                    - 1.19.x
+                platform:
+                    - ubuntu-latest
+                    - windows-latest
+        steps:
+        -   name: Install Azurite
+            run: |
+                docker run -d -p 10000:10000 -e AZURITE_ACCOUNTS="azurite:YXp1cml0ZWtleQo=" mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 --blobPort 10000
+        -   name: Install Go
+            uses: actions/setup-go@v3
+            with:
+                go-version: '${{ matrix.go-version }}'
+        -   name: Setup Golang caches
+            uses: actions/cache@v3
+            with:
+                # In order:
+                # * Module download cache
+                # * Build cache (Linux)
+                # * Build cache (Mac)
+                # * Build cache (Windows)
+                path: |
+                    ~/go/pkg/mod
+                    ~/.cache/go-build
+                    ~/Library/Caches/go-build
+                    ~\AppData\Local\go-build
+                key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}
+                restore-keys: |
+                    ${{ runner.os }}-go-${{ matrix.go-version }}-
+        -   name: Fetch Repository
+            uses: actions/checkout@v3
+        -   name: Run Test
+            run: cd ./azureblob && go test ./... -v -race
--- a/README.md
+++ b/README.md
@@ -54,6 +54,9 @@ type Storage interface {
 * [ArangoDB](/arangodb) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22ArangoDB%22">
    <img src="https://img.shields.io/github/workflow/status/gofiber/storage/ArangoDB?label=%F0%9F%A7%AA%20&style=flat&color=75C46B">
  </a>
+* [AzureBlob](/azureblob) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Azure%20Blob%22">
+  <img src="https://img.shields.io/github/workflow/status/gofiber/storage/Azure%20Blob?label=%F0%9F%A7%AA%20&style=flat&color=75C46B">
+  </a>
 * [Badger](/badger) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Local+Storage%22">
    <img src="https://img.shields.io/github/workflow/status/gofiber/storage/Local%20Storage?label=%F0%9F%A7%AA%20&style=flat&color=75C46B">
  </a>
--- a/azureblob/README.md
+++ b/azureblob/README.md
@@ -0,0 +1,102 @@
+# Azure blob
+
+[Azure Blob storage](https://azure.microsoft.com/en-us/products/storage/blobs/#overview) is Microsoft's object storage solution for the cloud.
+
+> NOTE: Go **1.18** or later is required. Source: [link](https://github.com/Azure/azure-sdk-for-go/blob/main/README.md)
+
+### Table of Contents
+
+- [Signatures](#signatures)
+- [Installation](#installation)
+- [Examples](#examples)
+- [Config](#config)
+- [Default Config](#default-config)
+
+### Signatures
+
+```go
+func New(config ...Config) Storage
+func (s *Storage) Get(key string) ([]byte, error)
+func (s *Storage) Set(key string, val []byte, exp time.Duration) error
+func (s *Storage) Delete(key string) error
+func (s *Storage) Reset() error
+func (s *Storage) Close() error
+```
+
+### Installation
+
+Azure blob storage driver is tested on the 2 last [Go versions](https://golang.org/dl/) with support for modules. So make sure to initialize one first if you didn't do that yet:
+
+```bash
+go mod init github.com/<user>/<repo>
+```
+
+And then install the azure blob implementation:
+
+```bash
+go get github.com/gofiber/storage/azureblob
+```
+
+### Examples
+
+Import the storage package.
+
+```go
+import "github.com/gofiber/storage/azureblob"
+```
+
+You can use the following possibilities to create a storage:
+
+```go
+// Initialize default config
+store := azureblob.New()
+
+// Initialize custom config
+store := azureblob.New(azureblob.Config{
+    Account:   "test",
+    Container: "test",
+    Credentials: Credentials{
+        Account: "test",
+        Key:     "YXp1cml0ZWtleQo=",
+    },
+})
+```
+
+### Config
+
+```go
+type Config struct {
+    // Storage account name.
+    Account string
+    // Container name.
+    Container string
+    // Storage endpoint.
+    // Optional. Default: "https://STORAGEACCOUNTNAME.blob.core.windows.net"
+    Endpoint string
+    // Request timeout.
+    // Optional. Default is 0 (no timeout)
+    RequestTimeout time.Duration
+    // Reset clears any existing keys in existing container.
+    // Optional. Default is false
+    Reset bool
+    // Credentials overrides AWS access key and AWS secret access key. Not recommended.
+    // Optional. Default is Credentials{}
+    Credentials Credentials
+    // The maximum number of times requests that encounter retryable failures should be attempted.
+    // Optional. Default is 3
+    MaxAttempts int
+}
+```
+
+### Default Config
+
+```go
+var ConfigDefault = Config{
+    Account:        "",
+    Container:      "",
+    Endpoint:       "",
+    RequestTimeout: 0,
+    Reset:          false,
+    MaxAttempts:    3,
+}
+```
--- a/azureblob/azureblob.go
+++ b/azureblob/azureblob.go
@@ -0,0 +1,133 @@
+package azureblob
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"io"
+	"time"
+)
+
+// Storage interface that is implemented by storage providers
+type Storage struct {
+	client         *azblob.Client
+	container      string
+	requestTimeout time.Duration
+}
+
+// New creates a new storage
+func New(config ...Config) *Storage {
+	// Set default config
+	cfg := configure(config...)
+	// Set the azure credentials
+	cred, err := azblob.NewSharedKeyCredential(cfg.Credentials.Account, cfg.Credentials.Key)
+	handleError(err)
+	client, err := azblob.NewClientWithSharedKeyCredential(cfg.Endpoint, cred, nil)
+	handleError(err)
+	_, err = client.CreateContainer(context.TODO(), cfg.Container, nil)
+	if err != nil {
+		if !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
+			panic(fmt.Sprintf("invalid config:, %v", err))
+		}
+	}
+	storage := &Storage{
+		client:         client,
+		container:      cfg.Container,
+		requestTimeout: cfg.RequestTimeout,
+	}
+
+	// Reset all entries if set to true
+	if cfg.Reset {
+		if err := storage.Reset(); err != nil {
+			panic(err)
+		}
+	}
+
+	return storage
+}
+
+// Get value by key
+func (s *Storage) Get(key string) ([]byte, error) {
+	if len(key) <= 0 {
+		return nil, nil
+	}
+	ctx, cancel := s.requestContext()
+	defer cancel()
+	resp, err := s.client.DownloadStream(ctx, s.container, key, nil)
+	if err != nil {
+		return []byte{}, err
+	}
+	data, err := io.ReadAll(resp.Body)
+	return data, err
+}
+
+// Set key with value
+func (s *Storage) Set(key string, val []byte, exp time.Duration) error {
+	if len(key) <= 0 {
+		return nil
+	}
+	ctx, cancel := s.requestContext()
+	defer cancel()
+	_, err := s.client.UploadBuffer(ctx, s.container, key, val, nil)
+	return err
+}
+
+// Delete entry by key
+func (s *Storage) Delete(key string) error {
+	if len(key) <= 0 {
+		return nil
+	}
+
+	ctx, cancel := s.requestContext()
+	defer cancel()
+	_, err := s.client.DeleteBlob(ctx, s.container, key, nil)
+	return err
+}
+
+// Reset all entries
+func (s *Storage) Reset() error {
+	ctx, cancel := s.requestContext()
+	defer cancel()
+	//_, err := s.client.DeleteContainer(ctx, s.container, nil)
+	//return err
+	pager := s.client.NewListBlobsFlatPager(s.container, nil)
+	errCounter := 0
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			errCounter = errCounter + 1
+		}
+		for _, v := range resp.Segment.BlobItems {
+			_, err = s.client.DeleteBlob(ctx, s.container, *v.Name, nil)
+			if err != nil {
+				errCounter = errCounter + 1
+			}
+		}
+	}
+	if errCounter > 0 {
+		return errors.New(fmt.Sprintf("%d errors occured while resetting", errCounter))
+	}
+	return nil
+}
+
+// Close the storage connextion
+func (s *Storage) Close() error {
+	return nil
+}
+
+// Context for making requests will timeout if a non-zero timeout is configured
+func (s *Storage) requestContext() (context.Context, context.CancelFunc) {
+	if s.requestTimeout > 0 {
+		return context.WithTimeout(context.Background(), s.requestTimeout)
+	}
+	return context.Background(), func() {}
+}
+
+// handleError is a helper to panic on error
+func handleError(err error) {
+	if err != nil {
+		panic(fmt.Sprintf("invalid config:, %v", err))
+	}
+}
--- a/azureblob/azureblob_test.go
+++ b/azureblob/azureblob_test.go
@@ -0,0 +1,132 @@
+package azureblob
+
+import (
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/gofiber/fiber/v2/utils"
+	"testing"
+)
+
+func newStore() *Storage {
+	return New(Config{
+		Account:   "azurite",
+		Container: "test",
+		Endpoint:  "http://127.0.0.1:10000/azurite",
+		Credentials: Credentials{
+			Account: "azurite",
+			Key:     "YXp1cml0ZWtleQo=",
+		},
+	})
+}
+func Test_AzureBlob_Get(t *testing.T) {
+	var (
+		key = "john"
+		val = []byte("doe")
+	)
+	testStore := newStore()
+
+	err := testStore.Set(key, val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get(key)
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, val, result)
+}
+
+func Test_AzureBlob_Set(t *testing.T) {
+	var (
+		key = "john"
+		val = []byte("doe")
+	)
+
+	testStore := newStore()
+	err := testStore.Set(key, val, 0)
+	utils.AssertEqual(t, nil, err)
+}
+
+func Test_AzureBlob_Delete(t *testing.T) {
+	var (
+		key = "john"
+		val = []byte("doe")
+	)
+	testStore := newStore()
+
+	err := testStore.Set(key, val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	err = testStore.Delete(key)
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get(key)
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			err = nil
+		}
+	}
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, true, len(result) == 0)
+}
+
+func Test_AzureBlob_Override(t *testing.T) {
+	var (
+		key = "john"
+		val = []byte("doe")
+	)
+	testStore := newStore()
+
+	err := testStore.Set(key, val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	err = testStore.Set(key, val, 0)
+	utils.AssertEqual(t, nil, err)
+}
+
+func Test_AzureBlob_Get_NotExist(t *testing.T) {
+	testStore := newStore()
+	result, err := testStore.Get("notexist")
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			err = nil
+		}
+	}
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, true, len(result) == 0)
+}
+
+func Test_AzureBlob_Reset(t *testing.T) {
+	var (
+		val = []byte("doe")
+	)
+	testStore := newStore()
+
+	err := testStore.Set("john1", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	err = testStore.Set("john2", val, 0)
+	utils.AssertEqual(t, nil, err)
+
+	err = testStore.Reset()
+	utils.AssertEqual(t, nil, err)
+
+	result, err := testStore.Get("john1")
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			err = nil
+		}
+	}
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, true, len(result) == 0)
+
+	result, err = testStore.Get("john2")
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			err = nil
+		}
+	}
+	utils.AssertEqual(t, nil, err)
+	utils.AssertEqual(t, true, len(result) == 0)
+}
+
+func Test_AzureBlob_Close(t *testing.T) {
+	testStore := newStore()
+	utils.AssertEqual(t, nil, testStore.Close())
+}
--- a/azureblob/config.go
+++ b/azureblob/config.go
@@ -0,0 +1,80 @@
+package azureblob
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+// Config defines the config for storage.
+type Config struct {
+	// Storage account name.
+	Account string
+	// Container name.
+	Container string
+	// Storage endpoint.
+	// Optional. Default: "https://STORAGEACCOUNTNAME.blob.core.windows.net"
+	Endpoint string
+	// Request timeout.
+	// Optional. Default is 0 (no timeout)
+	RequestTimeout time.Duration
+	// Reset clears any existing keys in existing container.
+	// Optional. Default is false
+	Reset bool
+	// Credentials overrides AWS access key and AWS secret access key. Not recommended.
+	// Optional. Default is Credentials{}
+	Credentials Credentials
+	// The maximum number of times requests that encounter retryable failures should be attempted.
+	// Optional. Default is 3
+	MaxAttempts int
+}
+
+// Credentials are the azure storage account access keys
+type Credentials struct {
+	Account string
+	Key     string
+}
+
+// ConfigDefault is the default config
+var ConfigDefault = Config{
+	Account:        "",
+	Container:      "",
+	Endpoint:       "",
+	RequestTimeout: 0,
+	Reset:          false,
+	MaxAttempts:    3,
+}
+
+// Helper function to set default values
+func configure(config ...Config) Config {
+	// Return default config if nothing provided
+	if len(config) < 1 {
+		return ConfigDefault
+	}
+	// Override default config
+	cfg := config[0]
+	valid, err := validateConfig(cfg)
+	if err != nil || !valid {
+		panic(fmt.Sprintf("invalid config:, %v", err))
+	}
+	if cfg.Endpoint == "" {
+		cfg.Endpoint = "https://" + cfg.Account + ".blob.core.windows.net"
+	}
+	return cfg
+}
+
+func validateConfig(config Config) (bool, error) {
+	if config.Credentials.Account == "" || config.Credentials.Key == "" {
+		err := errors.New("credentials must not be empty")
+		return false, err
+	}
+	if config.Account == "" || config.Container == "" {
+		err := errors.New("invalid account information provided")
+		return false, err
+	}
+	if config.Account != config.Credentials.Account {
+		err := errors.New("account configuration mismatch")
+		return false, err
+	}
+	return true, nil
+}
--- a/azureblob/go.mod
+++ b/azureblob/go.mod
@@ -0,0 +1,16 @@
+module github.com/gofiber/storage/azureblob
+
+go 1.19
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
+	github.com/gofiber/fiber/v2 v2.39.0
+)
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/text v0.3.7 // indirect
+)
--- a/azureblob/go.sum
+++ b/azureblob/go.sum
@@ -0,0 +1,27 @@
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
+github.com/gofiber/fiber/v2 v2.39.0 h1:uhWpYQ6EHN8J7FOPYbI2hrdBD/KNZBC5CjbuOd4QUt4=
+github.com/gofiber/fiber/v2 v2.39.0/go.mod h1:Cmuu+elPYGqlvQvdKyjtYsjGMi69PDp8a1AY2I5B2gM=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 h1:Tgea0cVUD0ivh5ADBX4WwuI12DUd2to3nCYe2eayMIw=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=