Add S3 storage support

config/config.go

@@ -6,11 +6,12 @@ import (
 	"net"
 	"time"
 
-	haikunator "github.com/atrox/haikunatorgo/v2"
 	"github.com/datarhei/core/v16/config/copy"
 	"github.com/datarhei/core/v16/config/value"
 	"github.com/datarhei/core/v16/config/vars"
 	"github.com/datarhei/core/v16/math/rand"
+
+	haikunator "github.com/atrox/haikunatorgo/v2"
 	"github.com/google/uuid"
 )
 
@@ -111,6 +112,7 @@ func (d *Config) Clone() *Config {
 	data.Storage.CORS.Origins = copy.Slice(d.Storage.CORS.Origins)
 	data.Storage.Disk.Cache.Types.Allow = copy.Slice(d.Storage.Disk.Cache.Types.Allow)
 	data.Storage.Disk.Cache.Types.Block = copy.Slice(d.Storage.Disk.Cache.Types.Block)
+	data.Storage.S3 = copy.Slice(d.Storage.S3)
 
 	data.FFmpeg.Access.Input.Allow = copy.Slice(d.FFmpeg.Access.Input.Allow)
 	data.FFmpeg.Access.Input.Block = copy.Slice(d.FFmpeg.Access.Input.Block)
@@ -195,6 +197,9 @@ func (d *Config) init() {
 	d.vars.Register(value.NewInt64(&d.Storage.Memory.Size, 0), "storage.memory.max_size_mbytes", "CORE_STORAGE_MEMORY_MAXSIZEMBYTES", nil, "Max. allowed megabytes for /memfs, 0 for unlimited", false, false)
 	d.vars.Register(value.NewBool(&d.Storage.Memory.Purge, false), "storage.memory.purge", "CORE_STORAGE_MEMORY_PURGE", nil, "Automatically remove the oldest files if /memfs is full", false, false)
 
+	// Storage (S3)
+	d.vars.Register(value.NewS3StorageListValue(&d.Storage.S3, []value.S3Storage{}, "|"), "storage.s3", "CORE_STORAGE_S3", nil, "List of S3 storage URLS", false, false)
+
 	// Storage (CORS)
 	d.vars.Register(value.NewCORSOrigins(&d.Storage.CORS.Origins, []string{"*"}, ","), "storage.cors.origins", "CORE_STORAGE_CORS_ORIGINS", nil, "Allowed CORS origins for /memfs and /data", false, false)
 

config/data.go

@@ -88,6 +88,7 @@ type Data struct {
 			Size  int64 `json:"max_size_mbytes" format:"int64"`
 			Purge bool  `json:"purge"`
 		} `json:"memory"`
+		S3   []value.S3Storage `json:"s3"`
 		CORS struct {
 			Origins []string `json:"origins"`
 		} `json:"cors"`
@@ -246,6 +247,8 @@ func MergeV2toV3(data *Data, d *v2.Data) (*Data, error) {
 	data.Storage.Disk.Cache.TTL = d.Storage.Disk.Cache.TTL
 	data.Storage.Disk.Cache.Types.Allow = copy.Slice(d.Storage.Disk.Cache.Types)
 
+	data.Storage.S3 = []value.S3Storage{}
+
 	data.Version = 3
 
 	return data, nil

config/value/s3.go

@@ -0,0 +1,179 @@
+package value
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/publicsuffix"
+)
+
+// array of s3 storages
+// https://access_key_id:secret_access_id@region.endpoint/bucket?name=aaa&mount=/abc&username=xxx&password=yyy
+
+type S3Storage struct {
+	Name       string `json:"name"`
+	Mountpoint string `json:"mountpoint"`
+	Auth       struct {
+		Enable   bool   `json:"enable"`
+		Username string `json:"username"`
+		Password string `json:"password"`
+	} `json:"auth"`
+	Endpoint        string `json:"endpoint"`
+	AccessKeyID     string `json:"access_key_id"`
+	SecretAccessKey string `json:"secret_access_key"`
+	Bucket          string `json:"bucket"`
+	Region          string `json:"region"`
+	UseSSL          bool   `json:"use_ssl"`
+}
+
+func (t *S3Storage) String() string {
+	u := url.URL{}
+
+	if t.UseSSL {
+		u.Scheme = "https"
+	} else {
+		u.Scheme = "http"
+	}
+
+	u.User = url.UserPassword(t.AccessKeyID, "---")
+
+	u.Host = t.Endpoint
+
+	if len(t.Region) != 0 {
+		u.Host = t.Region + "." + u.Host
+	}
+
+	if len(t.Bucket) != 0 {
+		u.Path = "/" + t.Bucket
+	}
+
+	v := url.Values{}
+	v.Set("name", t.Name)
+	v.Set("mountpoint", t.Mountpoint)
+
+	if t.Auth.Enable {
+		if len(t.Auth.Username) != 0 {
+			v.Set("username", t.Auth.Username)
+		}
+
+		if len(t.Auth.Password) != 0 {
+			v.Set("password", "---")
+		}
+	}
+
+	u.RawQuery = v.Encode()
+
+	return u.String()
+}
+
+type s3StorageListValue struct {
+	p         *[]S3Storage
+	separator string
+}
+
+func NewS3StorageListValue(p *[]S3Storage, val []S3Storage, separator string) *s3StorageListValue {
+	v := &s3StorageListValue{
+		p:         p,
+		separator: separator,
+	}
+
+	*p = val
+	return v
+}
+
+func (s *s3StorageListValue) Set(val string) error {
+	list := []S3Storage{}
+
+	for _, elm := range strings.Split(val, s.separator) {
+		u, err := url.Parse(elm)
+		if err != nil {
+			return fmt.Errorf("invalid S3 storage URL (%s): %w", elm, err)
+		}
+
+		t := S3Storage{
+			Name:        u.Query().Get("name"),
+			Mountpoint:  u.Query().Get("mountpoint"),
+			AccessKeyID: u.User.Username(),
+		}
+
+		hostname := u.Hostname()
+		port := u.Port()
+
+		domain, err := publicsuffix.EffectiveTLDPlusOne(hostname)
+		if err != nil {
+			return fmt.Errorf("invalid eTLD (%s): %w", hostname, err)
+		}
+
+		t.Endpoint = domain
+		if len(port) != 0 {
+			t.Endpoint += ":" + port
+		}
+
+		region := strings.TrimSuffix(hostname, domain)
+		if len(region) != 0 {
+			t.Region = strings.TrimSuffix(region, ".")
+		}
+
+		secret, ok := u.User.Password()
+		if ok {
+			t.SecretAccessKey = secret
+		}
+
+		t.Bucket = strings.TrimPrefix(u.Path, "/")
+
+		if u.Scheme == "https" {
+			t.UseSSL = true
+		}
+
+		if u.Query().Has("username") || u.Query().Has("password") {
+			t.Auth.Enable = true
+			t.Auth.Username = u.Query().Get("username")
+			t.Auth.Username = u.Query().Get("password")
+		}
+
+		list = append(list, t)
+	}
+
+	*s.p = list
+
+	return nil
+}
+
+func (s *s3StorageListValue) String() string {
+	if s.IsEmpty() {
+		return "(empty)"
+	}
+
+	list := []string{}
+
+	for _, t := range *s.p {
+		list = append(list, t.String())
+	}
+
+	return strings.Join(list, s.separator)
+}
+
+func (s *s3StorageListValue) Validate() error {
+	for i, t := range *s.p {
+		if len(t.Name) == 0 {
+			return fmt.Errorf("the name for s3 storage %d is missing", i)
+		}
+
+		if len(t.Mountpoint) == 0 {
+			return fmt.Errorf("the mountpoint for s3 storage %d is missing", i)
+		}
+
+		if t.Auth.Enable {
+			if len(t.Auth.Username) == 0 && len(t.Auth.Password) == 0 {
+				return fmt.Errorf("auth is enabled, but no username and password are set for s3 storage %d", i)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *s3StorageListValue) IsEmpty() bool {
+	return len(*s.p) == 0
+}