From a1682b7aa406633b286d2000fae73c905a69bd13 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Thu, 13 Apr 2023 12:19:20 +0200 Subject: [PATCH] Fix parsing S3 storage definition from environment variable --- config/value/s3.go | 64 +++++++++++++++++++---------------------- config/value/s3_test.go | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 config/value/s3_test.go diff --git a/config/value/s3.go b/config/value/s3.go index a85a0838..7d80c193 100644 --- a/config/value/s3.go +++ b/config/value/s3.go @@ -3,28 +3,29 @@ package value import ( "fmt" "net/url" + "regexp" "strings" - - "golang.org/x/net/publicsuffix" ) // array of s3 storages // https://access_key_id:secret_access_id@region.endpoint/bucket?name=aaa&mount=/abc&username=xxx&password=yyy type S3Storage struct { - Name string `json:"name"` - Mountpoint string `json:"mountpoint"` - Auth struct { - Enable bool `json:"enable"` - Username string `json:"username"` - Password string `json:"password"` - } `json:"auth"` - Endpoint string `json:"endpoint"` - AccessKeyID string `json:"access_key_id"` - SecretAccessKey string `json:"secret_access_key"` - Bucket string `json:"bucket"` - Region string `json:"region"` - UseSSL bool `json:"use_ssl"` + Name string `json:"name"` + Mountpoint string `json:"mountpoint"` + Auth S3StorageAuth `json:"auth"` + Endpoint string `json:"endpoint"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + Bucket string `json:"bucket"` + Region string `json:"region"` + UseSSL bool `json:"use_ssl"` +} + +type S3StorageAuth struct { + Enable bool `json:"enable"` + Username string `json:"username"` + Password string `json:"password"` } func (t *S3Storage) String() string { @@ -50,7 +51,7 @@ func (t *S3Storage) String() string { v := url.Values{} v.Set("name", t.Name) - v.Set("mountpoint", t.Mountpoint) + v.Set("mount", t.Mountpoint) if t.Auth.Enable { if len(t.Auth.Username) != 0 { @@ -70,12 +71,14 @@ func (t *S3Storage) String() string { type s3StorageListValue struct { p *[]S3Storage separator string + reName *regexp.Regexp } func NewS3StorageListValue(p *[]S3Storage, val []S3Storage, separator string) *s3StorageListValue { v := &s3StorageListValue{ p: p, separator: separator, + reName: regexp.MustCompile(`^[A-Za-z0-9_-]+$`), } *p = val @@ -93,27 +96,16 @@ func (s *s3StorageListValue) Set(val string) error { t := S3Storage{ Name: u.Query().Get("name"), - Mountpoint: u.Query().Get("mountpoint"), + Mountpoint: u.Query().Get("mount"), AccessKeyID: u.User.Username(), } - hostname := u.Hostname() - port := u.Port() + password, _ := u.User.Password() + t.SecretAccessKey = password - domain, err := publicsuffix.EffectiveTLDPlusOne(hostname) - if err != nil { - return fmt.Errorf("invalid eTLD (%s): %w", hostname, err) - } - - t.Endpoint = domain - if len(port) != 0 { - t.Endpoint += ":" + port - } - - region := strings.TrimSuffix(hostname, domain) - if len(region) != 0 { - t.Region = strings.TrimSuffix(region, ".") - } + region, endpoint, _ := strings.Cut(u.Host, ".") + t.Endpoint = endpoint + t.Region = region secret, ok := u.User.Password() if ok { @@ -129,7 +121,7 @@ func (s *s3StorageListValue) Set(val string) error { if u.Query().Has("username") || u.Query().Has("password") { t.Auth.Enable = true t.Auth.Username = u.Query().Get("username") - t.Auth.Username = u.Query().Get("password") + t.Auth.Password = u.Query().Get("password") } list = append(list, t) @@ -160,6 +152,10 @@ func (s *s3StorageListValue) Validate() error { return fmt.Errorf("the name for s3 storage %d is missing", i) } + if !s.reName.MatchString(t.Name) { + return fmt.Errorf("the name for s3 storage must match the pattern %s", s.reName.String()) + } + if len(t.Mountpoint) == 0 { return fmt.Errorf("the mountpoint for s3 storage %d is missing", i) } diff --git a/config/value/s3_test.go b/config/value/s3_test.go new file mode 100644 index 00000000..79cbee9d --- /dev/null +++ b/config/value/s3_test.go @@ -0,0 +1,53 @@ +package value + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestS3Value(t *testing.T) { + filesystems := []S3Storage{} + + v := NewS3StorageListValue(&filesystems, nil, " ") + require.Equal(t, "(empty)", v.String()) + + v.Set("https://access_key_id1:secret_access_id1@region1.subdomain.endpoint1.com/bucket1?name=aaa1&mount=/abc1&username=xxx1&password=yyy1 http://access_key_id2:secret_access_id2@region2.endpoint2.com/bucket2?name=aaa2&mount=/abc2&username=xxx2&password=yyy2") + require.Equal(t, []S3Storage{ + { + Name: "aaa1", + Mountpoint: "/abc1", + Auth: S3StorageAuth{ + Enable: true, + Username: "xxx1", + Password: "yyy1", + }, + Endpoint: "subdomain.endpoint1.com", + AccessKeyID: "access_key_id1", + SecretAccessKey: "secret_access_id1", + Bucket: "bucket1", + Region: "region1", + UseSSL: true, + }, + { + Name: "aaa2", + Mountpoint: "/abc2", + Auth: S3StorageAuth{ + Enable: true, + Username: "xxx2", + Password: "yyy2", + }, + Endpoint: "endpoint2.com", + AccessKeyID: "access_key_id2", + SecretAccessKey: "secret_access_id2", + Bucket: "bucket2", + Region: "region2", + UseSSL: false, + }, + }, filesystems) + require.Equal(t, "https://access_key_id1:---@region1.subdomain.endpoint1.com/bucket1?mount=%2Fabc1&name=aaa1&password=---&username=xxx1 http://access_key_id2:---@region2.endpoint2.com/bucket2?mount=%2Fabc2&name=aaa2&password=---&username=xxx2", v.String()) + require.NoError(t, v.Validate()) + + v.Set("https://access_key_id1:secret_access_id1@region1.endpoint1.com/bucket1?name=djk*;..&mount=/abc1&username=xxx1&password=yyy1") + require.Error(t, v.Validate()) +}