Merge branch 'dev' into cluster

config/value/s3.go

@@ -3,9 +3,8 @@ package value
 import (
 	"fmt"
 	"net/url"
+	"regexp"
 	"strings"
-
-	"golang.org/x/net/publicsuffix"
 )
 
 // array of s3 storages
@@ -14,11 +13,7 @@ import (
 type S3Storage struct {
 	Name            string        `json:"name"`
 	Mountpoint      string        `json:"mountpoint"`
-	Auth       struct {
+	Auth            S3StorageAuth `json:"auth"`
-		Enable   bool   `json:"enable"`
-		Username string `json:"username"`
-		Password string `json:"password"`
-	} `json:"auth"`
 	Endpoint        string        `json:"endpoint"`
 	AccessKeyID     string        `json:"access_key_id"`
 	SecretAccessKey string        `json:"secret_access_key"`
@@ -27,6 +22,12 @@ type S3Storage struct {
 	UseSSL          bool          `json:"use_ssl"`
 }
 
+type S3StorageAuth struct {
+	Enable   bool   `json:"enable"`
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
 func (t *S3Storage) String() string {
 	u := url.URL{}
 
@@ -50,7 +51,7 @@ func (t *S3Storage) String() string {
 
 	v := url.Values{}
 	v.Set("name", t.Name)
-	v.Set("mountpoint", t.Mountpoint)
+	v.Set("mount", t.Mountpoint)
 
 	if t.Auth.Enable {
 		if len(t.Auth.Username) != 0 {
@@ -70,12 +71,14 @@ func (t *S3Storage) String() string {
 type s3StorageListValue struct {
 	p         *[]S3Storage
 	separator string
+	reName    *regexp.Regexp
 }
 
 func NewS3StorageListValue(p *[]S3Storage, val []S3Storage, separator string) *s3StorageListValue {
 	v := &s3StorageListValue{
 		p:         p,
 		separator: separator,
+		reName:    regexp.MustCompile(`^[A-Za-z0-9_-]+$`),
 	}
 
 	*p = val
@@ -93,27 +96,16 @@ func (s *s3StorageListValue) Set(val string) error {
 
 		t := S3Storage{
 			Name:        u.Query().Get("name"),
-			Mountpoint:  u.Query().Get("mountpoint"),
+			Mountpoint:  u.Query().Get("mount"),
 			AccessKeyID: u.User.Username(),
 		}
 
-		hostname := u.Hostname()
+		password, _ := u.User.Password()
-		port := u.Port()
+		t.SecretAccessKey = password
 
-		domain, err := publicsuffix.EffectiveTLDPlusOne(hostname)
+		region, endpoint, _ := strings.Cut(u.Host, ".")
-		if err != nil {
+		t.Endpoint = endpoint
-			return fmt.Errorf("invalid eTLD (%s): %w", hostname, err)
+		t.Region = region
-		}
-
-		t.Endpoint = domain
-		if len(port) != 0 {
-			t.Endpoint += ":" + port
-		}
-
-		region := strings.TrimSuffix(hostname, domain)
-		if len(region) != 0 {
-			t.Region = strings.TrimSuffix(region, ".")
-		}
 
 		secret, ok := u.User.Password()
 		if ok {
@@ -129,7 +121,7 @@ func (s *s3StorageListValue) Set(val string) error {
 		if u.Query().Has("username") || u.Query().Has("password") {
 			t.Auth.Enable = true
 			t.Auth.Username = u.Query().Get("username")
-			t.Auth.Username = u.Query().Get("password")
+			t.Auth.Password = u.Query().Get("password")
 		}
 
 		list = append(list, t)
@@ -160,6 +152,10 @@ func (s *s3StorageListValue) Validate() error {
 			return fmt.Errorf("the name for s3 storage %d is missing", i)
 		}
 
+		if !s.reName.MatchString(t.Name) {
+			return fmt.Errorf("the name for s3 storage must match the pattern %s", s.reName.String())
+		}
+
 		if len(t.Mountpoint) == 0 {
 			return fmt.Errorf("the mountpoint for s3 storage %d is missing", i)
 		}

config/value/s3_test.go

@@ -0,0 +1,53 @@
+package value
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestS3Value(t *testing.T) {
+	filesystems := []S3Storage{}
+
+	v := NewS3StorageListValue(&filesystems, nil, " ")
+	require.Equal(t, "(empty)", v.String())
+
+	v.Set("https://access_key_id1:secret_access_id1@region1.subdomain.endpoint1.com/bucket1?name=aaa1&mount=/abc1&username=xxx1&password=yyy1 http://access_key_id2:secret_access_id2@region2.endpoint2.com/bucket2?name=aaa2&mount=/abc2&username=xxx2&password=yyy2")
+	require.Equal(t, []S3Storage{
+		{
+			Name:       "aaa1",
+			Mountpoint: "/abc1",
+			Auth: S3StorageAuth{
+				Enable:   true,
+				Username: "xxx1",
+				Password: "yyy1",
+			},
+			Endpoint:        "subdomain.endpoint1.com",
+			AccessKeyID:     "access_key_id1",
+			SecretAccessKey: "secret_access_id1",
+			Bucket:          "bucket1",
+			Region:          "region1",
+			UseSSL:          true,
+		},
+		{
+			Name:       "aaa2",
+			Mountpoint: "/abc2",
+			Auth: S3StorageAuth{
+				Enable:   true,
+				Username: "xxx2",
+				Password: "yyy2",
+			},
+			Endpoint:        "endpoint2.com",
+			AccessKeyID:     "access_key_id2",
+			SecretAccessKey: "secret_access_id2",
+			Bucket:          "bucket2",
+			Region:          "region2",
+			UseSSL:          false,
+		},
+	}, filesystems)
+	require.Equal(t, "https://access_key_id1:---@region1.subdomain.endpoint1.com/bucket1?mount=%2Fabc1&name=aaa1&password=---&username=xxx1 http://access_key_id2:---@region2.endpoint2.com/bucket2?mount=%2Fabc2&name=aaa2&password=---&username=xxx2", v.String())
+	require.NoError(t, v.Validate())
+
+	v.Set("https://access_key_id1:secret_access_id1@region1.endpoint1.com/bucket1?name=djk*;..&mount=/abc1&username=xxx1&password=yyy1")
+	require.Error(t, v.Validate())
+}

docs/docs.go

@@ -3647,6 +3647,20 @@ const docTemplate = `{
                     "type": "integer",
                     "format": "uint64"
                 },
+                "framerate": {
+                    "type": "object",
+                    "properties": {
+                        "avg": {
+                            "type": "number"
+                        },
+                        "max": {
+                            "type": "number"
+                        },
+                        "min": {
+                            "type": "number"
+                        }
+                    }
+                },
                 "height": {
                     "type": "integer",
                     "format": "uint64"
@@ -5021,18 +5035,7 @@ const docTemplate = `{
                     "type": "string"
                 },
                 "auth": {
-                    "type": "object",
+                    "$ref": "#/definitions/value.S3StorageAuth"
-                    "properties": {
-                        "enable": {
-                            "type": "boolean"
-                        },
-                        "password": {
-                            "type": "string"
-                        },
-                        "username": {
-                            "type": "string"
-                        }
-                    }
                 },
                 "bucket": {
                     "type": "string"
@@ -5056,6 +5059,20 @@ const docTemplate = `{
                     "type": "boolean"
                 }
             }
+        },
+        "value.S3StorageAuth": {
+            "type": "object",
+            "properties": {
+                "enable": {
+                    "type": "boolean"
+                },
+                "password": {
+                    "type": "string"
+                },
+                "username": {
+                    "type": "string"
+                }
+            }
         }
     },
     "securityDefinitions": {

docs/swagger.json

@@ -3640,6 +3640,20 @@
                     "type": "integer",
                     "format": "uint64"
                 },
+                "framerate": {
+                    "type": "object",
+                    "properties": {
+                        "avg": {
+                            "type": "number"
+                        },
+                        "max": {
+                            "type": "number"
+                        },
+                        "min": {
+                            "type": "number"
+                        }
+                    }
+                },
                 "height": {
                     "type": "integer",
                     "format": "uint64"
@@ -5014,18 +5028,7 @@
                     "type": "string"
                 },
                 "auth": {
-                    "type": "object",
+                    "$ref": "#/definitions/value.S3StorageAuth"
-                    "properties": {
-                        "enable": {
-                            "type": "boolean"
-                        },
-                        "password": {
-                            "type": "string"
-                        },
-                        "username": {
-                            "type": "string"
-                        }
-                    }
                 },
                 "bucket": {
                     "type": "string"
@@ -5049,6 +5052,20 @@
                     "type": "boolean"
                 }
             }
+        },
+        "value.S3StorageAuth": {
+            "type": "object",
+            "properties": {
+                "enable": {
+                    "type": "boolean"
+                },
+                "password": {
+                    "type": "string"
+                },
+                "username": {
+                    "type": "string"
+                }
+            }
         }
     },
     "securityDefinitions": {

docs/swagger.yaml

@@ -939,6 +939,15 @@ definitions:
       frame:
         format: uint64
         type: integer
+      framerate:
+        properties:
+          avg:
+            type: number
+          max:
+            type: number
+          min:
+            type: number
+        type: object
       height:
         format: uint64
         type: integer
@@ -1922,14 +1931,7 @@ definitions:
       access_key_id:
         type: string
       auth:
-        properties:
+        $ref: '#/definitions/value.S3StorageAuth'
-          enable:
-            type: boolean
-          password:
-            type: string
-          username:
-            type: string
-        type: object
       bucket:
         type: string
       endpoint:
@@ -1945,6 +1947,15 @@ definitions:
       use_ssl:
         type: boolean
     type: object
+  value.S3StorageAuth:
+    properties:
+      enable:
+        type: boolean
+      password:
+        type: string
+      username:
+        type: string
+    type: object
 info:
   contact:
     email: hello@datarhei.com

ffmpeg/parse/types.go

@@ -98,6 +98,11 @@ type ffmpegProgressIO struct {
 	Bitrate   float64 `json:"-"`          // bit/s
 	Frame     uint64  `json:"frame"`      // counter
 	Keyframe  uint64  `json:"keyframe"`   // counter
+	Framerate struct {
+		Min     float64 `json:"min"`
+		Max     float64 `json:"max"`
+		Average float64 `json:"avg"`
+	} `json:"framerate"`
 	Packet    uint64  `json:"packet"`               // counter
 	Extradata uint64  `json:"extradata_size_bytes"` // bytes
 	FPS       float64 `json:"-"`                    // rate, frames per second
@@ -112,6 +117,9 @@ func (io *ffmpegProgressIO) exportTo(progress *app.ProgressIO) {
 	progress.Stream = io.Stream
 	progress.Frame = io.Frame
 	progress.Keyframe = io.Keyframe
+	progress.Framerate.Min = io.Framerate.Min
+	progress.Framerate.Max = io.Framerate.Max
+	progress.Framerate.Average = io.Framerate.Average
 	progress.Packet = io.Packet
 	progress.FPS = io.FPS
 	progress.PPS = io.PPS

http/api/progress.go

@@ -7,6 +7,12 @@ import (
 	"github.com/datarhei/core/v16/restream/app"
 )
 
+type ProgressIOFramerate struct {
+	Min     json.Number `json:"min" swaggertype:"number" jsonschema:"type=number"`
+	Max     json.Number `json:"max" swaggertype:"number" jsonschema:"type=number"`
+	Average json.Number `json:"avg" swaggertype:"number" jsonschema:"type=number"`
+}
+
 // ProgressIO represents the progress of an ffmpeg input or output
 type ProgressIO struct {
 	ID      string `json:"id" jsonschema:"minLength=1"`
@@ -21,6 +27,7 @@ type ProgressIO struct {
 	Coder     string              `json:"coder"`
 	Frame     uint64              `json:"frame" format:"uint64"`
 	Keyframe  uint64              `json:"keyframe" format:"uint64"`
+	Framerate ProgressIOFramerate `json:"framerate"`
 	FPS       json.Number         `json:"fps" swaggertype:"number" jsonschema:"type=number"`
 	Packet    uint64              `json:"packet" format:"uint64"`
 	PPS       json.Number         `json:"pps" swaggertype:"number" jsonschema:"type=number"`
@@ -59,6 +66,9 @@ func (i *ProgressIO) Unmarshal(io *app.ProgressIO) {
 	i.Coder = io.Coder
 	i.Frame = io.Frame
 	i.Keyframe = io.Keyframe
+	i.Framerate.Min = json.Number(fmt.Sprintf("%.3f", io.Framerate.Min))
+	i.Framerate.Max = json.Number(fmt.Sprintf("%.3f", io.Framerate.Max))
+	i.Framerate.Average = json.Number(fmt.Sprintf("%.3f", io.Framerate.Average))
 	i.FPS = json.Number(fmt.Sprintf("%.3f", io.FPS))
 	i.Packet = io.Packet
 	i.PPS = json.Number(fmt.Sprintf("%.3f", io.PPS))

restream/app/progress.go

@@ -13,6 +13,11 @@ type ProgressIO struct {
 	Coder     string
 	Frame     uint64 // counter
 	Keyframe  uint64 // counter
+	Framerate struct {
+		Min     float64
+		Max     float64
+		Average float64
+	}
 	FPS       float64 // rate, frames per second
 	Packet    uint64  // counter
 	PPS       float64 // rate, packets per second