Support fps props for linux cams

go.mod

@@ -3,7 +3,7 @@ module github.com/pion/mediadevices
 go 1.13
 
 require (
-	github.com/blackjack/webcam v0.0.0-20230411204030-32744c21431f
+	github.com/blackjack/webcam v0.0.0-20230502173554-3b52e93e8607
 	github.com/gen2brain/malgo v0.11.10
 	github.com/google/uuid v1.3.0
 	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329

go.sum

@@ -1,5 +1,5 @@
-github.com/blackjack/webcam v0.0.0-20230411204030-32744c21431f h1:qBxp6Oz8y0AfeqjrYcHaYdfWQf+vUXAwgZ+GWnTtd/E=
+github.com/blackjack/webcam v0.0.0-20230502173554-3b52e93e8607 h1:KG44gkEm6X8qGbJnv9Ef02OSWYtP0pGnu5Pw8QiWxys=
-github.com/blackjack/webcam v0.0.0-20230411204030-32744c21431f/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
+github.com/blackjack/webcam v0.0.0-20230502173554-3b52e93e8607/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/driver/camera/camera_linux.go

@@ -6,9 +6,9 @@ import "C"
 import (
 	"context"
 	"errors"
-	"github.com/pion/mediadevices/pkg/driver/availability"
 	"image"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -16,6 +16,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/pion/mediadevices/pkg/driver/availability"
+
 	"github.com/blackjack/webcam"
 	"github.com/pion/mediadevices/pkg/driver"
 	"github.com/pion/mediadevices/pkg/frame"
@@ -314,13 +316,18 @@ func (c *camera) Properties() []prop.Media {
 			}
 
 			if frameSize.StepWidth == 0 || frameSize.StepHeight == 0 {
+				for _, framerate := range c.cam.GetSupportedFramerates(format, frameSize.MinWidth, frameSize.MinHeight) {
+					for _, fps := range enumFramerate(framerate) {
 						properties = append(properties, prop.Media{
 							Video: prop.Video{
 								Width:       int(frameSize.MaxWidth),
 								Height:      int(frameSize.MaxHeight),
 								FrameFormat: supportedFormat,
+								FrameRate:   fps,
 							},
 						})
+					}
+				}
 			} else {
 				// FIXME: we should probably use a custom data structure to capture all of the supported resolutions
 				for _, supportedResolution := range supportedResolutions {
@@ -339,17 +346,22 @@ func (c *camera) Properties() []prop.Media {
 						continue
 					}
 
+					for _, framerate := range c.cam.GetSupportedFramerates(format, uint32(width), uint32(height)) {
+						for _, fps := range enumFramerate(framerate) {
 							properties = append(properties, prop.Media{
 								Video: prop.Video{
 									Width:       width,
 									Height:      height,
 									FrameFormat: supportedFormat,
+									FrameRate:   fps,
 								},
 							})
 						}
 					}
 				}
 			}
+		}
+	}
 	return properties
 }
 
@@ -385,3 +397,37 @@ func (c *camera) IsAvailable() (bool, error) {
 		return false, availability.NewError(errno.Error())
 	}
 }
+
+// enumFramerate returns a list of fps options from a FrameRate struct.
+// discrete framerates will return a list of 1 fps element.
+// stepwise framerates will return a list of all possible fps options.
+func enumFramerate(framerate webcam.FrameRate) []float32 {
+	var framerates []float32
+	if framerate.StepNumerator == 0 && framerate.StepDenominator == 0 {
+		fr, err := calcFramerate(framerate.MaxNumerator, framerate.MaxDenominator)
+		if err != nil {
+			return framerates
+		}
+		framerates = append(framerates, fr)
+	} else {
+		for n := framerate.MinNumerator; n <= framerate.MaxNumerator; n += framerate.StepNumerator {
+			for d := framerate.MinDenominator; d <= framerate.MaxDenominator; d += framerate.StepDenominator {
+				fr, err := calcFramerate(n, d)
+				if err != nil {
+					continue
+				}
+				framerates = append(framerates, fr)
+			}
+		}
+	}
+	return framerates
+}
+
+// calcFramerate turns fraction into a float32 fps value.
+func calcFramerate(numerator uint32, denominator uint32) (float32, error) {
+	if denominator == 0 {
+		return 0, errors.New("framerate denominator is zero")
+	}
+	// round to three decimal places to avoid floating point precision issues
+	return float32(math.Round(1000.0/((float64(numerator))/float64(denominator))) / 1000), nil
+}

pkg/driver/camera/camera_linux_test.go

@@ -160,3 +160,34 @@ func TestGetCameraReadTimeout(t *testing.T) {
 		t.Errorf("Expected: %d, got: %d", expected, value)
 	}
 }
+
+func TestCalcFramerate(t *testing.T) {
+	framerates := []struct {
+		numerator   uint32
+		denominator uint32
+		expected    float32
+	}{
+		{1, 10, 10.0},
+		{1, 15, 15.0},
+		{1, 30, 30.0},
+		{1, 60, 60.0},
+		{1, 120, 120.0},
+	}
+
+	for _, framerate := range framerates {
+		value, err := calcFramerate(framerate.numerator, framerate.denominator)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// make sure we do not have any rounding errors
+		if value != framerate.expected {
+			t.Errorf("Expected: %f, got: %f", framerate.expected, value)
+		}
+	}
+
+	// divide by zero check
+	_, err := calcFramerate(1, 0)
+	if err == nil {
+		t.Errorf("Expected divide by zero error")
+	}
+}

pkg/prop/prop.go

@@ -145,11 +145,10 @@ func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
 	cmps.add(p.Width, o.Width)
 	cmps.add(p.Height, o.Height)
 	cmps.add(p.FrameFormat, o.FrameFormat)
-	// The next line is comment out for now to not include framerate in the fitness function.
+	// skip framerate if not available in media properties
-	// As camera.Properties does not have access to the list of available framerate at the moment,
+	if o.FrameRate > 0.0 {
-	// no driver can be matched with a framerate constraint.
+		cmps.add(p.FrameRate, o.FrameRate)
-	// Note this also affect screen caputre as screen.Properties does not fill in the Framerate field.
+	}
-	// cmps.add(p.FrameRate, o.FrameRate)
 	cmps.add(p.SampleRate, o.SampleRate)
 	cmps.add(p.Latency, o.Latency)
 	cmps.add(p.ChannelCount, o.ChannelCount)

pkg/prop/prop_test.go

@@ -85,6 +85,60 @@ func TestCompareMatch(t *testing.T) {
 			}},
 			true,
 		},
+		"FloatExactMatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: FloatExact(30),
+			}},
+			Media{Video: Video{
+				FrameRate: 30.0,
+			}},
+			true,
+		},
+		"FloatExactUnmatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: FloatExact(30),
+			}},
+			Media{Video: Video{
+				FrameRate: 30.1,
+			}},
+			false,
+		},
+		"FloatIdealMatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: Float(30),
+			}},
+			Media{Video: Video{
+				FrameRate: 30.0,
+			}},
+			true,
+		},
+		"FloatIdealUnmatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: Float(30),
+			}},
+			Media{Video: Video{
+				FrameRate: 10.0,
+			}},
+			true,
+		},
+		"FloatRangeMatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: FloatRanged{Min: 30, Max: 40},
+			}},
+			Media{Video: Video{
+				FrameRate: 35.0,
+			}},
+			true,
+		},
+		"FloatRangeUnmatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: FloatRanged{Min: 30, Max: 40},
+			}},
+			Media{Video: Video{
+				FrameRate: 50.0,
+			}},
+			false,
+		},
 		"FrameFormatOneOfUnmatch": {
 			MediaConstraints{VideoConstraints: VideoConstraints{
 				FrameFormat: FrameFormatOneOf{frame.FormatYUYV, frame.FormatUYVY},
@@ -366,3 +420,63 @@ func TestString(t *testing.T) {
 		})
 	})
 }
+
+func TestFrameRateProps(t *testing.T) {
+	testDataSet := map[string]struct {
+		a     MediaConstraints
+		b     Media
+		score float64
+		match bool
+	}{
+		"FrameRateIdealMatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: Float(30.0),
+			}},
+			Media{Video: Video{
+				FrameRate: 30.0,
+			}},
+			0.0,
+			true,
+		},
+		"FrameRateIdealUnmatch": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: Float(30.0),
+			}},
+			Media{Video: Video{
+				FrameRate: 60.0,
+			}},
+			0.5,
+			true,
+		},
+		"FrameRateConstraintMissing": {
+			// empty video fps constraint
+			MediaConstraints{VideoConstraints: VideoConstraints{}},
+			Media{Video: Video{
+				FrameRate: 30.0,
+			}},
+			0.0,
+			true,
+		},
+		"FrameRatePropMissing": {
+			MediaConstraints{VideoConstraints: VideoConstraints{
+				FrameRate: Float(30.0),
+			}},
+			// empty video fps property
+			Media{Video: Video{}},
+			0.0,
+			true,
+		},
+	}
+
+	for name, data := range testDataSet {
+		t.Run(name, func(t *testing.T) {
+			score, match := data.a.FitnessDistance(data.b)
+			if score != data.score {
+				t.Errorf("expected score %f, got %f", data.score, score)
+			}
+			if match != data.match {
+				t.Errorf("expected match %t, got %t", data.match, match)
+			}
+		})
+	}
+}