prop: support ranged/exact/oneof constraints

This commit is contained in:
Atsushi Watanabe
2020-05-21 11:44:35 +09:00
parent 305b7086e3
commit ecff5e63a5
13 changed files with 558 additions and 68 deletions

View File

@@ -11,6 +11,7 @@ import (
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
@@ -66,10 +67,10 @@ func main() {
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {
c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420
c.FrameFormat = prop.FrameFormatExact(frame.FormatI420) // most of the encoder accepts I420
c.Enabled = true
c.Width = 640
c.Height = 480
c.Width = prop.Int(640)
c.Height = prop.Int(480)
c.VideoTransform = markFacesTransformer
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},

View File

@@ -10,6 +10,7 @@ import (
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/rtp"
"github.com/pion/webrtc/v2"
"github.com/pion/webrtc/v2/pkg/media"
@@ -48,10 +49,10 @@ func main() {
_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {
c.FrameFormat = frame.FormatYUY2
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
c.Enabled = true
c.Width = 640
c.Height = 480
c.Width = prop.Int(640)
c.Height = prop.Int(480)
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})

View File

@@ -7,6 +7,7 @@ import (
"github.com/pion/mediadevices/examples/internal/signal"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
// This is required to use opus audio encoder
@@ -80,10 +81,10 @@ func main() {
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
},
Video: func(c *mediadevices.MediaTrackConstraints) {
c.FrameFormat = frame.FormatYUY2
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
c.Enabled = true
c.Width = 640
c.Height = 480
c.Width = prop.Int(640)
c.Height = prop.Int(480)
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})

View File

@@ -200,7 +200,11 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
for d, props := range driverProperties {
priority := float64(d.Info().Priority)
for _, p := range props {
fitnessDist := constraints.Media.FitnessDistance(p) - priority
fitnessDist, ok := constraints.MediaConstraints.FitnessDistance(p)
if !ok {
continue
}
fitnessDist -= priority
if fitnessDist < minFitnessDist {
minFitnessDist = fitnessDist
bestDriver = d
@@ -213,7 +217,8 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
return nil, MediaTrackConstraints{}, errNotFound
}
constraints.Merge(bestProp)
constraints.selectedMedia = bestProp
constraints.selectedMedia.Merge(constraints.MediaConstraints)
return bestDriver, constraints, nil
}

View File

@@ -50,8 +50,8 @@ func TestGetUserMedia(t *testing.T) {
constraints := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) {
c.Enabled = true
c.Width = 640
c.Height = 480
c.Width = prop.Int(640)
c.Height = prop.Int(480)
params := videoParams
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
},
@@ -64,8 +64,8 @@ func TestGetUserMedia(t *testing.T) {
constraintsWrong := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) {
c.Enabled = true
c.Width = 640
c.Height = 480
c.Width = prop.Int(640)
c.Height = prop.Int(480)
params := videoParams
params.BitRate = 0
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}

View File

@@ -14,7 +14,7 @@ type MediaStreamConstraints struct {
// MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
type MediaTrackConstraints struct {
prop.Media
prop.MediaConstraints
Enabled bool
// VideoEncoderBuilders are codec builders that are used for encoding the video
// and later being used for sending the appropriate RTP payload type.
@@ -34,6 +34,8 @@ type MediaTrackConstraints struct {
// AudioTransform will be used to transform the audio that's coming from the driver.
// So, basically it'll look like following: driver -> AudioTransform -> code
AudioTransform audio.TransformFunc
selectedMedia prop.Media
}
type MediaOption func(*MediaTrackConstraints)

83
pkg/prop/duration.go Normal file
View File

@@ -0,0 +1,83 @@
package prop
import (
"math"
"time"
)
type DurationConstraint interface {
Compare(time.Duration) (float64, bool)
Value() (time.Duration, bool)
}
type Duration time.Duration
func (d Duration) Compare(a time.Duration) (float64, bool) {
return math.Abs(float64(a-time.Duration(d))) / math.Max(math.Abs(float64(a)), math.Abs(float64(d))), true
}
func (d Duration) Value() (time.Duration, bool) { return time.Duration(d), true }
type DurationExact time.Duration
func (d DurationExact) Compare(a time.Duration) (float64, bool) {
if time.Duration(d) == a {
return 0.0, true
}
return 1.0, false
}
func (d DurationExact) Value() (time.Duration, bool) { return time.Duration(d), true }
type DurationOneOf []time.Duration
func (d DurationOneOf) Compare(a time.Duration) (float64, bool) {
for _, ii := range d {
if ii == a {
return 0.0, true
}
}
return 1.0, false
}
func (DurationOneOf) Value() (time.Duration, bool) { return 0, false }
type DurationRanged struct {
Min time.Duration
Max time.Duration
Ideal time.Duration
}
func (d DurationRanged) Compare(a time.Duration) (float64, bool) {
if d.Min != 0 && d.Min > a {
// Out of range
return 1.0, false
}
if d.Max != 0 && d.Max < a {
// Out of range
return 1.0, false
}
if d.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == d.Ideal:
return 0.0, true
case a < d.Ideal:
if d.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(d.Ideal-a) / float64(d.Ideal-d.Min), true
default:
if d.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-d.Ideal) / float64(d.Max-d.Ideal), true
}
}
func (DurationRanged) Value() (time.Duration, bool) { return 0, false }

82
pkg/prop/float.go Normal file
View File

@@ -0,0 +1,82 @@
package prop
import (
"math"
)
type FloatConstraint interface {
Compare(float32) (float64, bool)
Value() (float32, bool)
}
type Float float32
func (f Float) Compare(a float32) (float64, bool) {
return math.Abs(float64(a-float32(f))) / math.Max(math.Abs(float64(a)), math.Abs(float64(f))), true
}
func (f Float) Value() (float32, bool) { return float32(f), true }
type FloatExact float32
func (f FloatExact) Compare(a float32) (float64, bool) {
if float32(f) == a {
return 0.0, true
}
return 1.0, false
}
func (f FloatExact) Value() (float32, bool) { return float32(f), true }
type FloatOneOf []float32
func (f FloatOneOf) Compare(a float32) (float64, bool) {
for _, ff := range f {
if ff == a {
return 0.0, true
}
}
return 1.0, false
}
func (FloatOneOf) Value() (float32, bool) { return 0, false }
type FloatRanged struct {
Min float32
Max float32
Ideal float32
}
func (f FloatRanged) Compare(a float32) (float64, bool) {
if f.Min != 0 && f.Min > a {
// Out of range
return 1.0, false
}
if f.Max != 0 && f.Max < a {
// Out of range
return 1.0, false
}
if f.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == f.Ideal:
return 0.0, true
case a < f.Ideal:
if f.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(f.Ideal-a) / float64(f.Ideal-f.Min), true
default:
if f.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-f.Ideal) / float64(f.Max-f.Ideal), true
}
}
func (FloatRanged) Value() (float32, bool) { return 0, false }

45
pkg/prop/format.go Normal file
View File

@@ -0,0 +1,45 @@
package prop
import (
"github.com/pion/mediadevices/pkg/frame"
)
type FrameFormatConstraint interface {
Compare(frame.Format) (float64, bool)
Value() (frame.Format, bool)
}
type FrameFormat frame.Format
func (f FrameFormat) Compare(a frame.Format) (float64, bool) {
if frame.Format(f) == a {
return 0.0, true
}
return 1.0, true
}
func (f FrameFormat) Value() (frame.Format, bool) { return frame.Format(f), true }
type FrameFormatExact frame.Format
func (f FrameFormatExact) Compare(a frame.Format) (float64, bool) {
if frame.Format(f) == a {
return 0.0, true
}
return 1.0, false
}
func (f FrameFormatExact) Value() (frame.Format, bool) { return frame.Format(f), true }
type FrameFormatOneOf []frame.Format
func (f FrameFormatOneOf) Compare(a frame.Format) (float64, bool) {
for _, ff := range f {
if ff == a {
return 0.0, true
}
}
return 1.0, false
}
func (FrameFormatOneOf) Value() (frame.Format, bool) { return "", false }

82
pkg/prop/int.go Normal file
View File

@@ -0,0 +1,82 @@
package prop
import (
"math"
)
type IntConstraint interface {
Compare(int) (float64, bool)
Value() (int, bool)
}
type Int int
func (i Int) Compare(a int) (float64, bool) {
return math.Abs(float64(a-int(i))) / math.Max(math.Abs(float64(a)), math.Abs(float64(i))), true
}
func (i Int) Value() (int, bool) { return int(i), true }
type IntExact int
func (i IntExact) Compare(a int) (float64, bool) {
if int(i) == a {
return 0.0, true
}
return 1.0, false
}
func (i IntExact) Value() (int, bool) { return int(i), true }
type IntOneOf []int
func (i IntOneOf) Compare(a int) (float64, bool) {
for _, ii := range i {
if ii == a {
return 0.0, true
}
}
return 1.0, false
}
func (IntOneOf) Value() (int, bool) { return 0, false }
type IntRanged struct {
Min int
Max int
Ideal int
}
func (i IntRanged) Compare(a int) (float64, bool) {
if i.Min != 0 && i.Min > a {
// Out of range
return 1.0, false
}
if i.Max != 0 && i.Max < a {
// Out of range
return 1.0, false
}
if i.Ideal == 0 {
// If the value is in the range and Ideal is not specified,
// any value is evenly acceptable.
return 0.0, true
}
switch {
case a == i.Ideal:
return 0.0, true
case a < i.Ideal:
if i.Min == 0 {
// If Min is not specified, smaller values than Ideal are even.
return 0.0, true
}
return float64(i.Ideal-a) / float64(i.Ideal-i.Min), true
default:
if i.Max == 0 {
// If Max is not specified, larger values than Ideal are even.
return 0.0, true
}
return float64(a-i.Ideal) / float64(i.Max-i.Ideal), true
}
}
func (IntRanged) Value() (int, bool) { return 0, false }

View File

@@ -1,15 +1,18 @@
package prop
import (
"fmt"
"math"
"reflect"
"strconv"
"time"
"github.com/pion/mediadevices/pkg/frame"
)
type MediaConstraints struct {
DeviceID string
VideoConstraints
AudioConstraints
}
type Media struct {
DeviceID string
Video
@@ -17,7 +20,7 @@ type Media struct {
}
// Merge merges all the field values from o to p, except zero values.
func (p *Media) Merge(o Media) {
func (p *Media) Merge(o MediaConstraints) {
rp := reflect.ValueOf(p).Elem()
ro := reflect.ValueOf(o)
@@ -29,9 +32,9 @@ func (p *Media) Merge(o Media) {
fieldA := a.Field(i)
fieldB := b.Field(i)
// if a is a struct, b is also a struct. Then,
// if b is a struct, a is also a struct. Then,
// we recursively merge them
if fieldA.Kind() == reflect.Struct {
if fieldB.Kind() == reflect.Struct {
merge(fieldA, fieldB)
continue
}
@@ -43,67 +46,122 @@ func (p *Media) Merge(o Media) {
continue
}
fieldA.Set(fieldB)
switch c := fieldB.Interface().(type) {
case IntConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case FloatConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case DurationConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
case FrameFormatConstraint:
if v, ok := c.Value(); ok {
fieldA.Set(reflect.ValueOf(v))
}
default:
panic("unsupported property type")
}
}
}
merge(rp, ro)
}
func (p *Media) FitnessDistance(o Media) float64 {
func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
cmps := comparisons{}
cmps.add(p.Width, o.Width)
cmps.add(p.Height, o.Height)
cmps.add(p.FrameFormat, o.FrameFormat)
cmps.add(p.SampleRate, o.SampleRate)
cmps.add(p.Latency, o.Latency)
return cmps.fitnessDistance()
}
type comparisons map[string]string
type comparisons []struct {
desired, actual interface{}
}
func (c comparisons) add(actual, ideal interface{}) {
c[fmt.Sprint(actual)] = fmt.Sprint(ideal)
func (c *comparisons) add(desired, actual interface{}) {
if desired != nil {
*c = append(*c,
struct{ desired, actual interface{} }{
desired, actual,
},
)
}
}
// fitnessDistance is an implementation for https://w3c.github.io/mediacapture-main/#dfn-fitness-distance
func (c comparisons) fitnessDistance() float64 {
func (c *comparisons) fitnessDistance() (float64, bool) {
var dist float64
for actual, ideal := range c {
if actual == ideal {
continue
for _, field := range *c {
var d float64
var ok bool
switch c := field.desired.(type) {
case IntConstraint:
if actual, typeOK := field.actual.(int); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
actualF, err1 := strconv.ParseFloat(actual, 64)
idealF, err2 := strconv.ParseFloat(ideal, 64)
switch {
// If both of the values are numeric, we need to normalize the values to get the distance
case err1 == nil && err2 == nil:
dist += math.Abs(actualF-idealF) / math.Max(math.Abs(actualF), math.Abs(idealF))
// If both of the values are not numeric, the only comparison value is either 0 (matched) or 1 (not matched)
case err1 != nil && err2 != nil:
if actual != ideal {
dist++
case FloatConstraint:
if actual, typeOK := field.actual.(float32); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case DurationConstraint:
if actual, typeOK := field.actual.(time.Duration); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
case FrameFormatConstraint:
if actual, typeOK := field.actual.(frame.Format); typeOK {
d, ok = c.Compare(actual)
} else {
panic("wrong type of actual value")
}
// Comparing a numeric value with a non-numeric value is a an internal error, so panic.
default:
panic("fitnessDistance can't mix comparisons.")
panic("unsupported constraint type")
}
dist += d
if !ok {
return 0, false
}
}
return dist
return dist, true
}
// Video represents a video's properties
// VideoConstraints represents a video's constraints
type VideoConstraints struct {
Width, Height IntConstraint
FrameRate FloatConstraint
FrameFormat FrameFormatConstraint
}
// Video represents a video's constraints
type Video struct {
Width, Height int
FrameRate float32
FrameFormat frame.Format
}
// Audio represents an audio's properties
// AudioConstraints represents an audio's constraints
type AudioConstraints struct {
ChannelCount IntConstraint
Latency DurationConstraint
SampleRate IntConstraint
SampleSize IntConstraint
}
// Audio represents an audio's constraints
type Audio struct {
ChannelCount int
Latency time.Duration

View File

@@ -2,8 +2,138 @@ package prop
import (
"testing"
"time"
"github.com/pion/mediadevices/pkg/frame"
)
func TestCompareMatch(t *testing.T) {
testDataSet := map[string]struct {
a MediaConstraints
b Media
match bool
}{
"IntIdealUnmatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: Int(30),
}},
Media{Video: Video{
Width: 50,
}},
true,
},
"IntIdealMatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: Int(30),
}},
Media{Video: Video{
Width: 30,
}},
true,
},
"IntExactUnmatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: IntExact(30),
}},
Media{Video: Video{
Width: 50,
}},
false,
},
"IntExactMatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: IntExact(30),
}},
Media{Video: Video{
Width: 30,
}},
true,
},
"IntRangeUnmatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: IntRanged{Min: 30, Max: 40},
}},
Media{Video: Video{
Width: 50,
}},
false,
},
"IntRangeMatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
Width: IntRanged{Min: 30, Max: 40},
}},
Media{Video: Video{
Width: 35,
}},
true,
},
"FrameFormatOneOfUnmatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
FrameFormat: FrameFormatOneOf{frame.FormatYUYV, frame.FormatUYVY},
}},
Media{Video: Video{
FrameFormat: frame.FormatYUYV,
}},
true,
},
"FrameFormatOneOfMatch": {
MediaConstraints{VideoConstraints: VideoConstraints{
FrameFormat: FrameFormatOneOf{frame.FormatYUYV, frame.FormatUYVY},
}},
Media{Video: Video{
FrameFormat: frame.FormatMJPEG,
}},
false,
},
"DurationExactUnmatch": {
MediaConstraints{AudioConstraints: AudioConstraints{
Latency: DurationExact(time.Second),
}},
Media{Audio: Audio{
Latency: time.Second + time.Millisecond,
}},
false,
},
"DurationExactMatch": {
MediaConstraints{AudioConstraints: AudioConstraints{
Latency: DurationExact(time.Second),
}},
Media{Audio: Audio{
Latency: time.Second,
}},
true,
},
"DurationRangedUnmatch": {
MediaConstraints{AudioConstraints: AudioConstraints{
Latency: DurationRanged{Max: time.Second},
}},
Media{Audio: Audio{
Latency: time.Second + time.Millisecond,
}},
false,
},
"DurationRangedMatch": {
MediaConstraints{AudioConstraints: AudioConstraints{
Latency: DurationRanged{Max: time.Second},
}},
Media{Audio: Audio{
Latency: time.Millisecond,
}},
true,
},
}
for name, testData := range testDataSet {
testData := testData
t.Run(name, func(t *testing.T) {
_, match := testData.a.FitnessDistance(testData.b)
if match != testData.match {
t.Errorf("matching flag differs, expected: %v, got: %v", testData.match, match)
}
})
}
}
func TestMergeWithZero(t *testing.T) {
a := Media{
Video: Video{
@@ -11,9 +141,9 @@ func TestMergeWithZero(t *testing.T) {
},
}
b := Media{
Video: Video{
Height: 100,
b := MediaConstraints{
VideoConstraints: VideoConstraints{
Height: Int(100),
},
}
@@ -35,9 +165,9 @@ func TestMergeWithSameField(t *testing.T) {
},
}
b := Media{
Video: Video{
Width: 100,
b := MediaConstraints{
VideoConstraints: VideoConstraints{
Width: Int(100),
},
}
@@ -61,9 +191,9 @@ func TestMergeNested(t *testing.T) {
},
}
b := Media{
Video: Video{
Width: 100,
b := MediaConstraints{
VideoConstraints: VideoConstraints{
Width: Int(100),
},
}

View File

@@ -1,7 +1,7 @@
package mediadevices
import (
"fmt"
"errors"
"math/rand"
"sync"
@@ -62,11 +62,11 @@ func newTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrack
case driver.AudioRecorder:
rtpCodecs = opts.codecs[webrtc.RTPCodecTypeAudio]
buildSampler = func(t LocalTrack) samplerFunc {
return newAudioSampler(t, constraints.Latency)
return newAudioSampler(t, constraints.selectedMedia.Latency)
}
encoderBuilders, err = newAudioEncoderBuilders(r, constraints)
default:
err = fmt.Errorf("newTrack: invalid driver type")
err = errors.New("newTrack: invalid driver type")
}
if err != nil {
@@ -114,7 +114,7 @@ func newTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrack
}
d.Close()
return nil, fmt.Errorf("newTrack: failed to find a matching codec")
return nil, errors.New("newTrack: failed to find a matching codec")
}
// OnEnded sets an error handler. When a track has been created and started, if an
@@ -196,7 +196,7 @@ type encoderBuilder struct {
// newVideoEncoderBuilders transforms video given by VideoRecorder with the video transformer that is passed through
// constraints and create a list of generic encoder builders
func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
r, err := vr.VideoRecord(constraints.Media)
r, err := vr.VideoRecord(constraints.selectedMedia)
if err != nil {
return nil, err
}
@@ -209,7 +209,7 @@ func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackCons
for i, b := range constraints.VideoEncoderBuilders {
encoderBuilders[i].name = b.Name()
encoderBuilders[i].build = func() (codec.ReadCloser, error) {
return b.BuildVideoEncoder(r, constraints.Media)
return b.BuildVideoEncoder(r, constraints.selectedMedia)
}
}
return encoderBuilders, nil
@@ -218,7 +218,7 @@ func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackCons
// newAudioEncoderBuilders transforms audio given by AudioRecorder with the audio transformer that is passed through
// constraints and create a list of generic encoder builders
func newAudioEncoderBuilders(ar driver.AudioRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
r, err := ar.AudioRecord(constraints.Media)
r, err := ar.AudioRecord(constraints.selectedMedia)
if err != nil {
return nil, err
}
@@ -231,7 +231,7 @@ func newAudioEncoderBuilders(ar driver.AudioRecorder, constraints MediaTrackCons
for i, b := range constraints.AudioEncoderBuilders {
encoderBuilders[i].name = b.Name()
encoderBuilders[i].build = func() (codec.ReadCloser, error) {
return b.BuildAudioEncoder(r, constraints.Media)
return b.BuildAudioEncoder(r, constraints.selectedMedia)
}
}
return encoderBuilders, nil