WIP

examples/webrtc/main.go

@@ -13,15 +13,17 @@ import (
 	// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
 	// or you can also use openh264 for alternative h264 implementation
 	// "github.com/pion/mediadevices/pkg/codec/openh264"
-	"github.com/pion/mediadevices/pkg/codec/openh264" // This is required to use VP8/VP9 video encoder
+	// or if you use a raspberry pi like, you can use mmal for using its hardware encoder
+	// "github.com/pion/mediadevices/pkg/codec/mmal"
 	"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use VP8/VP9 video encoder
+	"github.com/pion/mediadevices/pkg/codec/vpx"  // This is required to use VP8/VP9 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
 	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
 	// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
-	_ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/camera"     // This is required to register camera adapter
+	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
 )
 
 const (
@@ -42,7 +44,23 @@ func main() {
 	signal.Decode(signal.MustReadStdin(), &offer)
 
 	// Create a new RTCPeerConnection
+	vp8Params, err := vpx.NewVP8Params()
+	if err != nil {
+		panic(err)
+	}
+	vp8Params.BitRate = 300_000 // 300kbps
+
+	opusParams, err := opus.NewParams()
+	if err != nil {
+		panic(err)
+	}
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&vp8Params),
+		mediadevices.WithAudioEncoders(&opusParams),
+	)
+
 	mediaEngine := webrtc.MediaEngine{}
+	codecSelector.Populate(&mediaEngine)
 	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
 		panic(err)
 	}
@@ -58,21 +76,6 @@ func main() {
 		fmt.Printf("Connection State has changed %s \n", connectionState.String())
 	})
 
-	vp8Params, err := openh264.NewParams()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 300_000 // 300kbps
-
-	opusParams, err := opus.NewParams()
-	if err != nil {
-		panic(err)
-	}
-	codecSelector := mediadevices.NewCodecSelector(
-		mediadevices.WithVideoEncoders(&vp8Params),
-		mediadevices.WithAudioEncoders(&opusParams),
-	)
-
 	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
 			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
 	github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
 	github.com/lherman-cs/opus v0.0.2
+	github.com/pion/logging v0.2.2
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5

internal/logging/logging.go

@@ -0,0 +1,11 @@
+package logging
+
+import (
+	"github.com/pion/logging"
+)
+
+var loggerFactory = logging.NewDefaultLoggerFactory()
+
+func NewLogger(scope string) logging.LeveledLogger {
+	return loggerFactory.NewLogger(scope)
+}

logging.go

@@ -0,0 +1,7 @@
+package mediadevices
+
+import (
+	"github.com/pion/mediadevices/internal/logging"
+)
+
+var logger = logging.NewLogger("mediadevices")

mediadevices.go

@@ -120,12 +120,15 @@ func queryDriverProperties(filter driver.FilterFn) map[driver.Driver][]prop.Medi
 func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints) (driver.Driver, MediaTrackConstraints, error) {
 	var bestDriver driver.Driver
 	var bestProp prop.Media
+	var foundPropertiesLog []string
 	minFitnessDist := math.Inf(1)
 
+	foundPropertiesLog = append(foundPropertiesLog, "\n============ Found Properties ============")
 	driverProperties := queryDriverProperties(filter)
 	for d, props := range driverProperties {
 		priority := float64(d.Info().Priority)
 		for _, p := range props {
+			foundPropertiesLog = append(foundPropertiesLog, p.String())
 			fitnessDist, ok := constraints.MediaConstraints.FitnessDistance(p)
 			if !ok {
 				continue
@@ -139,26 +142,18 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 		}
 	}
 
+	foundPropertiesLog = append(foundPropertiesLog, "=============== Constraints ==============")
+	foundPropertiesLog = append(foundPropertiesLog, constraints.String())
+	foundPropertiesLog = append(foundPropertiesLog, "================ Best Fit ================")
+
 	if bestDriver == nil {
-		var foundProperties []string
-		for _, props := range driverProperties {
-			for _, p := range props {
-				foundProperties = append(foundProperties, fmt.Sprint(&p))
-			}
-		}
-
-		err := fmt.Errorf(`%w:
-============ Found Properties ============
-
-%s
-
-=============== Constraints ==============
-
-%s
-`, errNotFound, strings.Join(foundProperties, "\n\n"), &constraints)
-		return nil, MediaTrackConstraints{}, err
+		foundPropertiesLog = append(foundPropertiesLog, "Not found")
+		logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
+		return nil, MediaTrackConstraints{}, errNotFound
 	}
 
+	foundPropertiesLog = append(foundPropertiesLog, bestProp.String())
+	logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
 	constraints.selectedMedia = prop.Media{}
 	constraints.selectedMedia.MergeConstraints(constraints.MediaConstraints)
 	constraints.selectedMedia.Merge(bestProp)

mediadevices_bench_test.go

@@ -0,0 +1,80 @@
+package mediadevices
+
+import (
+	"image"
+	"sync"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec/x264"
+	"github.com/pion/mediadevices/pkg/frame"
+)
+
+type mockVideoSource struct {
+	width, height int
+	pool          sync.Pool
+	decoder       frame.Decoder
+}
+
+func newMockVideoSource(width, height int) *mockVideoSource {
+	decoder, err := frame.NewDecoder(frame.FormatYUY2)
+	if err != nil {
+		panic(err)
+	}
+
+	return &mockVideoSource{
+		width:  width,
+		height: height,
+		pool: sync.Pool{
+			New: func() interface{} {
+				resolution := width * height
+				return make([]byte, resolution*2)
+			},
+		},
+		decoder: decoder,
+	}
+}
+
+func (source *mockVideoSource) ID() string   { return "" }
+func (source *mockVideoSource) Close() error { return nil }
+func (source *mockVideoSource) Read() (image.Image, func(), error) {
+	raw := source.pool.Get().([]byte)
+	decoded, release, err := source.decoder.Decode(raw, source.width, source.height)
+	source.pool.Put(raw)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return decoded, release, nil
+}
+
+func BenchmarkEndToEnd(b *testing.B) {
+	params, err := x264.NewParams()
+	if err != nil {
+		b.Fatal(err)
+	}
+	params.BitRate = 300_000
+
+	videoSource := newMockVideoSource(1920, 1080)
+	track := NewVideoTrack(videoSource, nil).(*VideoTrack)
+	defer track.Close()
+
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	encodedReader, err := params.BuildVideoEncoder(reader, inputProp)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer encodedReader.Close()
+
+	for i := 0; i < b.N; i++ {
+		_, release, err := encodedReader.Read()
+		if err != nil {
+			b.Fatal(err)
+		}
+		release()
+	}
+}

pkg/io/audio/detect.go

@@ -32,7 +32,10 @@ func DetectChanges(interval time.Duration, onChange func(prop.Media)) TransformF
 				dirty = true
 			}
 
-			latency := time.Duration(chunk.ChunkInfo().Len) * time.Second / time.Nanosecond / time.Duration(currentProp.SampleRate)
+			var latency time.Duration
+			if currentProp.SampleRate != 0 {
+				latency = time.Duration(chunk.ChunkInfo().Len) * time.Second / time.Nanosecond / time.Duration(currentProp.SampleRate)
+			}
 			if currentProp.Latency != latency {
 				currentProp.Latency = latency
 				dirty = true

pkg/prop/prop.go

@@ -42,12 +42,19 @@ func prettifyStruct(i interface{}) string {
 			value := obj.Field(i)
 
 			padding := strings.Repeat("  ", level)
-			if value.Kind() == reflect.Struct {
+			switch value.Kind() {
+			case reflect.Struct:
 				rows = append(rows, fmt.Sprintf("%s%v:", padding, field.Name))
 				addRows(level+1, value)
+			case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+				if value.IsNil() {
+					rows = append(rows, fmt.Sprintf("%s%v: any", padding, field.Name))
 				} else {
 					rows = append(rows, fmt.Sprintf("%s%v: %v", padding, field.Name, value))
 				}
+			default:
+				rows = append(rows, fmt.Sprintf("%s%v: %v", padding, field.Name, value))
+			}
 		}
 	}
 

track.go

@@ -2,7 +2,6 @@ package mediadevices
 
 import (
 	"errors"
-	"fmt"
 	"image"
 	"math/rand"
 	"sync"
@@ -123,8 +122,6 @@ func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.Read
 	signalCh := make(chan chan<- struct{})
 	track.activePeerConnections[pc] = signalCh
 
-	fmt.Println("Binding")
-
 	go func() {
 		var doneCh chan<- struct{}
 		defer func() {
@@ -245,8 +242,6 @@ func (track *VideoTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
 	}
 
 	wantCodecs := pc.GetRegisteredRTPCodecs(webrtc.RTPCodecTypeVideo)
-	fmt.Println(wantCodecs)
-	fmt.Println(&inputProp)
 	encodedReader, selectedCodec, err := track.selector.selectVideoCodec(wantCodecs, reader, inputProp)
 	if err != nil {
 		return nil, err