Add custom video/audio transforms

Resolves https://github.com/pion/mediadevices/issues/31

examples/transform/README.md

@@ -0,0 +1,29 @@
+## Instructions
+
+### Download transform example
+
+```
+go get github.com/pion/mediadevices/examples/transform
+```
+
+### Open example page
+
+[jsfiddle.net](https://jsfiddle.net/z7ms3u5r/) you should see two text-areas and a 'Start Session' button
+
+### Run transform with your browsers SessionDescription as stdin
+
+In the jsfiddle the top textarea is your browser, copy that and:
+
+#### Linux
+
+Run `echo $BROWSER_SDP | transform`
+
+### Input transform's SessionDescription into your browser
+
+Copy the text that `transform` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-WebRTC! Now start building something cool

examples/transform/main.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"fmt"
+	"image"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/examples/internal/signal"
+	_ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder
+	_ "github.com/pion/mediadevices/pkg/codec/opus"     // This is required to register opus audio encoder
+	_ "github.com/pion/mediadevices/pkg/codec/vpx"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/webrtc/v2"
+)
+
+const (
+	videoCodecName = webrtc.VP8
+)
+
+func removeBlue(r video.Reader) video.Reader {
+	return video.ReaderFunc(func() (img image.Image, err error) {
+		img, err = r.Read()
+		if err != nil {
+			return
+		}
+
+		yuvImg, ok := img.(*image.YCbCr)
+		if !ok {
+			return img, nil
+		}
+
+		yuvImg.Cb = make([]uint8, len(yuvImg.Cb))
+		return
+	})
+}
+
+func main() {
+	config := webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{"stun:stun.l.google.com:19302"},
+			},
+		},
+	}
+
+	// Wait for the offer to be pasted
+	offer := webrtc.SessionDescription{}
+	signal.Decode(signal.MustReadStdin(), &offer)
+
+	// Create a new RTCPeerConnection
+	mediaEngine := webrtc.MediaEngine{}
+	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
+		panic(err)
+	}
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	peerConnection, err := api.NewPeerConnection(config)
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+	})
+
+	md := mediadevices.NewMediaDevices(peerConnection)
+
+	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Audio: func(c *mediadevices.MediaTrackConstraints) {
+			c.CodecName = webrtc.Opus
+			c.Enabled = true
+			c.BitRate = 32000 // 32kbps
+		},
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.CodecName = videoCodecName
+			c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420
+			c.Enabled = true
+			c.Width = 640
+			c.Height = 480
+			c.BitRate = 100000 // 100kbps
+			c.VideoTransform = removeBlue
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	for _, tracker := range s.GetTracks() {
+		t := tracker.Track()
+		tracker.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+				t.ID(), t.Label(), err)
+		})
+		_, err = peerConnection.AddTrack(t)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Tweak transceiver direction to work with Firefox
+	for _, t := range peerConnection.GetTransceivers() {
+		t.Direction = webrtc.RTPTransceiverDirectionSendonly
+	}
+
+	// Set the remote SessionDescription
+	err = peerConnection.SetRemoteDescription(offer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// Sets the LocalDescription, and starts our UDP listeners
+	err = peerConnection.SetLocalDescription(answer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output the answer in base64 so we can paste it in browser
+	fmt.Println(signal.Encode(answer))
+	select {}
+}

mediadevices.go

@@ -135,8 +135,10 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 	// Reset Codec because bestProp only contains either audio.Prop or video.Prop
 	bestProp.Codec = constraints.Codec
 	bestConstraint := MediaTrackConstraints{
-		Media:   bestProp,
+		Media:          bestProp,
-		Enabled: true,
+		Enabled:        true,
+		AudioTransform: constraints.AudioTransform,
+		VideoTransform: constraints.VideoTransform,
 	}
 	return bestDriver, bestConstraint, nil
 }

mediastreamconstraints.go

@@ -1,6 +1,8 @@
 package mediadevices
 
 import (
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -13,6 +15,12 @@ type MediaStreamConstraints struct {
 type MediaTrackConstraints struct {
 	prop.Media
 	Enabled bool
+	// VideoTransform will be used to transform the video that's coming from the driver.
+	// So, basically it'll look like following: driver -> VideoTransform -> codec
+	VideoTransform video.TransformFunc
+	// AudioTransform will be used to transform the audio that's coming from the driver.
+	// So, basically it'll look like following: driver -> AudioTransform -> code
+	AudioTransform audio.TransformFunc
 }
 
 type MediaOption func(*MediaTrackConstraints)

pkg/io/audio/audio.go

@@ -9,3 +9,22 @@ type ReaderFunc func(samples [][2]float32) (n int, err error)
 func (rf ReaderFunc) Read(samples [][2]float32) (n int, err error) {
 	return rf(samples)
 }
+
+// TransformFunc produces a new Reader that will produces a transformed audio
+type TransformFunc func(r Reader) Reader
+
+// Merge merges transforms and produces a new TransformFunc that will execute
+// transforms in order
+func Merge(transforms ...TransformFunc) TransformFunc {
+	return func(r Reader) Reader {
+		for _, transform := range transforms {
+			if transform == nil {
+				continue
+			}
+
+			r = transform(r)
+		}
+
+		return r
+	}
+}

pkg/io/video/video.go

@@ -1,6 +1,8 @@
 package video
 
-import "image"
+import (
+	"image"
+)
 
 type Reader interface {
 	Read() (img image.Image, err error)
@@ -11,3 +13,22 @@ type ReaderFunc func() (img image.Image, err error)
 func (rf ReaderFunc) Read() (img image.Image, err error) {
 	return rf()
 }
+
+// TransformFunc produces a new Reader that will produces a transformed video
+type TransformFunc func(r Reader) Reader
+
+// Merge merges transforms and produces a new TransformFunc that will execute
+// transforms in order
+func Merge(transforms ...TransformFunc) TransformFunc {
+	return func(r Reader) Reader {
+		for _, transform := range transforms {
+			if transform == nil {
+				continue
+			}
+
+			r = transform(r)
+		}
+
+		return r
+	}
+}

track.go

@@ -99,6 +99,10 @@ func newVideoTrack(codecs []*webrtc.RTPCodec, d driver.Driver, constraints Media
 		return nil, err
 	}
 
+	if constraints.VideoTransform != nil {
+		r = constraints.VideoTransform(r)
+	}
+
 	encoder, err := codec.BuildVideoEncoder(r, constraints.Media)
 	if err != nil {
 		return nil, err
@@ -170,6 +174,10 @@ func newAudioTrack(codecs []*webrtc.RTPCodec, d driver.Driver, constraints Media
 		return nil, err
 	}
 
+	if constraints.AudioTransform != nil {
+		reader = constraints.AudioTransform(reader)
+	}
+
 	encoder, err := codec.BuildAudioEncoder(reader, constraints.Media)
 	if err != nil {
 		return nil, err