Update golang.org/x/image commit hash to e162460

Generated by Renovate Bot

.github/workflows/ci.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.14', '1.13' ]
+        go: [ '1.15', '1.14' ]
     name: Linux Go ${{ matrix.go }}
     steps:
       - name: Checkout
@@ -30,15 +30,15 @@ jobs:
             libvpx-dev \
             libx264-dev
       - name: go vet
-        run: go vet ./...
+        run: go vet -tags nolibopusfile ./...
       - name: go build
-        run: go build ./...
+        run: go build -tags nolibopusfile ./...
       - name: go build without CGO
         run: go build . pkg/...
         env:
           CGO_ENABLED: 0
       - name: go test
-        run: go test ./... -v -race
+        run: go test -tags nolibopusfile ./... -v -race
       - name: go test without CGO
         run: go test . pkg/... -v
         env:
@@ -47,7 +47,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        go: [ '1.14', '1.13' ]
+        go: [ '1.15', '1.14' ]
     name: Darwin Go ${{ matrix.go }}
     steps:
       - name: Checkout

examples/facedetection/README.md

@@ -0,0 +1,29 @@
+## Instructions
+
+### Download facedetection
+
+```
+go get github.com/pion/mediadevices/examples/facedetection
+```
+
+### Open example page
+
+[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/video) you should see two text-areas and a 'Start Session' button
+
+### Run facedetection with your browsers SessionDescription as stdin
+
+In the jsfiddle the top textarea is your browser, copy that and:
+
+#### Linux
+
+Run `echo $BROWSER_SDP | facedetection`
+
+### Input facedetection's SessionDescription into your browser
+
+Copy the text that `facedetection` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-WebRTC! Now start building something cool

examples/facedetection/detector.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"image"
+	"image/color"
+	"image/draw"
+	"io/ioutil"
+	"log"
+
+	"github.com/disintegration/imaging"
+	pigo "github.com/esimov/pigo/core"
+)
+
+var (
+	cascade    []byte
+	err        error
+	classifier *pigo.Pigo
+)
+
+func imgToGrayscale(img image.Image) []uint8 {
+	bounds := img.Bounds()
+	flatten := bounds.Dy() * bounds.Dx()
+	grayImg := make([]uint8, flatten)
+
+	i := 0
+	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+		for x := bounds.Min.X; x < bounds.Max.X; x++ {
+			pix := img.At(x, y)
+			grayPix := color.GrayModel.Convert(pix).(color.Gray)
+			grayImg[i] = grayPix.Y
+			i++
+		}
+	}
+	return grayImg
+}
+
+// clusterDetection runs Pigo face detector core methods
+// and returns a cluster with the detected faces coordinates.
+func clusterDetection(img image.Image) []pigo.Detection {
+	grayscale := imgToGrayscale(img)
+	bounds := img.Bounds()
+	cParams := pigo.CascadeParams{
+		MinSize:     100,
+		MaxSize:     600,
+		ShiftFactor: 0.15,
+		ScaleFactor: 1.1,
+		ImageParams: pigo.ImageParams{
+			Pixels: grayscale,
+			Rows:   bounds.Dy(),
+			Cols:   bounds.Dx(),
+			Dim:    bounds.Dx(),
+		},
+	}
+
+	if len(cascade) == 0 {
+		cascade, err = ioutil.ReadFile("facefinder")
+		if err != nil {
+			log.Fatalf("Error reading the cascade file: %s", err)
+		}
+		p := pigo.NewPigo()
+
+		// Unpack the binary file. This will return the number of cascade trees,
+		// the tree depth, the threshold and the prediction from tree's leaf nodes.
+		classifier, err = p.Unpack(cascade)
+		if err != nil {
+			log.Fatalf("Error unpacking the cascade file: %s", err)
+		}
+	}
+
+	// Run the classifier over the obtained leaf nodes and return the detection results.
+	// The result contains quadruplets representing the row, column, scale and detection score.
+	dets := classifier.RunCascade(cParams, 0.0)
+
+	// Calculate the intersection over union (IoU) of two clusters.
+	dets = classifier.ClusterDetections(dets, 0)
+
+	return dets
+}
+
+func drawCircle(img draw.Image, x0, y0, r int, c color.Color) {
+	x, y, dx, dy := r-1, 0, 1, 1
+	err := dx - (r * 2)
+
+	for x > y {
+		img.Set(x0+x, y0+y, c)
+		img.Set(x0+y, y0+x, c)
+		img.Set(x0-y, y0+x, c)
+		img.Set(x0-x, y0+y, c)
+		img.Set(x0-x, y0-y, c)
+		img.Set(x0-y, y0-x, c)
+		img.Set(x0+y, y0-x, c)
+		img.Set(x0+x, y0-y, c)
+
+		if err <= 0 {
+			y++
+			err += dy
+			dy += 2
+		}
+		if err > 0 {
+			x--
+			dx += 2
+			err += dx - (r * 2)
+		}
+	}
+}
+
+func markFaces(img image.Image) image.Image {
+	nrgba := imaging.Clone(img)
+	dets := clusterDetection(img)
+	for _, det := range dets {
+		if det.Q < 5.0 {
+			continue
+		}
+
+		drawCircle(nrgba, det.Col, det.Row, det.Scale/2, color.Black)
+	}
+	return nrgba
+}

examples/facedetection/main.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"fmt"
+	"image"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/examples/internal/signal"
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/webrtc/v2"
+)
+
+func markFacesTransformer(r video.Reader) video.Reader {
+	return video.ReaderFunc(func() (img image.Image, err error) {
+		img, err = r.Read()
+		if err != nil {
+			return
+		}
+
+		img = markFaces(img)
+		return
+	})
+}
+
+func main() {
+	config := webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{"stun:stun.l.google.com:19302"},
+			},
+		},
+	}
+
+	// Wait for the offer to be pasted
+	offer := webrtc.SessionDescription{}
+	signal.Decode(signal.MustReadStdin(), &offer)
+
+	// Create a new RTCPeerConnection
+	mediaEngine := webrtc.MediaEngine{}
+	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
+		panic(err)
+	}
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	peerConnection, err := api.NewPeerConnection(config)
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+	})
+
+	md := mediadevices.NewMediaDevices(peerConnection)
+
+	vp8Params, err := vpx.NewVP8Params()
+	if err != nil {
+		panic(err)
+	}
+	vp8Params.BitRate = 100000 // 100kbps
+
+	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormatExact(frame.FormatI420) // most of the encoder accepts I420
+			c.Enabled = true
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+			c.VideoTransform = markFacesTransformer
+			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	for _, tracker := range s.GetTracks() {
+		t := tracker.Track()
+		tracker.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+				t.ID(), t.Label(), err)
+		})
+		_, err = peerConnection.AddTransceiverFromTrack(t,
+			webrtc.RtpTransceiverInit{
+				Direction: webrtc.RTPTransceiverDirectionSendonly,
+			},
+		)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Set the remote SessionDescription
+	err = peerConnection.SetRemoteDescription(offer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// Sets the LocalDescription, and starts our UDP listeners
+	err = peerConnection.SetLocalDescription(answer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output the answer in base64 so we can paste it in browser
+	fmt.Println(signal.Encode(answer))
+	select {}
+}

examples/rtp-send/README.md

@@ -0,0 +1,29 @@
+## Instructions
+
+### Download rtp-send example
+
+```
+go get github.com/pion/mediadevices/examples/rtp-send
+```
+
+### Listen RTP
+
+Install GStreamer and run:
+```
+gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=VP8 \
+    ! rtpvp8depay ! vp8dec ! videoconvert ! autovideosink
+```
+
+Or run VLC media plyer:
+```
+vlc ./vp8.sdp
+```
+
+### Run rtp-send
+
+Run `rtp-send localhost:5000`
+
+A video should start playing in your GStreamer or VLC window.
+It's not WebRTC, but pure RTP.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/rtp-send/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v2/pkg/media"
+)
+
+const (
+	mtu = 1000
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+
+	md := mediadevices.NewMediaDevicesFromCodecs(
+		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
+			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
+				webrtc.NewRTPVP8Codec(100, 90000),
+			},
+		},
+		mediadevices.WithTrackGenerator(
+			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
+				mediadevices.LocalTrack, error,
+			) {
+				return newTrack(codec, id, os.Args[1]), nil
+			},
+		),
+	)
+
+	vp8Params, err := vpx.NewVP8Params()
+	if err != nil {
+		panic(err)
+	}
+	vp8Params.BitRate = 100000 // 100kbps
+
+	_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Enabled = true
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	select {}
+}
+
+type track struct {
+	codec      *webrtc.RTPCodec
+	packetizer rtp.Packetizer
+	id         string
+	conn       net.Conn
+}
+
+func newTrack(codec *webrtc.RTPCodec, id, dest string) *track {
+	addr, err := net.ResolveUDPAddr("udp", dest)
+	if err != nil {
+		panic(err)
+	}
+	conn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		panic(err)
+	}
+	return &track{
+		codec: codec,
+		packetizer: rtp.NewPacketizer(
+			mtu,
+			codec.PayloadType,
+			1,
+			codec.Payloader,
+			rtp.NewRandomSequencer(),
+			codec.ClockRate,
+		),
+		id:   id,
+		conn: conn,
+	}
+}
+
+func (t *track) WriteSample(s media.Sample) error {
+	buf := make([]byte, mtu)
+	pkts := t.packetizer.Packetize(s.Data, s.Samples)
+	for _, p := range pkts {
+		n, err := p.MarshalTo(buf)
+		if err != nil {
+			panic(err)
+		}
+		_, _ = t.conn.Write(buf[:n])
+	}
+	return nil
+}
+
+func (t *track) Codec() *webrtc.RTPCodec {
+	return t.codec
+}
+
+func (t *track) ID() string {
+	return t.id
+}
+
+func (t *track) Kind() webrtc.RTPCodecType {
+	return t.codec.Type
+}

examples/rtp-send/vp8.sdp

@@ -0,0 +1,9 @@
+v=0
+o=- 1234567890 1234567890 IN IP4 0.0.0.0
+s=RTP-Send Example
+i=Example
+c=IN IP4 0.0.0.0
+t=0 0
+a=recvonly
+m=video 5000 RTP/AVP 100
+a=rtpmap:100 VP8/90000

examples/screenshare/README.md

@@ -0,0 +1,29 @@
+## Instructions
+
+### Download screenshare
+
+```
+go get github.com/pion/mediadevices/examples/screenshare
+```
+
+### Open example page
+
+[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
+
+### Run screenshare with your browsers SessionDescription as stdin
+
+In the jsfiddle the top textarea is your browser, copy that and:
+
+#### Linux
+
+Run `echo $BROWSER_SDP | screenshare`
+
+### Input screenshare's SessionDescription into your browser
+
+Copy the text that `screenshare` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-WebRTC! Now start building something cool

examples/screenshare/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/examples/internal/signal"
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/webrtc/v2"
+)
+
+func main() {
+	config := webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{"stun:stun.l.google.com:19302"},
+			},
+		},
+	}
+
+	// Wait for the offer to be pasted
+	offer := webrtc.SessionDescription{}
+	signal.Decode(signal.MustReadStdin(), &offer)
+
+	// Create a new RTCPeerConnection
+	mediaEngine := webrtc.MediaEngine{}
+	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
+		panic(err)
+	}
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	peerConnection, err := api.NewPeerConnection(config)
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+	})
+
+	md := mediadevices.NewMediaDevices(peerConnection)
+
+	vp8Params, err := vpx.NewVP8Params()
+	if err != nil {
+		panic(err)
+	}
+	vp8Params.BitRate = 100000 // 100kbps
+
+	s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.Enabled = true
+			c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
+			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	for _, tracker := range s.GetTracks() {
+		t := tracker.Track()
+		tracker.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+				t.ID(), t.Label(), err)
+		})
+		_, err = peerConnection.AddTransceiverFromTrack(t,
+			webrtc.RtpTransceiverInit{
+				Direction: webrtc.RTPTransceiverDirectionSendonly,
+			},
+		)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Set the remote SessionDescription
+	err = peerConnection.SetRemoteDescription(offer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// Sets the LocalDescription, and starts our UDP listeners
+	err = peerConnection.SetLocalDescription(answer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output the answer in base64 so we can paste it in browser
+	fmt.Println(signal.Encode(answer))
+	select {}
+}

examples/simple/README.md

@@ -1,17 +1,29 @@
 ## Instructions
 
-### Download the example
+### Download gstreamer-send
 
 ```
 go get github.com/pion/mediadevices/examples/simple
 ```
 
-### Run the sample
+### Open example page
 
-```
-simple
-```
+[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
 
-### View yourself in the browser
+### Run simple with your browsers SessionDescription as stdin
 
-Open your browser and go to "http://localhost:1313"
+In the jsfiddle the top textarea is your browser, copy that and:
+
+#### Linux
+
+Run `echo $BROWSER_SDP | simple`
+
+### Input simple's SessionDescription into your browser
+
+Copy the text that `simple` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-WebRTC! Now start building something cool

examples/simple/main.go

@@ -2,68 +2,131 @@ package main
 
 import (
 	"fmt"
-	"image/jpeg"
-	"io"
-	"log"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
 
 	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/examples/internal/signal"
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/webrtc/v2"
+
+	// This is required to use opus audio encoder
+	"github.com/pion/mediadevices/pkg/codec/opus"
+
+	// If you don't like vpx, you can also use x264 by importing as below
+	// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
+	// or you can also use openh264 for alternative h264 implementation
+	// "github.com/pion/mediadevices/pkg/codec/openh264"
+	"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
 	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
+	_ "github.com/pion/mediadevices/pkg/driver/camera"     // This is required to register camera adapter
+	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
+)
+
+const (
+	videoCodecName = webrtc.VP8
 )
 
 func main() {
-	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(constraint *mediadevices.MediaTrackConstraints) {
-			constraint.Width = prop.Int(600)
-			constraint.Height = prop.Int(400)
+	config := webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{"stun:stun.l.google.com:19302"},
+			},
+		},
+	}
+
+	// Wait for the offer to be pasted
+	offer := webrtc.SessionDescription{}
+	signal.Decode(signal.MustReadStdin(), &offer)
+
+	// Create a new RTCPeerConnection
+	mediaEngine := webrtc.MediaEngine{}
+	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
+		panic(err)
+	}
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	peerConnection, err := api.NewPeerConnection(config)
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+	})
+
+	md := mediadevices.NewMediaDevices(peerConnection)
+
+	opusParams, err := opus.NewParams()
+	if err != nil {
+		panic(err)
+	}
+	opusParams.BitRate = 32000 // 32kbps
+
+	vp8Params, err := vpx.NewVP8Params()
+	if err != nil {
+		panic(err)
+	}
+	vp8Params.BitRate = 100000 // 100kbps
+
+	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Audio: func(c *mediadevices.MediaTrackConstraints) {
+			c.Enabled = true
+			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
+		},
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Enabled = true
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
 		},
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	t := s.GetVideoTracks()[0]
-	defer t.Stop()
-	videoTrack := t.(*mediadevices.VideoTrack)
-
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		videoReader := videoTrack.NewReader()
-		mimeWriter := multipart.NewWriter(w)
-
-		contentType := fmt.Sprintf("multipart/x-mixed-replace;boundary=%s", mimeWriter.Boundary())
-		w.Header().Add("Content-Type", contentType)
-
-		partHeader := make(textproto.MIMEHeader)
-		partHeader.Add("Content-Type", "image/jpeg")
-
-		for {
-			frame, err := videoReader.Read()
-			if err != nil {
-				if err == io.EOF {
-					return
-				}
-				panic(err)
-			}
-
-			partWriter, err := mimeWriter.CreatePart(partHeader)
-			if err != nil {
-				panic(err)
-			}
-
-			err = jpeg.Encode(partWriter, frame, nil)
-			if err != nil {
-				panic(err)
-			}
+	for _, tracker := range s.GetTracks() {
+		t := tracker.Track()
+		tracker.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+				t.ID(), t.Label(), err)
+		})
+		_, err = peerConnection.AddTransceiverFromTrack(t,
+			webrtc.RtpTransceiverInit{
+				Direction: webrtc.RTPTransceiverDirectionSendonly,
+			},
+		)
+		if err != nil {
+			panic(err)
 		}
-	})
+	}
 
-	fmt.Println("listening on http://localhost:1313")
-	log.Println(http.ListenAndServe("localhost:1313", nil))
+	// Set the remote SessionDescription
+	err = peerConnection.SetRemoteDescription(offer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// Sets the LocalDescription, and starts our UDP listeners
+	err = peerConnection.SetLocalDescription(answer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output the answer in base64 so we can paste it in browser
+	fmt.Println(signal.Encode(answer))
+	select {}
 }

go.mod

@@ -4,10 +4,10 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214
-	github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4
-	github.com/pion/webrtc/v2 v2.2.23
+	github.com/jfreymuth/pulse v0.0.0-20200817093420-a82ccdb5e8aa
+	github.com/lherman-cs/opus v0.0.0-20200925065115-26ea9d322d39
+	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
-	golang.org/x/image v0.0.0-20200801110659-972c09e46d76
-	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
+	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
+	golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a
 )

go.sum

@@ -15,15 +15,15 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214 h1:2xVJKIumEUWeV3vczQwn61SHjNZ94Bwk+4CTjmcePxk=
-github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
+github.com/jfreymuth/pulse v0.0.0-20200817093420-a82ccdb5e8aa h1:qUZIj5+D3UDgfshNe8Cz/9maOxe8ddt43qwQH9vEEC8=
+github.com/jfreymuth/pulse v0.0.0-20200817093420-a82ccdb5e8aa/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4 h1:2ydMA2KbxRkYmIw3R8Me8dn90bejxBR4MKYXJ5THK3I=
-github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
+github.com/lherman-cs/opus v0.0.0-20200925065115-26ea9d322d39 h1:WEYmSwg/uoPVmfmpXWPYplb1UUx/Jr4TXGNrPaI8Cj4=
+github.com/lherman-cs/opus v0.0.0-20200925065115-26ea9d322d39/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
 github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
 github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
 github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
@@ -33,8 +33,8 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pion/datachannel v1.4.19 h1:IcOmm5fdDzJVCMgFYDCMtFC+lrjG78KcMYXH+gOo6ys=
-github.com/pion/datachannel v1.4.19/go.mod h1:JzKF/zzeWgkOYwQ+KFb8JzbrUt8s63um+Qunu8VqTyw=
+github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
+github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
 github.com/pion/dtls/v2 v2.0.1 h1:ddE7+V0faYRbyh4uPsRZ2vLdRrjVZn+wmCfI7jlBfaA=
 github.com/pion/dtls/v2 v2.0.1/go.mod h1:uMQkz2W0cSqY00xav7WByQ4Hb+18xeQh2oH2fRezr5U=
 github.com/pion/dtls/v2 v2.0.2 h1:FHCHTiM182Y8e15aFTiORroiATUI16ryHiQh8AIOJ1E=
@@ -54,8 +54,8 @@ github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
 github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
 github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
 github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
-github.com/pion/sctp v1.7.8 h1:tEWel2BKXLZitU+LxY3GDeQXoKeTafYasiu/X+XBKNM=
-github.com/pion/sctp v1.7.8/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
+github.com/pion/sctp v1.7.10 h1:o3p3/hZB5Cx12RMGyWmItevJtZ6o2cpuxaw6GOS4x+8=
+github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=
 github.com/pion/sdp/v2 v2.4.0/go.mod h1:L2LxrOpSTJbAns244vfPChbciR/ReU1KWfG04OpkR7E=
 github.com/pion/srtp v1.5.1 h1:9Q3jAfslYZBt+C69SI/ZcONJh9049JUHZWYRRf5KEKw=
@@ -73,8 +73,8 @@ github.com/pion/turn/v2 v2.0.4 h1:oDguhEv2L/4rxwbL9clGLgtzQPjtuZwCdoM7Te8vQVk=
 github.com/pion/turn/v2 v2.0.4/go.mod h1:1812p4DcGVbYVBTiraUmP50XoKye++AMkbfp+N27mog=
 github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
 github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
-github.com/pion/webrtc/v2 v2.2.23 h1:rZdOC95fwUCoQFVjHooPAayx/vhs3SLHFz8J/iRkAuk=
-github.com/pion/webrtc/v2 v2.2.23/go.mod h1:1lN/3EcATkQxc7GJSQbISCGC2l64Xu2VSLpwEG3c/tM=
+github.com/pion/webrtc/v2 v2.2.26 h1:01hWE26pL3LgqfxvQ1fr6O4ZtyRFFJmQEZK39pHWfFc=
+github.com/pion/webrtc/v2 v2.2.26/go.mod h1:XMZbZRNHyPDe1gzTIHFcQu02283YO45CbiwFgKvXnmc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -98,8 +98,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/image v0.0.0-20200801110659-972c09e46d76 h1:U7GPaoQyQmX+CBRWXKrvRzWTbd+slqeSh8uARsIyhAw=
-golang.org/x/image v0.0.0-20200801110659-972c09e46d76/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200927104501-e162460cd6b5 h1:QelT11PB4FXiDEXucrfNckHoFxwt8USGY1ajP1ZF5lM=
+golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -119,8 +119,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c=
 golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 h1:sIky/MyNRSHTrdxfsiUSS4WIAMvInbeXljJz+jDjeYE=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

mediadevices.go

@@ -3,40 +3,110 @@ package mediadevices
 import (
 	"fmt"
 	"math"
+	"strings"
 
 	"github.com/pion/mediadevices/pkg/driver"
 	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/webrtc/v2"
 )
 
 var errNotFound = fmt.Errorf("failed to find the best driver that fits the constraints")
 
+// MediaDevices is an interface that's defined on https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices
+type MediaDevices interface {
+	GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error)
+	GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error)
+	EnumerateDevices() []MediaDeviceInfo
+}
+
+// NewMediaDevices creates MediaDevices interface that provides access to connected media input devices
+// like cameras and microphones, as well as screen sharing.
+// In essence, it lets you obtain access to any hardware source of media data.
+func NewMediaDevices(pc *webrtc.PeerConnection, opts ...MediaDevicesOption) MediaDevices {
+	codecs := make(map[webrtc.RTPCodecType][]*webrtc.RTPCodec)
+	for _, kind := range []webrtc.RTPCodecType{
+		webrtc.RTPCodecTypeAudio,
+		webrtc.RTPCodecTypeVideo,
+	} {
+		codecs[kind] = pc.GetRegisteredRTPCodecs(kind)
+	}
+	return NewMediaDevicesFromCodecs(codecs, opts...)
+}
+
+// NewMediaDevicesFromCodecs creates MediaDevices interface from lists of the available codecs
+// that provides access to connected media input devices like cameras and microphones,
+// as well as screen sharing.
+// In essence, it lets you obtain access to any hardware source of media data.
+func NewMediaDevicesFromCodecs(codecs map[webrtc.RTPCodecType][]*webrtc.RTPCodec, opts ...MediaDevicesOption) MediaDevices {
+	mdo := MediaDevicesOptions{
+		codecs:         codecs,
+		trackGenerator: defaultTrackGenerator,
+	}
+	for _, o := range opts {
+		o(&mdo)
+	}
+	return &mediaDevices{
+		MediaDevicesOptions: mdo,
+	}
+}
+
+// TrackGenerator is a function to create new track.
+type TrackGenerator func(payloadType uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error)
+
+var defaultTrackGenerator = TrackGenerator(func(pt uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error) {
+	return webrtc.NewTrack(pt, ssrc, id, label, codec)
+})
+
+type mediaDevices struct {
+	MediaDevicesOptions
+}
+
+// MediaDevicesOptions stores parameters used by MediaDevices.
+type MediaDevicesOptions struct {
+	codecs         map[webrtc.RTPCodecType][]*webrtc.RTPCodec
+	trackGenerator TrackGenerator
+}
+
+// MediaDevicesOption is a type of MediaDevices functional option.
+type MediaDevicesOption func(*MediaDevicesOptions)
+
+// WithTrackGenerator specifies a TrackGenerator to use customized track.
+func WithTrackGenerator(gen TrackGenerator) MediaDevicesOption {
+	return func(o *MediaDevicesOptions) {
+		o.trackGenerator = gen
+	}
+}
+
 // GetDisplayMedia prompts the user to select and grant permission to capture the contents
 // of a display or portion thereof (such as a window) as a MediaStream.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia
-func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	tracks := make([]Track, 0)
+func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+	trackers := make([]Tracker, 0)
 
-	cleanTracks := func() {
-		for _, t := range tracks {
+	cleanTrackers := func() {
+		for _, t := range trackers {
 			t.Stop()
 		}
 	}
 
+	var videoConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		var p MediaTrackConstraints
-		constraints.Video(&p)
-		track, err := selectScreen(p)
+		constraints.Video(&videoConstraints)
+	}
+
+	if videoConstraints.Enabled {
+		tracker, err := m.selectScreen(videoConstraints)
 		if err != nil {
-			cleanTracks()
+			cleanTrackers()
 			return nil, err
 		}
 
-		tracks = append(tracks, track)
+		trackers = append(trackers, tracker)
 	}
 
-	s, err := NewMediaStream(tracks...)
+	s, err := NewMediaStream(trackers...)
 	if err != nil {
-		cleanTracks()
+		cleanTrackers()
 		return nil, err
 	}
 
@@ -46,42 +116,48 @@ func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
 // GetUserMedia prompts the user for permission to use a media input which produces a MediaStream
 // with tracks containing the requested types of media.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
-func GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	tracks := make([]Track, 0)
+func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+	// TODO: It should return media stream based on constraints
+	trackers := make([]Tracker, 0)
 
-	cleanTracks := func() {
-		for _, t := range tracks {
+	cleanTrackers := func() {
+		for _, t := range trackers {
 			t.Stop()
 		}
 	}
 
+	var videoConstraints, audioConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		var p MediaTrackConstraints
-		constraints.Video(&p)
-		track, err := selectVideo(p)
-		if err != nil {
-			cleanTracks()
-			return nil, err
-		}
-
-		tracks = append(tracks, track)
+		constraints.Video(&videoConstraints)
 	}
 
 	if constraints.Audio != nil {
-		var p MediaTrackConstraints
-		constraints.Audio(&p)
-		track, err := selectAudio(p)
+		constraints.Audio(&audioConstraints)
+	}
+
+	if videoConstraints.Enabled {
+		tracker, err := m.selectVideo(videoConstraints)
 		if err != nil {
-			cleanTracks()
+			cleanTrackers()
 			return nil, err
 		}
 
-		tracks = append(tracks, track)
+		trackers = append(trackers, tracker)
 	}
 
-	s, err := NewMediaStream(tracks...)
+	if audioConstraints.Enabled {
+		tracker, err := m.selectAudio(audioConstraints)
+		if err != nil {
+			cleanTrackers()
+			return nil, err
+		}
+
+		trackers = append(trackers, tracker)
+	}
+
+	s, err := NewMediaStream(trackers...)
 	if err != nil {
-		cleanTracks()
+		cleanTrackers()
 		return nil, err
 	}
 
@@ -139,7 +215,23 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 	}
 
 	if bestDriver == nil {
-		return nil, MediaTrackConstraints{}, errNotFound
+		var foundProperties []string
+		for _, props := range driverProperties {
+			for _, p := range props {
+				foundProperties = append(foundProperties, fmt.Sprint(&p))
+			}
+		}
+
+		err := fmt.Errorf(`%w:
+============ Found Properties ============
+
+%s
+
+=============== Constraints ==============
+
+%s
+`, errNotFound, strings.Join(foundProperties, "\n\n"), &constraints)
+		return nil, MediaTrackConstraints{}, err
 	}
 
 	constraints.selectedMedia = prop.Media{}
@@ -148,7 +240,7 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 	return bestDriver, constraints, nil
 }
 
-func selectAudio(constraints MediaTrackConstraints) (Track, error) {
+func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker, error) {
 	typeFilter := driver.FilterAudioRecorder()
 
 	d, c, err := selectBestDriver(typeFilter, constraints)
@@ -156,10 +248,9 @@ func selectAudio(constraints MediaTrackConstraints) (Track, error) {
 		return nil, err
 	}
 
-	return newAudioTrack(d, c)
+	return newTrack(&m.MediaDevicesOptions, d, c)
 }
-
-func selectVideo(constraints MediaTrackConstraints) (Track, error) {
+func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	notScreenFilter := driver.FilterNot(driver.FilterDeviceType(driver.Screen))
 	filter := driver.FilterAnd(typeFilter, notScreenFilter)
@@ -169,10 +260,10 @@ func selectVideo(constraints MediaTrackConstraints) (Track, error) {
 		return nil, err
 	}
 
-	return newVideoTrack(d, c)
+	return newTrack(&m.MediaDevicesOptions, d, c)
 }
 
-func selectScreen(constraints MediaTrackConstraints) (Track, error) {
+func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	screenFilter := driver.FilterDeviceType(driver.Screen)
 	filter := driver.FilterAnd(typeFilter, screenFilter)
@@ -182,10 +273,10 @@ func selectScreen(constraints MediaTrackConstraints) (Track, error) {
 		return nil, err
 	}
 
-	return newVideoTrack(d, c)
+	return newTrack(&m.MediaDevicesOptions, d, c)
 }
 
-func EnumerateDevices() []MediaDeviceInfo {
+func (m *mediaDevices) EnumerateDevices() []MediaDeviceInfo {
 	drivers := driver.GetManager().Query(
 		driver.FilterFn(func(driver.Driver) bool { return true }))
 	info := make([]MediaDeviceInfo, 0, len(drivers))

mediadevices_test.go

@@ -19,25 +19,18 @@ import (
 )
 
 func TestGetUserMedia(t *testing.T) {
-	brokenVideoParams := mockParams{
+	videoParams := mockParams{
+		BaseParams: codec.BaseParams{
+			BitRate: 100000,
+		},
 		name: "MockVideo",
 	}
-	videoParams := brokenVideoParams
-	videoParams.BitRate = 100000
 	audioParams := mockParams{
 		BaseParams: codec.BaseParams{
 			BitRate: 32000,
 		},
 		name: "MockAudio",
 	}
-	constraints := MediaStreamConstraints{
-		Video: func(p *prop.Media) {
-			p.Width = 640
-			p.Height = 480
-		},
-		Audio: func(p *prop.Media) {},
-	}
-
 	md := NewMediaDevicesFromCodecs(
 		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
 			webrtc.RTPCodecTypeVideo: {
@@ -54,10 +47,7 @@ func TestGetUserMedia(t *testing.T) {
 				return newMockTrack(codec, id), nil
 			},
 		),
-		WithVideoEncoders(&brokenVideoParams),
-		WithAudioEncoders(&audioParams),
 	)
-<<<<<<< HEAD
 	constraints := MediaStreamConstraints{
 		Video: func(c *MediaTrackConstraints) {
 			c.Enabled = true
@@ -87,35 +77,13 @@ func TestGetUserMedia(t *testing.T) {
 			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
 		},
 	}
-=======
->>>>>>> ccd7985... Redesign GetUserMedia API
 
 	// GetUserMedia with broken parameters
-	ms, err := md.GetUserMedia(constraints)
+	ms, err := md.GetUserMedia(constraintsWrong)
 	if err == nil {
 		t.Fatal("Expected error, but got nil")
 	}
 
-	md = NewMediaDevicesFromCodecs(
-		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
-			},
-			webrtc.RTPCodecTypeAudio: []*webrtc.RTPCodec{
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
-			},
-		},
-		WithTrackGenerator(
-			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
-				LocalTrack, error,
-			) {
-				return newMockTrack(codec, id), nil
-			},
-		),
-		WithVideoEncoders(&videoParams),
-		WithAudioEncoders(&audioParams),
-	)
-
 	// GetUserMedia with correct parameters
 	ms, err = md.GetUserMedia(constraints)
 	if err != nil {

mediastream.go

@@ -9,82 +9,82 @@ import (
 // MediaStream is an interface that represents a collection of existing tracks.
 type MediaStream interface {
 	// GetAudioTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
-	GetAudioTracks() []Track
+	GetAudioTracks() []Tracker
 	// GetVideoTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
-	GetVideoTracks() []Track
+	GetVideoTracks() []Tracker
 	// GetTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
-	GetTracks() []Track
+	GetTracks() []Tracker
 	// AddTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
-	AddTrack(t Track)
+	AddTrack(t Tracker)
 	// RemoveTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
-	RemoveTrack(t Track)
+	RemoveTrack(t Tracker)
 }
 
 type mediaStream struct {
-	tracks map[string]Track
-	l      sync.RWMutex
+	trackers map[string]Tracker
+	l        sync.RWMutex
 }
 
 const rtpCodecTypeDefault webrtc.RTPCodecType = 0
 
 // NewMediaStream creates a MediaStream interface that's defined in
 // https://w3c.github.io/mediacapture-main/#dom-mediastream
-func NewMediaStream(tracks ...Track) (MediaStream, error) {
-	m := mediaStream{tracks: make(map[string]Track)}
+func NewMediaStream(trackers ...Tracker) (MediaStream, error) {
+	m := mediaStream{trackers: make(map[string]Tracker)}
 
-	for _, track := range tracks {
-		id := track.ID()
-		if _, ok := m.tracks[id]; !ok {
-			m.tracks[id] = track
+	for _, tracker := range trackers {
+		id := tracker.LocalTrack().ID()
+		if _, ok := m.trackers[id]; !ok {
+			m.trackers[id] = tracker
 		}
 	}
 
 	return &m, nil
 }
 
-func (m *mediaStream) GetAudioTracks() []Track {
-	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindAudio })
+func (m *mediaStream) GetAudioTracks() []Tracker {
+	return m.queryTracks(webrtc.RTPCodecTypeAudio)
 }
 
-func (m *mediaStream) GetVideoTracks() []Track {
-	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindVideo })
+func (m *mediaStream) GetVideoTracks() []Tracker {
+	return m.queryTracks(webrtc.RTPCodecTypeVideo)
 }
 
-func (m *mediaStream) GetTracks() []Track {
-	return m.queryTracks(func(t Track) bool { return true })
+func (m *mediaStream) GetTracks() []Tracker {
+	return m.queryTracks(rtpCodecTypeDefault)
 }
 
 // queryTracks returns all tracks that are the same kind as t.
 // If t is 0, which is the default, queryTracks will return all the tracks.
-func (m *mediaStream) queryTracks(filter func(track Track) bool) []Track {
+func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Tracker {
 	m.l.RLock()
 	defer m.l.RUnlock()
 
-	result := make([]Track, 0)
-	for _, track := range m.tracks {
-		if filter(track) {
-			result = append(result, track)
+	result := make([]Tracker, 0)
+	for _, tracker := range m.trackers {
+		if tracker.LocalTrack().Kind() == t || t == rtpCodecTypeDefault {
+			result = append(result, tracker)
 		}
 	}
 
 	return result
 }
 
-func (m *mediaStream) AddTrack(t Track) {
+func (m *mediaStream) AddTrack(t Tracker) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	id := t.ID()
-	if _, ok := m.tracks[id]; ok {
+	id := t.LocalTrack().ID()
+	if _, ok := m.trackers[id]; ok {
 		return
 	}
 
-	m.tracks[id] = t
+	m.trackers[id] = t
 }
 
-func (m *mediaStream) RemoveTrack(t Track) {
+func (m *mediaStream) RemoveTrack(t Tracker) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	delete(m.tracks, t.ID())
+	delete(m.trackers, t.LocalTrack().ID())
 }

mediastreamconstraints.go

@@ -1,6 +1,9 @@
 package mediadevices
 
 import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -12,6 +15,26 @@ type MediaStreamConstraints struct {
 // MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
 type MediaTrackConstraints struct {
 	prop.MediaConstraints
+	Enabled bool
+	// VideoEncoderBuilders are codec builders that are used for encoding the video
+	// and later being used for sending the appropriate RTP payload type.
+	//
+	// If one encoder builder fails to build the codec, the next builder will be used,
+	// repeating until a codec builds. If no builders build successfully, an error is returned.
+	VideoEncoderBuilders []codec.VideoEncoderBuilder
+	// AudioEncoderBuilders are codec builders that are used for encoding the audio
+	// and later being used for sending the appropriate RTP payload type.
+	//
+	// If one encoder builder fails to build the codec, the next builder will be used,
+	// repeating until a codec builds. If no builders build successfully, an error is returned.
+	AudioEncoderBuilders []codec.AudioEncoderBuilder
+	// VideoTransform will be used to transform the video that's coming from the driver.
+	// So, basically it'll look like following: driver -> VideoTransform -> codec
+	VideoTransform video.TransformFunc
+	// AudioTransform will be used to transform the audio that's coming from the driver.
+	// So, basically it'll look like following: driver -> AudioTransform -> code
+	AudioTransform audio.TransformFunc
+
 	selectedMedia prop.Media
 }
 

pkg/driver/camera/camera_linux.go

@@ -57,10 +57,11 @@ func init() {
 
 func newCamera(path string) *camera {
 	formats := map[webcam.PixelFormat]frame.Format{
-		webcam.PixelFormat(C.V4L2_PIX_FMT_YUYV):  frame.FormatYUYV,
-		webcam.PixelFormat(C.V4L2_PIX_FMT_UYVY):  frame.FormatUYVY,
-		webcam.PixelFormat(C.V4L2_PIX_FMT_NV12):  frame.FormatNV21,
-		webcam.PixelFormat(C.V4L2_PIX_FMT_MJPEG): frame.FormatMJPEG,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_YUV420): frame.FormatI420,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_YUYV):   frame.FormatYUYV,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_UYVY):   frame.FormatUYVY,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_NV12):   frame.FormatNV21,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_MJPEG):  frame.FormatMJPEG,
 	}
 
 	reversedFormats := make(map[frame.Format]webcam.PixelFormat)

pkg/frame/const.go

@@ -3,8 +3,6 @@ package frame
 type Format string
 
 const (
-	// YUV Formats
-
 	// FormatI420 https://www.fourcc.org/pixel-format/yuv-i420/
 	FormatI420 Format = "I420"
 	// FormatI444 is a YUV format without sub-sampling
@@ -16,18 +14,11 @@ const (
 	// FormatUYVY https://www.fourcc.org/pixel-format/yuv-uyvy/
 	FormatUYVY = "UYVY"
 
-	// RGB Formats
-
 	// FormatRGBA https://www.fourcc.org/pixel-format/rgb-rgba/
 	FormatRGBA Format = "RGBA"
 
-	// Compressed Formats
-
 	// FormatMJPEG https://www.fourcc.org/mjpg/
 	FormatMJPEG = "MJPEG"
 )
 
-// YUV aliases
-
-// FormatYUYV is an alias of FormatYUY2
 const FormatYUYV = FormatYUY2

pkg/frame/decode.go

@@ -5,7 +5,7 @@ import (
 )
 
 func NewDecoder(f Format) (Decoder, error) {
-	var decoder DecoderFunc
+	var decoder decoderFunc
 
 	switch f {
 	case FormatI420:

pkg/frame/frame.go

@@ -7,8 +7,8 @@ type Decoder interface {
 }
 
 // DecoderFunc is a proxy type for Decoder
-type DecoderFunc func(frame []byte, width, height int) (image.Image, error)
+type decoderFunc func(frame []byte, width, height int) (image.Image, error)
 
-func (f DecoderFunc) Decode(frame []byte, width, height int) (image.Image, error) {
+func (f decoderFunc) Decode(frame []byte, width, height int) (image.Image, error) {
 	return f(frame, width, height)
 }

pkg/io/broadcast.go

@@ -0,0 +1,162 @@
+package io
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	maskReading                = 1 << 63
+	defaultBroadcasterRingSize = 32
+	// TODO: If the data source has fps greater than 30, they'll see some
+	// 			 fps fluctuation. But, 30 fps should be enough for general cases.
+	defaultBroadcasterRingPollDuration = time.Millisecond * 33
+)
+
+var errEmptySource = fmt.Errorf("Source can't be nil")
+
+type broadcasterData struct {
+	data  interface{}
+	count uint32
+	err   error
+}
+
+type broadcasterRing struct {
+	// reading (1 bit) + reserved (31 bits) + data count (32 bits)
+	// IMPORTANT: state has to be the first element in struct, otherwise LoadUint64 will panic in 32 bits systems
+	//            due to unallignment
+	state        uint64
+	buffer       []atomic.Value
+	pollDuration time.Duration
+}
+
+func newBroadcasterRing(size uint, pollDuration time.Duration) *broadcasterRing {
+	return &broadcasterRing{buffer: make([]atomic.Value, size), pollDuration: pollDuration}
+}
+
+func (ring *broadcasterRing) index(count uint32) int {
+	return int(count) % len(ring.buffer)
+}
+
+func (ring *broadcasterRing) acquire(count uint32) func(*broadcasterData) {
+	// Reader has reached the latest data, should read from the source.
+	// Only allow 1 reader to read from the source. When there are more than 1 readers,
+	// the other readers will need to share the same data that the first reader gets from
+	// the source.
+	state := uint64(count)
+	if atomic.CompareAndSwapUint64(&ring.state, state, state|maskReading) {
+		return func(data *broadcasterData) {
+			i := ring.index(count)
+			ring.buffer[i].Store(data)
+			atomic.StoreUint64(&ring.state, uint64(count+1))
+		}
+	}
+
+	return nil
+}
+
+func (ring *broadcasterRing) get(count uint32) *broadcasterData {
+	for {
+		reading := uint64(count) | maskReading
+		// TODO: since it's lockless, it spends a lot of resources in the scheduling.
+		for atomic.LoadUint64(&ring.state) == reading {
+			// Yield current goroutine to let other goroutines to run instead
+			time.Sleep(ring.pollDuration)
+		}
+
+		i := ring.index(count)
+		data := ring.buffer[i].Load().(*broadcasterData)
+		if data.count == count {
+			return data
+		}
+
+		count++
+	}
+}
+
+func (ring *broadcasterRing) lastCount() uint32 {
+	// ring.state always keeps track the next count, so we need to subtract it by 1 to get the
+	// last count
+	return uint32(atomic.LoadUint64(&ring.state)) - 1
+}
+
+// Broadcaster is a generic pull-based broadcaster. Broadcaster is unique in a sense that
+// readers can come and go at anytime, and readers don't need to close or notify broadcaster.
+type Broadcaster struct {
+	source atomic.Value
+	buffer *broadcasterRing
+}
+
+// BroadcasterConfig is a config to control broadcaster behaviour
+type BroadcasterConfig struct {
+	// BufferSize configures the underlying ring buffer size that's being used
+	// to avoid data lost for late readers. The default value is 32.
+	BufferSize uint
+	// PollDuration configures the sleep duration in waiting for new data to come.
+	// The default value is 33 ms.
+	PollDuration time.Duration
+}
+
+// NewBroadcaster creates a new broadcaster. Source is expected to drop frames
+// when any of the readers is slower than the source.
+func NewBroadcaster(source Reader, config *BroadcasterConfig) *Broadcaster {
+	pollDuration := defaultBroadcasterRingPollDuration
+	var bufferSize uint = defaultBroadcasterRingSize
+	if config != nil {
+		if config.PollDuration != 0 {
+			pollDuration = config.PollDuration
+		}
+
+		if config.BufferSize != 0 {
+			bufferSize = config.BufferSize
+		}
+	}
+
+	var broadcaster Broadcaster
+	broadcaster.buffer = newBroadcasterRing(bufferSize, pollDuration)
+	broadcaster.ReplaceSource(source)
+
+	return &broadcaster
+}
+
+// NewReader creates a new reader. Each reader will retrieve the same data from the source.
+// copyFn is used to copy the data from the source to individual readers. Broadcaster uses a small ring
+// buffer, this means that slow readers might miss some data if they're really late and the data is no longer
+// in the ring buffer.
+func (broadcaster *Broadcaster) NewReader(copyFn func(interface{}) interface{}) Reader {
+	currentCount := broadcaster.buffer.lastCount()
+
+	return ReaderFunc(func() (data interface{}, err error) {
+		currentCount++
+		if push := broadcaster.buffer.acquire(currentCount); push != nil {
+			data, err = broadcaster.source.Load().(Reader).Read()
+			push(&broadcasterData{
+				data:  data,
+				err:   err,
+				count: currentCount,
+			})
+		} else {
+			ringData := broadcaster.buffer.get(currentCount)
+			data, err, currentCount = ringData.data, ringData.err, ringData.count
+		}
+
+		data = copyFn(data)
+		return
+	})
+}
+
+// ReplaceSource replaces the underlying source. This operation is thread safe.
+func (broadcaster *Broadcaster) ReplaceSource(source Reader) error {
+	if source == nil {
+		return errEmptySource
+	}
+
+	broadcaster.source.Store(source)
+	return nil
+}
+
+// ReplaceSource retrieves the underlying source. This operation is thread safe.
+func (broadcaster *Broadcaster) Source() Reader {
+	return broadcaster.source.Load().(Reader)
+}

pkg/io/reader.go

@@ -0,0 +1,14 @@
+package io
+
+// Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
+// to provide strong type.
+type Reader interface {
+	Read() (interface{}, error)
+}
+
+// ReaderFunc is a proxy type for Reader
+type ReaderFunc func() (interface{}, error)
+
+func (f ReaderFunc) Read() (interface{}, error) {
+	return f()
+}

pkg/io/video/broadcast.go

@@ -0,0 +1,76 @@
+package video
+
+import (
+	"fmt"
+	"image"
+
+	"github.com/pion/mediadevices/pkg/io"
+)
+
+var errEmptySource = fmt.Errorf("Source can't be nil")
+
+// Broadcaster is a specialized video broadcaster.
+type Broadcaster struct {
+	ioBroadcaster *io.Broadcaster
+}
+
+type BroadcasterConfig struct {
+	Core *io.BroadcasterConfig
+}
+
+// NewBroadcaster creates a new broadcaster. Source is expected to drop frames
+// when any of the readers is slower than the source.
+func NewBroadcaster(source Reader, config *BroadcasterConfig) *Broadcaster {
+	var coreConfig *io.BroadcasterConfig
+
+	if config != nil {
+		coreConfig = config.Core
+	}
+
+	broadcaster := io.NewBroadcaster(io.ReaderFunc(func() (interface{}, error) {
+		return source.Read()
+	}), coreConfig)
+
+	return &Broadcaster{broadcaster}
+}
+
+// NewReader creates a new reader. Each reader will retrieve the same data from the source.
+// copyFn is used to copy the data from the source to individual readers. Broadcaster uses a small ring
+// buffer, this means that slow readers might miss some data if they're really late and the data is no longer
+// in the ring buffer.
+func (broadcaster *Broadcaster) NewReader(copyFrame bool) Reader {
+	copyFn := func(src interface{}) interface{} { return src }
+
+	if copyFrame {
+		buffer := NewFrameBuffer(0)
+		copyFn = func(src interface{}) interface{} {
+			realSrc, _ := src.(image.Image)
+			buffer.StoreCopy(realSrc)
+			return buffer.Load()
+		}
+	}
+
+	reader := broadcaster.ioBroadcaster.NewReader(copyFn)
+	return ReaderFunc(func() (image.Image, error) {
+		data, err := reader.Read()
+		img, _ := data.(image.Image)
+		return img, err
+	})
+}
+
+// ReplaceSource replaces the underlying source. This operation is thread safe.
+func (broadcaster *Broadcaster) ReplaceSource(source Reader) error {
+	return broadcaster.ioBroadcaster.ReplaceSource(io.ReaderFunc(func() (interface{}, error) {
+		return source.Read()
+	}))
+}
+
+// Source retrieves the underlying source. This operation is thread safe.
+func (broadcaster *Broadcaster) Source() Reader {
+	source := broadcaster.ioBroadcaster.Source()
+	return ReaderFunc(func() (image.Image, error) {
+		data, err := source.Read()
+		img, _ := data.(image.Image)
+		return img, err
+	})
+}

pkg/io/video/broadcast_test.go

@@ -0,0 +1,187 @@
+package video
+
+import (
+	"fmt"
+	"image"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func BenchmarkBroadcast(b *testing.B) {
+	var src Reader
+	img := image.NewRGBA(image.Rect(0, 0, 1920, 1080))
+	interval := time.NewTicker(time.Millisecond * 33) // 30 fps
+	defer interval.Stop()
+	src = ReaderFunc(func() (image.Image, error) {
+		<-interval.C
+		return img, nil
+	})
+
+	for n := 1; n <= 4096; n *= 16 {
+		n := n
+
+		b.Run(fmt.Sprintf("Readers-%d", n), func(b *testing.B) {
+			b.SetParallelism(n)
+			broadcaster := NewBroadcaster(src, nil)
+			b.RunParallel(func(pb *testing.PB) {
+				reader := broadcaster.NewReader(false)
+				for pb.Next() {
+					reader.Read()
+				}
+			})
+		})
+	}
+}
+
+func TestBroadcast(t *testing.T) {
+	// https://github.com/pion/mediadevices/issues/198
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping because Darwin CI is not reliable for timing related tests.")
+	}
+	frames := make([]image.Image, 5*30) // 5 seconds worth of frames
+	resolution := image.Rect(0, 0, 1920, 1080)
+	for i := range frames {
+		rgba := image.NewRGBA(resolution)
+		rgba.Pix[0] = uint8(i >> 24)
+		rgba.Pix[1] = uint8(i >> 16)
+		rgba.Pix[2] = uint8(i >> 8)
+		rgba.Pix[3] = uint8(i)
+		frames[i] = rgba
+	}
+
+	routinePauseConds := []struct {
+		src          bool
+		dst          bool
+		expectedFPS  float64
+		expectedDrop float64
+	}{
+		{
+			src:         false,
+			dst:         false,
+			expectedFPS: 30,
+		},
+		{
+			src:          true,
+			dst:          false,
+			expectedFPS:  20,
+			expectedDrop: 10,
+		},
+		{
+			src:          false,
+			dst:          true,
+			expectedFPS:  20,
+			expectedDrop: 10,
+		},
+	}
+
+	for _, pauseCond := range routinePauseConds {
+		pauseCond := pauseCond
+		t.Run(fmt.Sprintf("SrcPause-%v/DstPause-%v", pauseCond.src, pauseCond.dst), func(t *testing.T) {
+			for n := 1; n <= 256; n *= 16 {
+				n := n
+
+				t.Run(fmt.Sprintf("Readers-%d", n), func(t *testing.T) {
+					var src Reader
+					interval := time.NewTicker(time.Millisecond * 33) // 30 fps
+					defer interval.Stop()
+					frameCount := 0
+					frameSent := 0
+					lastSend := time.Now()
+					src = ReaderFunc(func() (image.Image, error) {
+						if pauseCond.src && frameSent == 30 {
+							time.Sleep(time.Second)
+						}
+						<-interval.C
+
+						now := time.Now()
+						if interval := now.Sub(lastSend); interval > time.Millisecond*33*3/2 {
+							// Source reader should drop frames to catch up the latest frame.
+							drop := int(interval/(time.Millisecond*33)) - 1
+							frameCount += drop
+							t.Logf("Skipped %d frames", drop)
+						}
+						lastSend = now
+						frame := frames[frameCount]
+						frameCount++
+						frameSent++
+						return frame, nil
+					})
+					broadcaster := NewBroadcaster(src, nil)
+					var done uint32
+					duration := time.Second * 3
+					fpsChan := make(chan []float64)
+
+					var wg sync.WaitGroup
+					wg.Add(n)
+					for i := 0; i < n; i++ {
+						go func() {
+							reader := broadcaster.NewReader(false)
+							count := 0
+							lastFrameCount := -1
+							droppedFrames := 0
+							wg.Done()
+							wg.Wait()
+							for atomic.LoadUint32(&done) == 0 {
+								if pauseCond.dst && count == 30 {
+									time.Sleep(time.Second)
+								}
+								frame, err := reader.Read()
+								if err != nil {
+									t.Error(err)
+								}
+								rgba := frame.(*image.RGBA)
+								var frameCount int
+								frameCount |= int(rgba.Pix[0]) << 24
+								frameCount |= int(rgba.Pix[1]) << 16
+								frameCount |= int(rgba.Pix[2]) << 8
+								frameCount |= int(rgba.Pix[3])
+
+								droppedFrames += (frameCount - lastFrameCount - 1)
+								lastFrameCount = frameCount
+								count++
+							}
+
+							fps := float64(count) / duration.Seconds()
+							if fps < pauseCond.expectedFPS-2 || fps > pauseCond.expectedFPS+2 {
+								t.Fatal("Unexpected average FPS")
+							}
+
+							droppedFramesPerSecond := float64(droppedFrames) / duration.Seconds()
+							if droppedFramesPerSecond < pauseCond.expectedDrop-2 || droppedFramesPerSecond > pauseCond.expectedDrop+2 {
+								t.Fatal("Unexpected drop count")
+							}
+
+							fpsChan <- []float64{fps, droppedFramesPerSecond, float64(lastFrameCount)}
+						}()
+					}
+
+					time.Sleep(duration)
+					atomic.StoreUint32(&done, 1)
+
+					var fpsAvg float64
+					var droppedFramesPerSecondAvg float64
+					var lastFrameCountAvg float64
+					var count int
+					for metric := range fpsChan {
+						fps, droppedFramesPerSecond, lastFrameCount := metric[0], metric[1], metric[2]
+						fpsAvg += fps
+						droppedFramesPerSecondAvg += droppedFramesPerSecond
+						lastFrameCountAvg += lastFrameCount
+						count++
+						if count == n {
+							break
+						}
+					}
+
+					t.Log("Average FPS                      :", fpsAvg/float64(n))
+					t.Log("Average dropped frames per second:", droppedFramesPerSecondAvg/float64(n))
+					t.Log("Last frame count (src)           :", frameCount)
+					t.Log("Average last frame count (dst)   :", lastFrameCountAvg/float64(n))
+				})
+			}
+		})
+	}
+}

pkg/io/video/framebuffer.go

@@ -0,0 +1,214 @@
+package video
+
+import (
+	"image"
+)
+
+// FrameBuffer is a buffer that can store any image format.
+type FrameBuffer struct {
+	buffer []uint8
+	tmp    image.Image
+}
+
+// NewFrameBuffer creates a new FrameBuffer instance and initialize internal buffer
+// with initialSize
+func NewFrameBuffer(initialSize int) *FrameBuffer {
+	return &FrameBuffer{
+		buffer: make([]uint8, initialSize),
+	}
+}
+
+func (buff *FrameBuffer) storeInOrder(srcs ...[]uint8) {
+	var neededSize int
+
+	for _, src := range srcs {
+		neededSize += len(src)
+	}
+
+	if len(buff.buffer) < neededSize {
+		if cap(buff.buffer) >= neededSize {
+			buff.buffer = buff.buffer[:neededSize]
+		} else {
+			buff.buffer = make([]uint8, neededSize)
+		}
+	}
+
+	var currentLen int
+	for _, src := range srcs {
+		copy(buff.buffer[currentLen:], src)
+		currentLen += len(src)
+	}
+}
+
+// Load loads the current owned image
+func (buff *FrameBuffer) Load() image.Image {
+	return buff.tmp
+}
+
+// StoreCopy makes a copy of src and store its copy. StoreCopy will reuse as much memory as it can
+// from the previous copies. For example, if StoreCopy is given an image that has the same resolution
+// and format from the previous call, StoreCopy will not allocate extra memory and only copy the content
+// from src to the previous buffer.
+func (buff *FrameBuffer) StoreCopy(src image.Image) {
+	switch src := src.(type) {
+	case *image.Alpha:
+		clone, ok := buff.tmp.(*image.Alpha)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Alpha16:
+		clone, ok := buff.tmp.(*image.Alpha16)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.CMYK:
+		clone, ok := buff.tmp.(*image.CMYK)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Gray:
+		clone, ok := buff.tmp.(*image.Gray)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Gray16:
+		clone, ok := buff.tmp.(*image.Gray16)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NRGBA:
+		clone, ok := buff.tmp.(*image.NRGBA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NRGBA64:
+		clone, ok := buff.tmp.(*image.NRGBA64)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.RGBA:
+		clone, ok := buff.tmp.(*image.RGBA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.RGBA64:
+		clone, ok := buff.tmp.(*image.RGBA64)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NYCbCrA:
+		clone, ok := buff.tmp.(*image.NYCbCrA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		var currentLen int
+		buff.storeInOrder(src.Y, src.Cb, src.Cr, src.A)
+		clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
+		currentLen += len(src.Y)
+		clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
+		currentLen += len(src.Cb)
+		clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
+		currentLen += len(src.Cr)
+		clone.A = buff.buffer[currentLen : currentLen+len(src.A) : currentLen+len(src.A)]
+
+		buff.tmp = clone
+	case *image.YCbCr:
+		clone, ok := buff.tmp.(*image.YCbCr)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		var currentLen int
+		buff.storeInOrder(src.Y, src.Cb, src.Cr)
+		clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
+		currentLen += len(src.Y)
+		clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
+		currentLen += len(src.Cb)
+		clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
+
+		buff.tmp = clone
+	default:
+		var converted image.RGBA
+		imageToRGBA(&converted, src)
+		buff.StoreCopy(&converted)
+	}
+}

pkg/io/video/framebuffer_test.go

@@ -0,0 +1,195 @@
+package video
+
+import (
+	"image"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+func randomize(arr []uint8) {
+	for i := range arr {
+		arr[i] = uint8(rand.Uint32())
+	}
+}
+
+func BenchmarkFrameBufferCopyOptimized(b *testing.B) {
+	frameBuffer := NewFrameBuffer(0)
+	resolution := image.Rect(0, 0, 1920, 1080)
+	src := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+
+	for i := 0; i < b.N; i++ {
+		frameBuffer.StoreCopy(src)
+	}
+}
+
+func BenchmarkFrameBufferCopyNaive(b *testing.B) {
+	resolution := image.Rect(0, 0, 1920, 1080)
+	src := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+	var dst image.Image
+
+	for i := 0; i < b.N; i++ {
+		clone := *src
+		clone.Cb = make([]uint8, len(src.Cb))
+		clone.Cr = make([]uint8, len(src.Cr))
+		clone.Y = make([]uint8, len(src.Y))
+
+		copy(clone.Cb, src.Cb)
+		copy(clone.Cr, src.Cr)
+		copy(clone.Y, src.Y)
+		dst = &clone
+		_ = dst
+	}
+}
+
+func TestFrameBufferStoreCopyAndLoad(t *testing.T) {
+	resolution := image.Rect(0, 0, 16, 8)
+	rgbaLike := image.NewRGBA64(resolution)
+	randomize(rgbaLike.Pix)
+	testCases := map[string]struct {
+		New    func() image.Image
+		Update func(image.Image)
+	}{
+		"Alpha": {
+			New: func() image.Image {
+				return (*image.Alpha)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Alpha)
+				randomize(img.Pix)
+			},
+		},
+		"Alpha16": {
+			New: func() image.Image {
+				return (*image.Alpha16)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Alpha16)
+				randomize(img.Pix)
+			},
+		},
+		"CMYK": {
+			New: func() image.Image {
+				return (*image.CMYK)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.CMYK)
+				randomize(img.Pix)
+			},
+		},
+		"Gray": {
+			New: func() image.Image {
+				return (*image.Gray)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Gray)
+				randomize(img.Pix)
+			},
+		},
+		"Gray16": {
+			New: func() image.Image {
+				return (*image.Gray16)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Gray16)
+				randomize(img.Pix)
+			},
+		},
+		"NRGBA": {
+			New: func() image.Image {
+				return (*image.NRGBA)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NRGBA)
+				randomize(img.Pix)
+			},
+		},
+		"NRGBA64": {
+			New: func() image.Image {
+				return (*image.NRGBA64)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NRGBA64)
+				randomize(img.Pix)
+			},
+		},
+		"RGBA": {
+			New: func() image.Image {
+				return (*image.RGBA)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.RGBA)
+				randomize(img.Pix)
+			},
+		},
+		"RGBA64": {
+			New: func() image.Image {
+				return (*image.RGBA64)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.RGBA64)
+				randomize(img.Pix)
+			},
+		},
+		"NYCbCrA": {
+			New: func() image.Image {
+				img := image.NewNYCbCrA(resolution, image.YCbCrSubsampleRatio420)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				randomize(img.A)
+				img.CStride = 10
+				img.YStride = 5
+				return img
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NYCbCrA)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				randomize(img.A)
+				img.CStride = 3
+				img.YStride = 2
+			},
+		},
+		"YCbCr": {
+			New: func() image.Image {
+				img := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				img.CStride = 10
+				img.YStride = 5
+				return img
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.YCbCr)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				img.CStride = 3
+				img.YStride = 2
+			},
+		},
+	}
+
+	frameBuffer := NewFrameBuffer(0)
+
+	for name, testCase := range testCases {
+		// Since the test also wants to make sure that Copier can convert from 1 type to another,
+		// t.Run is not ideal since it'll run the tests separately
+		t.Log("Testing", name)
+
+		src := testCase.New()
+		frameBuffer.StoreCopy(src)
+		if !reflect.DeepEqual(frameBuffer.Load(), src) {
+			t.Fatal("Expected the copied image to be identical with the source")
+		}
+
+		testCase.Update(src)
+		frameBuffer.StoreCopy(src)
+		if !reflect.DeepEqual(frameBuffer.Load(), src) {
+			t.Fatal("Expected the copied image to be identical with the source after an update in source")
+		}
+	}
+}

pkg/prop/bool.go

@@ -1,5 +1,7 @@
 package prop
 
+import "fmt"
+
 // BoolConstraint is an interface to represent bool value constraint.
 type BoolConstraint interface {
 	Compare(bool) (float64, bool)
@@ -20,6 +22,11 @@ func (b BoolExact) Compare(o bool) (float64, bool) {
 // Value implements BoolConstraint.
 func (b BoolExact) Value() bool { return bool(b) }
 
+// String implements Stringify
+func (b BoolExact) String() string {
+	return fmt.Sprintf("%t (exact)", b)
+}
+
 // Bool specifies ideal bool value.
 type Bool BoolExact
 

pkg/prop/duration.go

@@ -1,7 +1,9 @@
 package prop
 
 import (
+	"fmt"
 	"math"
+	"strings"
 	"time"
 )
 
@@ -23,6 +25,11 @@ func (d Duration) Compare(a time.Duration) (float64, bool) {
 // Value implements DurationConstraint.
 func (d Duration) Value() (time.Duration, bool) { return time.Duration(d), true }
 
+// String implements Stringify
+func (d Duration) String() string {
+	return fmt.Sprintf("%v (ideal)", time.Duration(d))
+}
+
 // DurationExact specifies exact duration value.
 type DurationExact time.Duration
 
@@ -37,6 +44,11 @@ func (d DurationExact) Compare(a time.Duration) (float64, bool) {
 // Value implements DurationConstraint.
 func (d DurationExact) Value() (time.Duration, bool) { return time.Duration(d), true }
 
+// String implements Stringify
+func (d DurationExact) String() string {
+	return fmt.Sprintf("%v (exact)", time.Duration(d))
+}
+
 // DurationOneOf specifies list of expected duration values.
 type DurationOneOf []time.Duration
 
@@ -53,6 +65,16 @@ func (d DurationOneOf) Compare(a time.Duration) (float64, bool) {
 // Value implements DurationConstraint.
 func (DurationOneOf) Value() (time.Duration, bool) { return 0, false }
 
+// String implements Stringify
+func (d DurationOneOf) String() string {
+	var opts []string
+	for _, v := range d {
+		opts = append(opts, fmt.Sprint(v))
+	}
+
+	return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
+}
+
 // DurationRanged specifies range of expected duration value.
 // If Ideal is non-zero, closest value to Ideal takes priority.
 type DurationRanged struct {
@@ -96,3 +118,8 @@ func (d DurationRanged) Compare(a time.Duration) (float64, bool) {
 
 // Value implements DurationConstraint.
 func (DurationRanged) Value() (time.Duration, bool) { return 0, false }
+
+// String implements Stringify
+func (d DurationRanged) String() string {
+	return fmt.Sprintf("%s - %s (range), %s (ideal)", d.Min, d.Max, d.Ideal)
+}

pkg/prop/float.go

@@ -1,7 +1,9 @@
 package prop
 
 import (
+	"fmt"
 	"math"
+	"strings"
 )
 
 // FloatConstraint is an interface to represent float value constraint.
@@ -22,6 +24,11 @@ func (f Float) Compare(a float32) (float64, bool) {
 // Value implements FloatConstraint.
 func (f Float) Value() (float32, bool) { return float32(f), true }
 
+// String implements Stringify
+func (f Float) String() string {
+	return fmt.Sprintf("%.2f (ideal)", f)
+}
+
 // FloatExact specifies exact float value.
 type FloatExact float32
 
@@ -36,6 +43,11 @@ func (f FloatExact) Compare(a float32) (float64, bool) {
 // Value implements FloatConstraint.
 func (f FloatExact) Value() (float32, bool) { return float32(f), true }
 
+// String implements Stringify
+func (f FloatExact) String() string {
+	return fmt.Sprintf("%.2f (exact)", f)
+}
+
 // FloatOneOf specifies list of expected float values.
 type FloatOneOf []float32
 
@@ -52,6 +64,16 @@ func (f FloatOneOf) Compare(a float32) (float64, bool) {
 // Value implements FloatConstraint.
 func (FloatOneOf) Value() (float32, bool) { return 0, false }
 
+// String implements Stringify
+func (f FloatOneOf) String() string {
+	var opts []string
+	for _, v := range f {
+		opts = append(opts, fmt.Sprintf("%.2f", v))
+	}
+
+	return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
+}
+
 // FloatRanged specifies range of expected float value.
 // If Ideal is non-zero, closest value to Ideal takes priority.
 type FloatRanged struct {
@@ -95,3 +117,8 @@ func (f FloatRanged) Compare(a float32) (float64, bool) {
 
 // Value implements FloatConstraint.
 func (FloatRanged) Value() (float32, bool) { return 0, false }
+
+// String implements Stringify
+func (f FloatRanged) String() string {
+	return fmt.Sprintf("%.2f - %.2f (range), %.2f (ideal)", f.Min, f.Max, f.Ideal)
+}

pkg/prop/format.go

@@ -1,7 +1,9 @@
 package prop
 
 import (
+	"fmt"
 	"github.com/pion/mediadevices/pkg/frame"
+	"strings"
 )
 
 // FrameFormatConstraint is an interface to represent frame format constraint.
@@ -25,6 +27,11 @@ func (f FrameFormat) Compare(a frame.Format) (float64, bool) {
 // Value implements FrameFormatConstraint.
 func (f FrameFormat) Value() (frame.Format, bool) { return frame.Format(f), true }
 
+// String implements Stringify
+func (f FrameFormat) String() string {
+	return fmt.Sprintf("%s (ideal)", frame.Format(f))
+}
+
 // FrameFormatExact specifies exact frame format.
 type FrameFormatExact frame.Format
 
@@ -39,6 +46,11 @@ func (f FrameFormatExact) Compare(a frame.Format) (float64, bool) {
 // Value implements FrameFormatConstraint.
 func (f FrameFormatExact) Value() (frame.Format, bool) { return frame.Format(f), true }
 
+// String implements Stringify
+func (f FrameFormatExact) String() string {
+	return fmt.Sprintf("%s (exact)", frame.Format(f))
+}
+
 // FrameFormatOneOf specifies list of expected frame format.
 type FrameFormatOneOf []frame.Format
 
@@ -54,3 +66,13 @@ func (f FrameFormatOneOf) Compare(a frame.Format) (float64, bool) {
 
 // Value implements FrameFormatConstraint.
 func (FrameFormatOneOf) Value() (frame.Format, bool) { return "", false }
+
+// String implements Stringify
+func (f FrameFormatOneOf) String() string {
+	var opts []string
+	for _, v := range f {
+		opts = append(opts, fmt.Sprint(v))
+	}
+
+	return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
+}

pkg/prop/int.go

@@ -1,7 +1,9 @@
 package prop
 
 import (
+	"fmt"
 	"math"
+	"strings"
 )
 
 // IntConstraint is an interface to represent integer value constraint.
@@ -22,6 +24,11 @@ func (i Int) Compare(a int) (float64, bool) {
 // Value implements IntConstraint.
 func (i Int) Value() (int, bool) { return int(i), true }
 
+// String implements Stringify
+func (i Int) String() string {
+	return fmt.Sprintf("%d (ideal)", i)
+}
+
 // IntExact specifies exact int value.
 type IntExact int
 
@@ -33,6 +40,11 @@ func (i IntExact) Compare(a int) (float64, bool) {
 	return 1.0, false
 }
 
+// String implements Stringify
+func (i IntExact) String() string {
+	return fmt.Sprintf("%d (exact)", i)
+}
+
 // Value implements IntConstraint.
 func (i IntExact) Value() (int, bool) { return int(i), true }
 
@@ -52,6 +64,16 @@ func (i IntOneOf) Compare(a int) (float64, bool) {
 // Value implements IntConstraint.
 func (IntOneOf) Value() (int, bool) { return 0, false }
 
+// String implements Stringify
+func (i IntOneOf) String() string {
+	var opts []string
+	for _, v := range i {
+		opts = append(opts, fmt.Sprint(v))
+	}
+
+	return fmt.Sprintf("%s (one of values)", strings.Join(opts, ","))
+}
+
 // IntRanged specifies range of expected int value.
 // If Ideal is non-zero, closest value to Ideal takes priority.
 type IntRanged struct {
@@ -95,3 +117,8 @@ func (i IntRanged) Compare(a int) (float64, bool) {
 
 // Value implements IntConstraint.
 func (IntRanged) Value() (int, bool) { return 0, false }
+
+// String implements Stringify
+func (i IntRanged) String() string {
+	return fmt.Sprintf("%d - %d (range), %d (ideal)", i.Min, i.Max, i.Ideal)
+}

pkg/prop/prop.go

@@ -1,7 +1,9 @@
 package prop
 
 import (
+	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/pion/mediadevices/pkg/frame"
@@ -15,6 +17,10 @@ type MediaConstraints struct {
 	AudioConstraints
 }
 
+func (m *MediaConstraints) String() string {
+	return prettifyStruct(m)
+}
+
 // Media stores single set of media propaties.
 type Media struct {
 	DeviceID string
@@ -22,6 +28,33 @@ type Media struct {
 	Audio
 }
 
+func (m *Media) String() string {
+	return prettifyStruct(m)
+}
+
+func prettifyStruct(i interface{}) string {
+	var rows []string
+	var addRows func(int, reflect.Value)
+	addRows = func(level int, obj reflect.Value) {
+		typeOf := obj.Type()
+		for i := 0; i < obj.NumField(); i++ {
+			field := typeOf.Field(i)
+			value := obj.Field(i)
+
+			padding := strings.Repeat("  ", level)
+			if value.Kind() == reflect.Struct {
+				rows = append(rows, fmt.Sprintf("%s%v:", padding, field.Name))
+				addRows(level+1, value)
+			} else {
+				rows = append(rows, fmt.Sprintf("%s%v: %v", padding, field.Name, value))
+			}
+		}
+	}
+
+	addRows(0, reflect.ValueOf(i).Elem())
+	return strings.Join(rows, "\n")
+}
+
 // setterFn is a callback function to set value from fieldB to fieldA
 type setterFn func(fieldA, fieldB reflect.Value)
 

pkg/prop/prop_test.go

@@ -309,3 +309,60 @@ func TestMergeConstraintsNested(t *testing.T) {
 		t.Error("expected a.Width to be 100, but got 0")
 	}
 }
+
+func TestString(t *testing.T) {
+	t.Run("IdealValues", func(t *testing.T) {
+		t.Log("\n", &MediaConstraints{
+			DeviceID: String("one"),
+			VideoConstraints: VideoConstraints{
+				Width:       Int(1920),
+				FrameRate:   Float(30.0),
+				FrameFormat: FrameFormat(frame.FormatI420),
+			},
+			AudioConstraints: AudioConstraints{
+				Latency: Duration(time.Millisecond * 20),
+			},
+		})
+	})
+
+	t.Run("ExactValues", func(t *testing.T) {
+		t.Log("\n", &MediaConstraints{
+			DeviceID: StringExact("one"),
+			VideoConstraints: VideoConstraints{
+				Width:       IntExact(1920),
+				FrameRate:   FloatExact(30.0),
+				FrameFormat: FrameFormatExact(frame.FormatI420),
+			},
+			AudioConstraints: AudioConstraints{
+				Latency:     DurationExact(time.Millisecond * 20),
+				IsBigEndian: BoolExact(true),
+			},
+		})
+	})
+
+	t.Run("OneOfValues", func(t *testing.T) {
+		t.Log("\n", &MediaConstraints{
+			DeviceID: StringOneOf{"one", "two"},
+			VideoConstraints: VideoConstraints{
+				Width:       IntOneOf{1920, 1080},
+				FrameRate:   FloatOneOf{30.0, 60.1234},
+				FrameFormat: FrameFormatOneOf{frame.FormatI420, frame.FormatI444},
+			},
+			AudioConstraints: AudioConstraints{
+				Latency: DurationOneOf{time.Millisecond * 20, time.Millisecond * 40},
+			},
+		})
+	})
+
+	t.Run("RangedValues", func(t *testing.T) {
+		t.Log("\n", &MediaConstraints{
+			VideoConstraints: VideoConstraints{
+				Width:     &IntRanged{Min: 1080, Max: 1920, Ideal: 1500},
+				FrameRate: &FloatRanged{Min: 30.123, Max: 60.12321312, Ideal: 45.12312312},
+			},
+			AudioConstraints: AudioConstraints{
+				Latency: &DurationRanged{Min: time.Millisecond * 20, Max: time.Millisecond * 40, Ideal: time.Millisecond * 30},
+			},
+		})
+	})
+}

pkg/prop/string.go

@@ -1,5 +1,10 @@
 package prop
 
+import (
+	"fmt"
+	"strings"
+)
+
 // StringConstraint is an interface to represent string constraint.
 type StringConstraint interface {
 	Compare(string) (float64, bool)
@@ -21,6 +26,11 @@ func (f String) Compare(a string) (float64, bool) {
 // Value implements StringConstraint.
 func (f String) Value() (string, bool) { return string(f), true }
 
+// String implements Stringify
+func (f String) String() string {
+	return fmt.Sprintf("%s (ideal)", string(f))
+}
+
 // StringExact specifies exact string.
 type StringExact string
 
@@ -35,6 +45,11 @@ func (f StringExact) Compare(a string) (float64, bool) {
 // Value implements StringConstraint.
 func (f StringExact) Value() (string, bool) { return string(f), true }
 
+// String implements Stringify
+func (f StringExact) String() string {
+	return fmt.Sprintf("%s (exact)", string(f))
+}
+
 // StringOneOf specifies list of expected string.
 type StringOneOf []string
 
@@ -50,3 +65,8 @@ func (f StringOneOf) Compare(a string) (float64, bool) {
 
 // Value implements StringConstraint.
 func (StringOneOf) Value() (string, bool) { return "", false }
+
+// String implements Stringify
+func (f StringOneOf) String() string {
+	return fmt.Sprintf("%s (one of values)", strings.Join([]string(f), ","))
+}

rtp.go

@@ -1,101 +0,0 @@
-package mediadevices
-
-import (
-	"fmt"
-
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/rtcp"
-	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2"
-)
-
-type RTPTracker struct {
-	videoEncoders []codec.VideoEncoderBuilder
-	audioEncoders []codec.AudioEncoderBuilder
-}
-
-type RTPTrackerOption func(*RTPTracker)
-
-func WithVideoEncoders(codecs ...codec.VideoEncoderBuilder) func(*RTPTracker) {
-	return func(tracker *RTPTracker) {
-		tracker.videoEncoders = codecs
-	}
-}
-
-func WithAudioEncoders(codecs ...codec.AudioEncoderBuilder) func(*RTPTracker) {
-	return func(tracker *RTPTracker) {
-		tracker.audioEncoders = codecs
-	}
-}
-
-func NewRTPTracker(opts ...RTPTrackerOption) *RTPTracker {
-	var tracker RTPTracker
-
-	for _, opt := range opts {
-		opt(&tracker)
-	}
-
-	return &tracker
-}
-
-func (tracker *RTPTracker) Track(track Track) *RTPTrack {
-	rtpTrack := RTPTrack{
-		Track: track,
-	}
-
-	return &rtpTrack
-}
-
-type RTPTrack struct {
-	Track
-	tracker        *RTPTracker
-	currentEncoder codec.ReadCloser
-	currentParams  RTPParameters
-	lastProp       prop.Media
-}
-
-func (track *RTPTrack) SetParameters(params RTPParameters) error {
-	var err error
-
-	switch t := track.Track.(type) {
-	case *VideoTrack:
-		err = track.setParametersVideo(t, &params)
-	case *AudioTrack:
-		err = track.setParametersAudio(t, &params)
-	default:
-		err = fmt.Errorf("unsupported track type")
-	}
-
-	if err == nil {
-		track.currentParams = params
-	}
-
-	return err
-}
-
-func (track *RTPTrack) setParametersVideo(videoTrack *VideoTrack, params *RTPParameters) error {
-	if params.SelectedCodec.Type != webrtc.RTPCodecTypeVideo {
-		return fmt.Errorf("invalid selected RTP codec type. Expected video but got audio")
-	}
-
-	video.DetectChanges(interval time.Duration, onChange func(prop.Media))
-	return nil
-}
-
-func (track *RTPTrack) setParametersAudio(audioTrack *AudioTrack, params *RTPParameters) error {
-	return nil
-}
-
-func (track *RTPTrack) ReadRTP() (*rtp.Packet, error) {
-	if track.currentEncoder == nil {
-		return nil, fmt.Errorf("Encoder has not been specified. Please call SetParameters to specify.")
-	}
-
-	return nil, nil
-}
-
-func (track *RTPTrack) WriteRTCP(packet rtcp.Packet) error {
-	return nil
-}

sampler.go

@@ -0,0 +1,35 @@
+package mediadevices
+
+import (
+	"math"
+	"time"
+
+	"github.com/pion/webrtc/v2/pkg/media"
+)
+
+type samplerFunc func(b []byte) error
+
+// newVideoSampler creates a video sampler that uses the actual video frame rate and
+// the codec's clock rate to come up with a duration for each sample.
+func newVideoSampler(t LocalTrack) samplerFunc {
+	clockRate := float64(t.Codec().ClockRate)
+	lastTimestamp := time.Now()
+
+	return samplerFunc(func(b []byte) error {
+		now := time.Now()
+		duration := now.Sub(lastTimestamp).Seconds()
+		samples := uint32(math.Round(clockRate * duration))
+		lastTimestamp = now
+
+		return t.WriteSample(media.Sample{Data: b, Samples: samples})
+	})
+}
+
+// newAudioSampler creates a audio sampler that uses a fixed latency and
+// the codec's clock rate to come up with a duration for each sample.
+func newAudioSampler(t LocalTrack, latency time.Duration) samplerFunc {
+	samples := uint32(math.Round(float64(t.Codec().ClockRate) * latency.Seconds()))
+	return samplerFunc(func(b []byte) error {
+		return t.WriteSample(media.Sample{Data: b, Samples: samples})
+	})
+}

track.go

@@ -1,29 +1,22 @@
 package mediadevices
 
 import (
-	"fmt"
-	"image"
+	"errors"
+	"math/rand"
 	"sync"
 
+	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/driver"
-	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/mediadevices/pkg/wave"
+	mio "github.com/pion/mediadevices/pkg/io"
+	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v2/pkg/media"
 )
 
-// TrackKind represents content type of a track
-type TrackKind string
-
-const (
-	TrackKindVideo TrackKind = "video"
-	TrackKindAudio TrackKind = "audio"
-)
-
-// Track is an interface that represent MediaStreamTrack
+// Tracker is an interface that represent MediaStreamTrack
 // Reference: https://w3c.github.io/mediacapture-main/#mediastreamtrack
-type Track interface {
-	ID() string
-	Kind() TrackKind
+type Tracker interface {
+	Track() *webrtc.Track
+	LocalTrack() LocalTrack
 	Stop()
 	// OnEnded registers a handler to receive an error from the media stream track.
 	// If the error is already occured before registering, the handler will be
@@ -31,170 +24,18 @@ type Track interface {
 	OnEnded(func(error))
 }
 
-// VideoTrack is a specialized track for video
-type VideoTrack struct {
-	baseTrack
-	src         video.Reader
-	transformed video.Reader
-	mux         sync.Mutex
-	frameCount  int
-	lastFrame   image.Image
-	lastErr     error
+type LocalTrack interface {
+	WriteSample(s media.Sample) error
+	Codec() *webrtc.RTPCodec
+	ID() string
+	Kind() webrtc.RTPCodecType
 }
 
-func newVideoTrack(d driver.Driver, constraints MediaTrackConstraints) (*VideoTrack, error) {
-	err := d.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	recorder, ok := d.(driver.VideoRecorder)
-	if !ok {
-		d.Close()
-		return nil, fmt.Errorf("driver is not an video recorder")
-	}
-
-	r, err := recorder.VideoRecord(constraints.selectedMedia)
-	if err != nil {
-		d.Close()
-		return nil, err
-	}
-
-	return &VideoTrack{
-		baseTrack:   newBaseTrack(d, constraints),
-		src:         r,
-		transformed: r,
-	}, nil
-}
-
-// Kind returns track's kind
-func (track *VideoTrack) Kind() TrackKind {
-	return TrackKindVideo
-}
-
-// NewReader returns a reader to read frames from the source. You may create multiple
-// readers and read from them in different goroutines.
-//
-// In the case of multiple readers, reading from the source will only get triggered
-// when the reader has the latest frame from the source
-func (track *VideoTrack) NewReader() video.Reader {
-	var curFrameCount int
-	return video.ReaderFunc(func() (img image.Image, err error) {
-		track.mux.Lock()
-		defer track.mux.Unlock()
-
-		if curFrameCount != track.frameCount {
-			img = copyFrame(img, track.lastFrame)
-			err = track.lastErr
-		} else {
-			img, err = track.transformed.Read()
-			track.lastFrame = img
-			track.lastErr = err
-			track.frameCount++
-			if err != nil {
-				track.onErrorHandler(err)
-			}
-		}
-
-		curFrameCount = track.frameCount
-		return
-	})
-}
-
-// TODO: implement copy in place
-func copyFrame(dst, src image.Image) image.Image { return src }
-
-// Transform transforms the underlying source. The transformation will reflect to
-// all readers
-func (track *VideoTrack) Transform(fns ...video.TransformFunc) {
-	track.mux.Lock()
-	defer track.mux.Unlock()
-	track.transformed = video.Merge(fns...)(track.src)
-}
-
-// AudioTrack is a specialized track for audio
-type AudioTrack struct {
-	baseTrack
-	src         audio.Reader
-	transformed audio.Reader
-	mux         sync.Mutex
-	chunkCount  int
-	lastChunks  wave.Audio
-	lastErr     error
-}
-
-func newAudioTrack(d driver.Driver, constraints MediaTrackConstraints) (*AudioTrack, error) {
-	err := d.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	recorder, ok := d.(driver.AudioRecorder)
-	if !ok {
-		d.Close()
-		return nil, fmt.Errorf("driver is not an audio recorder")
-	}
-
-	r, err := recorder.AudioRecord(constraints.selectedMedia)
-	if err != nil {
-		d.Close()
-		return nil, err
-	}
-
-	return &AudioTrack{
-		baseTrack:   newBaseTrack(d, constraints),
-		src:         r,
-		transformed: r,
-	}, nil
-}
-
-func (track *AudioTrack) Kind() TrackKind {
-	return TrackKindAudio
-}
-
-// NewReader returns a reader to read audio chunks from the source. You may create multiple
-// readers and read from them in different goroutines.
-//
-// In the case of multiple readers, reading from the source will only get triggered
-// when the reader has the latest chunk from the source
-func (track *AudioTrack) NewReader() audio.Reader {
-	var currChunkCount int
-	return audio.ReaderFunc(func() (chunks wave.Audio, err error) {
-		track.mux.Lock()
-		defer track.mux.Unlock()
-
-		if currChunkCount != track.chunkCount {
-			chunks = copyChunks(chunks, track.lastChunks)
-			err = track.lastErr
-		} else {
-			chunks, err = track.transformed.Read()
-			track.lastChunks = chunks
-			track.lastErr = err
-			track.chunkCount++
-			if err != nil {
-				track.onErrorHandler(err)
-			}
-		}
-
-		currChunkCount = track.chunkCount
-		return
-	})
-}
-
-// TODO: implement copy in place
-func copyChunks(dst, src wave.Audio) wave.Audio { return src }
-
-// Transform transforms the underlying source. The transformation will reflect to
-// all readers
-func (track *AudioTrack) Transform(fns ...audio.TransformFunc) {
-	track.mux.Lock()
-	defer track.mux.Unlock()
-	track.transformed = audio.Merge(fns...)(track.src)
-}
-
-type baseTrack struct {
-	d           driver.Driver
-	constraints MediaTrackConstraints
+type track struct {
+	localTrack LocalTrack
+	d          driver.Driver
+	sample     samplerFunc
+	encoder    codec.ReadCloser
 
 	onErrorHandler func(error)
 	err            error
@@ -202,17 +43,83 @@ type baseTrack struct {
 	endOnce        sync.Once
 }
 
-func newBaseTrack(d driver.Driver, constraints MediaTrackConstraints) baseTrack {
-	return baseTrack{d: d, constraints: constraints}
-}
+func newTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*track, error) {
+	var encoderBuilders []encoderBuilder
+	var rtpCodecs []*webrtc.RTPCodec
+	var buildSampler func(t LocalTrack) samplerFunc
+	var err error
 
-func (t *baseTrack) ID() string {
-	return t.d.ID()
+	err = d.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	switch r := d.(type) {
+	case driver.VideoRecorder:
+		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeVideo]
+		buildSampler = newVideoSampler
+		encoderBuilders, err = newVideoEncoderBuilders(r, constraints)
+	case driver.AudioRecorder:
+		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeAudio]
+		buildSampler = func(t LocalTrack) samplerFunc {
+			return newAudioSampler(t, constraints.selectedMedia.Latency)
+		}
+		encoderBuilders, err = newAudioEncoderBuilders(r, constraints)
+	default:
+		err = errors.New("newTrack: invalid driver type")
+	}
+
+	if err != nil {
+		d.Close()
+		return nil, err
+	}
+
+	for _, builder := range encoderBuilders {
+		var matchedRTPCodec *webrtc.RTPCodec
+		for _, rtpCodec := range rtpCodecs {
+			if rtpCodec.Name == builder.name {
+				matchedRTPCodec = rtpCodec
+				break
+			}
+		}
+
+		if matchedRTPCodec == nil {
+			continue
+		}
+
+		localTrack, err := opts.trackGenerator(
+			matchedRTPCodec.PayloadType,
+			rand.Uint32(),
+			d.ID(),
+			matchedRTPCodec.Type.String(),
+			matchedRTPCodec,
+		)
+		if err != nil {
+			continue
+		}
+
+		encoder, err := builder.build()
+		if err != nil {
+			continue
+		}
+
+		t := track{
+			localTrack: localTrack,
+			sample:     buildSampler(localTrack),
+			d:          d,
+			encoder:    encoder,
+		}
+		go t.start()
+		return &t, nil
+	}
+
+	d.Close()
+	return nil, errors.New("newTrack: failed to find a matching codec")
 }
 
 // OnEnded sets an error handler. When a track has been created and started, if an
 // error occurs, handler will get called with the error given to the parameter.
-func (t *baseTrack) OnEnded(handler func(error)) {
+func (t *track) OnEnded(handler func(error)) {
 	t.mu.Lock()
 	t.onErrorHandler = handler
 	err := t.err
@@ -227,7 +134,7 @@ func (t *baseTrack) OnEnded(handler func(error)) {
 }
 
 // onError is a callback when an error occurs
-func (t *baseTrack) onError(err error) {
+func (t *track) onError(err error) {
 	t.mu.Lock()
 	t.err = err
 	handler := t.onErrorHandler
@@ -240,6 +147,92 @@ func (t *baseTrack) onError(err error) {
 	}
 }
 
-func (t *baseTrack) Stop() {
-	t.d.Close()
+// start starts the data flow from the driver all the way to the localTrack
+func (t *track) start() {
+	var n int
+	var err error
+	buff := make([]byte, 1024)
+	for {
+		n, err = t.encoder.Read(buff)
+		if err != nil {
+			if e, ok := err.(*mio.InsufficientBufferError); ok {
+				buff = make([]byte, 2*e.RequiredSize)
+				continue
+			}
+
+			t.onError(err)
+			return
+		}
+
+		if err := t.sample(buff[:n]); err != nil {
+			t.onError(err)
+			return
+		}
+	}
+}
+
+// Stop stops the underlying driver and encoder
+func (t *track) Stop() {
+	t.d.Close()
+	t.encoder.Close()
+}
+
+func (t *track) Track() *webrtc.Track {
+	return t.localTrack.(*webrtc.Track)
+}
+
+func (t *track) LocalTrack() LocalTrack {
+	return t.localTrack
+}
+
+// encoderBuilder is a generic encoder builder that acts as a delegator for codec.VideoEncoderBuilder and
+// codec.AudioEncoderBuilder. The idea of having a delegator is to reduce redundant codes that are being
+// duplicated for managing video and audio.
+type encoderBuilder struct {
+	name  string
+	build func() (codec.ReadCloser, error)
+}
+
+// newVideoEncoderBuilders transforms video given by VideoRecorder with the video transformer that is passed through
+// constraints and create a list of generic encoder builders
+func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
+	r, err := vr.VideoRecord(constraints.selectedMedia)
+	if err != nil {
+		return nil, err
+	}
+
+	if constraints.VideoTransform != nil {
+		r = constraints.VideoTransform(r)
+	}
+
+	encoderBuilders := make([]encoderBuilder, len(constraints.VideoEncoderBuilders))
+	for i, b := range constraints.VideoEncoderBuilders {
+		encoderBuilders[i].name = b.Name()
+		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
+			return b.BuildVideoEncoder(r, constraints.selectedMedia)
+		}
+	}
+	return encoderBuilders, nil
+}
+
+// newAudioEncoderBuilders transforms audio given by AudioRecorder with the audio transformer that is passed through
+// constraints and create a list of generic encoder builders
+func newAudioEncoderBuilders(ar driver.AudioRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
+	r, err := ar.AudioRecord(constraints.selectedMedia)
+	if err != nil {
+		return nil, err
+	}
+
+	if constraints.AudioTransform != nil {
+		r = constraints.AudioTransform(r)
+	}
+
+	encoderBuilders := make([]encoderBuilder, len(constraints.AudioEncoderBuilders))
+	for i, b := range constraints.AudioEncoderBuilders {
+		encoderBuilders[i].name = b.Name()
+		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
+			return b.BuildAudioEncoder(r, constraints.selectedMedia)
+		}
+	}
+	return encoderBuilders, nil
 }

track_test.go

@@ -10,7 +10,7 @@ func TestOnEnded(t *testing.T) {
 	errExpected := errors.New("an error")
 
 	t.Run("ErrorAfterRegister", func(t *testing.T) {
-		tr := &baseTrack{}
+		tr := &track{}
 
 		called := make(chan error, 1)
 		tr.OnEnded(func(error) {
@@ -35,7 +35,7 @@ func TestOnEnded(t *testing.T) {
 	})
 
 	t.Run("ErrorBeforeRegister", func(t *testing.T) {
-		tr := &baseTrack{}
+		tr := &track{}
 
 		tr.onError(errExpected)
 

webrtc.go

@@ -1,45 +0,0 @@
-package mediadevices
-
-import (
-	"github.com/pion/rtcp"
-	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2"
-)
-
-// == WebRTC v3 design ==
-
-// Reader is an interface to handle incoming RTP stream.
-type Reader interface {
-	ReadRTP() (*rtp.Packet, error)
-	WriteRTCP(rtcp.Packet) error
-}
-
-// TrackBase represents common MediaStreamTrack functionality of LocalTrack and RemoteTrack.
-type TrackBase interface {
-	ID() string
-}
-
-type LocalRTPTrack interface {
-	TrackBase
-	Reader
-
-	// SetParameters sets information about how the data is to be encoded.
-	// This will be called by PeerConnection according to the result of
-	// SDP based negotiation.
-	// It will be called via RTPSender.Parameters() by PeerConnection to
-	// tell the negotiated media codec information.
-	//
-	// This is pion's extension to process data without having encoder/decoder
-	// in webrtc package.
-	SetParameters(RTPParameters) error
-}
-
-// RTPParameters represents RTCRtpParameters which contains information about
-// how the RTC data is to be encoded/decoded.
-//
-// ref: https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSendParameters
-type RTPParameters struct {
-	SSRC          uint32
-	SelectedCodec *webrtc.RTPCodec
-	Codecs        []*webrtc.RTPCodec
-}