WIP

* Decouple encoding and webrtc from GetUserMedia
* Redesign VideoTrack and AudioTrack
* Remove mediadevices constructor
* Remove examples with old design
* Add a new simple example
* Remove Enable from constraints
* Implicitly disable a media when nil

.github/workflows/ci.yaml

@@ -8,12 +8,12 @@ on:
       - master
 
 jobs:
-  build:
+  build-linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         go: [ '1.14', '1.13' ]
-    name: Go ${{ matrix.go }}
+    name: Linux Go ${{ matrix.go }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -43,5 +43,37 @@ jobs:
         run: go test . pkg/... -v
         env:
           CGO_ENABLED: 0
-      #- name: golint
+  build-darwin:
-      #  run: go lint ./...
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        go: [ '1.14', '1.13' ]
+    name: Darwin Go ${{ matrix.go }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Install dependencies
+        run: |
+          brew install \
+            pkg-config \
+            opus \
+            libvpx \
+            x264
+      - name: go vet
+        run: go vet ./...
+      - name: go build
+        run: go build ./...
+      - name: go build without CGO
+        run: go build . pkg/...
+        env:
+          CGO_ENABLED: 0
+      - name: go test
+        run: go test ./... -v -race
+      - name: go test without CGO
+        run: go test . pkg/... -v
+        env:
+          CGO_ENABLED: 0

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Pion
+Copyright (c) 2019-2020 Pion
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -8,7 +8,7 @@ Go implementation of the [MediaDevices](https://developer.mozilla.org/en-US/docs
 
 | Interface  | Linux | Mac | Windows |
 | :--------: | :---: | :-: | :-----: |
-|   Camera   |  ✔️   | ✖️  |   ✔️    |
+|   Camera   |  ✔️   | ✔️  |   ✔️    |
 | Microphone |  ✔️   | ✖️  |   ✔️    |
 |   Screen   |  ✔️   | ✖️  |   ✖️    |
 
@@ -17,15 +17,15 @@ Go implementation of the [MediaDevices](https://developer.mozilla.org/en-US/docs
 |   OS    |                           Library/Interface                             |
 | :-----: | :---------------------------------------------------------------------: |
 |  Linux  |        [Video4Linux](https://en.wikipedia.org/wiki/Video4Linux)         |
-|   Mac   |                                  N/A                                    |
+|   Mac   |       [AVFoundation](https://developer.apple.com/av-foundation/)        |
 | Windows | [DirectShow](https://docs.microsoft.com/en-us/windows/win32/directshow) |
 
 |                     Pixel Format                      | Linux | Mac | Windows |
 | :---------------------------------------------------: | :---: | :-: | :-----: |
 | [YUY2](https://www.fourcc.org/pixel-format/yuv-yuy2/) |  ✔️   | ✖️  |   ✔️    |
-| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) |  ✔️   | ✖️  |   ✖️    |
+| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) |  ✔️   | ✔️  |   ✖️    |
 | [I420](https://www.fourcc.org/pixel-format/yuv-i420/) |  ✔️   | ✖️  |   ✖️    |
-| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) |  ✔️   | ✖️  |   ✖️    |
+| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) |  ✔️   | ✔️  |   ✖️    |
 |         [MJPEG](https://www.fourcc.org/mjpg/)         |  ✔️   | ✖️  |   ✖️    |
 
 ### Microphone

examples/facedetection/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download facedetection
-
-```
-go get github.com/pion/mediadevices/examples/facedetection
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/video) you should see two text-areas and a 'Start Session' button
-
-### Run facedetection with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | facedetection`
-
-### Input facedetection's SessionDescription into your browser
-
-Copy the text that `facedetection` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/facedetection/detector.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"image"
-	"image/color"
-	"image/draw"
-	"io/ioutil"
-	"log"
-
-	"github.com/disintegration/imaging"
-	pigo "github.com/esimov/pigo/core"
-)
-
-var (
-	cascade    []byte
-	err        error
-	classifier *pigo.Pigo
-)
-
-func imgToGrayscale(img image.Image) []uint8 {
-	bounds := img.Bounds()
-	flatten := bounds.Dy() * bounds.Dx()
-	grayImg := make([]uint8, flatten)
-
-	i := 0
-	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
-		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			pix := img.At(x, y)
-			grayPix := color.GrayModel.Convert(pix).(color.Gray)
-			grayImg[i] = grayPix.Y
-			i++
-		}
-	}
-	return grayImg
-}
-
-// clusterDetection runs Pigo face detector core methods
-// and returns a cluster with the detected faces coordinates.
-func clusterDetection(img image.Image) []pigo.Detection {
-	grayscale := imgToGrayscale(img)
-	bounds := img.Bounds()
-	cParams := pigo.CascadeParams{
-		MinSize:     100,
-		MaxSize:     600,
-		ShiftFactor: 0.15,
-		ScaleFactor: 1.1,
-		ImageParams: pigo.ImageParams{
-			Pixels: grayscale,
-			Rows:   bounds.Dy(),
-			Cols:   bounds.Dx(),
-			Dim:    bounds.Dx(),
-		},
-	}
-
-	if len(cascade) == 0 {
-		cascade, err = ioutil.ReadFile("facefinder")
-		if err != nil {
-			log.Fatalf("Error reading the cascade file: %s", err)
-		}
-		p := pigo.NewPigo()
-
-		// Unpack the binary file. This will return the number of cascade trees,
-		// the tree depth, the threshold and the prediction from tree's leaf nodes.
-		classifier, err = p.Unpack(cascade)
-		if err != nil {
-			log.Fatalf("Error unpacking the cascade file: %s", err)
-		}
-	}
-
-	// Run the classifier over the obtained leaf nodes and return the detection results.
-	// The result contains quadruplets representing the row, column, scale and detection score.
-	dets := classifier.RunCascade(cParams, 0.0)
-
-	// Calculate the intersection over union (IoU) of two clusters.
-	dets = classifier.ClusterDetections(dets, 0)
-
-	return dets
-}
-
-func drawCircle(img draw.Image, x0, y0, r int, c color.Color) {
-	x, y, dx, dy := r-1, 0, 1, 1
-	err := dx - (r * 2)
-
-	for x > y {
-		img.Set(x0+x, y0+y, c)
-		img.Set(x0+y, y0+x, c)
-		img.Set(x0-y, y0+x, c)
-		img.Set(x0-x, y0+y, c)
-		img.Set(x0-x, y0-y, c)
-		img.Set(x0-y, y0-x, c)
-		img.Set(x0+y, y0-x, c)
-		img.Set(x0+x, y0-y, c)
-
-		if err <= 0 {
-			y++
-			err += dy
-			dy += 2
-		}
-		if err > 0 {
-			x--
-			dx += 2
-			err += dx - (r * 2)
-		}
-	}
-}
-
-func markFaces(img image.Image) image.Image {
-	nrgba := imaging.Clone(img)
-	dets := clusterDetection(img)
-	for _, det := range dets {
-		if det.Q < 5.0 {
-			continue
-		}
-
-		drawCircle(nrgba, det.Col, det.Row, det.Scale/2, color.Black)
-	}
-	return nrgba
-}

examples/facedetection/main.go

@@ -1,119 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"image"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
-)
-
-func markFacesTransformer(r video.Reader) video.Reader {
-	return video.ReaderFunc(func() (img image.Image, err error) {
-		img, err = r.Read()
-		if err != nil {
-			return
-		}
-
-		img = markFaces(img)
-		return
-	})
-}
-
-func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormatExact(frame.FormatI420) // most of the encoder accepts I420
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoTransform = markFacesTransformer
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
-}

examples/rtp-send/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download rtp-send example
-
-```
-go get github.com/pion/mediadevices/examples/rtp-send
-```
-
-### Listen RTP
-
-Install GStreamer and run:
-```
-gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=VP8 \
-    ! rtpvp8depay ! vp8dec ! videoconvert ! autovideosink
-```
-
-Or run VLC media plyer:
-```
-vlc ./vp8.sdp
-```
-
-### Run rtp-send
-
-Run `rtp-send localhost:5000`
-
-A video should start playing in your GStreamer or VLC window.
-It's not WebRTC, but pure RTP.
-
-Congrats, you have used pion-MediaDevices! Now start building something cool

examples/rtp-send/main.go

@@ -1,120 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"net"
-	"os"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2"
-	"github.com/pion/webrtc/v2/pkg/media"
-)
-
-const (
-	mtu = 1000
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Printf("usage: %s host:port\n", os.Args[0])
-		return
-	}
-
-	md := mediadevices.NewMediaDevicesFromCodecs(
-		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
-				webrtc.NewRTPVP8Codec(100, 90000),
-			},
-		},
-		mediadevices.WithTrackGenerator(
-			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
-				mediadevices.LocalTrack, error,
-			) {
-				return newTrack(codec, id, os.Args[1]), nil
-			},
-		),
-	)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	select {}
-}
-
-type track struct {
-	codec      *webrtc.RTPCodec
-	packetizer rtp.Packetizer
-	id         string
-	conn       net.Conn
-}
-
-func newTrack(codec *webrtc.RTPCodec, id, dest string) *track {
-	addr, err := net.ResolveUDPAddr("udp", dest)
-	if err != nil {
-		panic(err)
-	}
-	conn, err := net.DialUDP("udp", nil, addr)
-	if err != nil {
-		panic(err)
-	}
-	return &track{
-		codec: codec,
-		packetizer: rtp.NewPacketizer(
-			mtu,
-			codec.PayloadType,
-			1,
-			codec.Payloader,
-			rtp.NewRandomSequencer(),
-			codec.ClockRate,
-		),
-		id:   id,
-		conn: conn,
-	}
-}
-
-func (t *track) WriteSample(s media.Sample) error {
-	buf := make([]byte, mtu)
-	pkts := t.packetizer.Packetize(s.Data, s.Samples)
-	for _, p := range pkts {
-		n, err := p.MarshalTo(buf)
-		if err != nil {
-			panic(err)
-		}
-		_, _ = t.conn.Write(buf[:n])
-	}
-	return nil
-}
-
-func (t *track) Codec() *webrtc.RTPCodec {
-	return t.codec
-}
-
-func (t *track) ID() string {
-	return t.id
-}
-
-func (t *track) Kind() webrtc.RTPCodecType {
-	return t.codec.Type
-}

examples/rtp-send/vp8.sdp

@@ -1,9 +0,0 @@
-v=0
-o=- 1234567890 1234567890 IN IP4 0.0.0.0
-s=RTP-Send Example
-i=Example
-c=IN IP4 0.0.0.0
-t=0 0
-a=recvonly
-m=video 5000 RTP/AVP 100
-a=rtpmap:100 VP8/90000

examples/screenshare/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download screenshare
-
-```
-go get github.com/pion/mediadevices/examples/screenshare
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
-
-### Run screenshare with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | screenshare`
-
-### Input screenshare's SessionDescription into your browser
-
-Copy the text that `screenshare` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/screenshare/main.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/webrtc/v2"
-)
-
-func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
-}

examples/simple/README.md

@@ -1,29 +1,17 @@
 ## Instructions
 
-### Download gstreamer-send
+### Download the example
 
 ```
 go get github.com/pion/mediadevices/examples/simple
 ```
 
-### Open example page
+### Run the sample
 
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
+```
+simple
+```
 
-### Run simple with your browsers SessionDescription as stdin
+### View yourself in the browser
 
-In the jsfiddle the top textarea is your browser, copy that and:
+Open your browser and go to "http://localhost:1313"
-
-#### Linux
-
-Run `echo $BROWSER_SDP | simple`
-
-### Input simple's SessionDescription into your browser
-
-Copy the text that `simple` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/simple/main.go

@@ -2,131 +2,68 @@ package main
 
 import (
 	"fmt"
+	"image/jpeg"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
 
 	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
-
-	// This is required to use opus audio encoder
-	"github.com/pion/mediadevices/pkg/codec/opus"
-
-	// If you don't like vpx, you can also use x264 by importing as below
-	// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
-	// or you can also use openh264 for alternative h264 implementation
-	// "github.com/pion/mediadevices/pkg/codec/openh264"
-	"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
 	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
-	// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
-)
-
-const (
-	videoCodecName = webrtc.VP8
 )
 
 func main() {
-	config := webrtc.Configuration{
+	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
-		ICEServers: []webrtc.ICEServer{
+		Video: func(constraint *mediadevices.MediaTrackConstraints) {
-			{
+			constraint.Width = prop.Int(600)
-				URLs: []string{"stun:stun.l.google.com:19302"},
+			constraint.Height = prop.Int(400)
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	opusParams, err := opus.NewParams()
-	if err != nil {
-		panic(err)
-	}
-	opusParams.BitRate = 32000 // 32kbps
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Audio: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
-		},
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
 		},
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	for _, tracker := range s.GetTracks() {
+	t := s.GetVideoTracks()[0]
-		t := tracker.Track()
+	defer t.Stop()
-		tracker.OnEnded(func(err error) {
+	videoTrack := t.(*mediadevices.VideoTrack)
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+
-				t.ID(), t.Label(), err)
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		videoReader := videoTrack.NewReader()
+		mimeWriter := multipart.NewWriter(w)
+
+		contentType := fmt.Sprintf("multipart/x-mixed-replace;boundary=%s", mimeWriter.Boundary())
+		w.Header().Add("Content-Type", contentType)
+
+		partHeader := make(textproto.MIMEHeader)
+		partHeader.Add("Content-Type", "image/jpeg")
+
+		for {
+			frame, err := videoReader.Read()
+			if err != nil {
+				if err == io.EOF {
+					return
+				}
+				panic(err)
+			}
+
+			partWriter, err := mimeWriter.CreatePart(partHeader)
+			if err != nil {
+				panic(err)
+			}
+
+			err = jpeg.Encode(partWriter, frame, nil)
+			if err != nil {
+				panic(err)
+			}
+		}
 	})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
 
-	// Set the remote SessionDescription
+	fmt.Println("listening on http://localhost:1313")
-	err = peerConnection.SetRemoteDescription(offer)
+	log.Println(http.ListenAndServe("localhost:1313", nil))
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
 }

go.mod

@@ -4,11 +4,10 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/faiface/beep v1.0.2
+	github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214
-	github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659
 	github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4
-	github.com/pion/webrtc/v2 v2.2.14
+	github.com/pion/webrtc/v2 v2.2.23
 	github.com/satori/go.uuid v1.2.0
-	golang.org/x/image v0.0.0-20200430140353-33d19683fad8
+	golang.org/x/image v0.0.0-20200801110659-972c09e46d76
-	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3
+	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
 )

go.sum

@@ -5,32 +5,18 @@ github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wX
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/faiface/beep v1.0.2 h1:UB5DiRNmA4erfUYnHbgU4UB6DlBOrsdEFRtcc8sCkdQ=
-github.com/faiface/beep v1.0.2/go.mod h1:1yLb5yRdHMsovYYWVqYLioXkVuziCSITW1oarTeduQM=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.1.1/go.mod h1:K1udHkiR3cOtlpKG5tZPD5XxrF7v2y7lDq7Whcj+xkQ=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
-github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherwasm v0.1.1/go.mod h1:kx4n9a+MzHH0BJJhvlsQ65hqLFXDO/m256AsaDPQ+/4=
-github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y=
-github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI=
-github.com/hajimehoshi/go-mp3 v0.1.1/go.mod h1:4i+c5pDNKDrxl1iu9iG90/+fhP37lio6gNhjCx9WBJw=
-github.com/hajimehoshi/oto v0.1.1/go.mod h1:hUiLWeBQnbDu4pZsAhOnGqMI1ZGibS6e2qhQdfpwz04=
-github.com/hajimehoshi/oto v0.3.1 h1:cpf/uIv4Q0oc5uf9loQn7PIehv+mZerh+0KKma6gzMk=
-github.com/hajimehoshi/oto v0.3.1/go.mod h1:e9eTLBB9iZto045HLbzfHJIc+jP3xaKrjZTghvb6fdM=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jfreymuth/oggvorbis v1.0.0/go.mod h1:abe6F9QRjuU9l+2jek3gj46lu40N4qlYxh2grqkLEDM=
+github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214 h1:2xVJKIumEUWeV3vczQwn61SHjNZ94Bwk+4CTjmcePxk=
-github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659 h1:DRA4BuRlhEILiud720WFWqqdADPzp1jTjQvyCr/PP80=
+github.com/jfreymuth/pulse v0.0.0-20200804114219-7d61c4938214/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
-github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
-github.com/jfreymuth/vorbis v1.0.0/go.mod h1:8zy3lUAm9K/rJJk223RKy6vjCZTWC61NA2QD06bfOE0=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -40,51 +26,55 @@ github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4 h1:2ydMA2KbxRkYmIw
 github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
 github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
 github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/lucasb-eyer/go-colorful v0.0.0-20181028223441-12d3b2882a08/go.mod h1:NXg0ArsFk0Y01623LgUqoqcouGDB+PwCCQlrwrG6xJ4=
 github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mewkiz/flac v1.0.5/go.mod h1:EHZNU32dMF6alpurYyKHDLYpW1lYpBZ5WrXi/VuNIGs=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pion/datachannel v1.4.17 h1:8CChK5VrJoGrwKCysoTscoWvshCAFpUkgY11Tqgz5hE=
+github.com/pion/datachannel v1.4.19 h1:IcOmm5fdDzJVCMgFYDCMtFC+lrjG78KcMYXH+gOo6ys=
-github.com/pion/datachannel v1.4.17/go.mod h1:+vPQfypU9vSsyPXogYj1hBThWQ6MNXEQoQAzxoPvjYM=
+github.com/pion/datachannel v1.4.19/go.mod h1:JzKF/zzeWgkOYwQ+KFb8JzbrUt8s63um+Qunu8VqTyw=
-github.com/pion/dtls/v2 v2.0.0 h1:Fk+MBhLZ/U1bImzAhmzwbO/pP2rKhtTw8iA934H3ybE=
+github.com/pion/dtls/v2 v2.0.1 h1:ddE7+V0faYRbyh4uPsRZ2vLdRrjVZn+wmCfI7jlBfaA=
-github.com/pion/dtls/v2 v2.0.0/go.mod h1:VkY5VL2wtsQQOG60xQ4lkV5pdn0wwBBTzCfRJqXhp3A=
+github.com/pion/dtls/v2 v2.0.1/go.mod h1:uMQkz2W0cSqY00xav7WByQ4Hb+18xeQh2oH2fRezr5U=
-github.com/pion/ice v0.7.15 h1:s1In+gnuyVq7WKWGVQL+1p+OcrMsbfL+VfSe2isH8Ag=
+github.com/pion/dtls/v2 v2.0.2 h1:FHCHTiM182Y8e15aFTiORroiATUI16ryHiQh8AIOJ1E=
-github.com/pion/ice v0.7.15/go.mod h1:Z6zybEQgky5mZkKcLfmvc266JukK2srz3VZBBD1iXBw=
+github.com/pion/dtls/v2 v2.0.2/go.mod h1:27PEO3MDdaCfo21heT59/vsdmZc0zMt9wQPcSlLu/1I=
+github.com/pion/ice v0.7.18 h1:KbAWlzWRUdX9SmehBh3gYpIFsirjhSQsCw6K2MjYMK0=
+github.com/pion/ice v0.7.18/go.mod h1:+Bvnm3nYC6Nnp7VV6glUkuOfToB/AtMRZpOU8ihuf4c=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
 github.com/pion/mdns v0.0.4 h1:O4vvVqr4DGX63vzmO6Fw9vpy3lfztVWHGCQfyw0ZLSY=
 github.com/pion/mdns v0.0.4/go.mod h1:R1sL0p50l42S5lJs91oNdUL58nm0QHrhxnSegr++qC0=
 github.com/pion/quic v0.1.1 h1:D951FV+TOqI9A0rTF7tHx0Loooqz+nyzjEyj8o3PuMA=
 github.com/pion/quic v0.1.1/go.mod h1:zEU51v7ru8Mp4AUBJvj6psrSth5eEFNnVQK5K48oV3k=
-github.com/pion/rtcp v1.2.1 h1:S3yG4KpYAiSmBVqKAfgRa5JdwBNj4zK3RLUa8JYdhak=
+github.com/pion/randutil v0.0.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.1/go.mod h1:a5dj2d6BKIKHl43EnAOIrCczcjESrtPuMgfmL6/K6QM=
+github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
-github.com/pion/rtp v1.5.4 h1:PuNg6xqV3brIUihatcKZj1YDUs+M45L0ZbrZWYtkDxY=
+github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtp v1.5.4/go.mod h1:bg60AL5GotNOlYZsqycbhDtEV3TkfbpXG0KBiUq29Mg=
+github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
-github.com/pion/sctp v1.7.6 h1:8qZTdJtbKfAns/Hv5L0PAj8FyXcsKhMH1pKUCGisQg4=
+github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
-github.com/pion/sctp v1.7.6/go.mod h1:ichkYQ5tlgCQwEwvgfdcAolqx1nHbYCxo4D7zK/K0X8=
+github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
-github.com/pion/sdp/v2 v2.3.7 h1:WUZHI3pfiYCaE8UGUYcabk863LCK+Bq3AklV5O0oInQ=
+github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
-github.com/pion/sdp/v2 v2.3.7/go.mod h1:+ZZf35r1+zbaWYiZLfPutWfx58DAWcGb2QsS3D/s9M8=
+github.com/pion/sctp v1.7.8 h1:tEWel2BKXLZitU+LxY3GDeQXoKeTafYasiu/X+XBKNM=
-github.com/pion/srtp v1.3.3 h1:8bjs9YaSNvSrbH0OfKxzPX+PTrCyAC2LoT9Qesugi+U=
+github.com/pion/sctp v1.7.8/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
-github.com/pion/srtp v1.3.3/go.mod h1:jNe0jmIOqksuurR9S/7yoKDalfPeluUFrNPCBqI4FOI=
+github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=
-github.com/pion/stun v0.3.3 h1:brYuPl9bN9w/VM7OdNzRSLoqsnwlyNvD9MVeJrHjDQw=
+github.com/pion/sdp/v2 v2.4.0/go.mod h1:L2LxrOpSTJbAns244vfPChbciR/ReU1KWfG04OpkR7E=
-github.com/pion/stun v0.3.3/go.mod h1:xrCld6XM+6GWDZdvjPlLMsTU21rNxnO6UO8XsAvHr/M=
+github.com/pion/srtp v1.5.1 h1:9Q3jAfslYZBt+C69SI/ZcONJh9049JUHZWYRRf5KEKw=
+github.com/pion/srtp v1.5.1/go.mod h1:B+QgX5xPeQTNc1CJStJPHzOlHK66ViMDWTT0HZTCkcA=
+github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
+github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
 github.com/pion/transport v0.6.0/go.mod h1:iWZ07doqOosSLMhZ+FXUTq+TamDoXSllxpbGcfkCmbE=
 github.com/pion/transport v0.8.10 h1:lTiobMEw2PG6BH/mgIVqTV2mBp/mPT+IJLaN8ZxgdHk=
 github.com/pion/transport v0.8.10/go.mod h1:tBmha/UCjpum5hqTWhfAEs3CO4/tHSg0MYRhSzR+CZ8=
 github.com/pion/transport v0.10.0 h1:9M12BSneJm6ggGhJyWpDveFOstJsTiQjkLf4M44rm80=
 github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE=
-github.com/pion/turn/v2 v2.0.3 h1:SJUUIbcPoehlyZgMyIUbBBDhI03sBx32x3JuSIBKBWA=
+github.com/pion/transport v0.10.1 h1:2W+yJT+0mOQ160ThZYUx5Zp2skzshiNgxrNE9GUfhJM=
-github.com/pion/turn/v2 v2.0.3/go.mod h1:kl1hmT3NxcLynpXVnwJgObL8C9NaCyPTeqI2DcCpSZs=
+github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
-github.com/pion/webrtc/v2 v2.2.14 h1:bRjnXTqMDJ3VERPF45z439Sv6QfDfjdYvdQk1QcIx8M=
+github.com/pion/turn/v2 v2.0.4 h1:oDguhEv2L/4rxwbL9clGLgtzQPjtuZwCdoM7Te8vQVk=
-github.com/pion/webrtc/v2 v2.2.14/go.mod h1:G+8lShCMbHhjpMF1ZJBkyuvrxXrvW4bxs3nOt+mJ2UI=
+github.com/pion/turn/v2 v2.0.4/go.mod h1:1812p4DcGVbYVBTiraUmP50XoKye++AMkbfp+N27mog=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
+github.com/pion/webrtc/v2 v2.2.23 h1:rZdOC95fwUCoQFVjHooPAayx/vhs3SLHFz8J/iRkAuk=
+github.com/pion/webrtc/v2 v2.2.23/go.mod h1:1lN/3EcATkQxc7GJSQbISCGC2l64Xu2VSLpwEG3c/tM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -98,41 +88,43 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
+golang.org/x/crypto v0.0.0-20200602180216-279210d13fed h1:g4KENRiCMEx58Q7/ecwfT0N2o8z35Fnbsjig/Alf2T4=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20180710024300-14dda7b62fcd h1:nLIcFw7GiqKXUS7HiChg6OAYWgASB2H97dZKd1GhDSs=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
-golang.org/x/exp v0.0.0-20180710024300-14dda7b62fcd/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
-golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/image v0.0.0-20200430140353-33d19683fad8 h1:6WW6V3x1P/jokJBpRQYUJnMHRP6isStQwCozxnU7XQw=
+golang.org/x/image v0.0.0-20200801110659-972c09e46d76 h1:U7GPaoQyQmX+CBRWXKrvRzWTbd+slqeSh8uARsIyhAw=
-golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200801110659-972c09e46d76/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/mobile v0.0.0-20180806140643-507816974b79 h1:t2JRgCWkY7Qaa1J2jal+wqC9OjbyHCHwIA9rVlRUSMo=
-golang.org/x/mobile v0.0.0-20180806140643-507816974b79/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
+golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 h1:sIky/MyNRSHTrdxfsiUSS4WIAMvInbeXljJz+jDjeYE=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -143,3 +135,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

mediadevices.go

@@ -6,106 +6,37 @@ import (
 
 	"github.com/pion/mediadevices/pkg/driver"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
 )
 
 var errNotFound = fmt.Errorf("failed to find the best driver that fits the constraints")
 
-// MediaDevices is an interface that's defined on https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices
-type MediaDevices interface {
-	GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	EnumerateDevices() []MediaDeviceInfo
-}
-
-// NewMediaDevices creates MediaDevices interface that provides access to connected media input devices
-// like cameras and microphones, as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevices(pc *webrtc.PeerConnection, opts ...MediaDevicesOption) MediaDevices {
-	codecs := make(map[webrtc.RTPCodecType][]*webrtc.RTPCodec)
-	for _, kind := range []webrtc.RTPCodecType{
-		webrtc.RTPCodecTypeAudio,
-		webrtc.RTPCodecTypeVideo,
-	} {
-		codecs[kind] = pc.GetRegisteredRTPCodecs(kind)
-	}
-	return NewMediaDevicesFromCodecs(codecs, opts...)
-}
-
-// NewMediaDevicesFromCodecs creates MediaDevices interface from lists of the available codecs
-// that provides access to connected media input devices like cameras and microphones,
-// as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevicesFromCodecs(codecs map[webrtc.RTPCodecType][]*webrtc.RTPCodec, opts ...MediaDevicesOption) MediaDevices {
-	mdo := MediaDevicesOptions{
-		codecs:         codecs,
-		trackGenerator: defaultTrackGenerator,
-	}
-	for _, o := range opts {
-		o(&mdo)
-	}
-	return &mediaDevices{
-		MediaDevicesOptions: mdo,
-	}
-}
-
-// TrackGenerator is a function to create new track.
-type TrackGenerator func(payloadType uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error)
-
-var defaultTrackGenerator = TrackGenerator(func(pt uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error) {
-	return webrtc.NewTrack(pt, ssrc, id, label, codec)
-})
-
-type mediaDevices struct {
-	MediaDevicesOptions
-}
-
-// MediaDevicesOptions stores parameters used by MediaDevices.
-type MediaDevicesOptions struct {
-	codecs         map[webrtc.RTPCodecType][]*webrtc.RTPCodec
-	trackGenerator TrackGenerator
-}
-
-// MediaDevicesOption is a type of MediaDevices functional option.
-type MediaDevicesOption func(*MediaDevicesOptions)
-
-// WithTrackGenerator specifies a TrackGenerator to use customized track.
-func WithTrackGenerator(gen TrackGenerator) MediaDevicesOption {
-	return func(o *MediaDevicesOptions) {
-		o.trackGenerator = gen
-	}
-}
-
 // GetDisplayMedia prompts the user to select and grant permission to capture the contents
 // of a display or portion thereof (such as a window) as a MediaStream.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia
-func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	trackers := make([]Tracker, 0)
+	tracks := make([]Track, 0)
 
-	cleanTrackers := func() {
+	cleanTracks := func() {
-		for _, t := range trackers {
+		for _, t := range tracks {
 			t.Stop()
 		}
 	}
 
-	var videoConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		constraints.Video(&videoConstraints)
+		var p MediaTrackConstraints
-	}
+		constraints.Video(&p)
-
+		track, err := selectScreen(p)
-	if videoConstraints.Enabled {
-		tracker, err := m.selectScreen(videoConstraints)
 		if err != nil {
-			cleanTrackers()
+			cleanTracks()
 			return nil, err
 		}
 
-		trackers = append(trackers, tracker)
+		tracks = append(tracks, track)
 	}
 
-	s, err := NewMediaStream(trackers...)
+	s, err := NewMediaStream(tracks...)
 	if err != nil {
-		cleanTrackers()
+		cleanTracks()
 		return nil, err
 	}
 
@@ -115,48 +46,42 @@ func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (Medi
 // GetUserMedia prompts the user for permission to use a media input which produces a MediaStream
 // with tracks containing the requested types of media.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
-func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+func GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	// TODO: It should return media stream based on constraints
+	tracks := make([]Track, 0)
-	trackers := make([]Tracker, 0)
 
-	cleanTrackers := func() {
+	cleanTracks := func() {
-		for _, t := range trackers {
+		for _, t := range tracks {
 			t.Stop()
 		}
 	}
 
-	var videoConstraints, audioConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		constraints.Video(&videoConstraints)
+		var p MediaTrackConstraints
+		constraints.Video(&p)
+		track, err := selectVideo(p)
+		if err != nil {
+			cleanTracks()
+			return nil, err
+		}
+
+		tracks = append(tracks, track)
 	}
 
 	if constraints.Audio != nil {
-		constraints.Audio(&audioConstraints)
+		var p MediaTrackConstraints
-	}
+		constraints.Audio(&p)
-
+		track, err := selectAudio(p)
-	if videoConstraints.Enabled {
-		tracker, err := m.selectVideo(videoConstraints)
 		if err != nil {
-			cleanTrackers()
+			cleanTracks()
 			return nil, err
 		}
 
-		trackers = append(trackers, tracker)
+		tracks = append(tracks, track)
 	}
 
-	if audioConstraints.Enabled {
+	s, err := NewMediaStream(tracks...)
-		tracker, err := m.selectAudio(audioConstraints)
 	if err != nil {
-			cleanTrackers()
+		cleanTracks()
-			return nil, err
-		}
-
-		trackers = append(trackers, tracker)
-	}
-
-	s, err := NewMediaStream(trackers...)
-	if err != nil {
-		cleanTrackers()
 		return nil, err
 	}
 
@@ -217,12 +142,13 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 		return nil, MediaTrackConstraints{}, errNotFound
 	}
 
-	constraints.selectedMedia = bestProp
+	constraints.selectedMedia = prop.Media{}
-	constraints.selectedMedia.Merge(constraints.MediaConstraints)
+	constraints.selectedMedia.MergeConstraints(constraints.MediaConstraints)
+	constraints.selectedMedia.Merge(bestProp)
 	return bestDriver, constraints, nil
 }
 
-func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker, error) {
+func selectAudio(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterAudioRecorder()
 
 	d, c, err := selectBestDriver(typeFilter, constraints)
@@ -230,9 +156,10 @@ func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newAudioTrack(d, c)
 }
-func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker, error) {
+
+func selectVideo(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	notScreenFilter := driver.FilterNot(driver.FilterDeviceType(driver.Screen))
 	filter := driver.FilterAnd(typeFilter, notScreenFilter)
@@ -242,10 +169,10 @@ func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newVideoTrack(d, c)
 }
 
-func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker, error) {
+func selectScreen(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	screenFilter := driver.FilterDeviceType(driver.Screen)
 	filter := driver.FilterAnd(typeFilter, screenFilter)
@@ -255,10 +182,10 @@ func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newVideoTrack(d, c)
 }
 
-func (m *mediaDevices) EnumerateDevices() []MediaDeviceInfo {
+func EnumerateDevices() []MediaDeviceInfo {
 	drivers := driver.GetManager().Query(
 		driver.FilterFn(func(driver.Driver) bool { return true }))
 	info := make([]MediaDeviceInfo, 0, len(drivers))

mediadevices_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pion/webrtc/v2/pkg/media"
 
 	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/driver"
 	_ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/videotest"
 	"github.com/pion/mediadevices/pkg/io/audio"
@@ -18,25 +19,32 @@ import (
 )
 
 func TestGetUserMedia(t *testing.T) {
-	videoParams := mockParams{
+	brokenVideoParams := mockParams{
-		BaseParams: codec.BaseParams{
-			BitRate: 100000,
-		},
 		name: "MockVideo",
 	}
+	videoParams := brokenVideoParams
+	videoParams.BitRate = 100000
 	audioParams := mockParams{
 		BaseParams: codec.BaseParams{
 			BitRate: 32000,
 		},
 		name: "MockAudio",
 	}
+	constraints := MediaStreamConstraints{
+		Video: func(p *prop.Media) {
+			p.Width = 640
+			p.Height = 480
+		},
+		Audio: func(p *prop.Media) {},
+	}
+
 	md := NewMediaDevicesFromCodecs(
 		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
+			webrtc.RTPCodecTypeVideo: {
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
+				{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
 			},
-			webrtc.RTPCodecTypeAudio: []*webrtc.RTPCodec{
+			webrtc.RTPCodecTypeAudio: {
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
+				{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
 			},
 		},
 		WithTrackGenerator(
@@ -46,7 +54,10 @@ func TestGetUserMedia(t *testing.T) {
 				return newMockTrack(codec, id), nil
 			},
 		),
+		WithVideoEncoders(&brokenVideoParams),
+		WithAudioEncoders(&audioParams),
 	)
+<<<<<<< HEAD
 	constraints := MediaStreamConstraints{
 		Video: func(c *MediaTrackConstraints) {
 			c.Enabled = true
@@ -76,13 +87,35 @@ func TestGetUserMedia(t *testing.T) {
 			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
 		},
 	}
+=======
+>>>>>>> ccd7985... Redesign GetUserMedia API
 
 	// GetUserMedia with broken parameters
-	ms, err := md.GetUserMedia(constraintsWrong)
+	ms, err := md.GetUserMedia(constraints)
 	if err == nil {
 		t.Fatal("Expected error, but got nil")
 	}
 
+	md = NewMediaDevicesFromCodecs(
+		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
+			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
+				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
+			},
+			webrtc.RTPCodecTypeAudio: []*webrtc.RTPCodec{
+				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
+			},
+		},
+		WithTrackGenerator(
+			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
+				LocalTrack, error,
+			) {
+				return newMockTrack(codec, id), nil
+			},
+		),
+		WithVideoEncoders(&videoParams),
+		WithAudioEncoders(&audioParams),
+	)
+
 	// GetUserMedia with correct parameters
 	ms, err = md.GetUserMedia(constraints)
 	if err != nil {
@@ -122,6 +155,9 @@ func TestGetUserMedia(t *testing.T) {
 		})
 	}
 	time.Sleep(50 * time.Millisecond)
+	for _, track := range tracks {
+		track.Stop()
+	}
 }
 
 type mockTrack struct {
@@ -217,3 +253,56 @@ func (m *mockAudioCodec) Read(b []byte) (int, error) {
 	return len(b), nil
 }
 func (m *mockAudioCodec) Close() error { return nil }
+
+func TestSelectBestDriverConstraintsResultIsSetProperly(t *testing.T) {
+	filterFn := driver.FilterVideoRecorder()
+	drivers := driver.GetManager().Query(filterFn)
+	if len(drivers) == 0 {
+		t.Fatal("expect to get at least 1 driver")
+	}
+
+	driver := drivers[0]
+	err := driver.Open()
+	if err != nil {
+		t.Fatal("expect to open driver successfully")
+	}
+	defer driver.Close()
+
+	if len(driver.Properties()) == 0 {
+		t.Fatal("expect to get at least 1 property")
+	}
+	expectedProp := driver.Properties()[0]
+	// Since this is a continuous value, bestConstraints should be set with the value that user specified
+	expectedProp.FrameRate = 30.0
+
+	wantConstraints := MediaTrackConstraints{
+		MediaConstraints: prop.MediaConstraints{
+			VideoConstraints: prop.VideoConstraints{
+				// By reducing the width from the driver by a tiny amount, this property should be chosen.
+				// At the same time, we'll be able to find out if the return constraints will be properly set
+				// to the best constraints.
+				Width:       prop.Int(expectedProp.Width - 1),
+				Height:      prop.Int(expectedProp.Width),
+				FrameFormat: prop.FrameFormat(expectedProp.FrameFormat),
+				FrameRate:   prop.Float(30.0),
+			},
+		},
+	}
+
+	bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if driver != bestDriver {
+		t.Fatal("best driver is not expected")
+	}
+
+	s := bestConstraints.selectedMedia
+	if s.Width != expectedProp.Width ||
+		s.Height != expectedProp.Height ||
+		s.FrameFormat != expectedProp.FrameFormat ||
+		s.FrameRate != expectedProp.FrameRate {
+		t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
+	}
+}

mediastream.go

@@ -9,19 +9,19 @@ import (
 // MediaStream is an interface that represents a collection of existing tracks.
 type MediaStream interface {
 	// GetAudioTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
-	GetAudioTracks() []Tracker
+	GetAudioTracks() []Track
 	// GetVideoTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
-	GetVideoTracks() []Tracker
+	GetVideoTracks() []Track
 	// GetTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
-	GetTracks() []Tracker
+	GetTracks() []Track
 	// AddTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
-	AddTrack(t Tracker)
+	AddTrack(t Track)
 	// RemoveTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
-	RemoveTrack(t Tracker)
+	RemoveTrack(t Track)
 }
 
 type mediaStream struct {
-	trackers map[string]Tracker
+	tracks map[string]Track
 	l      sync.RWMutex
 }
 
@@ -29,62 +29,62 @@ const rtpCodecTypeDefault webrtc.RTPCodecType = 0
 
 // NewMediaStream creates a MediaStream interface that's defined in
 // https://w3c.github.io/mediacapture-main/#dom-mediastream
-func NewMediaStream(trackers ...Tracker) (MediaStream, error) {
+func NewMediaStream(tracks ...Track) (MediaStream, error) {
-	m := mediaStream{trackers: make(map[string]Tracker)}
+	m := mediaStream{tracks: make(map[string]Track)}
 
-	for _, tracker := range trackers {
+	for _, track := range tracks {
-		id := tracker.LocalTrack().ID()
+		id := track.ID()
-		if _, ok := m.trackers[id]; !ok {
+		if _, ok := m.tracks[id]; !ok {
-			m.trackers[id] = tracker
+			m.tracks[id] = track
 		}
 	}
 
 	return &m, nil
 }
 
-func (m *mediaStream) GetAudioTracks() []Tracker {
+func (m *mediaStream) GetAudioTracks() []Track {
-	return m.queryTracks(webrtc.RTPCodecTypeAudio)
+	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindAudio })
 }
 
-func (m *mediaStream) GetVideoTracks() []Tracker {
+func (m *mediaStream) GetVideoTracks() []Track {
-	return m.queryTracks(webrtc.RTPCodecTypeVideo)
+	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindVideo })
 }
 
-func (m *mediaStream) GetTracks() []Tracker {
+func (m *mediaStream) GetTracks() []Track {
-	return m.queryTracks(rtpCodecTypeDefault)
+	return m.queryTracks(func(t Track) bool { return true })
 }
 
 // queryTracks returns all tracks that are the same kind as t.
 // If t is 0, which is the default, queryTracks will return all the tracks.
-func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Tracker {
+func (m *mediaStream) queryTracks(filter func(track Track) bool) []Track {
 	m.l.RLock()
 	defer m.l.RUnlock()
 
-	result := make([]Tracker, 0)
+	result := make([]Track, 0)
-	for _, tracker := range m.trackers {
+	for _, track := range m.tracks {
-		if tracker.LocalTrack().Kind() == t || t == rtpCodecTypeDefault {
+		if filter(track) {
-			result = append(result, tracker)
+			result = append(result, track)
 		}
 	}
 
 	return result
 }
 
-func (m *mediaStream) AddTrack(t Tracker) {
+func (m *mediaStream) AddTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	id := t.LocalTrack().ID()
+	id := t.ID()
-	if _, ok := m.trackers[id]; ok {
+	if _, ok := m.tracks[id]; ok {
 		return
 	}
 
-	m.trackers[id] = t
+	m.tracks[id] = t
 }
 
-func (m *mediaStream) RemoveTrack(t Tracker) {
+func (m *mediaStream) RemoveTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	delete(m.trackers, t.LocalTrack().ID())
+	delete(m.tracks, t.ID())
 }

mediastreamconstraints.go

@@ -1,9 +1,6 @@
 package mediadevices
 
 import (
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -15,26 +12,6 @@ type MediaStreamConstraints struct {
 // MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
 type MediaTrackConstraints struct {
 	prop.MediaConstraints
-	Enabled bool
-	// VideoEncoderBuilders are codec builders that are used for encoding the video
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	VideoEncoderBuilders []codec.VideoEncoderBuilder
-	// AudioEncoderBuilders are codec builders that are used for encoding the audio
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	AudioEncoderBuilders []codec.AudioEncoderBuilder
-	// VideoTransform will be used to transform the video that's coming from the driver.
-	// So, basically it'll look like following: driver -> VideoTransform -> codec
-	VideoTransform video.TransformFunc
-	// AudioTransform will be used to transform the audio that's coming from the driver.
-	// So, basically it'll look like following: driver -> AudioTransform -> code
-	AudioTransform audio.TransformFunc
-
 	selectedMedia prop.Media
 }
 

pkg/avfoundation/.gitignore

@@ -0,0 +1,25 @@
+#User settings
+xcuserdata/
+
+## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
+*.xcscmblueprint
+*.xccheckout
+
+## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
+build/
+DerivedData/
+*.moved-aside
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+
+## Gcc Patch
+/*.gcno
+.DS_STORE
+
+Build/

pkg/avfoundation/AVFoundationBind.xcodeproj/project.pbxproj

@@ -0,0 +1,294 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		F0143CC12479F78E00EC29C9 /* AVFoundationBind.h in Headers */ = {isa = PBXBuildFile; fileRef = F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */; };
+		F0143CC32479F78E00EC29C9 /* AVFoundationBind.m in Sources */ = {isa = PBXBuildFile; fileRef = F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */; settings = {COMPILER_FLAGS = "-fno-objc-arc"; }; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libAVFoundationBind.a; sourceTree = BUILT_PRODUCTS_DIR; };
+		F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AVFoundationBind.h; sourceTree = "<group>"; };
+		F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AVFoundationBind.m; sourceTree = "<group>"; };
+		F0FDDA0B247E15D900A3429D /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		F0143CBB2479F78E00EC29C9 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		F0143CB42479F78E00EC29C9 = {
+			isa = PBXGroup;
+			children = (
+				F0143CBF2479F78E00EC29C9 /* AVFoundationBind */,
+				F0143CBE2479F78E00EC29C9 /* Products */,
+				F0FDDA0A247E15D900A3429D /* Frameworks */,
+			);
+			sourceTree = "<group>";
+		};
+		F0143CBE2479F78E00EC29C9 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		F0143CBF2479F78E00EC29C9 /* AVFoundationBind */ = {
+			isa = PBXGroup;
+			children = (
+				F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */,
+				F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */,
+			);
+			path = AVFoundationBind;
+			sourceTree = "<group>";
+		};
+		F0FDDA0A247E15D900A3429D /* Frameworks */ = {
+			isa = PBXGroup;
+			children = (
+				F0FDDA0B247E15D900A3429D /* AVFoundation.framework */,
+			);
+			name = Frameworks;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXHeadersBuildPhase section */
+		F0143CB92479F78E00EC29C9 /* Headers */ = {
+			isa = PBXHeadersBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				F0143CC12479F78E00EC29C9 /* AVFoundationBind.h in Headers */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXHeadersBuildPhase section */
+
+/* Begin PBXNativeTarget section */
+		F0143CBC2479F78E00EC29C9 /* AVFoundationBind */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = F0143CC62479F78E00EC29C9 /* Build configuration list for PBXNativeTarget "AVFoundationBind" */;
+			buildPhases = (
+				F0143CB92479F78E00EC29C9 /* Headers */,
+				F0143CBA2479F78E00EC29C9 /* Sources */,
+				F0143CBB2479F78E00EC29C9 /* Frameworks */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = AVFoundationBind;
+			productName = AVFoundationBind;
+			productReference = F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */;
+			productType = "com.apple.product-type.library.static";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		F0143CB52479F78E00EC29C9 /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastUpgradeCheck = 1150;
+				ORGANIZATIONNAME = "Herman, Lukas";
+				TargetAttributes = {
+					F0143CBC2479F78E00EC29C9 = {
+						CreatedOnToolsVersion = 11.5;
+					};
+				};
+			};
+			buildConfigurationList = F0143CB82479F78E00EC29C9 /* Build configuration list for PBXProject "AVFoundationBind" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = F0143CB42479F78E00EC29C9;
+			productRefGroup = F0143CBE2479F78E00EC29C9 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				F0143CBC2479F78E00EC29C9 /* AVFoundationBind */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXSourcesBuildPhase section */
+		F0143CBA2479F78E00EC29C9 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				F0143CC32479F78E00EC29C9 /* AVFoundationBind.m in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin XCBuildConfiguration section */
+		F0143CC42479F78E00EC29C9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
+				MTL_FAST_MATH = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = macosx;
+			};
+			name = Debug;
+		};
+		F0143CC52479F78E00EC29C9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				MTL_FAST_MATH = YES;
+				SDKROOT = macosx;
+			};
+			name = Release;
+		};
+		F0143CC72479F78E00EC29C9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CODE_SIGN_STYLE = Automatic;
+				EXECUTABLE_PREFIX = lib;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SKIP_INSTALL = YES;
+			};
+			name = Debug;
+		};
+		F0143CC82479F78E00EC29C9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CODE_SIGN_STYLE = Automatic;
+				EXECUTABLE_PREFIX = lib;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SKIP_INSTALL = YES;
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		F0143CB82479F78E00EC29C9 /* Build configuration list for PBXProject "AVFoundationBind" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				F0143CC42479F78E00EC29C9 /* Debug */,
+				F0143CC52479F78E00EC29C9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		F0143CC62479F78E00EC29C9 /* Build configuration list for PBXNativeTarget "AVFoundationBind" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				F0143CC72479F78E00EC29C9 /* Debug */,
+				F0143CC82479F78E00EC29C9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = F0143CB52479F78E00EC29C9 /* Project object */;
+}

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:AVFoundationBind.xcodeproj">
+   </FileRef>
+</Workspace>

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>PreviewsEnabled</key>
+	<false/>
+</dict>
+</plist>

pkg/avfoundation/AVFoundationBind.xcodeproj/xcshareddata/xcschemes/AVFoundationBind.xcscheme

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1150"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "F0143CBC2479F78E00EC29C9"
+               BuildableName = "libAVFoundationBind.a"
+               BlueprintName = "AVFoundationBind"
+               ReferencedContainer = "container:AVFoundationBind.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Release"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "F0143CBC2479F78E00EC29C9"
+            BuildableName = "libAVFoundationBind.a"
+            BlueprintName = "AVFoundationBind"
+            ReferencedContainer = "container:AVFoundationBind.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

pkg/avfoundation/AVFoundationBind/AVFoundationBind.h

@@ -0,0 +1,78 @@
+// MIT License
+// 
+// Copyright (c) 2019-2020 Pion
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <stddef.h>
+
+#define MAX_DEVICES                      8
+#define MAX_PROPERTIES                   64
+#define MAX_DEVICE_UID_CHARS             64
+
+typedef const char* STATUS;
+static STATUS STATUS_OK                       = (STATUS) NULL;
+static STATUS STATUS_NULL_ARG                 = (STATUS) "One of the arguments was null";
+static STATUS STATUS_DEVICE_INIT_FAILED       = (STATUS) "Failed to init device";
+static STATUS STATUS_UNSUPPORTED_FRAME_FORMAT = (STATUS) "Unsupported frame format";
+static STATUS STATUS_UNSUPPORTED_MEDIA_TYPE   = (STATUS) "Unsupported media type";
+static STATUS STATUS_FAILED_TO_ACQUIRE_LOCK   = (STATUS) "Failed to acquire a lock";
+static STATUS STATUS_UNSUPPORTED_FORMAT       = (STATUS) "Unsupported device format";
+
+typedef enum AVBindMediaType {
+    AVBindMediaTypeVideo,
+    AVBindMediaTypeAudio,
+} AVBindMediaType;
+
+typedef enum AVBindFrameFormat {
+    AVBindFrameFormatI420,
+    AVBindFrameFormatNV21,
+    AVBindFrameFormatYUY2,
+    AVBindFrameFormatUYVY,
+} AVBindFrameFormat;
+
+typedef void (*AVBindDataCallback)(void *userData, void *buf, int len);
+
+typedef struct AVBindMediaProperty {
+    // video property
+    int width, height;
+    AVBindFrameFormat frameFormat;
+    
+    // audio property
+    
+} AVBindMediaProperty, *PAVBindMediaProperty;
+
+typedef struct AVBindSession AVBindSession, *PAVBindSession;
+
+typedef struct {
+    char uid[MAX_DEVICE_UID_CHARS + 1];
+} AVBindDevice, *PAVBindDevice;
+
+// AVBindDevices returns a list of AVBindDevices. The result array is pointing to a static
+// memory. The caller is expected to not hold on to the address for a long time and make a copy.
+// Everytime this function gets called, the array will be overwritten and the memory will be reused.
+STATUS AVBindDevices(AVBindMediaType, PAVBindDevice*, int*);
+
+STATUS AVBindSessionInit(AVBindDevice, PAVBindSession*);
+STATUS AVBindSessionFree(PAVBindSession*);
+STATUS AVBindSessionOpen(PAVBindSession, AVBindMediaProperty, AVBindDataCallback, void*);
+STATUS AVBindSessionClose(PAVBindSession);
+STATUS AVBindSessionProperties(PAVBindSession, PAVBindMediaProperty*, int*);

pkg/avfoundation/AVFoundationBind/AVFoundationBind.m

@@ -0,0 +1,350 @@
+// MIT License
+// 
+// Copyright (c) 2019-2020 Pion
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// Naming Convention (let "name" as an actual variable name):
+//   - mName: "name" is a member of an Objective C object
+//   - pName: "name" is a C pointer
+//   - refName: "name" is an Objective C object reference
+
+#import <Foundation/Foundation.h>
+#import <AVFoundation/AVFoundation.h>
+#import "AVFoundationBind.h"
+#include <string.h>
+
+#define CHK(condition, status) \
+    do { \
+        if(!(condition)) { \
+            retStatus = status; \
+            goto cleanup; \
+        } \
+    } while(0)
+
+#define CHK_STATUS(status) \
+    do { \
+        if(status != STATUS_OK) { \
+            retStatus = status; \
+            goto cleanup; \
+        } \
+    } while(0)
+
+@interface VideoDataDelegate : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
+
+@property (readonly) AVBindDataCallback mCallback;
+@property (readonly) void *mPUserData;
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection;
+
+@end
+
+@implementation VideoDataDelegate
+
+- (id) init: (AVBindDataCallback) callback
+withUserData: (void*) pUserData {
+    self = [super init];
+    _mCallback = callback;
+    _mPUserData = pUserData;
+
+    return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection {
+    if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 ||
+        !CMSampleBufferIsValid(sampleBuffer) ||
+        !CMSampleBufferDataIsReady(sampleBuffer)) {
+      return;
+    }
+    
+    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+    if (imageBuffer == NULL) {
+      return;
+    }
+    
+    imageBuffer = CVBufferRetain(imageBuffer);
+    CVReturn ret =
+        CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
+    if (ret != kCVReturnSuccess) {
+      return;
+    }
+    
+    size_t heightY = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
+    size_t bytesPerRowY = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
+    
+    size_t heightUV = CVPixelBufferGetHeightOfPlane(imageBuffer, 1);
+    size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 1);
+    
+    int len = (int)((heightY * bytesPerRowY) + (2 * heightUV * bytesPerRowUV));
+    void *buf = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
+    _mCallback(_mPUserData, buf, len);
+    
+    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
+    CVBufferRelease(imageBuffer);
+}
+
+@end
+
+@interface AudioDataDelegate : NSObject<AVCaptureAudioDataOutputSampleBufferDelegate>
+
+@property (readonly) AVBindDataCallback mCallback;
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection;
+
+@end
+
+@implementation AudioDataDelegate
+
+- (id) init: (AVBindDataCallback) callback {
+    self = [super init];
+    _mCallback = callback;
+    return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection {
+    // TODO
+}
+
+@end
+
+STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
+    STATUS retStatus = STATUS_OK;
+    switch (format) {
+        case AVBindFrameFormatNV21:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+            break;
+        case AVBindFrameFormatUYVY:
+            *pFourCC = kCVPixelFormatType_422YpCbCr8;
+            break;
+        // TODO: Add the rest of frame formats
+        default:
+            retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
+    }
+    return retStatus;
+}
+
+STATUS frameFormatFromFourCC(FourCharCode fourCC, AVBindFrameFormat *pFormat) {
+    STATUS retStatus = STATUS_OK;
+    switch (fourCC) {
+         case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+             *pFormat = AVBindFrameFormatNV21;
+             break;
+         case kCVPixelFormatType_422YpCbCr8:
+             *pFormat = AVBindFrameFormatUYVY;
+             break;
+         // TODO: Add the rest of frame formats
+         default:
+             retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
+     }
+    return retStatus;
+}
+
+
+STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *pLen) {
+    static AVBindDevice devices[MAX_DEVICES];
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(mediaType == AVBindMediaTypeVideo || mediaType == AVBindMediaTypeAudio, STATUS_UNSUPPORTED_MEDIA_TYPE);
+    CHK(ppDevices != NULL && pLen != NULL, STATUS_NULL_ARG);
+    
+    PAVBindDevice pDevice;
+    AVMediaType _mediaType = mediaType == AVBindMediaTypeVideo ? AVMediaTypeVideo : AVMediaTypeAudio;
+    NSArray *refAllTypes = @[
+        AVCaptureDeviceTypeBuiltInWideAngleCamera,
+        AVCaptureDeviceTypeBuiltInMicrophone,
+        AVCaptureDeviceTypeExternalUnknown
+    ];
+    AVCaptureDeviceDiscoverySession *refSession = [AVCaptureDeviceDiscoverySession
+        discoverySessionWithDeviceTypes: refAllTypes
+        mediaType: _mediaType
+        position: AVCaptureDevicePositionUnspecified];
+    
+    int i = 0;
+    for (AVCaptureDevice *refDevice in refSession.devices) {
+        if (i >= MAX_DEVICES) {
+            break;
+        }
+        
+        pDevice = devices + i;
+        strncpy(pDevice->uid, refDevice.uniqueID.UTF8String, MAX_DEVICE_UID_CHARS);
+        pDevice->uid[MAX_DEVICE_UID_CHARS] = '\0';
+        i++;
+    }
+    
+    *ppDevices = devices;
+    *pLen = i;
+    
+cleanup:
+    [refPool drain];
+    return retStatus;
+}
+
+struct AVBindSession {
+    AVBindDevice device;
+    AVCaptureSession *refCaptureSession;
+    AVBindMediaProperty properties[MAX_PROPERTIES];
+};
+
+
+STATUS AVBindSessionInit(AVBindDevice device, PAVBindSession *ppSessionResult) {
+    STATUS retStatus = STATUS_OK;
+    CHK(ppSessionResult != NULL, STATUS_NULL_ARG);
+    PAVBindSession pSession = malloc(sizeof(AVBindSession));
+    pSession->device = device;
+    pSession->refCaptureSession = NULL;
+    *ppSessionResult = pSession;
+    
+cleanup:
+    return retStatus;
+}
+
+STATUS AVBindSessionFree(PAVBindSession *ppSession) {
+    STATUS retStatus = STATUS_OK;
+    CHK(ppSession != NULL, STATUS_NULL_ARG);
+    PAVBindSession pSession = *ppSession;
+    if (pSession->refCaptureSession != NULL) {
+        [pSession->refCaptureSession release];
+        pSession->refCaptureSession = NULL;
+    }
+    free(pSession);
+    *ppSession = NULL;
+
+cleanup:
+    return retStatus;
+}
+
+STATUS AVBindSessionOpen(PAVBindSession pSession,
+                  AVBindMediaProperty property,
+                  AVBindDataCallback dataCallback,
+                  void *pUserData) {
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(pSession != NULL && dataCallback != NULL, STATUS_NULL_ARG);
+    
+    AVCaptureDeviceInput *refInput;
+    NSError *refErr = NULL;
+    NSString *refUID = [NSString stringWithUTF8String: pSession->device.uid];
+    AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refUID];
+    
+    refInput = [[AVCaptureDeviceInput alloc] initWithDevice: refDevice error: &refErr];
+    CHK(refErr == NULL, STATUS_DEVICE_INIT_FAILED);
+    
+    AVCaptureSession *refCaptureSession = [[AVCaptureSession alloc] init];
+    refCaptureSession.sessionPreset = AVCaptureSessionPresetMedium;
+    [refCaptureSession addInput: refInput];
+
+    if ([refDevice hasMediaType: AVMediaTypeVideo]) {
+        VideoDataDelegate *pDelegate = [[VideoDataDelegate alloc]
+                                        init: dataCallback
+                                        withUserData: pUserData];
+        
+        AVCaptureVideoDataOutput *pOutput = [[AVCaptureVideoDataOutput alloc] init];
+        FourCharCode fourCC;
+        CHK_STATUS(frameFormatToFourCC(property.frameFormat, &fourCC));
+
+        pOutput.videoSettings = @{
+            (id)kCVPixelBufferWidthKey: @(property.width),
+            (id)kCVPixelBufferHeightKey: @(property.height),
+            (id)kCVPixelBufferPixelFormatTypeKey: @(fourCC),
+        };
+        pOutput.alwaysDiscardsLateVideoFrames = YES;
+        dispatch_queue_t queue =
+            dispatch_queue_create("captureQueue", DISPATCH_QUEUE_SERIAL);
+        [pOutput setSampleBufferDelegate:pDelegate queue:queue];
+        [refCaptureSession addOutput: pOutput];
+    } else {
+        // TODO: implement audio pipeline
+    }
+    
+    pSession->refCaptureSession = [refCaptureSession retain];
+    [refCaptureSession startRunning];
+    
+cleanup:
+    [refPool drain];
+    return retStatus;
+}
+
+
+STATUS AVBindSessionClose(PAVBindSession pSession) {
+    STATUS retStatus = STATUS_OK;
+    CHK(pSession != NULL, STATUS_NULL_ARG);
+    CHK(pSession->refCaptureSession != NULL, STATUS_OK);
+    
+    [pSession->refCaptureSession stopRunning];
+    [pSession->refCaptureSession release];
+    pSession->refCaptureSession = NULL;
+        
+cleanup:
+    return retStatus;
+}
+
+STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *ppProperties, int *pLen) {
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(pSession != NULL && ppProperties != NULL && pLen != NULL, STATUS_NULL_ARG);
+    
+    NSString *refDeviceUID = [NSString stringWithUTF8String: pSession->device.uid];
+    AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refDeviceUID];
+    FourCharCode fourCC;
+    CMVideoFormatDescriptionRef videoFormat;
+    CMVideoDimensions videoDimensions;
+
+    memset(pSession->properties, 0, sizeof(pSession->properties));
+    PAVBindMediaProperty pProperty = pSession->properties;
+    int len = 0;
+    for (AVCaptureDeviceFormat *refFormat in refDevice.formats) {
+        // TODO: Probably gives a warn to the user
+        if (len >= MAX_PROPERTIES) {
+            break;
+        }
+        
+        if ([refFormat.mediaType isEqual:AVMediaTypeVideo]) {
+            fourCC = CMFormatDescriptionGetMediaSubType(refFormat.formatDescription);
+            if (frameFormatFromFourCC(fourCC, &pProperty->frameFormat) != STATUS_OK) {
+                continue;
+            }
+            
+            videoFormat = (CMVideoFormatDescriptionRef) refFormat.formatDescription;
+            videoDimensions = CMVideoFormatDescriptionGetDimensions(videoFormat);
+            pProperty->height = videoDimensions.height;
+            pProperty->width = videoDimensions.width;
+        } else {
+            // TODO: Get audio properties
+        }
+        
+        pProperty++;
+        len++;
+    }
+    
+    *ppProperties = pSession->properties;
+    *pLen = len;
+    
+cleanup:
+    
+    [refPool drain];
+    return retStatus;
+}

pkg/avfoundation/avfoundation_callback_darwin.go

@@ -0,0 +1,56 @@
+package avfoundation
+
+// extern void onData(void*, void*, int);
+import "C"
+import (
+	"sync"
+	"unsafe"
+)
+
+var mu sync.Mutex
+var nextID handleID
+
+type dataCb func(data []byte)
+
+var handles = make(map[handleID]dataCb)
+
+type handleID int
+
+//export onData
+func onData(userData unsafe.Pointer, buf unsafe.Pointer, length C.int) {
+	data := C.GoBytes(buf, length)
+
+	handleNum := (*C.int)(userData)
+	cb, ok := lookup(handleID(*handleNum))
+	if ok {
+		cb(data)
+	}
+}
+
+func register(fn dataCb) handleID {
+	mu.Lock()
+	defer mu.Unlock()
+
+	nextID++
+	for handles[nextID] != nil {
+		nextID++
+	}
+	handles[nextID] = fn
+
+	return nextID
+}
+
+func lookup(i handleID) (cb dataCb, ok bool) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	cb, ok = handles[i]
+	return
+}
+
+func unregister(i handleID) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	delete(handles, i)
+}

pkg/avfoundation/avfoundation_darwin.go

@@ -0,0 +1,217 @@
+// Package avfoundation provides AVFoundation binding for Go
+package avfoundation
+
+// #cgo CFLAGS: -x objective-c
+// #cgo LDFLAGS: -framework AVFoundation -framework Foundation -framework CoreMedia -framework CoreVideo
+// #include "AVFoundationBind/AVFoundationBind.h"
+// #include "AVFoundationBind/AVFoundationBind.m"
+// extern void onData(void*, void*, int);
+// void onDataBridge(void *userData, void *buf, int len) {
+// 	onData(userData, buf, len);
+// }
+import "C"
+import (
+	"fmt"
+	"io"
+	"unsafe"
+
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+type MediaType C.AVBindMediaType
+
+const (
+	Video = MediaType(C.AVBindMediaTypeVideo)
+	Audio = MediaType(C.AVBindMediaTypeAudio)
+)
+
+// Device represents a metadata that later can be used to retrieve back the
+// underlying device given by AVFoundation
+type Device struct {
+	// UID is a unique identifier for a device
+	UID     string
+	cDevice C.AVBindDevice
+}
+
+func frameFormatToAVBind(f frame.Format) (C.AVBindFrameFormat, bool) {
+	switch f {
+	case frame.FormatI420:
+		return C.AVBindFrameFormatI420, true
+	case frame.FormatNV21:
+		return C.AVBindFrameFormatNV21, true
+	case frame.FormatYUY2:
+		return C.AVBindFrameFormatYUY2, true
+	case frame.FormatUYVY:
+		return C.AVBindFrameFormatUYVY, true
+	default:
+		return 0, false
+	}
+}
+
+func frameFormatFromAVBind(f C.AVBindFrameFormat) (frame.Format, bool) {
+	switch f {
+	case C.AVBindFrameFormatI420:
+		return frame.FormatI420, true
+	case C.AVBindFrameFormatNV21:
+		return frame.FormatNV21, true
+	case C.AVBindFrameFormatYUY2:
+		return frame.FormatYUY2, true
+	case C.AVBindFrameFormatUYVY:
+		return frame.FormatUYVY, true
+	default:
+		return "", false
+	}
+}
+
+// Devices uses AVFoundation to query a list of devices based on the media type
+func Devices(mediaType MediaType) ([]Device, error) {
+	var cDevicesPtr C.PAVBindDevice
+	var cDevicesLen C.int
+
+	status := C.AVBindDevices(C.AVBindMediaType(mediaType), &cDevicesPtr, &cDevicesLen)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	cDevices := (*[1 << 28]C.AVBindDevice)(unsafe.Pointer(cDevicesPtr))[:cDevicesLen:cDevicesLen]
+	devices := make([]Device, cDevicesLen)
+
+	for i := range devices {
+		devices[i].UID = C.GoString(&cDevices[i].uid[0])
+		devices[i].cDevice = cDevices[i]
+	}
+
+	return devices, nil
+}
+
+// ReadCloser is a wrapper around the data callback from AVFoundation. The data received from the
+// the underlying callback can be retrieved by calling Read.
+type ReadCloser struct {
+	dataChan chan []byte
+	id       handleID
+	onClose  func()
+}
+
+func newReadCloser(onClose func()) *ReadCloser {
+	var rc ReadCloser
+	rc.dataChan = make(chan []byte, 1)
+	rc.onClose = onClose
+	rc.id = register(rc.dataCb)
+	return &rc
+}
+
+func (rc *ReadCloser) dataCb(data []byte) {
+	// TODO: add a policy for slow reader
+	rc.dataChan <- data
+}
+
+// Read reads raw data, the format is determined by the media type and property:
+//   - For video, each call will return a frame.
+//   - For audio, each call will return a chunk which its size configured by Latency
+func (rc *ReadCloser) Read() ([]byte, error) {
+	data, ok := <-rc.dataChan
+	if !ok {
+		return nil, io.EOF
+	}
+	return data, nil
+}
+
+// Close closes the capturing session, and no data will flow anymore
+func (rc *ReadCloser) Close() {
+	if rc.onClose != nil {
+		rc.onClose()
+	}
+	close(rc.dataChan)
+	unregister(rc.id)
+}
+
+// Session represents a capturing session.
+type Session struct {
+	device   Device
+	cSession C.PAVBindSession
+}
+
+// NewSession creates a new capturing session
+func NewSession(device Device) (*Session, error) {
+	var session Session
+
+	status := C.AVBindSessionInit(device.cDevice, &session.cSession)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+
+	session.device = device
+	return &session, nil
+}
+
+// Close stops capturing session and frees up resources
+func (session *Session) Close() error {
+	if session.cSession == nil {
+		return nil
+	}
+
+	status := C.AVBindSessionFree(&session.cSession)
+	if status != nil {
+		return fmt.Errorf("%s", C.GoString(status))
+	}
+	return nil
+}
+
+// Open start capturing session. As soon as it returns successfully, the data will start
+// flowing. The raw data can be retrieved by using ReadCloser's Read method.
+func (session *Session) Open(property prop.Media) (*ReadCloser, error) {
+	frameFormat, ok := frameFormatToAVBind(property.FrameFormat)
+	if !ok {
+		return nil, fmt.Errorf("Unsupported frame format")
+	}
+
+	cProperty := C.AVBindMediaProperty{
+		width:       C.int(property.Width),
+		height:      C.int(property.Height),
+		frameFormat: frameFormat,
+	}
+
+	rc := newReadCloser(func() {
+		C.AVBindSessionClose(session.cSession)
+	})
+	status := C.AVBindSessionOpen(
+		session.cSession,
+		cProperty,
+		C.AVBindDataCallback(unsafe.Pointer(C.onDataBridge)),
+		unsafe.Pointer(&rc.id),
+	)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+	return rc, nil
+}
+
+// Properties queries a list of properties that device supports
+func (session *Session) Properties() []prop.Media {
+	var cPropertiesPtr C.PAVBindMediaProperty
+	var cPropertiesLen C.int
+
+	status := C.AVBindSessionProperties(session.cSession, &cPropertiesPtr, &cPropertiesLen)
+	if status != nil {
+		return nil
+	}
+
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	cProperties := (*[1 << 28]C.AVBindMediaProperty)(unsafe.Pointer(cPropertiesPtr))[:cPropertiesLen:cPropertiesLen]
+	var properties []prop.Media
+	for _, cProperty := range cProperties {
+		frameFormat, ok := frameFormatFromAVBind(cProperty.frameFormat)
+		if ok {
+			properties = append(properties, prop.Media{
+				Video: prop.Video{
+					Width:       int(cProperty.width),
+					Height:      int(cProperty.height),
+					FrameFormat: frameFormat,
+				},
+			})
+		}
+	}
+	return properties
+}

pkg/codec/opus/opus.go

@@ -1,6 +1,7 @@
 package opus
 
 import (
+	"errors"
 	"fmt"
 	"math"
 
@@ -9,11 +10,12 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/prop"
 	"github.com/pion/mediadevices/pkg/wave"
+	"github.com/pion/mediadevices/pkg/wave/mixer"
 )
 
 type encoder struct {
 	engine *opus.Encoder
-	inBuff *wave.Float32Interleaved
+	inBuff wave.Audio
 	reader audio.Reader
 }
 
@@ -32,6 +34,10 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 		params.BitRate = 32000
 	}
 
+	if params.ChannelMixer == nil {
+		params.ChannelMixer = &mixer.MonoMixer{}
+	}
+
 	// Select the nearest supported latency
 	var targetLatency float64
 	// TODO: use p.Latency.Milliseconds() after Go 1.12 EOL
@@ -47,8 +53,7 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 		targetLatency = latency
 	}
 
-	// Since audio.Reader only supports stereo mode, channels is always 2
+	channels := p.ChannelCount
-	channels := 2
 
 	engine, err := opus.NewEncoder(p.SampleRate, channels, opus.AppVoIP)
 	if err != nil {
@@ -58,47 +63,37 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 		return nil, err
 	}
 
-	inBuffSize := int(targetLatency * float64(p.SampleRate) / 1000)
+	rMix := audio.NewChannelMixer(channels, params.ChannelMixer)
-	inBuff := wave.NewFloat32Interleaved(
+	rBuf := audio.NewBuffer(int(targetLatency * float64(p.SampleRate) / 1000))
-		wave.ChunkInfo{Channels: channels, Len: inBuffSize},
+	e := encoder{
-	)
+		engine: engine,
-	inBuff.Data = inBuff.Data[:0]
+		reader: rMix(rBuf(r)),
-	e := encoder{engine, inBuff, r}
+	}
 	return &e, nil
 }
 
-func (e *encoder) Read(p []byte) (n int, err error) {
+func (e *encoder) Read(p []byte) (int, error) {
-	// While the buffer is not full, keep reading so that we meet the latency requirement
-	nLatency := e.inBuff.ChunkInfo().Len * e.inBuff.ChunkInfo().Channels
-	for len(e.inBuff.Data) < nLatency {
 	buff, err := e.reader.Read()
 	if err != nil {
 		return 0, err
 	}
-		// TODO: convert audio format
-		b, ok := buff.(*wave.Float32Interleaved)
-		if !ok {
-			panic("unsupported audio format")
-		}
-		switch {
-		case b.Size.Channels == 1 && e.inBuff.ChunkInfo().Channels != 1:
-			for _, d := range b.Data {
-				for ch := 0; ch < e.inBuff.ChunkInfo().Channels; ch++ {
-					e.inBuff.Data = append(e.inBuff.Data, d)
-				}
-			}
-		case b.Size.Channels == e.inBuff.ChunkInfo().Channels:
-			e.inBuff.Data = append(e.inBuff.Data, b.Data...)
-		}
-	}
 
-	n, err = e.engine.EncodeFloat32(e.inBuff.Data[:nLatency], p)
+	switch b := buff.(type) {
+	case *wave.Int16Interleaved:
+		n, err := e.engine.Encode(b.Data, p)
 		if err != nil {
 			return n, err
 		}
-	e.inBuff.Data = e.inBuff.Data[nLatency:]
-
 		return n, nil
+	case *wave.Float32Interleaved:
+		n, err := e.engine.EncodeFloat32(b.Data, p)
+		if err != nil {
+			return n, err
+		}
+		return n, nil
+	default:
+		return 0, errors.New("unknown type of audio buffer")
+	}
 }
 
 func (e *encoder) SetBitRate(b int) error {

pkg/codec/opus/params.go

@@ -4,12 +4,15 @@ import (
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/mediadevices/pkg/wave/mixer"
 	"github.com/pion/webrtc/v2"
 )
 
 // Params stores opus specific encoding parameters.
 type Params struct {
 	codec.BaseParams
+	// ChannelMixer is a mixer to be used if number of given and expected channels differ.
+	ChannelMixer mixer.ChannelMixer
 }
 
 // NewParams returns default opus codec specific parameters.

pkg/driver/camera/camera_darwin.go

@@ -0,0 +1,71 @@
+package camera
+
+import (
+	"image"
+
+	"github.com/pion/mediadevices/pkg/avfoundation"
+	"github.com/pion/mediadevices/pkg/driver"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+type camera struct {
+	device  avfoundation.Device
+	session *avfoundation.Session
+}
+
+func init() {
+	devices, err := avfoundation.Devices(avfoundation.Video)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, device := range devices {
+		cam := newCamera(device)
+		driver.GetManager().Register(cam, driver.Info{
+			Label:      device.UID,
+			DeviceType: driver.Camera,
+		})
+	}
+}
+
+func newCamera(device avfoundation.Device) *camera {
+	return &camera{
+		device: device,
+	}
+}
+
+func (cam *camera) Open() error {
+	var err error
+	cam.session, err = avfoundation.NewSession(cam.device)
+	return err
+}
+
+func (cam *camera) Close() error {
+	return cam.session.Close()
+}
+
+func (cam *camera) VideoRecord(property prop.Media) (video.Reader, error) {
+	decoder, err := frame.NewDecoder(property.FrameFormat)
+	if err != nil {
+		return nil, err
+	}
+
+	rc, err := cam.session.Open(property)
+	if err != nil {
+		return nil, err
+	}
+	r := video.ReaderFunc(func() (image.Image, error) {
+		frame, err := rc.Read()
+		if err != nil {
+			return nil, err
+		}
+		return decoder.Decode(frame, property.Width, property.Height)
+	})
+	return r, nil
+}
+
+func (cam *camera) Properties() []prop.Media {
+	return cam.session.Properties()
+}

pkg/driver/microphone/microphone_linux.go

@@ -14,7 +14,7 @@ import (
 type microphone struct {
 	c           *pulse.Client
 	id          string
-	samplesChan chan<- []float32
+	samplesChan chan<- []int16
 }
 
 func init() {
@@ -85,14 +85,14 @@ func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
 		pulse.RecordSource(src),
 	)
 
-	samplesChan := make(chan []float32, 1)
+	samplesChan := make(chan []int16, 1)
 
-	handler := func(b []float32) (int, error) {
+	handler := func(b []int16) (int, error) {
 		samplesChan <- b
 		return len(b), nil
 	}
 
-	stream, err := m.c.NewRecord(pulse.Float32Writer(handler), options...)
+	stream, err := m.c.NewRecord(pulse.Int16Writer(handler), options...)
 	if err != nil {
 		return nil, err
 	}
@@ -104,7 +104,7 @@ func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
 			return nil, io.EOF
 		}
 
-		a := wave.NewFloat32Interleaved(
+		a := wave.NewInt16Interleaved(
 			wave.ChunkInfo{
 				Channels: p.ChannelCount,
 				Len:      len(buff) / p.ChannelCount,

pkg/driver/microphone/microphone_windows.go

@@ -214,7 +214,7 @@ func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
 			}
 		}
 
-		a := wave.NewFloat32Interleaved(
+		a := wave.NewInt16Interleaved(
 			wave.ChunkInfo{
 				Channels: p.ChannelCount,
 				Len:      (int(b.waveHdr.dwBytesRecorded) / 2) / p.ChannelCount,
@@ -224,7 +224,7 @@ func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
 		j := 0
 		for i := 0; i < a.Size.Len; i++ {
 			for ch := 0; ch < a.Size.Channels; ch++ {
-				a.SetFloat32(i, ch, wave.Float32Sample(float32(b.data[j])/0x8000))
+				a.SetInt16(i, ch, wave.Int16Sample(b.data[j]))
 				j++
 			}
 		}

pkg/io/audio/beep.go

@@ -1,89 +0,0 @@
-package audio
-
-import (
-	"io"
-
-	"github.com/faiface/beep"
-
-	"github.com/pion/mediadevices/pkg/wave"
-)
-
-type beepStreamer struct {
-	err error
-	r   Reader
-}
-
-func ToBeep(r Reader) beep.Streamer {
-	if r == nil {
-		panic("FromReader requires a non-nil Reader")
-	}
-
-	return &beepStreamer{r: r}
-}
-
-func (b *beepStreamer) Stream(samples [][2]float64) (int, bool) {
-	// Since there was an error, the stream has to be drained
-	if b.err != nil {
-		return 0, false
-	}
-
-	d, err := b.r.Read()
-	if err != nil {
-		b.err = err
-		if err != io.EOF {
-			return 0, false
-		}
-	}
-
-	n := d.ChunkInfo().Len
-	for i := 0; i < n; i++ {
-		samples[i][0] = float64(wave.Float32SampleFormat.Convert(d.At(i, 0)).(wave.Float32Sample))
-		samples[i][1] = float64(wave.Float32SampleFormat.Convert(d.At(i, 1)).(wave.Float32Sample))
-	}
-
-	return n, true
-}
-
-func (b *beepStreamer) Err() error {
-	return b.err
-}
-
-type beepReader struct {
-	s    beep.Streamer
-	buff [][2]float64
-	size int
-}
-
-func FromBeep(s beep.Streamer) Reader {
-	if s == nil {
-		panic("FromStreamer requires a non-nil beep.Streamer")
-	}
-
-	return &beepReader{
-		s:    s,
-		buff: make([][2]float64, 1024), // TODO: configure chunk size
-	}
-}
-
-func (r *beepReader) Read() (wave.Audio, error) {
-	out := wave.NewFloat32Interleaved(
-		wave.ChunkInfo{Len: len(r.buff), Channels: 2, SamplingRate: 48000},
-	)
-
-	n, ok := r.s.Stream(r.buff)
-	if !ok {
-		err := r.s.Err()
-		if err == nil {
-			err = io.EOF
-		}
-
-		return nil, err
-	}
-
-	for i := 0; i < n; i++ {
-		out.SetFloat32(i, 0, wave.Float32Sample(r.buff[i][0]))
-		out.SetFloat32(i, 1, wave.Float32Sample(r.buff[i][1]))
-	}
-
-	return out, nil
-}

pkg/io/audio/buffer.go

@@ -0,0 +1,89 @@
+package audio
+
+import (
+	"errors"
+
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+var errUnsupported = errors.New("unsupported audio format")
+
+// NewBuffer creates audio transform to buffer signal to have exact nSample samples.
+func NewBuffer(nSamples int) TransformFunc {
+	var inBuff wave.Audio
+
+	return func(r Reader) Reader {
+		return ReaderFunc(func() (wave.Audio, error) {
+			for {
+				if inBuff != nil && inBuff.ChunkInfo().Len >= nSamples {
+					break
+				}
+
+				buff, err := r.Read()
+				if err != nil {
+					return nil, err
+				}
+				switch b := buff.(type) {
+				case *wave.Float32Interleaved:
+					ib, ok := inBuff.(*wave.Float32Interleaved)
+					if !ok || ib.Size.Channels != b.Size.Channels {
+						ib = wave.NewFloat32Interleaved(
+							wave.ChunkInfo{
+								SamplingRate: b.Size.SamplingRate,
+								Channels:     b.Size.Channels,
+								Len:          nSamples,
+							},
+						)
+						ib.Data = ib.Data[:0]
+						ib.Size.Len = 0
+						inBuff = ib
+					}
+					ib.Data = append(ib.Data, b.Data...)
+					ib.Size.Len += b.Size.Len
+
+				case *wave.Int16Interleaved:
+					ib, ok := inBuff.(*wave.Int16Interleaved)
+					if !ok || ib.Size.Channels != b.Size.Channels {
+						ib = wave.NewInt16Interleaved(
+							wave.ChunkInfo{
+								SamplingRate: b.Size.SamplingRate,
+								Channels:     b.Size.Channels,
+								Len:          nSamples,
+							},
+						)
+						ib.Data = ib.Data[:0]
+						ib.Size.Len = 0
+						inBuff = ib
+					}
+					ib.Data = append(ib.Data, b.Data...)
+					ib.Size.Len += b.Size.Len
+
+				default:
+					return nil, errUnsupported
+				}
+			}
+			switch ib := inBuff.(type) {
+			case *wave.Int16Interleaved:
+				ibCopy := *ib
+				ibCopy.Size.Len = nSamples
+				n := nSamples * ib.Size.Channels
+				ibCopy.Data = make([]int16, n)
+				copy(ibCopy.Data, ib.Data)
+				ib.Data = ib.Data[n:]
+				ib.Size.Len -= nSamples
+				return &ibCopy, nil
+
+			case *wave.Float32Interleaved:
+				ibCopy := *ib
+				ibCopy.Size.Len = nSamples
+				n := nSamples * ib.Size.Channels
+				ibCopy.Data = make([]float32, n)
+				copy(ibCopy.Data, ib.Data)
+				ib.Data = ib.Data[n:]
+				ib.Size.Len -= nSamples
+				return &ibCopy, nil
+			}
+			return nil, errUnsupported
+		})
+	}
+}

pkg/io/audio/buffer_test.go

@@ -0,0 +1,72 @@
+package audio
+
+import (
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+func TestBuffer(t *testing.T) {
+	input := []wave.Audio{
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 1, Channels: 2, SamplingRate: 1234},
+			Data: []int16{1, 2},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{3, 4, 5, 6, 7, 8},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 2, Channels: 2, SamplingRate: 1234},
+			Data: []int16{9, 10, 11, 12},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 7, Channels: 2, SamplingRate: 1234},
+			Data: []int16{13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26},
+		},
+	}
+	expected := []wave.Audio{
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{1, 2, 3, 4, 5, 6},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{7, 8, 9, 10, 11, 12},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{13, 14, 15, 16, 17, 18},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{19, 20, 21, 22, 23, 24},
+		},
+	}
+
+	trans := NewBuffer(3)
+
+	var iSent int
+	r := trans(ReaderFunc(func() (wave.Audio, error) {
+		if iSent < len(input) {
+			iSent++
+			return input[iSent-1], nil
+		}
+		return nil, io.EOF
+	}))
+
+	for i := 0; ; i++ {
+		a, err := r.Read()
+		if err != nil {
+			if err == io.EOF && i >= len(expected) {
+				break
+			}
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(expected[i], a) {
+			t.Errorf("Expected wave[%d]: %v, got: %v", i, expected[i], a)
+		}
+	}
+}

pkg/io/audio/mixer.go

@@ -0,0 +1,40 @@
+package audio
+
+import (
+	"github.com/pion/mediadevices/pkg/wave"
+	"github.com/pion/mediadevices/pkg/wave/mixer"
+)
+
+// NewChannelMixer creates audio transform to mix audio channels.
+func NewChannelMixer(channels int, mixer mixer.ChannelMixer) TransformFunc {
+	return func(r Reader) Reader {
+		return ReaderFunc(func() (wave.Audio, error) {
+			buff, err := r.Read()
+			if err != nil {
+				return nil, err
+			}
+			ci := buff.ChunkInfo()
+			if ci.Channels == channels {
+				return buff, nil
+			}
+
+			ci.Channels = channels
+
+			var mixed wave.Audio
+			switch buff.(type) {
+			case *wave.Int16Interleaved:
+				mixed = wave.NewInt16Interleaved(ci)
+			case *wave.Int16NonInterleaved:
+				mixed = wave.NewInt16NonInterleaved(ci)
+			case *wave.Float32Interleaved:
+				mixed = wave.NewFloat32Interleaved(ci)
+			case *wave.Float32NonInterleaved:
+				mixed = wave.NewFloat32NonInterleaved(ci)
+			}
+			if err := mixer.Mix(mixed, buff); err != nil {
+				return nil, err
+			}
+			return mixed, nil
+		})
+	}
+}

pkg/io/audio/mixer_test.go

@@ -0,0 +1,57 @@
+package audio
+
+import (
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/wave"
+	"github.com/pion/mediadevices/pkg/wave/mixer"
+)
+
+func TestMixer(t *testing.T) {
+	input := []wave.Audio{
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 1, Channels: 2, SamplingRate: 1234},
+			Data: []int16{1, 3},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 2, SamplingRate: 1234},
+			Data: []int16{2, 4, 3, 5, 4, 6},
+		},
+	}
+	expected := []wave.Audio{
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 1, Channels: 1, SamplingRate: 1234},
+			Data: []int16{2},
+		},
+		&wave.Int16Interleaved{
+			Size: wave.ChunkInfo{Len: 3, Channels: 1, SamplingRate: 1234},
+			Data: []int16{3, 4, 5},
+		},
+	}
+
+	trans := NewChannelMixer(1, &mixer.MonoMixer{})
+
+	var iSent int
+	r := trans(ReaderFunc(func() (wave.Audio, error) {
+		if iSent < len(input) {
+			iSent++
+			return input[iSent-1], nil
+		}
+		return nil, io.EOF
+	}))
+
+	for i := 0; ; i++ {
+		a, err := r.Read()
+		if err != nil {
+			if err == io.EOF && i >= len(expected) {
+				break
+			}
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(expected[i], a) {
+			t.Errorf("Expected wave[%d]: %v, got: %v", i, expected[i], a)
+		}
+	}
+}

pkg/io/video/detect.go

@@ -0,0 +1,58 @@
+package video
+
+import (
+	"image"
+	"time"
+
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+// DetectChanges will detect frame and video property changes. For video property detection,
+// since it's time related, interval will be used to determine the sample rate.
+func DetectChanges(interval time.Duration, onChange func(prop.Media)) TransformFunc {
+	return func(r Reader) Reader {
+		var currentProp prop.Media
+		var lastTaken time.Time
+		var frames uint
+		return ReaderFunc(func() (image.Image, error) {
+			var dirty bool
+
+			img, err := r.Read()
+			if err != nil {
+				return nil, err
+			}
+
+			bounds := img.Bounds()
+			if currentProp.Width != bounds.Dx() {
+				currentProp.Width = bounds.Dx()
+				dirty = true
+			}
+
+			if currentProp.Height != bounds.Dy() {
+				currentProp.Height = bounds.Dy()
+				dirty = true
+			}
+
+			// TODO: maybe detect frame format? It probably doesn't make sense since some
+			// formats only are about memory layout, e.g. YUV2 vs NV12.
+
+			now := time.Now()
+			elapsed := now.Sub(lastTaken)
+			if elapsed >= interval {
+				fps := float32(float64(frames) / elapsed.Seconds())
+				// TODO: maybe add some epsilon so that small changes will not mark as dirty
+				currentProp.FrameRate = fps
+				frames = 0
+				lastTaken = now
+				dirty = true
+			}
+
+			if dirty {
+				onChange(currentProp)
+			}
+
+			frames++
+			return img, nil
+		})
+	}
+}

pkg/io/video/detect_test.go

@@ -0,0 +1,158 @@
+package video
+
+import (
+	"fmt"
+	"image"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func BenchmarkDetectChanges(b *testing.B) {
+	var src Reader
+	src = ReaderFunc(func() (image.Image, error) {
+		return image.NewRGBA(image.Rect(0, 0, 1920, 1080)), nil
+	})
+
+	b.Run("WithoutDetectChanges", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			src.Read()
+		}
+	})
+
+	ns := []int{1, 8, 64, 256}
+	for _, n := range ns {
+		n := n
+		src := src
+		b.Run(fmt.Sprintf("WithDetectChanges%d", n), func(b *testing.B) {
+			for i := 0; i < n; i++ {
+				src = DetectChanges(time.Microsecond, func(p prop.Media) {})(src)
+			}
+
+			for i := 0; i < b.N; i++ {
+				src.Read()
+			}
+		})
+	}
+}
+
+func TestDetectChanges(t *testing.T) {
+	buildSource := func(p prop.Media) (Reader, func(prop.Media)) {
+		return ReaderFunc(func() (image.Image, error) {
+				return image.NewRGBA(image.Rect(0, 0, p.Width, p.Height)), nil
+			}), func(newProp prop.Media) {
+				p = newProp
+			}
+	}
+
+	assertEq := func(t *testing.T, actual prop.Media, expected prop.Media, output image.Image, assertFrameRate bool) {
+		if actual.Height != expected.Height {
+			t.Fatalf("expected height from to be %d but got %d", expected.Height, actual.Height)
+		}
+
+		if actual.Width != expected.Width {
+			t.Fatalf("expected width from to be %d but got %d", expected.Width, actual.Width)
+		}
+
+		if assertFrameRate {
+			diff := actual.FrameRate - expected.FrameRate
+			// TODO: reduce this eps. Darwin CI keeps failing if we use a lower value
+			var eps float32 = 1.5
+			if diff < -eps || diff > eps {
+				t.Fatalf("expected frame rate to be %f (+-%f) but got %f", expected.FrameRate, eps, actual.FrameRate)
+			}
+		}
+
+		if output.Bounds().Dy() != expected.Height {
+			t.Fatalf("expected output height from to be %d but got %d", expected.Height, output.Bounds().Dy())
+		}
+
+		if output.Bounds().Dx() != expected.Width {
+			t.Fatalf("expected output width from to be %d but got %d", expected.Width, output.Bounds().Dx())
+		}
+	}
+
+	t.Run("OnChangeCalledBeforeFirstFrame", func(t *testing.T) {
+		var detectBeforeFirstFrame bool
+		var expected prop.Media
+		var actual prop.Media
+		expected.Width = 1920
+		expected.Height = 1080
+		src, _ := buildSource(expected)
+		src = DetectChanges(time.Second, func(p prop.Media) {
+			actual = p
+			detectBeforeFirstFrame = true
+		})(src)
+
+		frame, err := src.Read()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !detectBeforeFirstFrame {
+			t.Fatal("on change callback should have called before first frame")
+		}
+
+		assertEq(t, actual, expected, frame, false)
+	})
+
+	t.Run("DetectChangesOnEveryUpdate", func(t *testing.T) {
+		var expected prop.Media
+		var actual prop.Media
+		expected.Width = 1920
+		expected.Height = 1080
+		src, update := buildSource(expected)
+		src = DetectChanges(time.Second, func(p prop.Media) {
+			actual = p
+		})(src)
+
+		for width := 1920; width < 4000; width += 100 {
+			for height := 1080; height < 2000; height += 100 {
+				expected.Width = width
+				expected.Height = height
+				update(expected)
+				frame, err := src.Read()
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				assertEq(t, actual, expected, frame, false)
+			}
+		}
+	})
+
+	t.Run("FrameRateAccuracy", func(t *testing.T) {
+		// https://github.com/pion/mediadevices/issues/198
+		if runtime.GOOS == "darwin" {
+			t.Skip("Skipping because Darwin CI is not reliable for timing related tests.")
+		}
+
+		var expected prop.Media
+		var actual prop.Media
+		var count int
+		expected.Width = 1920
+		expected.Height = 1080
+		expected.FrameRate = 30
+		src, _ := buildSource(expected)
+		src = Throttle(expected.FrameRate)(src)
+		src = DetectChanges(time.Second*5, func(p prop.Media) {
+			actual = p
+			count++
+		})(src)
+
+		for count < 3 {
+			frame, err := src.Read()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			checkFrameRate := false
+			if actual.FrameRate != 0.0 {
+				checkFrameRate = true
+			}
+			assertEq(t, actual, expected, frame, checkFrameRate)
+		}
+	})
+}

pkg/io/video/throttle_test.go

@@ -2,32 +2,37 @@ package video
 
 import (
 	"image"
+	"runtime"
 	"testing"
 	"time"
 )
 
 func TestThrottle(t *testing.T) {
+	// https://github.com/pion/mediadevices/issues/198
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping because Darwin CI is not reliable for timing related tests.")
+	}
 	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
 
-	ticker := time.NewTicker(time.Millisecond)
+	ticker := time.NewTicker(20 * time.Millisecond)
 	defer ticker.Stop()
 
 	var cntPush int
-	trans := Throttle(100)
+	trans := Throttle(50)
 	r := trans(ReaderFunc(func() (image.Image, error) {
 		<-ticker.C
 		cntPush++
 		return img, nil
 	}))
 
-	for i := 0; i < 50; i++ {
+	for i := 0; i < 20; i++ {
 		_, err := r.Read()
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
 		}
 	}
-	cntExpected := 500
+	cntExpected := 20
-	if cntPush < cntExpected*9/10 || cntExpected*11/10 < cntPush {
+	if cntPush < cntExpected*8/10 || cntExpected*12/10 < cntPush {
 		t.Fatalf("Number of pushed images is expected to be %d, but pushed %d", cntExpected, cntPush)
 	}
 	t.Log(cntPush)

pkg/prop/bool.go

@@ -0,0 +1,30 @@
+package prop
+
+// BoolConstraint is an interface to represent bool value constraint.
+type BoolConstraint interface {
+	Compare(bool) (float64, bool)
+	Value() bool
+}
+
+// BoolExact specifies exact bool value.
+type BoolExact bool
+
+// Compare implements BoolConstraint.
+func (b BoolExact) Compare(o bool) (float64, bool) {
+	if bool(b) == o {
+		return 0.0, true
+	}
+	return 1.0, false
+}
+
+// Value implements BoolConstraint.
+func (b BoolExact) Value() bool { return bool(b) }
+
+// Bool specifies ideal bool value.
+type Bool BoolExact
+
+// Compare implements BoolConstraint.
+func (b Bool) Compare(o bool) (float64, bool) {
+	dist, _ := BoolExact(b).Compare(o)
+	return dist, true
+}

pkg/prop/prop.go

@@ -22,8 +22,12 @@ type Media struct {
 	Audio
 }
 
-// Merge merges all the field values from o to p, except zero values.
+// setterFn is a callback function to set value from fieldB to fieldA
-func (p *Media) Merge(o MediaConstraints) {
+type setterFn func(fieldA, fieldB reflect.Value)
+
+// merge merges all the field values from o to p, except zero values. It's guaranteed that setterFn will be called
+// when fieldA and fieldB are not struct.
+func (p *Media) merge(o interface{}, set setterFn) {
 	rp := reflect.ValueOf(p).Elem()
 	ro := reflect.ValueOf(o)
 
@@ -49,6 +53,21 @@ func (p *Media) Merge(o MediaConstraints) {
 				continue
 			}
 
+			set(fieldA, fieldB)
+		}
+	}
+
+	merge(rp, ro)
+}
+
+func (p *Media) Merge(o Media) {
+	p.merge(o, func(fieldA, fieldB reflect.Value) {
+		fieldA.Set(fieldB)
+	})
+}
+
+func (p *Media) MergeConstraints(o MediaConstraints) {
+	p.merge(o, func(fieldA, fieldB reflect.Value) {
 		switch c := fieldB.Interface().(type) {
 		case IntConstraint:
 			if v, ok := c.Value(); ok {
@@ -70,13 +89,12 @@ func (p *Media) Merge(o MediaConstraints) {
 			if v, ok := c.Value(); ok {
 				fieldA.Set(reflect.ValueOf(v))
 			}
+		case BoolConstraint:
+			fieldA.Set(reflect.ValueOf(c.Value()))
 		default:
 			panic("unsupported property type")
 		}
-		}
+	})
-	}
-
-	merge(rp, ro)
 }
 
 // FitnessDistance calculates fitness of media property and media constraints.
@@ -89,6 +107,10 @@ func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
 	cmps.add(p.FrameFormat, o.FrameFormat)
 	cmps.add(p.SampleRate, o.SampleRate)
 	cmps.add(p.Latency, o.Latency)
+	cmps.add(p.ChannelCount, o.ChannelCount)
+	cmps.add(p.IsBigEndian, o.IsBigEndian)
+	cmps.add(p.IsFloat, o.IsFloat)
+	cmps.add(p.IsInterleaved, o.IsInterleaved)
 
 	return cmps.fitnessDistance()
 }
@@ -144,6 +166,12 @@ func (c *comparisons) fitnessDistance() (float64, bool) {
 			} else {
 				panic("wrong type of actual value")
 			}
+		case BoolConstraint:
+			if actual, typeOK := field.actual.(bool); typeOK {
+				d, ok = c.Compare(actual)
+			} else {
+				panic("wrong type of actual value")
+			}
 		default:
 			panic("unsupported constraint type")
 		}
@@ -175,6 +203,9 @@ type AudioConstraints struct {
 	Latency       DurationConstraint
 	SampleRate    IntConstraint
 	SampleSize    IntConstraint
+	IsBigEndian   BoolConstraint
+	IsFloat       BoolConstraint
+	IsInterleaved BoolConstraint
 }
 
 // Audio represents an audio's constraints
@@ -183,4 +214,7 @@ type Audio struct {
 	Latency       time.Duration
 	SampleRate    int
 	SampleSize    int
+	IsBigEndian   bool
+	IsFloat       bool
+	IsInterleaved bool
 }

pkg/prop/prop_test.go

@@ -139,6 +139,24 @@ func TestCompareMatch(t *testing.T) {
 			}},
 			true,
 		},
+		"BoolExactUnmatch": {
+			MediaConstraints{AudioConstraints: AudioConstraints{
+				IsFloat: BoolExact(true),
+			}},
+			Media{Audio: Audio{
+				IsFloat: false,
+			}},
+			false,
+		},
+		"BoolExactMatch": {
+			MediaConstraints{AudioConstraints: AudioConstraints{
+				IsFloat: BoolExact(true),
+			}},
+			Media{Audio: Audio{
+				IsFloat: true,
+			}},
+			true,
+		},
 	}
 
 	for name, testData := range testDataSet {
@@ -159,9 +177,9 @@ func TestMergeWithZero(t *testing.T) {
 		},
 	}
 
-	b := MediaConstraints{
+	b := Media{
-		VideoConstraints: VideoConstraints{
+		Video: Video{
-			Height: Int(100),
+			Height: 100,
 		},
 	}
 
@@ -183,9 +201,9 @@ func TestMergeWithSameField(t *testing.T) {
 		},
 	}
 
-	b := MediaConstraints{
+	b := Media{
-		VideoConstraints: VideoConstraints{
+		Video: Video{
-			Width: Int(100),
+			Width: 100,
 		},
 	}
 
@@ -209,9 +227,9 @@ func TestMergeNested(t *testing.T) {
 		},
 	}
 
-	b := MediaConstraints{
+	b := Media{
-		VideoConstraints: VideoConstraints{
+		Video: Video{
-			Width: Int(100),
+			Width: 100,
 		},
 	}
 
@@ -221,3 +239,73 @@ func TestMergeNested(t *testing.T) {
 		t.Error("expected a.Width to be 100, but got 0")
 	}
 }
+
+func TestMergeConstraintsWithZero(t *testing.T) {
+	a := Media{
+		Video: Video{
+			Width: 30,
+		},
+	}
+
+	b := MediaConstraints{
+		VideoConstraints: VideoConstraints{
+			Height: Int(100),
+		},
+	}
+
+	a.MergeConstraints(b)
+
+	if a.Width == 0 {
+		t.Error("expected a.Width to be 30, but got 0")
+	}
+
+	if a.Height == 0 {
+		t.Error("expected a.Height to be 100, but got 0")
+	}
+}
+
+func TestMergeConstraintsWithSameField(t *testing.T) {
+	a := Media{
+		Video: Video{
+			Width: 30,
+		},
+	}
+
+	b := MediaConstraints{
+		VideoConstraints: VideoConstraints{
+			Width: Int(100),
+		},
+	}
+
+	a.MergeConstraints(b)
+
+	if a.Width != 100 {
+		t.Error("expected a.Width to be 100, but got 0")
+	}
+}
+
+func TestMergeConstraintsNested(t *testing.T) {
+	type constraints struct {
+		Media
+	}
+
+	a := constraints{
+		Media{
+			Video: Video{
+				Width: 30,
+			},
+		},
+	}
+
+	b := MediaConstraints{
+		VideoConstraints: VideoConstraints{
+			Width: Int(100),
+		},
+	}
+
+	a.MergeConstraints(b)
+
+	if a.Width != 100 {
+		t.Error("expected a.Width to be 100, but got 0")
+	}
+}

pkg/wave/decoder.go

@@ -0,0 +1,297 @@
+package wave
+
+import (
+	"encoding/binary"
+	"fmt"
+	"math"
+	"reflect"
+	"unsafe"
+)
+
+// Format represents how audio is formatted in memory
+type Format fmt.Stringer
+
+type RawFormat struct {
+	SampleSize  int
+	IsFloat     bool
+	Interleaved bool
+}
+
+func (f *RawFormat) String() string {
+	sampleSizeInBits := f.SampleSize * 8
+	dataTypeStr := "Int"
+	if f.IsFloat {
+		dataTypeStr = "Float"
+	}
+	interleavedStr := "NonInterleaved"
+	if f.Interleaved {
+		interleavedStr = "Interleaved"
+	}
+
+	return fmt.Sprintf("%s%d%s", dataTypeStr, sampleSizeInBits, interleavedStr)
+}
+
+var hostEndian binary.ByteOrder
+var registeredDecoders = map[string]Decoder{}
+
+func init() {
+	switch v := *(*uint16)(unsafe.Pointer(&([]byte{0x12, 0x34}[0]))); v {
+	case 0x1234:
+		hostEndian = binary.BigEndian
+	case 0x3412:
+		hostEndian = binary.LittleEndian
+	default:
+		panic(fmt.Sprintf("failed to determine host endianness: %x", v))
+	}
+
+	decoderBuilders := []DecoderBuilderFunc{
+		newInt16InterleavedDecoder,
+		newInt16NonInterleavedDecoder,
+		newFloat32InterleavedDecoder,
+		newFloat32NonInterleavedDecoder,
+	}
+
+	for _, decoderBuilder := range decoderBuilders {
+		err := RegisterDecoder(decoderBuilder)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+// Decoder decodes raw chunk to Audio
+type Decoder interface {
+	// Decode decodes raw chunk in endian byte order
+	Decode(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error)
+}
+
+// DecoderFunc is a proxy type for Decoder
+type DecoderFunc func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error)
+
+func (f DecoderFunc) Decode(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+	return f(endian, chunk, channels)
+}
+
+// DecoderBuilder builds raw audio decoder
+type DecoderBuilder interface {
+	// NewDecoder creates a new decoder for specified format
+	NewDecoder() (Decoder, Format)
+}
+
+// DecoderBuilderFunc is a proxy type for DecoderBuilder
+type DecoderBuilderFunc func() (Decoder, Format)
+
+func (builderFunc DecoderBuilderFunc) NewDecoder() (Decoder, Format) {
+	return builderFunc()
+}
+
+func RegisterDecoder(builder DecoderBuilder) error {
+	decoder, format := builder.NewDecoder()
+	formatStr := format.String()
+	if _, ok := registeredDecoders[formatStr]; ok {
+		return fmt.Errorf("%v has already been registered", format)
+	}
+
+	registeredDecoders[formatStr] = decoder
+	return nil
+}
+
+// NewDecoder creates a decoder to decode raw audio data in the given format
+func NewDecoder(format Format) (Decoder, error) {
+	decoder, ok := registeredDecoders[format.String()]
+	if !ok {
+		return nil, fmt.Errorf("%s format is not supported", format)
+	}
+
+	return decoder, nil
+}
+
+func calculateChunkInfo(chunk []byte, channels int, sampleSize int) (ChunkInfo, error) {
+	if channels <= 0 {
+		return ChunkInfo{}, fmt.Errorf("channels has to be greater than 0")
+	}
+
+	if sampleSize <= 0 {
+		return ChunkInfo{}, fmt.Errorf("sample size has to be greater than 0")
+	}
+
+	sampleLen := channels * sampleSize
+	if len(chunk)%sampleLen != 0 {
+		expectedLen := len(chunk) + (sampleLen - len(chunk)%sampleLen)
+		return ChunkInfo{}, fmt.Errorf("expected chunk to have a length of %d, but got %d", expectedLen, len(chunk))
+	}
+
+	return ChunkInfo{
+		Channels: channels,
+		Len:      len(chunk) / (channels * sampleSize),
+	}, nil
+}
+
+func newInt16InterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  2,
+		IsFloat:     false,
+		Interleaved: true,
+	}
+
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewInt16Interleaved(chunkInfo)
+
+		if endian == hostEndian {
+			n := len(chunk)
+			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
+			dst := *(*[]byte)(unsafe.Pointer(&h))
+			copy(dst, chunk)
+			return container, nil
+		}
+
+		sampleLen := sampleSize * channels
+		var i int
+		for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
+			for ch := 0; ch < channels; ch++ {
+				flatOffset := offset + ch*sampleSize
+				sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
+				container.SetInt16(i, ch, Int16Sample(sample))
+			}
+			i++
+		}
+
+		return container, nil
+
+	})
+
+	return decoder, format
+}
+
+func newInt16NonInterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  2,
+		IsFloat:     false,
+		Interleaved: false,
+	}
+
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewInt16NonInterleaved(chunkInfo)
+		chunkLen := len(chunk) / channels
+
+		if endian == hostEndian {
+			for ch := 0; ch < channels; ch++ {
+				offset := ch * chunkLen
+				h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
+				dst := *(*[]byte)(unsafe.Pointer(&h))
+				copy(dst, chunk[offset:offset+chunkLen])
+			}
+			return container, nil
+		}
+
+		for ch := 0; ch < channels; ch++ {
+			offset := ch * chunkLen
+			for i := 0; i < chunkInfo.Len; i++ {
+				flatOffset := offset + i*sampleSize
+				sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
+				container.SetInt16(i, ch, Int16Sample(sample))
+			}
+		}
+
+		return container, nil
+	})
+
+	return decoder, format
+}
+
+func newFloat32InterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  4,
+		IsFloat:     true,
+		Interleaved: true,
+	}
+
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewFloat32Interleaved(chunkInfo)
+
+		if endian == hostEndian {
+			n := len(chunk)
+			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
+			dst := *(*[]byte)(unsafe.Pointer(&h))
+			copy(dst, chunk)
+			return container, nil
+		}
+
+		sampleLen := sampleSize * channels
+		var i int
+		for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
+			for ch := 0; ch < channels; ch++ {
+				flatOffset := offset + ch*sampleSize
+				sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
+				sampleF := math.Float32frombits(sample)
+				container.SetFloat32(i, ch, Float32Sample(sampleF))
+			}
+			i++
+		}
+
+		return container, nil
+	})
+
+	return decoder, format
+}
+
+func newFloat32NonInterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  4,
+		IsFloat:     true,
+		Interleaved: false,
+	}
+
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewFloat32NonInterleaved(chunkInfo)
+		chunkLen := len(chunk) / channels
+
+		if endian == hostEndian {
+			for ch := 0; ch < channels; ch++ {
+				offset := ch * chunkLen
+				h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
+				dst := *(*[]byte)(unsafe.Pointer(&h))
+				copy(dst, chunk[offset:offset+chunkLen])
+			}
+			return container, nil
+		}
+
+		for ch := 0; ch < channels; ch++ {
+			offset := ch * chunkLen
+			for i := 0; i < chunkInfo.Len; i++ {
+				flatOffset := offset + i*sampleSize
+				sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
+				sampleF := math.Float32frombits(sample)
+				container.SetFloat32(i, ch, Float32Sample(sampleF))
+			}
+		}
+
+		return container, nil
+	})
+
+	return decoder, format
+}

pkg/wave/decoder_bench_test.go

@@ -0,0 +1,39 @@
+package wave
+
+import (
+	"encoding/binary"
+	"fmt"
+	"testing"
+)
+
+func BenchmarkDecoder(b *testing.B) {
+	var nonHostEndian binary.ByteOrder
+	if hostEndian == binary.BigEndian {
+		nonHostEndian = binary.LittleEndian
+	} else {
+		nonHostEndian = binary.BigEndian
+	}
+
+	for format, decoder := range registeredDecoders {
+		format := format
+		decoder := decoder
+
+		b.Run(fmt.Sprintf("%sHostEndian", format), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				_, err := decoder.Decode(hostEndian, make([]byte, 800), 2)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+
+		b.Run(fmt.Sprintf("%sNonHostEndian", format), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				_, err := decoder.Decode(nonHostEndian, make([]byte, 800), 2)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/wave/decoder_test.go

@@ -0,0 +1,353 @@
+package wave
+
+import (
+	"encoding/binary"
+	"math"
+	"reflect"
+	"testing"
+)
+
+func TestCalculateChunkInfo(t *testing.T) {
+	testCases := map[string]struct {
+		chunk      []byte
+		channels   int
+		sampleSize int
+		expected   ChunkInfo
+		expectErr  bool
+	}{
+		"InvalidChunkSize1": {
+			chunk:      make([]byte, 3),
+			channels:   2,
+			sampleSize: 2,
+			expected:   ChunkInfo{},
+			expectErr:  true,
+		},
+		"InvalidChunkSize2": {
+			chunk:      make([]byte, 4),
+			channels:   2,
+			sampleSize: 4,
+			expected:   ChunkInfo{},
+			expectErr:  true,
+		},
+		"InvalidChannels": {
+			chunk:      nil,
+			channels:   0,
+			sampleSize: 2,
+			expected:   ChunkInfo{},
+			expectErr:  true,
+		},
+		"InvalidSampleSize": {
+			chunk:      nil,
+			channels:   2,
+			sampleSize: 0,
+			expected:   ChunkInfo{},
+			expectErr:  true,
+		},
+		"Valid1": {
+			chunk:      nil,
+			channels:   2,
+			sampleSize: 2,
+			expected: ChunkInfo{
+				Len:          0,
+				Channels:     2,
+				SamplingRate: 0,
+			},
+			expectErr: false,
+		},
+		"Valid2": {
+			chunk:      make([]byte, 8),
+			channels:   2,
+			sampleSize: 4,
+			expected: ChunkInfo{
+				Len:          1,
+				Channels:     2,
+				SamplingRate: 0,
+			},
+			expectErr: false,
+		},
+		"Valid3": {
+			chunk:      make([]byte, 4),
+			channels:   1,
+			sampleSize: 2,
+			expected: ChunkInfo{
+				Len:          2,
+				Channels:     1,
+				SamplingRate: 0,
+			},
+			expectErr: false,
+		},
+	}
+
+	for testCaseName, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCaseName, func(t *testing.T) {
+			actual, err := calculateChunkInfo(testCase.chunk, testCase.channels, testCase.sampleSize)
+			if testCase.expectErr && err == nil {
+				t.Fatal("expected an error, but got nil")
+			} else if !testCase.expectErr && err != nil {
+				t.Fatalf("expected no error, but got %s", err)
+			} else if !testCase.expectErr && !reflect.DeepEqual(actual, testCase.expected) {
+				t.Errorf("Wrong chunk info calculation result,\nexpected:\n%+v\ngot:\n%+v", testCase.expected, actual)
+			}
+		})
+	}
+}
+
+func TestNewDecoder(t *testing.T) {
+	rawFormats := []RawFormat{
+		{
+			SampleSize:  2,
+			IsFloat:     false,
+			Interleaved: false,
+		},
+		{
+			SampleSize:  4,
+			IsFloat:     true,
+			Interleaved: false,
+		},
+		{
+			SampleSize:  2,
+			IsFloat:     false,
+			Interleaved: true,
+		},
+		{
+			SampleSize:  4,
+			IsFloat:     true,
+			Interleaved: true,
+		},
+	}
+
+	for _, rawFormat := range rawFormats {
+		_, err := NewDecoder(&rawFormat)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestDecodeInt16Interleaved(t *testing.T) {
+	raw := []byte{
+		// 16 bits per channel
+		0x01, 0x02, 0x03, 0x04,
+		0x05, 0x06, 0x07, 0x08,
+	}
+	decoder, _ := newInt16InterleavedDecoder()
+
+	t.Run("BigEndian", func(t *testing.T) {
+		expected := &Int16Interleaved{
+			Data: []int16{
+				int16(binary.BigEndian.Uint16([]byte{0x01, 0x02})),
+				int16(binary.BigEndian.Uint16([]byte{0x03, 0x04})),
+				int16(binary.BigEndian.Uint16([]byte{0x05, 0x06})),
+				int16(binary.BigEndian.Uint16([]byte{0x07, 0x08})),
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+
+	t.Run("LittleEndian", func(t *testing.T) {
+		expected := &Int16Interleaved{
+			Data: []int16{
+				int16(binary.LittleEndian.Uint16([]byte{0x01, 0x02})),
+				int16(binary.LittleEndian.Uint16([]byte{0x03, 0x04})),
+				int16(binary.LittleEndian.Uint16([]byte{0x05, 0x06})),
+				int16(binary.LittleEndian.Uint16([]byte{0x07, 0x08})),
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+}
+
+func TestDecodeInt16NonInterleaved(t *testing.T) {
+	raw := []byte{
+		// 16 bits per channel
+		0x01, 0x02, 0x03, 0x04,
+		0x05, 0x06, 0x07, 0x08,
+	}
+
+	decoder, _ := newInt16NonInterleavedDecoder()
+
+	t.Run("BigEndian", func(t *testing.T) {
+		expected := &Int16NonInterleaved{
+			Data: [][]int16{
+				{int16(binary.BigEndian.Uint16([]byte{0x01, 0x02})), int16(binary.BigEndian.Uint16([]byte{0x03, 0x04}))},
+				{int16(binary.BigEndian.Uint16([]byte{0x05, 0x06})), int16(binary.BigEndian.Uint16([]byte{0x07, 0x08}))},
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+
+	t.Run("LittleEndian", func(t *testing.T) {
+		expected := &Int16NonInterleaved{
+			Data: [][]int16{
+				{int16(binary.LittleEndian.Uint16([]byte{0x01, 0x02})), int16(binary.LittleEndian.Uint16([]byte{0x03, 0x04}))},
+				{int16(binary.LittleEndian.Uint16([]byte{0x05, 0x06})), int16(binary.LittleEndian.Uint16([]byte{0x07, 0x08}))},
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+}
+
+func TestDecodeFloat32Interleaved(t *testing.T) {
+	raw := []byte{
+		// 32 bits per channel
+		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+		0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+	}
+
+	decoder, _ := newFloat32InterleavedDecoder()
+
+	t.Run("BigEndian", func(t *testing.T) {
+		expected := &Float32Interleaved{
+			Data: []float32{
+				math.Float32frombits(binary.BigEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04})),
+				math.Float32frombits(binary.BigEndian.Uint32([]byte{0x05, 0x06, 0x07, 0x08})),
+				math.Float32frombits(binary.BigEndian.Uint32([]byte{0x09, 0x0a, 0x0b, 0x0c})),
+				math.Float32frombits(binary.BigEndian.Uint32([]byte{0x0d, 0x0e, 0x0f, 0x10})),
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+
+	t.Run("LittleEndian", func(t *testing.T) {
+		expected := &Float32Interleaved{
+			Data: []float32{
+				math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04})),
+				math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x05, 0x06, 0x07, 0x08})),
+				math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x09, 0x0a, 0x0b, 0x0c})),
+				math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x0d, 0x0e, 0x0f, 0x10})),
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+}
+
+func TestDecodeFloat32NonInterleaved(t *testing.T) {
+	raw := []byte{
+		// 32 bits per channel
+		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+		0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+	}
+
+	decoder, _ := newFloat32NonInterleavedDecoder()
+
+	t.Run("BigEndian", func(t *testing.T) {
+		expected := &Float32NonInterleaved{
+			Data: [][]float32{
+				{
+					math.Float32frombits(binary.BigEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04})),
+					math.Float32frombits(binary.BigEndian.Uint32([]byte{0x05, 0x06, 0x07, 0x08})),
+				},
+				{
+					math.Float32frombits(binary.BigEndian.Uint32([]byte{0x09, 0x0a, 0x0b, 0x0c})),
+					math.Float32frombits(binary.BigEndian.Uint32([]byte{0x0d, 0x0e, 0x0f, 0x10})),
+				},
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+
+	t.Run("LittleEndian", func(t *testing.T) {
+		expected := &Float32NonInterleaved{
+			Data: [][]float32{
+				{
+					math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04})),
+					math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x05, 0x06, 0x07, 0x08})),
+				},
+				{
+					math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x09, 0x0a, 0x0b, 0x0c})),
+					math.Float32frombits(binary.LittleEndian.Uint32([]byte{0x0d, 0x0e, 0x0f, 0x10})),
+				},
+			},
+			Size: ChunkInfo{
+				Len:      2,
+				Channels: 2,
+			},
+		}
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(expected, actual) {
+			t.Errorf("Wrong decode result,\nexpected:\n%+v\ngot:\n%+v", expected, actual)
+		}
+	})
+}

pkg/wave/float32.go

@@ -34,6 +34,16 @@ func (a *Float32Interleaved) SetFloat32(i, ch int, s Float32Sample) {
 	a.Data[i*a.Size.Channels+ch] = float32(s)
 }
 
+// SubAudio returns part of the original audio sharing the buffer.
+func (a *Float32Interleaved) SubAudio(offsetSamples, nSamples int) *Float32Interleaved {
+	ret := *a
+	offset := offsetSamples * a.Size.Channels
+	n := nSamples * a.Size.Channels
+	ret.Data = ret.Data[offset : offset+n]
+	ret.Size.Len = nSamples
+	return &ret
+}
+
 func NewFloat32Interleaved(size ChunkInfo) *Float32Interleaved {
 	return &Float32Interleaved{
 		Data: make([]float32, size.Channels*size.Len),
@@ -68,6 +78,16 @@ func (a *Float32NonInterleaved) SetFloat32(i, ch int, s Float32Sample) {
 	a.Data[ch][i] = float32(s)
 }
 
+// SubAudio returns part of the original audio sharing the buffer.
+func (a *Float32NonInterleaved) SubAudio(offsetSamples, nSamples int) *Float32NonInterleaved {
+	ret := *a
+	for i := range a.Data {
+		ret.Data[i] = ret.Data[i][offsetSamples : offsetSamples+nSamples]
+	}
+	ret.Size.Len = nSamples
+	return &ret
+}
+
 func NewFloat32NonInterleaved(size ChunkInfo) *Float32NonInterleaved {
 	d := make([][]float32, size.Channels)
 	for i := 0; i < size.Channels; i++ {

pkg/wave/float32_test.go

@@ -51,3 +51,44 @@ func TestFloat32(t *testing.T) {
 		})
 	}
 }
+
+func TestFloat32SubAudio(t *testing.T) {
+	t.Run("Interleaved", func(t *testing.T) {
+		in := &Float32Interleaved{
+			Data: []float32{
+				0.1, -0.5, 0.2, -0.6, 0.3, -0.7, 0.4, -0.8, 0.5, -0.9, 0.6, -1.0, 0.7, -1.1, 0.8, -1.2,
+			},
+			Size: ChunkInfo{8, 2, 48000},
+		}
+		expected := &Float32Interleaved{
+			Data: []float32{
+				0.3, -0.7, 0.4, -0.8, 0.5, -0.9,
+			},
+			Size: ChunkInfo{3, 2, 48000},
+		}
+		out := in.SubAudio(2, 3)
+		if !reflect.DeepEqual(expected, out) {
+			t.Errorf("SubAudio differs, expected: %v, got: %v", expected, out)
+		}
+	})
+	t.Run("NonInterleaved", func(t *testing.T) {
+		in := &Float32NonInterleaved{
+			Data: [][]float32{
+				{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8},
+				{-0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1, -1.2},
+			},
+			Size: ChunkInfo{8, 2, 48000},
+		}
+		expected := &Float32NonInterleaved{
+			Data: [][]float32{
+				{0.3, 0.4, 0.5},
+				{-0.7, -0.8, -0.9},
+			},
+			Size: ChunkInfo{3, 2, 48000},
+		}
+		out := in.SubAudio(2, 3)
+		if !reflect.DeepEqual(expected, out) {
+			t.Errorf("SubAudio differs, expected: %v, got: %v", expected, out)
+		}
+	})
+}

pkg/wave/int16.go

@@ -34,6 +34,16 @@ func (a *Int16Interleaved) SetInt16(i, ch int, s Int16Sample) {
 	a.Data[i*a.Size.Channels+ch] = int16(s)
 }
 
+// SubAudio returns part of the original audio sharing the buffer.
+func (a *Int16Interleaved) SubAudio(offsetSamples, nSamples int) *Int16Interleaved {
+	ret := *a
+	offset := offsetSamples * a.Size.Channels
+	n := nSamples * a.Size.Channels
+	ret.Data = ret.Data[offset : offset+n]
+	ret.Size.Len = nSamples
+	return &ret
+}
+
 func NewInt16Interleaved(size ChunkInfo) *Int16Interleaved {
 	return &Int16Interleaved{
 		Data: make([]int16, size.Channels*size.Len),
@@ -68,6 +78,16 @@ func (a *Int16NonInterleaved) SetInt16(i, ch int, s Int16Sample) {
 	a.Data[ch][i] = int16(s)
 }
 
+// SubAudio returns part of the original audio sharing the buffer.
+func (a *Int16NonInterleaved) SubAudio(offsetSamples, nSamples int) *Int16NonInterleaved {
+	ret := *a
+	for i := range a.Data {
+		ret.Data[i] = ret.Data[i][offsetSamples : offsetSamples+nSamples]
+	}
+	ret.Size.Len = nSamples
+	return &ret
+}
+
 func NewInt16NonInterleaved(size ChunkInfo) *Int16NonInterleaved {
 	d := make([][]int16, size.Channels)
 	for i := 0; i < size.Channels; i++ {

pkg/wave/int16_test.go

@@ -51,3 +51,44 @@ func TestInt16(t *testing.T) {
 		})
 	}
 }
+
+func TestInt32SubAudio(t *testing.T) {
+	t.Run("Interleaved", func(t *testing.T) {
+		in := &Int16Interleaved{
+			Data: []int16{
+				1, -5, 2, -6, 3, -7, 4, -8, 5, -9, 6, -10, 7, -11, 8, -12,
+			},
+			Size: ChunkInfo{8, 2, 48000},
+		}
+		expected := &Int16Interleaved{
+			Data: []int16{
+				3, -7, 4, -8, 5, -9,
+			},
+			Size: ChunkInfo{3, 2, 48000},
+		}
+		out := in.SubAudio(2, 3)
+		if !reflect.DeepEqual(expected, out) {
+			t.Errorf("SubAudio differs, expected: %v, got: %v", expected, out)
+		}
+	})
+	t.Run("NonInterleaved", func(t *testing.T) {
+		in := &Int16NonInterleaved{
+			Data: [][]int16{
+				{1, 2, 3, 4, 5, 6, 7, 8},
+				{-5, -6, -7, -8, -9, -10, -11, -12},
+			},
+			Size: ChunkInfo{8, 2, 48000},
+		}
+		expected := &Int16NonInterleaved{
+			Data: [][]int16{
+				{3, 4, 5},
+				{-7, -8, -9},
+			},
+			Size: ChunkInfo{3, 2, 48000},
+		}
+		out := in.SubAudio(2, 3)
+		if !reflect.DeepEqual(expected, out) {
+			t.Errorf("SubAudio differs, expected: %v, got: %v", expected, out)
+		}
+	})
+}

pkg/wave/int64.go

@@ -0,0 +1,8 @@
+package wave
+
+// Int64Sample is a 64-bits signed integer audio sample.
+type Int64Sample int64
+
+func (s Int64Sample) Int() int64 {
+	return int64(s)
+}

pkg/wave/mixer/mixer.go

@@ -0,0 +1,42 @@
+package mixer
+
+import (
+	"errors"
+
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+// ChannelMixer mixes audio into specifix channels.
+type ChannelMixer interface {
+	Mix(dst wave.Audio, src wave.Audio) error
+}
+
+// MonoMixer mixes channels into monaural audio.
+type MonoMixer struct {
+}
+
+func (m *MonoMixer) Mix(dst wave.Audio, src wave.Audio) error {
+	if dst.ChunkInfo().Len != src.ChunkInfo().Len {
+		return errors.New("buffer size mismatch")
+	}
+	dstSetter, ok := dst.(wave.EditableAudio)
+	if !ok {
+		return errors.New("destination buffer is not settable")
+	}
+
+	n := src.ChunkInfo().Len
+	channels := src.ChunkInfo().Channels
+	dstChannels := dst.ChunkInfo().Channels
+	for i := 0; i < n; i++ {
+		var mean int64
+		for ch := 0; ch < channels; ch++ {
+			mean += src.At(i, ch).Int()
+		}
+		mean /= int64(channels)
+
+		for ch := 0; ch < dstChannels; ch++ {
+			dstSetter.Set(i, ch, wave.Int64Sample(mean))
+		}
+	}
+	return nil
+}

pkg/wave/mixer/mixer_test.go

@@ -0,0 +1,80 @@
+package mixer
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+func TestMonoMixer(t *testing.T) {
+	testCases := map[string]struct {
+		src      wave.Audio
+		dst      wave.Audio
+		expected wave.Audio
+	}{
+		"MultiToMono": {
+			src: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 3,
+				},
+				Data: []int16{
+					0, 2, 4,
+					1, -2, 1,
+					3, 3, 6,
+				},
+			},
+			dst: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 1,
+				},
+				Data: make([]int16, 3),
+			},
+			expected: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 1,
+				},
+				Data: []int16{2, 0, 4},
+			},
+		},
+		"MonoToStereo": {
+			src: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 1,
+				},
+				Data: []int16{0, 2, 4},
+			},
+			dst: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 2,
+				},
+				Data: make([]int16, 6),
+			},
+			expected: &wave.Int16Interleaved{
+				Size: wave.ChunkInfo{
+					Len:      3,
+					Channels: 2,
+				},
+				Data: []int16{0, 0, 2, 2, 4, 4},
+			},
+		},
+	}
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			m := &MonoMixer{}
+			err := m.Mix(testCase.dst, testCase.src)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(testCase.expected, testCase.dst) {
+				t.Errorf("Mix result is wrong\nexpected: %v\ngot: %v", testCase.expected, testCase.dst)
+			}
+		})
+	}
+}

pkg/wave/wave.go

@@ -8,6 +8,12 @@ type Audio interface {
 	At(i, ch int) Sample
 }
 
+// EditableAudio is an editable finite series of audio Sample values.
+type EditableAudio interface {
+	Audio
+	Set(i, ch int, s Sample)
+}
+
 // ChunkInfo contains size of the audio chunk.
 type ChunkInfo struct {
 	Len          int

rtp.go

@@ -0,0 +1,101 @@
+package mediadevices
+
+import (
+	"fmt"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v2"
+)
+
+type RTPTracker struct {
+	videoEncoders []codec.VideoEncoderBuilder
+	audioEncoders []codec.AudioEncoderBuilder
+}
+
+type RTPTrackerOption func(*RTPTracker)
+
+func WithVideoEncoders(codecs ...codec.VideoEncoderBuilder) func(*RTPTracker) {
+	return func(tracker *RTPTracker) {
+		tracker.videoEncoders = codecs
+	}
+}
+
+func WithAudioEncoders(codecs ...codec.AudioEncoderBuilder) func(*RTPTracker) {
+	return func(tracker *RTPTracker) {
+		tracker.audioEncoders = codecs
+	}
+}
+
+func NewRTPTracker(opts ...RTPTrackerOption) *RTPTracker {
+	var tracker RTPTracker
+
+	for _, opt := range opts {
+		opt(&tracker)
+	}
+
+	return &tracker
+}
+
+func (tracker *RTPTracker) Track(track Track) *RTPTrack {
+	rtpTrack := RTPTrack{
+		Track: track,
+	}
+
+	return &rtpTrack
+}
+
+type RTPTrack struct {
+	Track
+	tracker        *RTPTracker
+	currentEncoder codec.ReadCloser
+	currentParams  RTPParameters
+	lastProp       prop.Media
+}
+
+func (track *RTPTrack) SetParameters(params RTPParameters) error {
+	var err error
+
+	switch t := track.Track.(type) {
+	case *VideoTrack:
+		err = track.setParametersVideo(t, &params)
+	case *AudioTrack:
+		err = track.setParametersAudio(t, &params)
+	default:
+		err = fmt.Errorf("unsupported track type")
+	}
+
+	if err == nil {
+		track.currentParams = params
+	}
+
+	return err
+}
+
+func (track *RTPTrack) setParametersVideo(videoTrack *VideoTrack, params *RTPParameters) error {
+	if params.SelectedCodec.Type != webrtc.RTPCodecTypeVideo {
+		return fmt.Errorf("invalid selected RTP codec type. Expected video but got audio")
+	}
+
+	video.DetectChanges(interval time.Duration, onChange func(prop.Media))
+	return nil
+}
+
+func (track *RTPTrack) setParametersAudio(audioTrack *AudioTrack, params *RTPParameters) error {
+	return nil
+}
+
+func (track *RTPTrack) ReadRTP() (*rtp.Packet, error) {
+	if track.currentEncoder == nil {
+		return nil, fmt.Errorf("Encoder has not been specified. Please call SetParameters to specify.")
+	}
+
+	return nil, nil
+}
+
+func (track *RTPTrack) WriteRTCP(packet rtcp.Packet) error {
+	return nil
+}

sampler.go

@@ -1,35 +0,0 @@
-package mediadevices
-
-import (
-	"math"
-	"time"
-
-	"github.com/pion/webrtc/v2/pkg/media"
-)
-
-type samplerFunc func(b []byte) error
-
-// newVideoSampler creates a video sampler that uses the actual video frame rate and
-// the codec's clock rate to come up with a duration for each sample.
-func newVideoSampler(t LocalTrack) samplerFunc {
-	clockRate := float64(t.Codec().ClockRate)
-	lastTimestamp := time.Now()
-
-	return samplerFunc(func(b []byte) error {
-		now := time.Now()
-		duration := now.Sub(lastTimestamp).Seconds()
-		samples := uint32(math.Round(clockRate * duration))
-		lastTimestamp = now
-
-		return t.WriteSample(media.Sample{Data: b, Samples: samples})
-	})
-}
-
-// newAudioSampler creates a audio sampler that uses a fixed latency and
-// the codec's clock rate to come up with a duration for each sample.
-func newAudioSampler(t LocalTrack, latency time.Duration) samplerFunc {
-	samples := uint32(math.Round(float64(t.Codec().ClockRate) * latency.Seconds()))
-	return samplerFunc(func(b []byte) error {
-		return t.WriteSample(media.Sample{Data: b, Samples: samples})
-	})
-}

track.go

@@ -1,22 +1,29 @@
 package mediadevices
 
 import (
-	"errors"
+	"fmt"
-	"math/rand"
+	"image"
 	"sync"
 
-	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/driver"
-	mio "github.com/pion/mediadevices/pkg/io"
+	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/webrtc/v2/pkg/media"
+	"github.com/pion/mediadevices/pkg/wave"
 )
 
-// Tracker is an interface that represent MediaStreamTrack
+// TrackKind represents content type of a track
+type TrackKind string
+
+const (
+	TrackKindVideo TrackKind = "video"
+	TrackKindAudio TrackKind = "audio"
+)
+
+// Track is an interface that represent MediaStreamTrack
 // Reference: https://w3c.github.io/mediacapture-main/#mediastreamtrack
-type Tracker interface {
+type Track interface {
-	Track() *webrtc.Track
+	ID() string
-	LocalTrack() LocalTrack
+	Kind() TrackKind
 	Stop()
 	// OnEnded registers a handler to receive an error from the media stream track.
 	// If the error is already occured before registering, the handler will be
@@ -24,18 +31,170 @@ type Tracker interface {
 	OnEnded(func(error))
 }
 
-type LocalTrack interface {
+// VideoTrack is a specialized track for video
-	WriteSample(s media.Sample) error
+type VideoTrack struct {
-	Codec() *webrtc.RTPCodec
+	baseTrack
-	ID() string
+	src         video.Reader
-	Kind() webrtc.RTPCodecType
+	transformed video.Reader
+	mux         sync.Mutex
+	frameCount  int
+	lastFrame   image.Image
+	lastErr     error
 }
 
-type track struct {
+func newVideoTrack(d driver.Driver, constraints MediaTrackConstraints) (*VideoTrack, error) {
-	localTrack LocalTrack
+	err := d.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	recorder, ok := d.(driver.VideoRecorder)
+	if !ok {
+		d.Close()
+		return nil, fmt.Errorf("driver is not an video recorder")
+	}
+
+	r, err := recorder.VideoRecord(constraints.selectedMedia)
+	if err != nil {
+		d.Close()
+		return nil, err
+	}
+
+	return &VideoTrack{
+		baseTrack:   newBaseTrack(d, constraints),
+		src:         r,
+		transformed: r,
+	}, nil
+}
+
+// Kind returns track's kind
+func (track *VideoTrack) Kind() TrackKind {
+	return TrackKindVideo
+}
+
+// NewReader returns a reader to read frames from the source. You may create multiple
+// readers and read from them in different goroutines.
+//
+// In the case of multiple readers, reading from the source will only get triggered
+// when the reader has the latest frame from the source
+func (track *VideoTrack) NewReader() video.Reader {
+	var curFrameCount int
+	return video.ReaderFunc(func() (img image.Image, err error) {
+		track.mux.Lock()
+		defer track.mux.Unlock()
+
+		if curFrameCount != track.frameCount {
+			img = copyFrame(img, track.lastFrame)
+			err = track.lastErr
+		} else {
+			img, err = track.transformed.Read()
+			track.lastFrame = img
+			track.lastErr = err
+			track.frameCount++
+			if err != nil {
+				track.onErrorHandler(err)
+			}
+		}
+
+		curFrameCount = track.frameCount
+		return
+	})
+}
+
+// TODO: implement copy in place
+func copyFrame(dst, src image.Image) image.Image { return src }
+
+// Transform transforms the underlying source. The transformation will reflect to
+// all readers
+func (track *VideoTrack) Transform(fns ...video.TransformFunc) {
+	track.mux.Lock()
+	defer track.mux.Unlock()
+	track.transformed = video.Merge(fns...)(track.src)
+}
+
+// AudioTrack is a specialized track for audio
+type AudioTrack struct {
+	baseTrack
+	src         audio.Reader
+	transformed audio.Reader
+	mux         sync.Mutex
+	chunkCount  int
+	lastChunks  wave.Audio
+	lastErr     error
+}
+
+func newAudioTrack(d driver.Driver, constraints MediaTrackConstraints) (*AudioTrack, error) {
+	err := d.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	recorder, ok := d.(driver.AudioRecorder)
+	if !ok {
+		d.Close()
+		return nil, fmt.Errorf("driver is not an audio recorder")
+	}
+
+	r, err := recorder.AudioRecord(constraints.selectedMedia)
+	if err != nil {
+		d.Close()
+		return nil, err
+	}
+
+	return &AudioTrack{
+		baseTrack:   newBaseTrack(d, constraints),
+		src:         r,
+		transformed: r,
+	}, nil
+}
+
+func (track *AudioTrack) Kind() TrackKind {
+	return TrackKindAudio
+}
+
+// NewReader returns a reader to read audio chunks from the source. You may create multiple
+// readers and read from them in different goroutines.
+//
+// In the case of multiple readers, reading from the source will only get triggered
+// when the reader has the latest chunk from the source
+func (track *AudioTrack) NewReader() audio.Reader {
+	var currChunkCount int
+	return audio.ReaderFunc(func() (chunks wave.Audio, err error) {
+		track.mux.Lock()
+		defer track.mux.Unlock()
+
+		if currChunkCount != track.chunkCount {
+			chunks = copyChunks(chunks, track.lastChunks)
+			err = track.lastErr
+		} else {
+			chunks, err = track.transformed.Read()
+			track.lastChunks = chunks
+			track.lastErr = err
+			track.chunkCount++
+			if err != nil {
+				track.onErrorHandler(err)
+			}
+		}
+
+		currChunkCount = track.chunkCount
+		return
+	})
+}
+
+// TODO: implement copy in place
+func copyChunks(dst, src wave.Audio) wave.Audio { return src }
+
+// Transform transforms the underlying source. The transformation will reflect to
+// all readers
+func (track *AudioTrack) Transform(fns ...audio.TransformFunc) {
+	track.mux.Lock()
+	defer track.mux.Unlock()
+	track.transformed = audio.Merge(fns...)(track.src)
+}
+
+type baseTrack struct {
 	d           driver.Driver
-	sample     samplerFunc
+	constraints MediaTrackConstraints
-	encoder    codec.ReadCloser
 
 	onErrorHandler func(error)
 	err            error
@@ -43,83 +202,17 @@ type track struct {
 	endOnce        sync.Once
 }
 
-func newTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*track, error) {
+func newBaseTrack(d driver.Driver, constraints MediaTrackConstraints) baseTrack {
-	var encoderBuilders []encoderBuilder
+	return baseTrack{d: d, constraints: constraints}
-	var rtpCodecs []*webrtc.RTPCodec
-	var buildSampler func(t LocalTrack) samplerFunc
-	var err error
-
-	err = d.Open()
-	if err != nil {
-		return nil, err
 }
 
-	switch r := d.(type) {
+func (t *baseTrack) ID() string {
-	case driver.VideoRecorder:
+	return t.d.ID()
-		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeVideo]
-		buildSampler = newVideoSampler
-		encoderBuilders, err = newVideoEncoderBuilders(r, constraints)
-	case driver.AudioRecorder:
-		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeAudio]
-		buildSampler = func(t LocalTrack) samplerFunc {
-			return newAudioSampler(t, constraints.selectedMedia.Latency)
-		}
-		encoderBuilders, err = newAudioEncoderBuilders(r, constraints)
-	default:
-		err = errors.New("newTrack: invalid driver type")
-	}
-
-	if err != nil {
-		d.Close()
-		return nil, err
-	}
-
-	for _, builder := range encoderBuilders {
-		var matchedRTPCodec *webrtc.RTPCodec
-		for _, rtpCodec := range rtpCodecs {
-			if rtpCodec.Name == builder.name {
-				matchedRTPCodec = rtpCodec
-				break
-			}
-		}
-
-		if matchedRTPCodec == nil {
-			continue
-		}
-
-		localTrack, err := opts.trackGenerator(
-			matchedRTPCodec.PayloadType,
-			rand.Uint32(),
-			d.ID(),
-			matchedRTPCodec.Type.String(),
-			matchedRTPCodec,
-		)
-		if err != nil {
-			continue
-		}
-
-		encoder, err := builder.build()
-		if err != nil {
-			continue
-		}
-
-		t := track{
-			localTrack: localTrack,
-			sample:     buildSampler(localTrack),
-			d:          d,
-			encoder:    encoder,
-		}
-		go t.start()
-		return &t, nil
-	}
-
-	d.Close()
-	return nil, errors.New("newTrack: failed to find a matching codec")
 }
 
 // OnEnded sets an error handler. When a track has been created and started, if an
 // error occurs, handler will get called with the error given to the parameter.
-func (t *track) OnEnded(handler func(error)) {
+func (t *baseTrack) OnEnded(handler func(error)) {
 	t.mu.Lock()
 	t.onErrorHandler = handler
 	err := t.err
@@ -134,7 +227,7 @@ func (t *track) OnEnded(handler func(error)) {
 }
 
 // onError is a callback when an error occurs
-func (t *track) onError(err error) {
+func (t *baseTrack) onError(err error) {
 	t.mu.Lock()
 	t.err = err
 	handler := t.onErrorHandler
@@ -147,92 +240,6 @@ func (t *track) onError(err error) {
 	}
 }
 
-// start starts the data flow from the driver all the way to the localTrack
+func (t *baseTrack) Stop() {
-func (t *track) start() {
-	var n int
-	var err error
-	buff := make([]byte, 1024)
-	for {
-		n, err = t.encoder.Read(buff)
-		if err != nil {
-			if e, ok := err.(*mio.InsufficientBufferError); ok {
-				buff = make([]byte, 2*e.RequiredSize)
-				continue
-			}
-
-			t.onError(err)
-			return
-		}
-
-		if err := t.sample(buff[:n]); err != nil {
-			t.onError(err)
-			return
-		}
-	}
-}
-
-// Stop stops the underlying driver and encoder
-func (t *track) Stop() {
 	t.d.Close()
-	t.encoder.Close()
-}
-
-func (t *track) Track() *webrtc.Track {
-	return t.localTrack.(*webrtc.Track)
-}
-
-func (t *track) LocalTrack() LocalTrack {
-	return t.localTrack
-}
-
-// encoderBuilder is a generic encoder builder that acts as a delegator for codec.VideoEncoderBuilder and
-// codec.AudioEncoderBuilder. The idea of having a delegator is to reduce redundant codes that are being
-// duplicated for managing video and audio.
-type encoderBuilder struct {
-	name  string
-	build func() (codec.ReadCloser, error)
-}
-
-// newVideoEncoderBuilders transforms video given by VideoRecorder with the video transformer that is passed through
-// constraints and create a list of generic encoder builders
-func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
-	r, err := vr.VideoRecord(constraints.selectedMedia)
-	if err != nil {
-		return nil, err
-	}
-
-	if constraints.VideoTransform != nil {
-		r = constraints.VideoTransform(r)
-	}
-
-	encoderBuilders := make([]encoderBuilder, len(constraints.VideoEncoderBuilders))
-	for i, b := range constraints.VideoEncoderBuilders {
-		encoderBuilders[i].name = b.Name()
-		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
-			return b.BuildVideoEncoder(r, constraints.selectedMedia)
-		}
-	}
-	return encoderBuilders, nil
-}
-
-// newAudioEncoderBuilders transforms audio given by AudioRecorder with the audio transformer that is passed through
-// constraints and create a list of generic encoder builders
-func newAudioEncoderBuilders(ar driver.AudioRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
-	r, err := ar.AudioRecord(constraints.selectedMedia)
-	if err != nil {
-		return nil, err
-	}
-
-	if constraints.AudioTransform != nil {
-		r = constraints.AudioTransform(r)
-	}
-
-	encoderBuilders := make([]encoderBuilder, len(constraints.AudioEncoderBuilders))
-	for i, b := range constraints.AudioEncoderBuilders {
-		encoderBuilders[i].name = b.Name()
-		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
-			return b.BuildAudioEncoder(r, constraints.selectedMedia)
-		}
-	}
-	return encoderBuilders, nil
 }

track_test.go

@@ -10,7 +10,7 @@ func TestOnEnded(t *testing.T) {
 	errExpected := errors.New("an error")
 
 	t.Run("ErrorAfterRegister", func(t *testing.T) {
-		tr := &track{}
+		tr := &baseTrack{}
 
 		called := make(chan error, 1)
 		tr.OnEnded(func(error) {
@@ -35,7 +35,7 @@ func TestOnEnded(t *testing.T) {
 	})
 
 	t.Run("ErrorBeforeRegister", func(t *testing.T) {
-		tr := &track{}
+		tr := &baseTrack{}
 
 		tr.onError(errExpected)
 

webrtc.go

@@ -0,0 +1,45 @@
+package mediadevices
+
+import (
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v2"
+)
+
+// == WebRTC v3 design ==
+
+// Reader is an interface to handle incoming RTP stream.
+type Reader interface {
+	ReadRTP() (*rtp.Packet, error)
+	WriteRTCP(rtcp.Packet) error
+}
+
+// TrackBase represents common MediaStreamTrack functionality of LocalTrack and RemoteTrack.
+type TrackBase interface {
+	ID() string
+}
+
+type LocalRTPTrack interface {
+	TrackBase
+	Reader
+
+	// SetParameters sets information about how the data is to be encoded.
+	// This will be called by PeerConnection according to the result of
+	// SDP based negotiation.
+	// It will be called via RTPSender.Parameters() by PeerConnection to
+	// tell the negotiated media codec information.
+	//
+	// This is pion's extension to process data without having encoder/decoder
+	// in webrtc package.
+	SetParameters(RTPParameters) error
+}
+
+// RTPParameters represents RTCRtpParameters which contains information about
+// how the RTC data is to be encoded/decoded.
+//
+// ref: https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSendParameters
+type RTPParameters struct {
+	SSRC          uint32
+	SelectedCodec *webrtc.RTPCodec
+	Codecs        []*webrtc.RTPCodec
+}