Fixes avfoundation panic when using NV12 frame (#302)

* Fixes frame.FormatI420 in avfoundation

* Fixes avfoundation: frame size

which causes panic when using NV12

* avfoundation: uses CVPixelBufferGetDataSize() as frame buffer size

frame: fix frame size checking for YUY2, UYVY

* add camera_darwin_test

.github/workflows/ci.yaml

@@ -8,17 +8,18 @@ on:
       - master
 
 jobs:
-  build:
+  build-linux:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        go: [ '1.14', '1.13' ]
-    name: Go ${{ matrix.go }}
+        go: [ '1.15', '1.14' ]
+    name: Linux Go ${{ matrix.go }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Setup Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -29,19 +30,44 @@ jobs:
             libva-dev \
             libvpx-dev \
             libx264-dev
-      - name: go vet
-        run: go vet ./...
-      - name: go build
-        run: go build ./...
-      - name: go build without CGO
-        run: go build . pkg/...
-        env:
-          CGO_ENABLED: 0
-      - name: go test
-        run: go test ./... -v -race
-      - name: go test without CGO
-        run: go test . pkg/... -v
-        env:
-          CGO_ENABLED: 0
-      #- name: golint
-      #  run: go lint ./...
+      - name: Run Test Suite
+        run: make test
+      - uses: codecov/codecov-action@v1	
+        if: matrix.go == '1.15'
+  build-darwin:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [ '1.15', '1.14' ]
+    name: Darwin Go ${{ matrix.go }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Install dependencies
+        run: |
+          brew install \
+            pkg-config \
+            opus \
+            libvpx \
+            x264
+      - name: Run Test Suite
+        run: make test
+  check-licenses:
+    runs-on: ubuntu-latest
+    name: Check Licenses
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.15'
+      - name: Installing go-licenses
+        run: go get github.com/google/go-licenses
+      - name: Checking licenses
+        run: go-licenses check ./...

.gitignore

@@ -10,3 +10,6 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+scripts/cross
+coverage.txt

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "cvendor/src/openh264"]
-	path = cvendor/src/openh264
-	url = https://github.com/cisco/openh264.git

CODEOWNERS

@@ -1 +0,0 @@
-* @lherman-cs @at-wat

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Pion
+Copyright (c) 2019-2020 Pion
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,56 +1,81 @@
-vendor_dir = cvendor
-src_dir = $(vendor_dir)/src
-lib_dir = $(vendor_dir)/lib
-include_dir = $(vendor_dir)/include
+docker_owner := lherman
+docker_prefix := cross
+toolchain_dockerfiles := dockerfiles
+script_path := $(realpath scripts)
+toolchain_path := $(script_path)/$(docker_prefix)
+os_list := \
+	linux \
+	windows \
+	darwin
+arch_list := \
+	armv7 \
+	arm64 \
+	x64
+supported_platforms := \
+  linux-armv7 \
+  linux-arm64 \
+  linux-x64 \
+  windows-x64 \
+  darwin-x64
+cmd_build := build
+cmd_test := test
+examples_dir := examples
+codec_dir := pkg/codec
+codec_list := $(shell ls $(codec_dir)/*/Makefile)
+codec_list := $(codec_list:$(codec_dir)/%/Makefile=%)
+targets := $(foreach codec, $(codec_list), $(addprefix $(cmd_build)-$(codec)-, $(supported_platforms)))
+pkgs_without_mmal := $(shell go list ./... | grep -v mmal)
 
-make_args.x86_64-windows = \
-	CC=x86_64-w64-mingw32-gcc \
-	CXX=x86_64-w64-mingw32-g++ \
-	ARCH=x86_64 \
-	OS=mingw_nt
-make_args.x86_64-darwin = \
-	CC=o64-clang \
-	CXX=o64-clang++ \
-	AR=llvm-ar \
-	ARCH=x86_64 \
-	OS=darwin
+define BUILD_TEMPLATE
+ifneq (,$$(findstring $(2)-$(3),$$(supported_platforms)))
+$$(cmd_build)-$(1)-$(2)-$(3): toolchain-$(2)-$(3)
+	$$(MAKE) --directory=$$(codec_dir)/$(1) \
+		MEDIADEVICES_TOOLCHAIN_BIN=$$(toolchain_path)/$(docker_prefix)-$(2)-$(3) \
+		MEDIADEVICES_TARGET_PLATFORM=$(2)-$(3) \
+		MEDIADEVICES_TARGET_OS=$(2) \
+		MEDIADEVICES_TARGET_ARCH=$(3)
+endif
+endef
 
-.PHONY: vendor
-vendor: \
-	$(include_dir)/openh264 \
-	cross-libraries
+.PHONY: all
+all: $(cmd_test) $(cmd_build)
 
-$(include_dir)/openh264: $(src_dir)/openh264
-	mkdir -p $@
-	cp $^/codec/api/svc/*.h $@
+# Subcommand:
+# 	make build[-<codec_name>-<os>-<arch>]
+#
+# Description:
+# 	Build codec dependencies to multiple platforms.
+#
+# Examples:
+#   * make build: build all codecs for all supported platforms
+#   * make build-opus-darwin-x64: only build opus for darwin-x64 platform
+$(cmd_build): $(targets)
 
-$(lib_dir)/openh264/libopenh264.x86_64-linux.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
+toolchain-%: $(toolchain_dockerfiles)
+	$(MAKE) --directory=$< "$*" \
+		MEDIADEVICES_DOCKER_OWNER=$(docker_owner) \
+		MEDIADEVICES_DOCKER_PREFIX=$(docker_prefix)
+	@mkdir -p $(toolchain_path)
+	@docker run $(docker_owner)/$(docker_prefix)-$* > \
+		$(toolchain_path)/$(docker_prefix)-$*
+	@chmod +x $(toolchain_path)/$(docker_prefix)-$*
 
-$(lib_dir)/openh264/libopenh264.x86_64-windows.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ $(make_args.x86_64-windows) libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
+$(foreach codec, $(codec_list), \
+	$(foreach os, $(os_list), \
+		$(foreach arch, $(arch_list), \
+			$(eval $(call BUILD_TEMPLATE,$(codec),$(os),$(arch))))))
 
-$(lib_dir)/openh264/libopenh264.x86_64-darwin.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ $(make_args.x86_64-darwin) libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
-
-.PHONY: cross-libraries
-cross-libraries:
-	docker build -t mediadevices-libs-builder -f libs-builder.Dockerfile .
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-linux.a
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-windows.a
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-darwin.a
+# Subcommand:
+# 	make test
+#
+# Description:
+# 	Run a series of tests
+$(cmd_test):
+	go vet $(pkgs_without_mmal)
+	go build $(pkgs_without_mmal)
+	# go build without CGO
+	CGO_ENABLED=0 go build . pkg/...
+	# go build with CGO
+	CGO_ENABLED=1 go build $(pkgs_without_mmal)
+	$(MAKE) --directory=$(examples_dir)
+	go test -v -race -coverprofile=coverage.txt -covermode=atomic $(pkgs_without_mmal)

README.md

@@ -1,75 +1,237 @@
-# mediadevices
+<h1 align="center">
+  <br>
+  Pion MediaDevices
+  <br>
+</h1>
+<h4 align="center">Go implementation of the <a href="https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices">MediaDevices</a> API</h4>
+<p align="center">
+  <a href="https://pion.ly/slack"><img src="https://img.shields.io/badge/join-us%20on%20slack-gray.svg?longCache=true&logo=slack&colorB=brightgreen" alt="Slack Widget"></a>
+  <a href="https://github.com/pion/mediadevices/actions"><img src="https://github.com/pion/mediadevices/workflows/CI/badge.svg?branch=master" alt="Build status"></a> 
+  <a href="https://pkg.go.dev/github.com/pion/mediadevices"><img src="https://godoc.org/github.com/pion/mediadevices?status.svg" alt="GoDoc"></a>
+  <a href="https://codecov.io/gh/pion/mediadevices"><img src="https://codecov.io/gh/pion/mediadevices/branch/master/graph/badge.svg" alt="Coverage Status"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
+</p>
+<br>
 
-Go implementation of the [MediaDevices](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices) API.
+`mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
 
-![](img/demo.gif)
+## Install
 
-## Interfaces
-
-| Interface  | Linux | Mac | Windows |
-| :--------: | :---: | :-: | :-----: |
-|   Camera   |  ✔️   | ✖️  |   ✔️    |
-| Microphone |  ✔️   | ✖️  |   ✔️    |
-|   Screen   |  ✔️   | ✖️  |   ✖️    |
-
-### Camera
-
-|   OS    |                           Library/Interface                             |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |        [Video4Linux](https://en.wikipedia.org/wiki/Video4Linux)         |
-|   Mac   |                                  N/A                                    |
-| Windows | [DirectShow](https://docs.microsoft.com/en-us/windows/win32/directshow) |
-
-|                     Pixel Format                      | Linux | Mac | Windows |
-| :---------------------------------------------------: | :---: | :-: | :-----: |
-| [YUY2](https://www.fourcc.org/pixel-format/yuv-yuy2/) |  ✔️   | ✖️  |   ✔️    |
-| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) |  ✔️   | ✖️  |   ✖️    |
-| [I420](https://www.fourcc.org/pixel-format/yuv-i420/) |  ✔️   | ✖️  |   ✖️    |
-| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) |  ✔️   | ✖️  |   ✖️    |
-|         [MJPEG](https://www.fourcc.org/mjpg/)         |  ✔️   | ✖️  |   ✖️    |
-
-### Microphone
-
-|   OS    |                            Library/Interface                            |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |         [PulseAudio](https://en.wikipedia.org/wiki/PulseAudio)          |
-|   Mac   |                                   N/A                                   |
-| Windows |  [waveIn](https://docs.microsoft.com/en-us/windows/win32/api/mmeapi/)   |
-
-### Screen casting
-
-|   OS    |                            Library/Interface                            |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |          [X11](https://en.wikipedia.org/wiki/X_Window_System)           |
-|   Mac   |                                   N/A                                   |
-| Windows |                                   N/A                                   |
-
-## Codecs
-
-| Audio Codec |                    Library/Interface                     |
-| :---------: | :------------------------------------------------------: |
-|    OPUS     | [libopus](http://opus-codec.org/)                        |
-
-| Video Codec |                    Library/Interface                     |
-| :---------: | :------------------------------------------------------: |
-|    H.264    | [OpenH264](https://www.openh264.org/)                    |
-|     VP8     | [libvpx](https://www.webmproject.org/code/)              |
-|     VP9     | [libvpx](https://www.webmproject.org/code/)              |
+`go get -u github.com/pion/mediadevices`
 
 ## Usage
 
-[Wiki](https://github.com/pion/mediadevices/wiki)
+The following snippet shows how to capture a camera stream and store a frame as a jpeg image:
+
+```go
+package main
+
+import (
+	"image/jpeg"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/prop"
+
+	// This is required to register camera adapter
+	_ "github.com/pion/mediadevices/pkg/driver/camera" 
+	// Note: If you don't have a camera or your adapters are not supported,
+	//       you can always swap your adapters with our dummy adapters below.
+	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
+)
+
+func main() {
+	stream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(constraint *mediadevices.MediaTrackConstraints) {
+			// Query for ideal resolutions
+			constraint.Width = prop.Int(600)
+			constraint.Height = prop.Int(400)
+		},
+	})
+
+	// Since track can represent audio as well, we need to cast it to 
+	// *mediadevices.VideoTrack to get video specific functionalities
+	track := stream.GetVideoTracks()[0]
+	videoTrack := track.(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	// Create a new video reader to get the decoded frames. Release is used 
+	// to return the buffer to hold frame back to the source so that the buffer 
+	// can be reused for the next frames.
+	videoReader := videoTrack.NewReader(false)
+	frame, release, _ := videoReader.Read()
+	defer release()
+
+	// Since frame is the standard image.Image, it's compatible with Go standard 
+	// library. For example, capturing the first frame and store it as a jpeg image.
+	output, _ := os.Create("frame.jpg")
+	jpeg.Encode(output, frame, nil)
+}
+
+
+```
+
+## More Examples
+
+* [Webrtc](/examples/webrtc) - Use Webrtc to create a realtime peer-to-peer video call
+* [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
+* [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
+* [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
+* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
+
+## Available Media Inputs
+
+| Input  | Linux | Mac | Windows |
+| :--------: | :---: | :-: | :-----: |
+|   Camera   |  ✔️   | ✔️  |   ✔️    |
+| Microphone |  ✔️   | ✔️  |   ✔️    |
+|   Screen   |  ✔️   | ✔️  |   ✔️    |
+
+By default, there's no media input registered. This decision was made to allow you to play only what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
+
+```go
+import (
+	...
+	_ "github.com/pion/mediadevices/pkg/driver/camera"
+)
+```
+
+## Available Codecs
+
+In order to encode your video/audio, `mediadevices` needs to know what codecs that you want to use and their parameters. To do this, you need to import the associated packages for the codecs, and add them to the codec selector that you'll pass to `GetUserMedia`:
+
+```go
+package main
+
+import (
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+)
+
+func main() {
+	// configure codec specific parameters
+	x264Params, _ := x264.NewParams()
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+	
+	mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {},
+		Codec: codecSelector, // let GetUsermedia know available codecs
+	})
+}
+```
+
+Since `mediadevices` doesn't implement the video/audio codecs, it needs to call the codec libraries from the system through cgo. Therefore, you're required to install the codec libraries before you can use them in `mediadevices`. In the next section, it shows a list of available codecs, where the packages are defined (documentation linked), and installation instructions.
+
+Note: we do not provide recommendations on choosing one codec or another as it is very complex and can be subjective.
+
+### Video Codecs
+
+#### x264
+A free software library and application for encoding video streams into the H.264/MPEG-4 AVC compression format.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/x264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/x264)
+* Installation:
+  * Mac: `brew install x264`
+  * Ubuntu: `apt install libx264-dev`
+  
+#### mmal
+A framework to enable H264 hardware encoding for Raspberry Pi or boards that use VideoCore GPUs.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/mmal](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/mmal)
+* Installation: no installation needed, mmal should come built in Raspberry Pi devices
+
+#### openh264
+A codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/openh264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/openh264)
+* Installation: no installation needed, included as a static binary
+
+#### vpx
+A free software video codec library from Google and the Alliance for Open Media that implements VP8/VP9 video coding formats.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/vpx](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vpx)
+* Installation:
+  * Mac: `brew install libvpx`
+  * Ubuntu: `apt install libvpx-dev`
+  
+#### vaapi
+An open source API that allows applications such as VLC media player or GStreamer to use hardware video acceleration capabilities (currently support VP8/VP9).
+
+* Package: [github.com/pion/mediadevices/pkg/codec/vaapi](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vaapi)
+* Installation:
+  * Ubuntu: `apt install libva-dev`
+
+
+### Audio Codecs
+
+#### opus
+A totally open, royalty-free, highly versatile audio codec.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/opus](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/opus)
+* Installation:
+  * Mac: `brew install opus`
+  * Ubuntu: `apt install libopus-dev`
+
+## Benchmark
+
+Result as of Nov 4, 2020 with Go 1.14 on a Raspberry pi 3, `mediadevices` can produce video, encode, send across network, and decode at **720p, 30 fps with < 500 ms latency**.  
+
+The test was taken by capturing a camera stream, decoding the raw frames, encoding the video stream with mmal, and sending the stream through Webrtc.
+
+## FAQ
+
+### Failed to find the best driver that fits the constraints
+
+`mediadevices` provides an automated driver discovery through `GetUserMedia` and `GetDisplayMedia`. The driver discover algorithm works something like:
+
+1. Open all registered drivers
+2. Get all properties (property describes what a driver is capable of, e.g. resolution, frame rate, etc.) from opened drivers
+3. Find the best property that meets the criteria
+
+So, when `mediadevices` returns `failed to find the best driver that fits the constraints` error, one of the following conditions might have occured:
+* Driver was not imported as a side effect in your program, e.g. `import _ github.com/pion/mediadevices/pkg/driver/camera`
+* Your constraint is too strict that there's no driver can fullfil your requirements. In this case, you can try to turn up the debug level by specifying the following environment variable: `export PION_LOG_DEBUG=all` to see what was too strict and tune that.
+* Your driver is not supported/implemented. In this case, you can either let us know (file an issue) and wait for the maintainers to implement it. Or, you can implement it yourself and register it through `RegisterDriverAdapter`
+
+### Failed to find vpx/x264/mmal/opus codecs
+
+Since `mediadevices` uses cgo to access video/audio codecs, it needs to find these libraries from the system. To accomplish this, [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) is used for library discovery.
+
+If you see the following error message at compile time:
+```
+# pkg-config --cflags  -- vpx
+Package vpx was not found in the pkg-config search path.
+Perhaps you should add the directory containing `vpx.pc'
+to the PKG_CONFIG_PATH environment variable
+No package 'vpx' found
+pkg-config: exit status 1
+```
+
+There are 2 common problems:
+
+* The required codec library is not installed (vpx in this example). In this case, please refer to the [available codecs](#available-codecs).
+* Pkg-config fails to find the `.pc` files for this codec ([reference](https://people.freedesktop.org/~dbn/pkg-config-guide.html#using)). In this case, you need to find where the codec library's `.pc` is stored, and let pkg-config knows with: `export PKG_CONFIG_PATH=/path/to/directory`.
+
+
+## Community
+Pion has an active community on the [Slack](https://pion.ly/slack).
+
+Follow the [Pion Twitter](https://twitter.com/_pion) for project updates and important WebRTC news.
+
+We are always looking to support **your projects**. Please reach out if you have something to build!
+If you need commercial support or don't want to use public methods you can contact us at [team@pion.ly](mailto:team@pion.ly)
 
 ## Contributing
+Check out the **[contributing wiki](https://github.com/pion/webrtc/wiki/Contributing)** to join the group of amazing people making this project possible:
 
-- [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
+* [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
 * [Atsushi Watanabe](https://github.com/at-wat) - _VP8, Screencast, etc._
 
-## Project Status
-
-[![Stargazers over time](https://starchart.cc/pion/mediadevices.svg)](https://starchart.cc/pion/mediadevices)
-
-## References
-
-- https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs
-- https://tools.ietf.org/html/rfc7742
+## License
+MIT License - see [LICENSE](LICENSE) for full text

codec.go

@@ -0,0 +1,141 @@
+package mediadevices
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/webrtc/v3"
+)
+
+// CodecSelector is a container of video and audio encoder builders, which later will be used
+// for codec matching.
+type CodecSelector struct {
+	videoEncoders []codec.VideoEncoderBuilder
+	audioEncoders []codec.AudioEncoderBuilder
+}
+
+// CodecSelectorOption is a type for specifying CodecSelector options
+type CodecSelectorOption func(*CodecSelector)
+
+// WithVideoEncoders replace current video codecs with listed encoders
+func WithVideoEncoders(encoders ...codec.VideoEncoderBuilder) CodecSelectorOption {
+	return func(t *CodecSelector) {
+		t.videoEncoders = encoders
+	}
+}
+
+// WithVideoEncoders replace current audio codecs with listed encoders
+func WithAudioEncoders(encoders ...codec.AudioEncoderBuilder) CodecSelectorOption {
+	return func(t *CodecSelector) {
+		t.audioEncoders = encoders
+	}
+}
+
+// NewCodecSelector constructs CodecSelector with given variadic options
+func NewCodecSelector(opts ...CodecSelectorOption) *CodecSelector {
+	var track CodecSelector
+
+	for _, opt := range opts {
+		opt(&track)
+	}
+
+	return &track
+}
+
+// Populate lets the webrtc engine be aware of supported codecs that are contained in CodecSelector
+func (selector *CodecSelector) Populate(setting *webrtc.MediaEngine) {
+	for _, encoder := range selector.videoEncoders {
+		setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeVideo)
+	}
+
+	for _, encoder := range selector.audioEncoders {
+		setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeAudio)
+	}
+}
+
+// selectVideoCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "video/<codecName>" or "<codecName>"
+func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
+	var selectedEncoder codec.VideoEncoderBuilder
+	var encodedReader codec.ReadCloser
+	var errReasons []string
+	var err error
+
+outer:
+	for _, wantCodec := range codecNames {
+		wantCodecLower := strings.ToLower(wantCodec)
+		for _, encoder := range selector.videoEncoders {
+			// MimeType is formated as "video/<codecName>"
+			if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
+				encodedReader, err = encoder.BuildVideoEncoder(reader, inputProp)
+				if err == nil {
+					selectedEncoder = encoder
+					break outer
+				}
+			}
+
+			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
+		}
+	}
+
+	if selectedEncoder == nil {
+		return nil, nil, errors.New(strings.Join(errReasons, "\n\n"))
+	}
+
+	return encodedReader, selectedEncoder.RTPCodec(), nil
+}
+
+func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
+	var codecNames []string
+
+	for _, codec := range codecs {
+		codecNames = append(codecNames, codec.MimeType)
+	}
+
+	return selector.selectVideoCodecByNames(reader, inputProp, codecNames...)
+}
+
+// selectAudioCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "audio/<codecName>" or "<codecName>"
+func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
+	var selectedEncoder codec.AudioEncoderBuilder
+	var encodedReader codec.ReadCloser
+	var errReasons []string
+	var err error
+
+outer:
+	for _, wantCodec := range codecNames {
+		wantCodecLower := strings.ToLower(wantCodec)
+		for _, encoder := range selector.audioEncoders {
+			// MimeType is formated as "audio/<codecName>"
+			if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
+				encodedReader, err = encoder.BuildAudioEncoder(reader, inputProp)
+				if err == nil {
+					selectedEncoder = encoder
+					break outer
+				}
+			}
+
+			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
+		}
+	}
+
+	if selectedEncoder == nil {
+		return nil, nil, errors.New(strings.Join(errReasons, "\n\n"))
+	}
+
+	return encodedReader, selectedEncoder.RTPCodec(), nil
+}
+
+func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
+	var codecNames []string
+
+	for _, codec := range codecs {
+		codecNames = append(codecNames, codec.MimeType)
+	}
+
+	return selector.selectAudioCodecByNames(reader, inputProp, codecNames...)
+}

codecov.yml

@@ -0,0 +1,10 @@
+coverage:
+  status:
+    project:
+      default:
+        # Allow decreasing 2% of total coverage to avoid noise.
+        threshold: 2%
+    patch: off
+
+ignore:
+  - "examples/*"

cvendor/include/openh264/codec_ver.h

@@ -1,15 +0,0 @@
-//The current file is auto-generated by script: generate_codec_ver.sh
-#ifndef CODEC_VER_H
-#define CODEC_VER_H
-
-#include "codec_app_def.h"
-
-static const OpenH264Version g_stCodecVersion  = {2, 0, 0, 1905};
-static const char* const g_strCodecVer  = "OpenH264 version:2.0.0.1905";
-
-#define OPENH264_MAJOR (2)
-#define OPENH264_MINOR (0)
-#define OPENH264_REVISION (0)
-#define OPENH264_RESERVED (1905)
-
-#endif  // CODEC_VER_H

dockerfiles/Makefile

@@ -0,0 +1,11 @@
+dockerfiles := $(wildcard *.Dockerfile)
+supported_platforms := $(dockerfiles:.Dockerfile=)
+
+.PHONY: all
+all: $(supported_platforms)
+
+%: %.Dockerfile guard-MEDIADEVICES_DOCKER_OWNER guard-MEDIADEVICES_DOCKER_PREFIX
+	docker build -t "$(MEDIADEVICES_DOCKER_OWNER)/$(MEDIADEVICES_DOCKER_PREFIX)-$@" -f "$<" . 
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

dockerfiles/darwin-x64.Dockerfile

@@ -0,0 +1,23 @@
+FROM dockcross/base
+
+ENV OSX_CROSS_PATH=/osxcross
+
+COPY --from=dockercore/golang-cross "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}/"
+ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+
+ENV CC=x86_64-apple-darwin14-clang \
+    CXX=x86_64-apple-darwin14-clang++ \
+    CPP=x86_64-apple-darwin14-clang++ \
+    AR=x86_64-apple-darwin14-ar \
+    AS=x86_64-apple-darwin14-as \
+    LD=x86_64-apple-darwin14-ld
+
+COPY darwin-x64.cmake ${OSX_CROSS_PATH}/
+ENV CMAKE_TOOLCHAIN_FILE ${OSX_CROSS_PATH}/darwin-x64.cmake
+
+ARG IMAGE=lherman/cross-darwin-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}

dockerfiles/darwin-x64.cmake

@@ -0,0 +1,8 @@
+set(CMAKE_SYSTEM_NAME Darwin)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR x86_64)
+
+set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_CXX_COMPILER $ENV{CXX})
+set(CMAKE_AR $ENV{AR})
+set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})

dockerfiles/init.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+apt-get update 
+apt-get install -y nasm clang llvm
+
+curl -L https://golang.org/dl/go1.15.6.linux-amd64.tar.gz | tar -C /usr/local -xzf -
+ln -s /usr/local/go/bin/go /usr/local/bin/go

dockerfiles/linux-arm64.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-arm64
+
+ARG IMAGE=lherman/cross-linux-arm64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/linux-armv7.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-armv7
+
+ARG IMAGE=lherman/cross-linux-armv7
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/linux-x64.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-x64
+
+ARG IMAGE=lherman/cross-linux-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/windows-x64.Dockerfile

@@ -0,0 +1,11 @@
+FROM dockcross/windows-static-x64-posix
+
+ARG IMAGE=lherman/cross-windows-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+RUN ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-gcc /usr/bin/x86_64-w64-mingw32-gcc && \
+    ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-g++ /usr/bin/x86_64-w64-mingw32-g++ && \
+    ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-ar /usr/bin/x86_64-w64-mingw32-ar

examples/Makefile

@@ -0,0 +1,8 @@
+examples := $(shell find * -maxdepth 0 -type d)
+examples := $(filter-out internal,$(examples))
+
+.PHONY: all $(examples)
+all: $(examples)
+
+$(examples):
+	cd $@ && go build

examples/archive/.gitignore

@@ -0,0 +1 @@
+archive

examples/archive/README.md

@@ -0,0 +1,38 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download archive examplee
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Run archive example
+
+Run `cd mediadevices/examples/archive && go build && ./archive recorded.h264`
+
+To stop recording, press `Ctrl+c` or send a SIGINT signal.
+
+### Playback recorded video
+
+Install GStreamer and run:
+```
+gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
+```
+
+Or run VLC media plyer:
+```
+vlc recorded.h264
+```
+
+A video should start playing in your GStreamer or VLC window.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/archive/main.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s <path/to/file.h264>\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT)
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
+		return video.ReaderFunc(func() (img image.Image, release func(), err error) {
+			// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
+			// will finish its execution and the program will finish
+			select {
+			case <-sigs:
+				return nil, func() {}, io.EOF
+			default:
+			}
+
+			return r.Read()
+		})
+	}))
+
+	reader, err := videoTrack.NewEncodedIOReader(x264Params.RTPCodec().MimeType)
+	must(err)
+	defer reader.Close()
+
+	out, err := os.Create(dest)
+	must(err)
+
+	fmt.Println("Recording... Press Ctrl+c to stop")
+	_, err = io.Copy(out, reader)
+	must(err)
+	fmt.Println("Your video has been recorded to", dest)
+}

examples/facedetection/.gitignore

@@ -0,0 +1 @@
+facedetection

examples/facedetection/README.md

@@ -1,29 +1,15 @@
 ## Instructions
 
-### Download facedetection
+### Download facedetection example
 
 ```
-go get github.com/pion/mediadevices/examples/facedetection
+git clone https://github.com/pion/mediadevices.git
 ```
 
-### Open example page
+### Compile and Run facedetection
 
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/video) you should see two text-areas and a 'Start Session' button
+Run `cd mediadevices/examples/facedetection && go build && ./facedetection`
 
-### Run facedetection with your browsers SessionDescription as stdin
+You should be able to see some loggings when it can see faces.
 
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | facedetection`
-
-### Input facedetection's SessionDescription into your browser
-
-Copy the text that `facedetection` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/facedetection/detector.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"image"
-	"image/color"
-	"image/draw"
-	"io/ioutil"
-	"log"
-
-	"github.com/disintegration/imaging"
-	pigo "github.com/esimov/pigo/core"
-)
-
-var (
-	cascade    []byte
-	err        error
-	classifier *pigo.Pigo
-)
-
-func imgToGrayscale(img image.Image) []uint8 {
-	bounds := img.Bounds()
-	flatten := bounds.Dy() * bounds.Dx()
-	grayImg := make([]uint8, flatten)
-
-	i := 0
-	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
-		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			pix := img.At(x, y)
-			grayPix := color.GrayModel.Convert(pix).(color.Gray)
-			grayImg[i] = grayPix.Y
-			i++
-		}
-	}
-	return grayImg
-}
-
-// clusterDetection runs Pigo face detector core methods
-// and returns a cluster with the detected faces coordinates.
-func clusterDetection(img image.Image) []pigo.Detection {
-	grayscale := imgToGrayscale(img)
-	bounds := img.Bounds()
-	cParams := pigo.CascadeParams{
-		MinSize:     100,
-		MaxSize:     600,
-		ShiftFactor: 0.15,
-		ScaleFactor: 1.1,
-		ImageParams: pigo.ImageParams{
-			Pixels: grayscale,
-			Rows:   bounds.Dy(),
-			Cols:   bounds.Dx(),
-			Dim:    bounds.Dx(),
-		},
-	}
-
-	if len(cascade) == 0 {
-		cascade, err = ioutil.ReadFile("facefinder")
-		if err != nil {
-			log.Fatalf("Error reading the cascade file: %s", err)
-		}
-		p := pigo.NewPigo()
-
-		// Unpack the binary file. This will return the number of cascade trees,
-		// the tree depth, the threshold and the prediction from tree's leaf nodes.
-		classifier, err = p.Unpack(cascade)
-		if err != nil {
-			log.Fatalf("Error unpacking the cascade file: %s", err)
-		}
-	}
-
-	// Run the classifier over the obtained leaf nodes and return the detection results.
-	// The result contains quadruplets representing the row, column, scale and detection score.
-	dets := classifier.RunCascade(cParams, 0.0)
-
-	// Calculate the intersection over union (IoU) of two clusters.
-	dets = classifier.ClusterDetections(dets, 0)
-
-	return dets
-}
-
-func drawCircle(img draw.Image, x0, y0, r int, c color.Color) {
-	x, y, dx, dy := r-1, 0, 1, 1
-	err := dx - (r * 2)
-
-	for x > y {
-		img.Set(x0+x, y0+y, c)
-		img.Set(x0+y, y0+x, c)
-		img.Set(x0-y, y0+x, c)
-		img.Set(x0-x, y0+y, c)
-		img.Set(x0-x, y0-y, c)
-		img.Set(x0-y, y0-x, c)
-		img.Set(x0+y, y0-x, c)
-		img.Set(x0+x, y0-y, c)
-
-		if err <= 0 {
-			y++
-			err += dy
-			dy += 2
-		}
-		if err > 0 {
-			x--
-			dx += 2
-			err += dx - (r * 2)
-		}
-	}
-}
-
-func markFaces(img image.Image) image.Image {
-	nrgba := imaging.Clone(img)
-	dets := clusterDetection(img)
-	for _, det := range dets {
-		if det.Q < 5.0 {
-			continue
-		}
-
-		drawCircle(nrgba, det.Col, det.Row, det.Scale/2, color.Black)
-	}
-	return nrgba
-}

examples/facedetection/main.go

@@ -1,119 +1,107 @@
 package main
 
 import (
-	"fmt"
 	"image"
+	"io/ioutil"
+	"log"
+	"time"
 
+	pigo "github.com/esimov/pigo/core"
 	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
 	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/io/video"
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
 )
 
-func markFacesTransformer(r video.Reader) video.Reader {
-	return video.ReaderFunc(func() (img image.Image, err error) {
-		img, err = r.Read()
-		if err != nil {
-			return
-		}
+const (
+	confidenceLevel = 5.0
+)
 
-		img = markFaces(img)
-		return
-	})
+var (
+	cascade    []byte
+	classifier *pigo.Pigo
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func detectFace(frame *image.YCbCr) bool {
+	bounds := frame.Bounds()
+	cascadeParams := pigo.CascadeParams{
+		MinSize:     100,
+		MaxSize:     600,
+		ShiftFactor: 0.15,
+		ScaleFactor: 1.1,
+		ImageParams: pigo.ImageParams{
+			Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
+			Rows:   bounds.Dy(),
+			Cols:   bounds.Dx(),
+			Dim:    bounds.Dx(),
+		},
+	}
+
+	// Run the classifier over the obtained leaf nodes and return the detection results.
+	// The result contains quadruplets representing the row, column, scale and detection score.
+	dets := classifier.RunCascade(cascadeParams, 0.0)
+
+	// Calculate the intersection over union (IoU) of two clusters.
+	dets = classifier.ClusterDetections(dets, 0)
+
+	for _, det := range dets {
+		if det.Q >= confidenceLevel {
+			return true
+		}
+	}
+
+	return false
 }
 
 func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
+	// prepare face detector
+	var err error
+	cascade, err = ioutil.ReadFile("facefinder")
 	if err != nil {
-		panic(err)
+		log.Fatalf("Error reading the cascade file: %s", err)
 	}
+	p := pigo.NewPigo()
 
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
+	// Unpack the binary file. This will return the number of cascade trees,
+	// the tree depth, the threshold and the prediction from tree's leaf nodes.
+	classifier, err = p.Unpack(cascade)
 	if err != nil {
-		panic(err)
+		log.Fatalf("Error unpacking the cascade file: %s", err)
 	}
-	vp8Params.BitRate = 100000 // 100kbps
 
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormatExact(frame.FormatI420) // most of the encoder accepts I420
-			c.Enabled = true
+			c.FrameFormat = prop.FrameFormatOneOf{frame.FormatI420, frame.FormatYUY2}
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
-			c.VideoTransform = markFacesTransformer
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
 		},
 	})
-	if err != nil {
-		panic(err)
-	}
+	must(err)
 
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
+	// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoReader := videoTrack.NewReader(false)
+	// To save resources, we can simply use 4 fps to detect faces.
+	ticker := time.NewTicker(time.Millisecond * 250)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		frame, release, err := videoReader.Read()
+		must(err)
+
+		// Since we asked the frame format to be exactly I420/YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
+		if detectFace(frame.(*image.YCbCr)) {
+			log.Println("Detect a face")
 		}
-	}
 
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
+		release()
 	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
 }

examples/go.mod

@@ -2,8 +2,8 @@ module github.com/pion/mediadevices/examples
 
 go 1.14
 
-replace github.com/pion/mediadevices => ../
-
 // Please don't commit require entries of examples.
 // `git checkout master examples/go.mod` to revert this file.
-require github.com/pion/mediadevices v0.0.0-00010101000000-000000000000
+require github.com/pion/mediadevices v0.0.0
+
+replace github.com/pion/mediadevices v0.0.0 => ../

examples/http/.gitignore

@@ -0,0 +1 @@
+http

examples/http/README.md

@@ -0,0 +1,19 @@
+## Instructions
+
+### Download http example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Compile and Run HTTP server
+
+Run `cd mediadevices/examples/http && go build && ./http :1313`
+
+
+### Access the camera stream from the browser
+
+Go to "http://localhost:1313"
+
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/http/main.go

@@ -0,0 +1,85 @@
+// This is an example of using mediadevices to broadcast your camera through http.
+// The example doesn't aim to be performant, but rather it strives to be simple.
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"image/jpeg"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/prop"
+
+	// Note: If you don't have a camera or microphone or your adapters are not supported,
+	//       you can always swap your adapters with our dummy adapters below.
+	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(constraint *mediadevices.MediaTrackConstraints) {
+			constraint.Width = prop.Int(600)
+			constraint.Height = prop.Int(400)
+		},
+	})
+	must(err)
+
+	track := mediaStream.GetVideoTracks()[0]
+	videoTrack := track.(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		videoReader := videoTrack.NewReader(false)
+		mimeWriter := multipart.NewWriter(w)
+
+		contentType := fmt.Sprintf("multipart/x-mixed-replace;boundary=%s", mimeWriter.Boundary())
+		w.Header().Add("Content-Type", contentType)
+
+		partHeader := make(textproto.MIMEHeader)
+		partHeader.Add("Content-Type", "image/jpeg")
+
+		for {
+			frame, release, err := videoReader.Read()
+			if err == io.EOF {
+				return
+			}
+			must(err)
+
+			err = jpeg.Encode(&buf, frame, nil)
+			// Since we're done with img, we need to release img so that that the original owner can reuse
+			// this memory.
+			release()
+			must(err)
+
+			partWriter, err := mimeWriter.CreatePart(partHeader)
+			must(err)
+
+			_, err = partWriter.Write(buf.Bytes())
+			buf.Reset()
+			must(err)
+		}
+	})
+
+	fmt.Printf("listening on %s\n", dest)
+	log.Println(http.ListenAndServe(dest, nil))
+}

examples/rtp-send/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download rtp-send example
-
-```
-go get github.com/pion/mediadevices/examples/rtp-send
-```
-
-### Listen RTP
-
-Install GStreamer and run:
-```
-gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=VP8 \
-    ! rtpvp8depay ! vp8dec ! videoconvert ! autovideosink
-```
-
-Or run VLC media plyer:
-```
-vlc ./vp8.sdp
-```
-
-### Run rtp-send
-
-Run `rtp-send localhost:5000`
-
-A video should start playing in your GStreamer or VLC window.
-It's not WebRTC, but pure RTP.
-
-Congrats, you have used pion-MediaDevices! Now start building something cool

examples/rtp-send/main.go

@@ -1,120 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"net"
-	"os"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2"
-	"github.com/pion/webrtc/v2/pkg/media"
-)
-
-const (
-	mtu = 1000
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Printf("usage: %s host:port\n", os.Args[0])
-		return
-	}
-
-	md := mediadevices.NewMediaDevicesFromCodecs(
-		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
-				webrtc.NewRTPVP8Codec(100, 90000),
-			},
-		},
-		mediadevices.WithTrackGenerator(
-			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
-				mediadevices.LocalTrack, error,
-			) {
-				return newTrack(codec, id, os.Args[1]), nil
-			},
-		),
-	)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	select {}
-}
-
-type track struct {
-	codec      *webrtc.RTPCodec
-	packetizer rtp.Packetizer
-	id         string
-	conn       net.Conn
-}
-
-func newTrack(codec *webrtc.RTPCodec, id, dest string) *track {
-	addr, err := net.ResolveUDPAddr("udp", dest)
-	if err != nil {
-		panic(err)
-	}
-	conn, err := net.DialUDP("udp", nil, addr)
-	if err != nil {
-		panic(err)
-	}
-	return &track{
-		codec: codec,
-		packetizer: rtp.NewPacketizer(
-			mtu,
-			codec.PayloadType,
-			1,
-			codec.Payloader,
-			rtp.NewRandomSequencer(),
-			codec.ClockRate,
-		),
-		id:   id,
-		conn: conn,
-	}
-}
-
-func (t *track) WriteSample(s media.Sample) error {
-	buf := make([]byte, mtu)
-	pkts := t.packetizer.Packetize(s.Data, s.Samples)
-	for _, p := range pkts {
-		n, err := p.MarshalTo(buf)
-		if err != nil {
-			panic(err)
-		}
-		_, _ = t.conn.Write(buf[:n])
-	}
-	return nil
-}
-
-func (t *track) Codec() *webrtc.RTPCodec {
-	return t.codec
-}
-
-func (t *track) ID() string {
-	return t.id
-}
-
-func (t *track) Kind() webrtc.RTPCodecType {
-	return t.codec.Type
-}

examples/rtp/.gitignore

@@ -0,0 +1 @@
+rtp

examples/rtp/README.md

@@ -0,0 +1,38 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download rtp example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Listen RTP
+
+Install GStreamer and run:
+```
+gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=H264 \
+    ! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
+```
+
+Or run VLC media plyer:
+```
+vlc ./h264.sdp
+```
+
+### Run rtp
+
+Run `cd mediadevices/examples/archive && go build && ./rtp localhost:5000`
+
+A video should start playing in your GStreamer or VLC window.
+It's not WebRTC, but pure RTP.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/rtp/h264.sdp

@@ -6,4 +6,4 @@ c=IN IP4 0.0.0.0
 t=0 0
 a=recvonly
 m=video 5000 RTP/AVP 100
-a=rtpmap:100 VP8/90000
+a=rtpmap:100 H264/90000

examples/rtp/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+const (
+	mtu = 1000
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0]
+	defer videoTrack.Close()
+
+	rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().MimeType, rand.Uint32(), mtu)
+	must(err)
+
+	addr, err := net.ResolveUDPAddr("udp", dest)
+	must(err)
+	conn, err := net.DialUDP("udp", nil, addr)
+	must(err)
+
+	buff := make([]byte, mtu)
+	for {
+		pkts, release, err := rtpReader.Read()
+		must(err)
+
+		for _, pkt := range pkts {
+			n, err := pkt.MarshalTo(buff)
+			must(err)
+
+			_, err = conn.Write(buff[:n])
+			must(err)
+		}
+
+		release()
+	}
+}

examples/screenshare/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download screenshare
-
-```
-go get github.com/pion/mediadevices/examples/screenshare
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
-
-### Run screenshare with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | screenshare`
-
-### Input screenshare's SessionDescription into your browser
-
-Copy the text that `screenshare` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/screenshare/main.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/webrtc/v2"
-)
-
-func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
-}

examples/simple/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download gstreamer-send
-
-```
-go get github.com/pion/mediadevices/examples/simple
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
-
-### Run simple with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | simple`
-
-### Input simple's SessionDescription into your browser
-
-Copy the text that `simple` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/webrtc/.gitignore

@@ -0,0 +1 @@
+webrtc

examples/webrtc/README.md

@@ -0,0 +1,42 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+* [opus](https://github.com/pion/mediadevices#opus)
+
+### Download webrtc example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+#### Compile webrtc example
+
+```
+cd mediadevices/examples/webrtc && go build
+```
+
+### Open example page
+
+[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
+
+### Run the webrtc example with your browsers SessionDescription as stdin
+
+In the jsfiddle the top textarea is your browser, copy that, and store the session description in an environment variable, `export SDP=<put_the_sdp_here>`
+
+Run `echo $SDP | ./webrtc`
+
+### Input webrtc's SessionDescription into your browser
+
+Copy the text that `./webrtc` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/webrtc/main.go

@@ -5,19 +5,18 @@ import (
 
 	"github.com/pion/mediadevices"
 	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 
-	// This is required to use opus audio encoder
-	"github.com/pion/mediadevices/pkg/codec/opus"
-
-	// If you don't like vpx, you can also use x264 by importing as below
-	// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
+	// If you don't like x264, you can also use vpx by importing as below
+	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
 	// or you can also use openh264 for alternative h264 implementation
 	// "github.com/pion/mediadevices/pkg/codec/openh264"
-	"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
+	// or if you use a raspberry pi like, you can use mmal for using its hardware encoder
+	// "github.com/pion/mediadevices/pkg/codec/mmal"
+	"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use opus audio encoder
+	"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
@@ -27,10 +26,6 @@ import (
 	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
 )
 
-const (
-	videoCodecName = webrtc.VP8
-)
-
 func main() {
 	config := webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
@@ -45,11 +40,24 @@ func main() {
 	signal.Decode(signal.MustReadStdin(), &offer)
 
 	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
+	x264Params, err := x264.NewParams()
+	if err != nil {
 		panic(err)
 	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	x264Params.BitRate = 500_000 // 500kbps
+
+	opusParams, err := opus.NewParams()
+	if err != nil {
+		panic(err)
+	}
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+		mediadevices.WithAudioEncoders(&opusParams),
+	)
+
+	mediaEngine := webrtc.MediaEngine{}
+	codecSelector.Populate(&mediaEngine)
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(&mediaEngine))
 	peerConnection, err := api.NewPeerConnection(config)
 	if err != nil {
 		panic(err)
@@ -61,44 +69,27 @@ func main() {
 		fmt.Printf("Connection State has changed %s \n", connectionState.String())
 	})
 
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	opusParams, err := opus.NewParams()
-	if err != nil {
-		panic(err)
-	}
-	opusParams.BitRate = 32000 // 32kbps
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Audio: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
-		},
+	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
 		},
+		Audio: func(c *mediadevices.MediaTrackConstraints) {
+		},
+		Codec: codecSelector,
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
+	for _, track := range s.GetTracks() {
+		track.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s) ended with error: %v\n",
+				track.ID(), err)
 		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
+
+		_, err = peerConnection.AddTransceiverFromTrack(track,
 			webrtc.RtpTransceiverInit{
 				Direction: webrtc.RTPTransceiverDirectionSendonly,
 			},
@@ -120,13 +111,23 @@ func main() {
 		panic(err)
 	}
 
+	// Create channel that is blocked until ICE Gathering is complete
+	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+
 	// Sets the LocalDescription, and starts our UDP listeners
 	err = peerConnection.SetLocalDescription(answer)
 	if err != nil {
 		panic(err)
 	}
 
+	// Block until ICE Gathering is complete, disabling trickle ICE
+	// we do this because we only can exchange one signaling message
+	// in a production application you should exchange ICE Candidates via OnICECandidate
+	<-gatherComplete
+
 	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
+	fmt.Println(signal.Encode(*peerConnection.LocalDescription()))
+
+	// Block forever
 	select {}
 }

go.mod

@@ -3,12 +3,17 @@ module github.com/pion/mediadevices
 go 1.13
 
 require (
+	github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd // indirect
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/faiface/beep v1.0.2
-	github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659
-	github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4
-	github.com/pion/webrtc/v2 v2.2.14
+	github.com/gen2brain/malgo v0.10.29
+	github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
+	github.com/google/uuid v1.2.0
+	github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
+	github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 // indirect
+	github.com/pion/logging v0.2.2
+	github.com/pion/rtp v1.6.2
+	github.com/pion/webrtc/v3 v3.0.5
 	github.com/satori/go.uuid v1.2.0
-	golang.org/x/image v0.0.0-20200430140353-33d19683fad8
-	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3
+	golang.org/x/image v0.0.0-20201208152932-35266b937fa6
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -1,90 +1,86 @@
+github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd h1:u7K2oMFMd8APDV3fM1j2rO3U/XJf1g1qC3DDTKou8iM=
+github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
 github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
-github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
-github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/faiface/beep v1.0.2 h1:UB5DiRNmA4erfUYnHbgU4UB6DlBOrsdEFRtcc8sCkdQ=
-github.com/faiface/beep v1.0.2/go.mod h1:1yLb5yRdHMsovYYWVqYLioXkVuziCSITW1oarTeduQM=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.1.1/go.mod h1:K1udHkiR3cOtlpKG5tZPD5XxrF7v2y7lDq7Whcj+xkQ=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gen2brain/malgo v0.10.29 h1:bTYiUTUKJsEomNby+W0hgyLrOttUXIk4lTEnKA54iqM=
+github.com/gen2brain/malgo v0.10.29/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
+github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
+github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw=
-github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherwasm v0.1.1/go.mod h1:kx4n9a+MzHH0BJJhvlsQ65hqLFXDO/m256AsaDPQ+/4=
-github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y=
-github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI=
-github.com/hajimehoshi/go-mp3 v0.1.1/go.mod h1:4i+c5pDNKDrxl1iu9iG90/+fhP37lio6gNhjCx9WBJw=
-github.com/hajimehoshi/oto v0.1.1/go.mod h1:hUiLWeBQnbDu4pZsAhOnGqMI1ZGibS6e2qhQdfpwz04=
-github.com/hajimehoshi/oto v0.3.1 h1:cpf/uIv4Q0oc5uf9loQn7PIehv+mZerh+0KKma6gzMk=
-github.com/hajimehoshi/oto v0.3.1/go.mod h1:e9eTLBB9iZto045HLbzfHJIc+jP3xaKrjZTghvb6fdM=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jfreymuth/oggvorbis v1.0.0/go.mod h1:abe6F9QRjuU9l+2jek3gj46lu40N4qlYxh2grqkLEDM=
-github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659 h1:DRA4BuRlhEILiud720WFWqqdADPzp1jTjQvyCr/PP80=
-github.com/jfreymuth/pulse v0.0.0-20200506145638-1534c4af9659/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
-github.com/jfreymuth/vorbis v1.0.0/go.mod h1:8zy3lUAm9K/rJJk223RKy6vjCZTWC61NA2QD06bfOE0=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f h1:5hWo+DzJQSOBl6X+TDac0SPWffRonuRJ2///OYtYRT8=
+github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4 h1:2ydMA2KbxRkYmIw3R8Me8dn90bejxBR4MKYXJ5THK3I=
-github.com/lherman-cs/opus v0.0.0-20200223204610-6a4b98199ea4/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
-github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
-github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/lucasb-eyer/go-colorful v0.0.0-20181028223441-12d3b2882a08/go.mod h1:NXg0ArsFk0Y01623LgUqoqcouGDB+PwCCQlrwrG6xJ4=
-github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mewkiz/flac v1.0.5/go.mod h1:EHZNU32dMF6alpurYyKHDLYpW1lYpBZ5WrXi/VuNIGs=
+github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 h1:4BxFx5XCtXc+nFtXDGDW+Uu5sPtsAbvPh6RObj3fG9o=
+github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pion/datachannel v1.4.17 h1:8CChK5VrJoGrwKCysoTscoWvshCAFpUkgY11Tqgz5hE=
-github.com/pion/datachannel v1.4.17/go.mod h1:+vPQfypU9vSsyPXogYj1hBThWQ6MNXEQoQAzxoPvjYM=
-github.com/pion/dtls/v2 v2.0.0 h1:Fk+MBhLZ/U1bImzAhmzwbO/pP2rKhtTw8iA934H3ybE=
-github.com/pion/dtls/v2 v2.0.0/go.mod h1:VkY5VL2wtsQQOG60xQ4lkV5pdn0wwBBTzCfRJqXhp3A=
-github.com/pion/ice v0.7.15 h1:s1In+gnuyVq7WKWGVQL+1p+OcrMsbfL+VfSe2isH8Ag=
-github.com/pion/ice v0.7.15/go.mod h1:Z6zybEQgky5mZkKcLfmvc266JukK2srz3VZBBD1iXBw=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
+github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
+github.com/pion/dtls/v2 v2.0.4 h1:WuUcqi6oYMu/noNTz92QrF1DaFj4eXbhQ6dzaaAwOiI=
+github.com/pion/dtls/v2 v2.0.4/go.mod h1:qAkFscX0ZHoI1E07RfYPoRw3manThveu+mlTDdOxoGI=
+github.com/pion/ice/v2 v2.0.15 h1:KZrwa2ciL9od8+TUVJiYTNsCW9J5lktBjGwW1MacEnQ=
+github.com/pion/ice/v2 v2.0.15/go.mod h1:ZIiVGevpgAxF/cXiIVmuIUtCb3Xs4gCzCbXB6+nFkSI=
+github.com/pion/interceptor v0.0.9 h1:fk5hTdyLO3KURQsf/+RjMpEm4NE3yeTY9Kh97b5BvwA=
+github.com/pion/interceptor v0.0.9/go.mod h1:dHgEP5dtxOTf21MObuBAjJeAayPxLUAZjerGH8Xr07c=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
 github.com/pion/mdns v0.0.4 h1:O4vvVqr4DGX63vzmO6Fw9vpy3lfztVWHGCQfyw0ZLSY=
 github.com/pion/mdns v0.0.4/go.mod h1:R1sL0p50l42S5lJs91oNdUL58nm0QHrhxnSegr++qC0=
-github.com/pion/quic v0.1.1 h1:D951FV+TOqI9A0rTF7tHx0Loooqz+nyzjEyj8o3PuMA=
-github.com/pion/quic v0.1.1/go.mod h1:zEU51v7ru8Mp4AUBJvj6psrSth5eEFNnVQK5K48oV3k=
-github.com/pion/rtcp v1.2.1 h1:S3yG4KpYAiSmBVqKAfgRa5JdwBNj4zK3RLUa8JYdhak=
-github.com/pion/rtcp v1.2.1/go.mod h1:a5dj2d6BKIKHl43EnAOIrCczcjESrtPuMgfmL6/K6QM=
-github.com/pion/rtp v1.5.4 h1:PuNg6xqV3brIUihatcKZj1YDUs+M45L0ZbrZWYtkDxY=
-github.com/pion/rtp v1.5.4/go.mod h1:bg60AL5GotNOlYZsqycbhDtEV3TkfbpXG0KBiUq29Mg=
-github.com/pion/sctp v1.7.6 h1:8qZTdJtbKfAns/Hv5L0PAj8FyXcsKhMH1pKUCGisQg4=
-github.com/pion/sctp v1.7.6/go.mod h1:ichkYQ5tlgCQwEwvgfdcAolqx1nHbYCxo4D7zK/K0X8=
-github.com/pion/sdp/v2 v2.3.7 h1:WUZHI3pfiYCaE8UGUYcabk863LCK+Bq3AklV5O0oInQ=
-github.com/pion/sdp/v2 v2.3.7/go.mod h1:+ZZf35r1+zbaWYiZLfPutWfx58DAWcGb2QsS3D/s9M8=
-github.com/pion/srtp v1.3.3 h1:8bjs9YaSNvSrbH0OfKxzPX+PTrCyAC2LoT9Qesugi+U=
-github.com/pion/srtp v1.3.3/go.mod h1:jNe0jmIOqksuurR9S/7yoKDalfPeluUFrNPCBqI4FOI=
-github.com/pion/stun v0.3.3 h1:brYuPl9bN9w/VM7OdNzRSLoqsnwlyNvD9MVeJrHjDQw=
-github.com/pion/stun v0.3.3/go.mod h1:xrCld6XM+6GWDZdvjPlLMsTU21rNxnO6UO8XsAvHr/M=
-github.com/pion/transport v0.6.0/go.mod h1:iWZ07doqOosSLMhZ+FXUTq+TamDoXSllxpbGcfkCmbE=
-github.com/pion/transport v0.8.10 h1:lTiobMEw2PG6BH/mgIVqTV2mBp/mPT+IJLaN8ZxgdHk=
+github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
+github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
+github.com/pion/rtcp v1.2.6 h1:1zvwBbyd0TeEuuWftrd/4d++m+/kZSeiguxU61LFWpo=
+github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0=
+github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U=
+github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
+github.com/pion/sctp v1.7.11 h1:UCnj7MsobLKLuP/Hh+JMiI/6W5Bs/VF45lWKgHFjSIE=
+github.com/pion/sctp v1.7.11/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
+github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
+github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk=
+github.com/pion/srtp/v2 v2.0.1 h1:kgfh65ob3EcnFYA4kUBvU/menCp9u7qaJLXwWgpobzs=
+github.com/pion/srtp/v2 v2.0.1/go.mod h1:c8NWHhhkFf/drmHTAblkdu8++lsISEBBdAuiyxgqIsE=
+github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
+github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
 github.com/pion/transport v0.8.10/go.mod h1:tBmha/UCjpum5hqTWhfAEs3CO4/tHSg0MYRhSzR+CZ8=
-github.com/pion/transport v0.10.0 h1:9M12BSneJm6ggGhJyWpDveFOstJsTiQjkLf4M44rm80=
 github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE=
-github.com/pion/turn/v2 v2.0.3 h1:SJUUIbcPoehlyZgMyIUbBBDhI03sBx32x3JuSIBKBWA=
-github.com/pion/turn/v2 v2.0.3/go.mod h1:kl1hmT3NxcLynpXVnwJgObL8C9NaCyPTeqI2DcCpSZs=
-github.com/pion/webrtc/v2 v2.2.14 h1:bRjnXTqMDJ3VERPF45z439Sv6QfDfjdYvdQk1QcIx8M=
-github.com/pion/webrtc/v2 v2.2.14/go.mod h1:G+8lShCMbHhjpMF1ZJBkyuvrxXrvW4bxs3nOt+mJ2UI=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
+github.com/pion/transport v0.12.1/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
+github.com/pion/transport v0.12.2 h1:WYEjhloRHt1R86LhUKjC5y+P52Y11/QqEUalvtzVoys=
+github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
+github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
+github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
+github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
+github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
+github.com/pion/webrtc/v3 v3.0.5 h1:utennp7RwX+2mtyMzoOXE03IUIckiHBigjarRJZ2DqY=
+github.com/pion/webrtc/v3 v3.0.5/go.mod h1:/EDCREM8y+JrJSkoCRHpoz//qtuBCOYV4E96vEK3bz0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -93,53 +89,65 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
-golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20180710024300-14dda7b62fcd h1:nLIcFw7GiqKXUS7HiChg6OAYWgASB2H97dZKd1GhDSs=
-golang.org/x/exp v0.0.0-20180710024300-14dda7b62fcd/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI=
-golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
-golang.org/x/image v0.0.0-20200430140353-33d19683fad8 h1:6WW6V3x1P/jokJBpRQYUJnMHRP6isStQwCozxnU7XQw=
-golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/mobile v0.0.0-20180806140643-507816974b79 h1:t2JRgCWkY7Qaa1J2jal+wqC9OjbyHCHwIA9rVlRUSMo=
-golang.org/x/mobile v0.0.0-20180806140643-507816974b79/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
+golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/image v0.0.0-20201208152932-35266b937fa6 h1:nfeHNc1nAqecKCy2FCy4HY+soOOe5sDLJ/gZLbx6GYI=
+golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 h1:3uJsdck53FDIpWwLeAXlia9p4C8j0BO2xZrqzKpL0D8=
+golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/codec/measurement_test.go

@@ -2,12 +2,18 @@ package codec
 
 import (
 	"io"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
 )
 
 func TestMeasureBitRateStatic(t *testing.T) {
+	// https://github.com/pion/mediadevices/issues/198
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping because Darwin CI is not reliable for timing related tests.")
+	}
+
 	r, w := io.Pipe()
 	const (
 		dataSize       = 1000
@@ -54,6 +60,11 @@ func TestMeasureBitRateStatic(t *testing.T) {
 }
 
 func TestMeasureBitRateDynamic(t *testing.T) {
+	// https://github.com/pion/mediadevices/issues/198
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping because Darwin CI is not reliable for timing related tests.")
+	}
+
 	r, w := io.Pipe()
 	const (
 		dataSize       = 1000

internal/logging/logging.go

@@ -0,0 +1,11 @@
+package logging
+
+import (
+	"github.com/pion/logging"
+)
+
+var loggerFactory = logging.NewDefaultLoggerFactory()
+
+func NewLogger(scope string) logging.LeveledLogger {
+	return loggerFactory.NewLogger(scope)
+}

ioreader.go

@@ -0,0 +1,61 @@
+package mediadevices
+
+type EncodedBuffer struct {
+	Data    []byte
+	Samples uint32
+}
+
+type EncodedReadCloser interface {
+	Read() (EncodedBuffer, func(), error)
+	Close() error
+}
+
+type encodedReadCloserImpl struct {
+	readFn  func() (EncodedBuffer, func(), error)
+	closeFn func() error
+}
+
+func (r *encodedReadCloserImpl) Read() (EncodedBuffer, func(), error) {
+	return r.readFn()
+}
+
+func (r *encodedReadCloserImpl) Close() error {
+	return r.closeFn()
+}
+
+type encodedIOReadCloserImpl struct {
+	readFn  func([]byte) (int, error)
+	closeFn func() error
+}
+
+func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserImpl {
+	var encoded EncodedBuffer
+	release := func() {}
+	return &encodedIOReadCloserImpl{
+		readFn: func(b []byte) (int, error) {
+			var err error
+
+			if len(encoded.Data) == 0 {
+				release()
+				encoded, release, err = reader.Read()
+				if err != nil {
+					reader.Close()
+					return 0, err
+				}
+			}
+
+			n := copy(b, encoded.Data)
+			encoded.Data = encoded.Data[n:]
+			return n, nil
+		},
+		closeFn: reader.Close,
+	}
+}
+
+func (r *encodedIOReadCloserImpl) Read(b []byte) (int, error) {
+	return r.readFn(b)
+}
+
+func (r *encodedIOReadCloserImpl) Close() error {
+	return r.closeFn()
+}

libs-builder.Dockerfile

@@ -1,10 +0,0 @@
-FROM dockercore/golang-cross
-
-RUN apt-get update -qq \
-  && apt-get install -y \
-    g++-mingw-w64 \
-    nasm \
-  && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-VOLUME /go/src/github.com/pion/mediadevices
-WORKDIR /go/src/github.com/pion/mediadevices

logging.go

@@ -0,0 +1,7 @@
+package mediadevices
+
+import (
+	"github.com/pion/mediadevices/internal/logging"
+)
+
+var logger = logging.NewLogger("mediadevices")

mediadeviceinfo.go

@@ -7,7 +7,7 @@ type MediaDeviceType int
 
 // MediaDeviceType definitions.
 const (
-	VideoInput MediaDeviceType = iota
+	VideoInput MediaDeviceType = iota + 1
 	AudioInput
 	AudioOutput
 )

mediadevices.go

@@ -3,98 +3,30 @@ package mediadevices
 import (
 	"fmt"
 	"math"
+	"strings"
 
 	"github.com/pion/mediadevices/pkg/driver"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
 )
 
 var errNotFound = fmt.Errorf("failed to find the best driver that fits the constraints")
 
-// MediaDevices is an interface that's defined on https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices
-type MediaDevices interface {
-	GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	EnumerateDevices() []MediaDeviceInfo
-}
-
-// NewMediaDevices creates MediaDevices interface that provides access to connected media input devices
-// like cameras and microphones, as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevices(pc *webrtc.PeerConnection, opts ...MediaDevicesOption) MediaDevices {
-	codecs := make(map[webrtc.RTPCodecType][]*webrtc.RTPCodec)
-	for _, kind := range []webrtc.RTPCodecType{
-		webrtc.RTPCodecTypeAudio,
-		webrtc.RTPCodecTypeVideo,
-	} {
-		codecs[kind] = pc.GetRegisteredRTPCodecs(kind)
-	}
-	return NewMediaDevicesFromCodecs(codecs, opts...)
-}
-
-// NewMediaDevicesFromCodecs creates MediaDevices interface from lists of the available codecs
-// that provides access to connected media input devices like cameras and microphones,
-// as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevicesFromCodecs(codecs map[webrtc.RTPCodecType][]*webrtc.RTPCodec, opts ...MediaDevicesOption) MediaDevices {
-	mdo := MediaDevicesOptions{
-		codecs:         codecs,
-		trackGenerator: defaultTrackGenerator,
-	}
-	for _, o := range opts {
-		o(&mdo)
-	}
-	return &mediaDevices{
-		MediaDevicesOptions: mdo,
-	}
-}
-
-// TrackGenerator is a function to create new track.
-type TrackGenerator func(payloadType uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error)
-
-var defaultTrackGenerator = TrackGenerator(func(pt uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error) {
-	return webrtc.NewTrack(pt, ssrc, id, label, codec)
-})
-
-type mediaDevices struct {
-	MediaDevicesOptions
-}
-
-// MediaDevicesOptions stores parameters used by MediaDevices.
-type MediaDevicesOptions struct {
-	codecs         map[webrtc.RTPCodecType][]*webrtc.RTPCodec
-	trackGenerator TrackGenerator
-}
-
-// MediaDevicesOption is a type of MediaDevices functional option.
-type MediaDevicesOption func(*MediaDevicesOptions)
-
-// WithTrackGenerator specifies a TrackGenerator to use customized track.
-func WithTrackGenerator(gen TrackGenerator) MediaDevicesOption {
-	return func(o *MediaDevicesOptions) {
-		o.trackGenerator = gen
-	}
-}
-
 // GetDisplayMedia prompts the user to select and grant permission to capture the contents
 // of a display or portion thereof (such as a window) as a MediaStream.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia
-func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	trackers := make([]Tracker, 0)
+func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+	trackers := make([]Track, 0)
 
 	cleanTrackers := func() {
 		for _, t := range trackers {
-			t.Stop()
+			t.Close()
 		}
 	}
 
 	var videoConstraints MediaTrackConstraints
 	if constraints.Video != nil {
 		constraints.Video(&videoConstraints)
-	}
-
-	if videoConstraints.Enabled {
-		tracker, err := m.selectScreen(videoConstraints)
+		tracker, err := selectScreen(videoConstraints, constraints.Codec)
 		if err != nil {
 			cleanTrackers()
 			return nil, err
@@ -115,27 +47,20 @@ func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (Medi
 // GetUserMedia prompts the user for permission to use a media input which produces a MediaStream
 // with tracks containing the requested types of media.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
-func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+func GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
 	// TODO: It should return media stream based on constraints
-	trackers := make([]Tracker, 0)
+	trackers := make([]Track, 0)
 
 	cleanTrackers := func() {
 		for _, t := range trackers {
-			t.Stop()
+			t.Close()
 		}
 	}
 
 	var videoConstraints, audioConstraints MediaTrackConstraints
 	if constraints.Video != nil {
 		constraints.Video(&videoConstraints)
-	}
-
-	if constraints.Audio != nil {
-		constraints.Audio(&audioConstraints)
-	}
-
-	if videoConstraints.Enabled {
-		tracker, err := m.selectVideo(videoConstraints)
+		tracker, err := selectVideo(videoConstraints, constraints.Codec)
 		if err != nil {
 			cleanTrackers()
 			return nil, err
@@ -144,8 +69,9 @@ func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaSt
 		trackers = append(trackers, tracker)
 	}
 
-	if audioConstraints.Enabled {
-		tracker, err := m.selectAudio(audioConstraints)
+	if constraints.Audio != nil {
+		constraints.Audio(&audioConstraints)
+		tracker, err := selectAudio(audioConstraints, constraints.Codec)
 		if err != nil {
 			cleanTrackers()
 			return nil, err
@@ -194,12 +120,15 @@ func queryDriverProperties(filter driver.FilterFn) map[driver.Driver][]prop.Medi
 func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints) (driver.Driver, MediaTrackConstraints, error) {
 	var bestDriver driver.Driver
 	var bestProp prop.Media
+	var foundPropertiesLog []string
 	minFitnessDist := math.Inf(1)
 
+	foundPropertiesLog = append(foundPropertiesLog, "\n============ Found Properties ============")
 	driverProperties := queryDriverProperties(filter)
 	for d, props := range driverProperties {
 		priority := float64(d.Info().Priority)
 		for _, p := range props {
+			foundPropertiesLog = append(foundPropertiesLog, p.String())
 			fitnessDist, ok := constraints.MediaConstraints.FitnessDistance(p)
 			if !ok {
 				continue
@@ -213,16 +142,25 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 		}
 	}
 
+	foundPropertiesLog = append(foundPropertiesLog, "=============== Constraints ==============")
+	foundPropertiesLog = append(foundPropertiesLog, constraints.String())
+	foundPropertiesLog = append(foundPropertiesLog, "================ Best Fit ================")
+
 	if bestDriver == nil {
+		foundPropertiesLog = append(foundPropertiesLog, "Not found")
+		logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
 		return nil, MediaTrackConstraints{}, errNotFound
 	}
 
-	constraints.selectedMedia = bestProp
-	constraints.selectedMedia.Merge(constraints.MediaConstraints)
+	foundPropertiesLog = append(foundPropertiesLog, bestProp.String())
+	logger.Debug(strings.Join(foundPropertiesLog, "\n\n"))
+	constraints.selectedMedia = prop.Media{}
+	constraints.selectedMedia.MergeConstraints(constraints.MediaConstraints)
+	constraints.selectedMedia.Merge(bestProp)
 	return bestDriver, constraints, nil
 }
 
-func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker, error) {
+func selectAudio(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
 	typeFilter := driver.FilterAudioRecorder()
 
 	d, c, err := selectBestDriver(typeFilter, constraints)
@@ -230,9 +168,9 @@ func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newTrackFromDriver(d, c, selector)
 }
-func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker, error) {
+func selectVideo(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	notScreenFilter := driver.FilterNot(driver.FilterDeviceType(driver.Screen))
 	filter := driver.FilterAnd(typeFilter, notScreenFilter)
@@ -242,10 +180,10 @@ func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newTrackFromDriver(d, c, selector)
 }
 
-func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker, error) {
+func selectScreen(constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	screenFilter := driver.FilterDeviceType(driver.Screen)
 	filter := driver.FilterAnd(typeFilter, screenFilter)
@@ -255,10 +193,10 @@ func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newTrackFromDriver(d, c, selector)
 }
 
-func (m *mediaDevices) EnumerateDevices() []MediaDeviceInfo {
+func EnumerateDevices() []MediaDeviceInfo {
 	drivers := driver.GetManager().Query(
 		driver.FilterFn(func(driver.Driver) bool { return true }))
 	info := make([]MediaDeviceInfo, 0, len(drivers))

mediadevices_bench_test.go

@@ -0,0 +1,82 @@
+// +build e2e
+
+package mediadevices
+
+import (
+	"image"
+	"sync"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec/x264"
+	"github.com/pion/mediadevices/pkg/frame"
+)
+
+type mockVideoSource struct {
+	width, height int
+	pool          sync.Pool
+	decoder       frame.Decoder
+}
+
+func newMockVideoSource(width, height int) *mockVideoSource {
+	decoder, err := frame.NewDecoder(frame.FormatYUY2)
+	if err != nil {
+		panic(err)
+	}
+
+	return &mockVideoSource{
+		width:  width,
+		height: height,
+		pool: sync.Pool{
+			New: func() interface{} {
+				resolution := width * height
+				return make([]byte, resolution*2)
+			},
+		},
+		decoder: decoder,
+	}
+}
+
+func (source *mockVideoSource) ID() string   { return "" }
+func (source *mockVideoSource) Close() error { return nil }
+func (source *mockVideoSource) Read() (image.Image, func(), error) {
+	raw := source.pool.Get().([]byte)
+	decoded, release, err := source.decoder.Decode(raw, source.width, source.height)
+	source.pool.Put(raw)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return decoded, release, nil
+}
+
+func BenchmarkEndToEnd(b *testing.B) {
+	params, err := x264.NewParams()
+	if err != nil {
+		b.Fatal(err)
+	}
+	params.BitRate = 300_000
+
+	videoSource := newMockVideoSource(1920, 1080)
+	track := NewVideoTrack(videoSource, nil).(*VideoTrack)
+	defer track.Close()
+
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	encodedReader, err := params.BuildVideoEncoder(reader, inputProp)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer encodedReader.Close()
+
+	for i := 0; i < b.N; i++ {
+		_, release, err := encodedReader.Read()
+		if err != nil {
+			b.Fatal(err)
+		}
+		release()
+	}
+}

mediadevices_test.go

@@ -1,90 +1,42 @@
 package mediadevices
 
 import (
-	"errors"
 	"io"
 	"testing"
 	"time"
 
-	"github.com/pion/webrtc/v2"
-	"github.com/pion/webrtc/v2/pkg/media"
-
-	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/driver"
 	_ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/videotest"
-	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
 func TestGetUserMedia(t *testing.T) {
-	videoParams := mockParams{
-		BaseParams: codec.BaseParams{
-			BitRate: 100000,
-		},
-		name: "MockVideo",
-	}
-	audioParams := mockParams{
-		BaseParams: codec.BaseParams{
-			BitRate: 32000,
-		},
-		name: "MockAudio",
-	}
-	md := NewMediaDevicesFromCodecs(
-		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
-			},
-			webrtc.RTPCodecTypeAudio: []*webrtc.RTPCodec{
-				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
-			},
-		},
-		WithTrackGenerator(
-			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
-				LocalTrack, error,
-			) {
-				return newMockTrack(codec, id), nil
-			},
-		),
-	)
 	constraints := MediaStreamConstraints{
 		Video: func(c *MediaTrackConstraints) {
-			c.Enabled = true
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
-			params := videoParams
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
 		},
 		Audio: func(c *MediaTrackConstraints) {
-			c.Enabled = true
-			params := audioParams
-			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
 		},
 	}
 	constraintsWrong := MediaStreamConstraints{
 		Video: func(c *MediaTrackConstraints) {
-			c.Enabled = true
-			c.Width = prop.Int(640)
+			c.Width = prop.IntExact(10000)
 			c.Height = prop.Int(480)
-			params := videoParams
-			params.BitRate = 0
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
 		},
 		Audio: func(c *MediaTrackConstraints) {
-			c.Enabled = true
-			params := audioParams
-			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
 		},
 	}
 
 	// GetUserMedia with broken parameters
-	ms, err := md.GetUserMedia(constraintsWrong)
+	ms, err := GetUserMedia(constraintsWrong)
 	if err == nil {
 		t.Fatal("Expected error, but got nil")
 	}
 
 	// GetUserMedia with correct parameters
-	ms, err = md.GetUserMedia(constraints)
+	ms, err = GetUserMedia(constraints)
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
 	}
@@ -102,11 +54,11 @@ func TestGetUserMedia(t *testing.T) {
 	time.Sleep(50 * time.Millisecond)
 
 	for _, track := range tracks {
-		track.Stop()
+		track.Close()
 	}
 
 	// Stop and retry GetUserMedia
-	ms, err = md.GetUserMedia(constraints)
+	ms, err = GetUserMedia(constraints)
 	if err != nil {
 		t.Fatalf("Failed to GetUserMedia after the previsous tracks stopped: %v", err)
 	}
@@ -122,98 +74,60 @@ func TestGetUserMedia(t *testing.T) {
 		})
 	}
 	time.Sleep(50 * time.Millisecond)
-}
-
-type mockTrack struct {
-	codec *webrtc.RTPCodec
-	id    string
-}
-
-func newMockTrack(codec *webrtc.RTPCodec, id string) *mockTrack {
-	return &mockTrack{
-		codec: codec,
-		id:    id,
+	for _, track := range tracks {
+		track.Close()
 	}
 }
 
-func (t *mockTrack) WriteSample(s media.Sample) error {
-	return nil
-}
-
-func (t *mockTrack) Codec() *webrtc.RTPCodec {
-	return t.codec
-}
-
-func (t *mockTrack) ID() string {
-	return t.id
-}
-
-func (t *mockTrack) Kind() webrtc.RTPCodecType {
-	return t.codec.Type
-}
-
-type mockParams struct {
-	codec.BaseParams
-	name string
-}
-
-func (params *mockParams) Name() string {
-	return params.name
-}
-
-func (params *mockParams) BuildVideoEncoder(r video.Reader, p prop.Media) (codec.ReadCloser, error) {
-	if params.BitRate == 0 {
-		// This is a dummy error to test the failure condition.
-		return nil, errors.New("wrong codec parameter")
+func TestSelectBestDriverConstraintsResultIsSetProperly(t *testing.T) {
+	filterFn := driver.FilterVideoRecorder()
+	drivers := driver.GetManager().Query(filterFn)
+	if len(drivers) == 0 {
+		t.Fatal("expect to get at least 1 driver")
 	}
-	return &mockVideoCodec{
-		r:      r,
-		closed: make(chan struct{}),
-	}, nil
-}
 
-func (params *mockParams) BuildAudioEncoder(r audio.Reader, p prop.Media) (codec.ReadCloser, error) {
-	return &mockAudioCodec{
-		r:      r,
-		closed: make(chan struct{}),
-	}, nil
-}
-
-type mockCodec struct{}
-
-func (e *mockCodec) SetBitRate(b int) error {
-	return nil
-}
-
-func (e *mockCodec) ForceKeyFrame() error {
-	return nil
-}
-
-type mockVideoCodec struct {
-	mockCodec
-	r      video.Reader
-	closed chan struct{}
-}
-
-func (m *mockVideoCodec) Read(b []byte) (int, error) {
-	if _, err := m.r.Read(); err != nil {
-		return 0, err
+	driver := drivers[0]
+	err := driver.Open()
+	if err != nil {
+		t.Fatal("expect to open driver successfully")
 	}
-	return len(b), nil
-}
+	defer driver.Close()
 
-func (m *mockVideoCodec) Close() error { return nil }
-
-type mockAudioCodec struct {
-	mockCodec
-	r      audio.Reader
-	closed chan struct{}
-}
-
-func (m *mockAudioCodec) Read(b []byte) (int, error) {
-	if _, err := m.r.Read(); err != nil {
-		return 0, err
+	if len(driver.Properties()) == 0 {
+		t.Fatal("expect to get at least 1 property")
+	}
+	expectedProp := driver.Properties()[0]
+	// Since this is a continuous value, bestConstraints should be set with the value that user specified
+	expectedProp.FrameRate = 30.0
+
+	wantConstraints := MediaTrackConstraints{
+		MediaConstraints: prop.MediaConstraints{
+			VideoConstraints: prop.VideoConstraints{
+				// By reducing the width from the driver by a tiny amount, this property should be chosen.
+				// At the same time, we'll be able to find out if the return constraints will be properly set
+				// to the best constraints.
+				Width:       prop.Int(expectedProp.Width - 1),
+				Height:      prop.Int(expectedProp.Width),
+				FrameFormat: prop.FrameFormat(expectedProp.FrameFormat),
+				FrameRate:   prop.Float(30.0),
+			},
+		},
+	}
+
+	bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if driver != bestDriver {
+		t.Fatal("best driver is not expected")
+	}
+
+	s := bestConstraints.selectedMedia
+	if s.Width != expectedProp.Width ||
+		s.Height != expectedProp.Height ||
+		s.FrameFormat != expectedProp.FrameFormat ||
+		s.FrameRate != expectedProp.FrameRate {
+		t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
 	}
-	return len(b), nil
 }
-func (m *mockAudioCodec) Close() error { return nil }

mediastream.go

@@ -3,88 +3,86 @@ package mediadevices
 import (
 	"sync"
 
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 // MediaStream is an interface that represents a collection of existing tracks.
 type MediaStream interface {
 	// GetAudioTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
-	GetAudioTracks() []Tracker
+	GetAudioTracks() []Track
 	// GetVideoTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
-	GetVideoTracks() []Tracker
+	GetVideoTracks() []Track
 	// GetTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
-	GetTracks() []Tracker
+	GetTracks() []Track
 	// AddTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
-	AddTrack(t Tracker)
+	AddTrack(t Track)
 	// RemoveTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
-	RemoveTrack(t Tracker)
+	RemoveTrack(t Track)
 }
 
 type mediaStream struct {
-	trackers map[string]Tracker
-	l        sync.RWMutex
+	tracks map[Track]struct{}
+	l      sync.RWMutex
 }
 
-const rtpCodecTypeDefault webrtc.RTPCodecType = 0
+const trackTypeDefault webrtc.RTPCodecType = 0
 
 // NewMediaStream creates a MediaStream interface that's defined in
 // https://w3c.github.io/mediacapture-main/#dom-mediastream
-func NewMediaStream(trackers ...Tracker) (MediaStream, error) {
-	m := mediaStream{trackers: make(map[string]Tracker)}
+func NewMediaStream(tracks ...Track) (MediaStream, error) {
+	m := mediaStream{tracks: make(map[Track]struct{})}
 
-	for _, tracker := range trackers {
-		id := tracker.LocalTrack().ID()
-		if _, ok := m.trackers[id]; !ok {
-			m.trackers[id] = tracker
+	for _, track := range tracks {
+		if _, ok := m.tracks[track]; !ok {
+			m.tracks[track] = struct{}{}
 		}
 	}
 
 	return &m, nil
 }
 
-func (m *mediaStream) GetAudioTracks() []Tracker {
+func (m *mediaStream) GetAudioTracks() []Track {
 	return m.queryTracks(webrtc.RTPCodecTypeAudio)
 }
 
-func (m *mediaStream) GetVideoTracks() []Tracker {
+func (m *mediaStream) GetVideoTracks() []Track {
 	return m.queryTracks(webrtc.RTPCodecTypeVideo)
 }
 
-func (m *mediaStream) GetTracks() []Tracker {
-	return m.queryTracks(rtpCodecTypeDefault)
+func (m *mediaStream) GetTracks() []Track {
+	return m.queryTracks(trackTypeDefault)
 }
 
 // queryTracks returns all tracks that are the same kind as t.
 // If t is 0, which is the default, queryTracks will return all the tracks.
-func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Tracker {
+func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Track {
 	m.l.RLock()
 	defer m.l.RUnlock()
 
-	result := make([]Tracker, 0)
-	for _, tracker := range m.trackers {
-		if tracker.LocalTrack().Kind() == t || t == rtpCodecTypeDefault {
-			result = append(result, tracker)
+	result := make([]Track, 0)
+	for track := range m.tracks {
+		if track.Kind() == t || t == trackTypeDefault {
+			result = append(result, track)
 		}
 	}
 
 	return result
 }
 
-func (m *mediaStream) AddTrack(t Tracker) {
+func (m *mediaStream) AddTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	id := t.LocalTrack().ID()
-	if _, ok := m.trackers[id]; ok {
+	if _, ok := m.tracks[t]; ok {
 		return
 	}
 
-	m.trackers[id] = t
+	m.tracks[t] = struct{}{}
 }
 
-func (m *mediaStream) RemoveTrack(t Tracker) {
+func (m *mediaStream) RemoveTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	delete(m.trackers, t.LocalTrack().ID())
+	delete(m.tracks, t)
 }

mediastream_test.go

@@ -0,0 +1,112 @@
+package mediadevices
+
+import (
+	"io"
+	"testing"
+
+	"github.com/pion/webrtc/v3"
+)
+
+type mockMediaStreamTrack struct {
+	kind MediaDeviceType
+}
+
+func (track *mockMediaStreamTrack) ID() string {
+	return ""
+}
+
+func (track *mockMediaStreamTrack) StreamID() string {
+	return ""
+}
+
+func (track *mockMediaStreamTrack) Close() error {
+	return nil
+}
+
+func (track *mockMediaStreamTrack) Kind() webrtc.RTPCodecType {
+	switch track.kind {
+	case AudioInput:
+		return webrtc.RTPCodecTypeAudio
+	case VideoInput:
+		return webrtc.RTPCodecTypeVideo
+	default:
+		panic("invalid track kind")
+	}
+}
+
+func (track *mockMediaStreamTrack) OnEnded(handler func(error)) {
+}
+
+func (track *mockMediaStreamTrack) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) {
+	return webrtc.RTPCodecParameters{}, nil
+}
+
+func (track *mockMediaStreamTrack) Unbind(ctx webrtc.TrackLocalContext) error {
+	return nil
+}
+
+func (track *mockMediaStreamTrack) NewRTPReader(codecName string, ssrc uint32, mtu int) (RTPReadCloser, error) {
+	return nil, nil
+}
+
+func (track *mockMediaStreamTrack) NewEncodedReader(codecName string) (EncodedReadCloser, error) {
+	return nil, nil
+}
+
+func (track *mockMediaStreamTrack) NewEncodedIOReader(codecName string) (io.ReadCloser, error) {
+	return nil, nil
+}
+
+func TestMediaStreamFilters(t *testing.T) {
+	audioTracks := []Track{
+		&mockMediaStreamTrack{AudioInput},
+		&mockMediaStreamTrack{AudioInput},
+		&mockMediaStreamTrack{AudioInput},
+		&mockMediaStreamTrack{AudioInput},
+		&mockMediaStreamTrack{AudioInput},
+	}
+
+	videoTracks := []Track{
+		&mockMediaStreamTrack{VideoInput},
+		&mockMediaStreamTrack{VideoInput},
+		&mockMediaStreamTrack{VideoInput},
+	}
+
+	tracks := append(audioTracks, videoTracks...)
+	stream, err := NewMediaStream(tracks...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expect := func(t *testing.T, actual, expected []Track) {
+		if len(actual) != len(expected) {
+			t.Fatalf("%s: Expected to get %d trackers, but got %d trackers", t.Name(), len(expected), len(actual))
+		}
+
+		for _, a := range actual {
+			found := false
+			for _, e := range expected {
+				if e == a {
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				t.Fatalf("%s: Expected to find %p in the query results", t.Name(), a)
+			}
+		}
+	}
+
+	t.Run("GetAudioTracks", func(t *testing.T) {
+		expect(t, stream.GetAudioTracks(), audioTracks)
+	})
+
+	t.Run("GetVideoTracks", func(t *testing.T) {
+		expect(t, stream.GetVideoTracks(), videoTracks)
+	})
+
+	t.Run("GetTracks", func(t *testing.T) {
+		expect(t, stream.GetTracks(), tracks)
+	})
+}

mediastreamconstraints.go

@@ -1,40 +1,18 @@
 package mediadevices
 
 import (
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
 type MediaStreamConstraints struct {
 	Audio MediaOption
 	Video MediaOption
+	Codec *CodecSelector
 }
 
 // MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
 type MediaTrackConstraints struct {
 	prop.MediaConstraints
-	Enabled bool
-	// VideoEncoderBuilders are codec builders that are used for encoding the video
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	VideoEncoderBuilders []codec.VideoEncoderBuilder
-	// AudioEncoderBuilders are codec builders that are used for encoding the audio
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	AudioEncoderBuilders []codec.AudioEncoderBuilder
-	// VideoTransform will be used to transform the video that's coming from the driver.
-	// So, basically it'll look like following: driver -> VideoTransform -> codec
-	VideoTransform video.TransformFunc
-	// AudioTransform will be used to transform the audio that's coming from the driver.
-	// So, basically it'll look like following: driver -> AudioTransform -> code
-	AudioTransform audio.TransformFunc
-
 	selectedMedia prop.Media
 }
 

meta.go

@@ -0,0 +1,35 @@
+package mediadevices
+
+import (
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+// detectCurrentVideoProp is a small helper to get current video property
+func detectCurrentVideoProp(broadcaster *video.Broadcaster) (prop.Media, error) {
+	var currentProp prop.Media
+
+	// Since broadcaster has a ring buffer internally, a new reader will either read the last
+	// buffered frame or a new frame from the source. This also implies that no frame will be lost
+	// in any case.
+	metaReader := broadcaster.NewReader(false)
+	metaReader = video.DetectChanges(0, 0, func(p prop.Media) { currentProp = p })(metaReader)
+	_, _, err := metaReader.Read()
+
+	return currentProp, err
+}
+
+// detectCurrentAudioProp is a small helper to get current audio property
+func detectCurrentAudioProp(broadcaster *audio.Broadcaster) (prop.Media, error) {
+	var currentProp prop.Media
+
+	// Since broadcaster has a ring buffer internally, a new reader will either read the last
+	// buffered frame or a new frame from the source. This also implies that no frame will be lost
+	// in any case.
+	metaReader := broadcaster.NewReader(false)
+	metaReader = audio.DetectChanges(0, func(p prop.Media) { currentProp = p })(metaReader)
+	_, _, err := metaReader.Read()
+
+	return currentProp, err
+}

meta_test.go

@@ -0,0 +1,98 @@
+package mediadevices
+
+import (
+	"image"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+func TestDetectCurrentVideoProp(t *testing.T) {
+	resolution := image.Rect(0, 0, 4, 4)
+	first := image.NewRGBA(resolution)
+	first.Pix[0] = 1
+	second := image.NewRGBA(resolution)
+	second.Pix[0] = 2
+
+	isFirst := true
+	source := video.ReaderFunc(func() (image.Image, func(), error) {
+		if isFirst {
+			isFirst = true
+			return first, func() {}, nil
+		} else {
+			return second, func() {}, nil
+		}
+	})
+
+	broadcaster := video.NewBroadcaster(source, nil)
+
+	currentProp, err := detectCurrentVideoProp(broadcaster)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if currentProp.Width != resolution.Dx() {
+		t.Fatalf("Expect the actual width to be %d, but got %d", currentProp.Width, resolution.Dx())
+	}
+
+	if currentProp.Height != resolution.Dy() {
+		t.Fatalf("Expect the actual height to be %d, but got %d", currentProp.Height, resolution.Dy())
+	}
+
+	reader := broadcaster.NewReader(false)
+	img, _, err := reader.Read()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rgba := img.(*image.RGBA)
+	if rgba.Pix[0] != 1 {
+		t.Fatal("Expect the frame after reading the current prop is not the first frame")
+	}
+}
+
+func TestDetectCurrentAudioProp(t *testing.T) {
+	info := wave.ChunkInfo{
+		Len:          4,
+		Channels:     2,
+		SamplingRate: 48000,
+	}
+	first := wave.NewInt16Interleaved(info)
+	first.Data[0] = 1
+	second := wave.NewInt16Interleaved(info)
+	second.Data[0] = 2
+
+	isFirst := true
+	source := audio.ReaderFunc(func() (wave.Audio, func(), error) {
+		if isFirst {
+			isFirst = true
+			return first, func() {}, nil
+		} else {
+			return second, func() {}, nil
+		}
+	})
+
+	broadcaster := audio.NewBroadcaster(source, nil)
+
+	currentProp, err := detectCurrentAudioProp(broadcaster)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if currentProp.ChannelCount != info.Channels {
+		t.Fatalf("Expect the actual channel count to be %d, but got %d", currentProp.ChannelCount, info.Channels)
+	}
+
+	reader := broadcaster.NewReader(false)
+	chunk, _, err := reader.Read()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	realChunk := chunk.(*wave.Int16Interleaved)
+	if realChunk.Data[0] != 1 {
+		t.Fatal("Expect the chunk after reading the current prop is not the first chunk")
+	}
+}

pkg/avfoundation/.gitignore

@@ -0,0 +1,25 @@
+#User settings
+xcuserdata/
+
+## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
+*.xcscmblueprint
+*.xccheckout
+
+## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
+build/
+DerivedData/
+*.moved-aside
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+
+## Gcc Patch
+/*.gcno
+.DS_STORE
+
+Build/

pkg/avfoundation/AVFoundationBind.xcodeproj/project.pbxproj

@@ -0,0 +1,294 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		F0143CC12479F78E00EC29C9 /* AVFoundationBind.h in Headers */ = {isa = PBXBuildFile; fileRef = F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */; };
+		F0143CC32479F78E00EC29C9 /* AVFoundationBind.m in Sources */ = {isa = PBXBuildFile; fileRef = F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */; settings = {COMPILER_FLAGS = "-fno-objc-arc"; }; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libAVFoundationBind.a; sourceTree = BUILT_PRODUCTS_DIR; };
+		F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AVFoundationBind.h; sourceTree = "<group>"; };
+		F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AVFoundationBind.m; sourceTree = "<group>"; };
+		F0FDDA0B247E15D900A3429D /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		F0143CBB2479F78E00EC29C9 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		F0143CB42479F78E00EC29C9 = {
+			isa = PBXGroup;
+			children = (
+				F0143CBF2479F78E00EC29C9 /* AVFoundationBind */,
+				F0143CBE2479F78E00EC29C9 /* Products */,
+				F0FDDA0A247E15D900A3429D /* Frameworks */,
+			);
+			sourceTree = "<group>";
+		};
+		F0143CBE2479F78E00EC29C9 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		F0143CBF2479F78E00EC29C9 /* AVFoundationBind */ = {
+			isa = PBXGroup;
+			children = (
+				F0143CC02479F78E00EC29C9 /* AVFoundationBind.h */,
+				F0143CC22479F78E00EC29C9 /* AVFoundationBind.m */,
+			);
+			path = AVFoundationBind;
+			sourceTree = "<group>";
+		};
+		F0FDDA0A247E15D900A3429D /* Frameworks */ = {
+			isa = PBXGroup;
+			children = (
+				F0FDDA0B247E15D900A3429D /* AVFoundation.framework */,
+			);
+			name = Frameworks;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXHeadersBuildPhase section */
+		F0143CB92479F78E00EC29C9 /* Headers */ = {
+			isa = PBXHeadersBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				F0143CC12479F78E00EC29C9 /* AVFoundationBind.h in Headers */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXHeadersBuildPhase section */
+
+/* Begin PBXNativeTarget section */
+		F0143CBC2479F78E00EC29C9 /* AVFoundationBind */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = F0143CC62479F78E00EC29C9 /* Build configuration list for PBXNativeTarget "AVFoundationBind" */;
+			buildPhases = (
+				F0143CB92479F78E00EC29C9 /* Headers */,
+				F0143CBA2479F78E00EC29C9 /* Sources */,
+				F0143CBB2479F78E00EC29C9 /* Frameworks */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = AVFoundationBind;
+			productName = AVFoundationBind;
+			productReference = F0143CBD2479F78E00EC29C9 /* libAVFoundationBind.a */;
+			productType = "com.apple.product-type.library.static";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		F0143CB52479F78E00EC29C9 /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastUpgradeCheck = 1150;
+				ORGANIZATIONNAME = "Herman, Lukas";
+				TargetAttributes = {
+					F0143CBC2479F78E00EC29C9 = {
+						CreatedOnToolsVersion = 11.5;
+					};
+				};
+			};
+			buildConfigurationList = F0143CB82479F78E00EC29C9 /* Build configuration list for PBXProject "AVFoundationBind" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = F0143CB42479F78E00EC29C9;
+			productRefGroup = F0143CBE2479F78E00EC29C9 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				F0143CBC2479F78E00EC29C9 /* AVFoundationBind */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXSourcesBuildPhase section */
+		F0143CBA2479F78E00EC29C9 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				F0143CC32479F78E00EC29C9 /* AVFoundationBind.m in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin XCBuildConfiguration section */
+		F0143CC42479F78E00EC29C9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
+				MTL_FAST_MATH = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = macosx;
+			};
+			name = Debug;
+		};
+		F0143CC52479F78E00EC29C9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				MTL_FAST_MATH = YES;
+				SDKROOT = macosx;
+			};
+			name = Release;
+		};
+		F0143CC72479F78E00EC29C9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CODE_SIGN_STYLE = Automatic;
+				EXECUTABLE_PREFIX = lib;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SKIP_INSTALL = YES;
+			};
+			name = Debug;
+		};
+		F0143CC82479F78E00EC29C9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				CODE_SIGN_STYLE = Automatic;
+				EXECUTABLE_PREFIX = lib;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SKIP_INSTALL = YES;
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		F0143CB82479F78E00EC29C9 /* Build configuration list for PBXProject "AVFoundationBind" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				F0143CC42479F78E00EC29C9 /* Debug */,
+				F0143CC52479F78E00EC29C9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		F0143CC62479F78E00EC29C9 /* Build configuration list for PBXNativeTarget "AVFoundationBind" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				F0143CC72479F78E00EC29C9 /* Debug */,
+				F0143CC82479F78E00EC29C9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = F0143CB52479F78E00EC29C9 /* Project object */;
+}

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:AVFoundationBind.xcodeproj">
+   </FileRef>
+</Workspace>

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

pkg/avfoundation/AVFoundationBind.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>PreviewsEnabled</key>
+	<false/>
+</dict>
+</plist>

pkg/avfoundation/AVFoundationBind.xcodeproj/xcshareddata/xcschemes/AVFoundationBind.xcscheme

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1150"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "F0143CBC2479F78E00EC29C9"
+               BuildableName = "libAVFoundationBind.a"
+               BlueprintName = "AVFoundationBind"
+               ReferencedContainer = "container:AVFoundationBind.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Release"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "F0143CBC2479F78E00EC29C9"
+            BuildableName = "libAVFoundationBind.a"
+            BlueprintName = "AVFoundationBind"
+            ReferencedContainer = "container:AVFoundationBind.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

pkg/avfoundation/AVFoundationBind/AVFoundationBind.h

@@ -0,0 +1,79 @@
+// MIT License
+// 
+// Copyright (c) 2019-2020 Pion
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <stddef.h>
+
+#define MAX_DEVICES                      8
+#define MAX_PROPERTIES                   64
+#define MAX_DEVICE_UID_CHARS             64
+
+typedef const char* STATUS;
+static STATUS STATUS_OK                       = (STATUS) NULL;
+static STATUS STATUS_NULL_ARG                 = (STATUS) "One of the arguments was null";
+static STATUS STATUS_DEVICE_INIT_FAILED       = (STATUS) "Failed to init device";
+static STATUS STATUS_UNSUPPORTED_FRAME_FORMAT = (STATUS) "Unsupported frame format";
+static STATUS STATUS_UNSUPPORTED_MEDIA_TYPE   = (STATUS) "Unsupported media type";
+static STATUS STATUS_FAILED_TO_ACQUIRE_LOCK   = (STATUS) "Failed to acquire a lock";
+static STATUS STATUS_UNSUPPORTED_FORMAT       = (STATUS) "Unsupported device format";
+
+typedef enum AVBindMediaType {
+    AVBindMediaTypeVideo,
+    AVBindMediaTypeAudio,
+} AVBindMediaType;
+
+typedef enum AVBindFrameFormat {
+    AVBindFrameFormatI420,
+    AVBindFrameFormatNV21,
+    AVBindFrameFormatNV12,
+    AVBindFrameFormatYUY2,
+    AVBindFrameFormatUYVY,
+} AVBindFrameFormat;
+
+typedef void (*AVBindDataCallback)(void *userData, void *buf, int len);
+
+typedef struct AVBindMediaProperty {
+    // video property
+    int width, height;
+    AVBindFrameFormat frameFormat;
+    
+    // audio property
+    
+} AVBindMediaProperty, *PAVBindMediaProperty;
+
+typedef struct AVBindSession AVBindSession, *PAVBindSession;
+
+typedef struct {
+    char uid[MAX_DEVICE_UID_CHARS + 1];
+} AVBindDevice, *PAVBindDevice;
+
+// AVBindDevices returns a list of AVBindDevices. The result array is pointing to a static
+// memory. The caller is expected to not hold on to the address for a long time and make a copy.
+// Everytime this function gets called, the array will be overwritten and the memory will be reused.
+STATUS AVBindDevices(AVBindMediaType, PAVBindDevice*, int*);
+
+STATUS AVBindSessionInit(AVBindDevice, PAVBindSession*);
+STATUS AVBindSessionFree(PAVBindSession*);
+STATUS AVBindSessionOpen(PAVBindSession, AVBindMediaProperty, AVBindDataCallback, void*);
+STATUS AVBindSessionClose(PAVBindSession);
+STATUS AVBindSessionProperties(PAVBindSession, PAVBindMediaProperty*, int*);

pkg/avfoundation/AVFoundationBind/AVFoundationBind.m

@@ -0,0 +1,376 @@
+// MIT License
+//
+// Copyright (c) 2019-2020 Pion
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// Naming Convention (let "name" as an actual variable name):
+//   - mName: "name" is a member of an Objective C object
+//   - pName: "name" is a C pointer
+//   - refName: "name" is an Objective C object reference
+
+#import <Foundation/Foundation.h>
+#import <AVFoundation/AVFoundation.h>
+#import "AVFoundationBind.h"
+#include <string.h>
+
+#define CHK(condition, status) \
+    do { \
+        if(!(condition)) { \
+            retStatus = status; \
+            goto cleanup; \
+        } \
+    } while(0)
+
+#define CHK_STATUS(status) \
+    do { \
+        if(status != STATUS_OK) { \
+            retStatus = status; \
+            goto cleanup; \
+        } \
+    } while(0)
+
+@interface VideoDataDelegate : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
+
+@property (readonly) AVBindDataCallback mCallback;
+@property (readonly) void *mPUserData;
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection;
+
+@end
+
+@implementation VideoDataDelegate
+
+- (id) init: (AVBindDataCallback) callback
+withUserData: (void*) pUserData {
+    self = [super init];
+    _mCallback = callback;
+    _mPUserData = pUserData;
+
+    return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection {
+    if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 ||
+        !CMSampleBufferIsValid(sampleBuffer) ||
+        !CMSampleBufferDataIsReady(sampleBuffer)) {
+      return;
+    }
+
+    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+    if (imageBuffer == NULL) {
+      return;
+    }
+
+    imageBuffer = CVBufferRetain(imageBuffer);
+    CVReturn ret =
+        CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
+    if (ret != kCVReturnSuccess) {
+      return;
+    }
+
+    void *buf = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
+    size_t dataSize = CVPixelBufferGetDataSize(imageBuffer);
+    _mCallback(_mPUserData, buf, (int)dataSize);
+
+    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
+    CVBufferRelease(imageBuffer);
+}
+
+@end
+
+@interface AudioDataDelegate : NSObject<AVCaptureAudioDataOutputSampleBufferDelegate>
+
+@property (readonly) AVBindDataCallback mCallback;
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection;
+
+@end
+
+@implementation AudioDataDelegate
+
+- (id) init: (AVBindDataCallback) callback {
+    self = [super init];
+    _mCallback = callback;
+    return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+       fromConnection:(AVCaptureConnection *)connection {
+    // TODO
+}
+
+@end
+
+STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
+    STATUS retStatus = STATUS_OK;
+    // Useful mapping reference from ffmpeg:
+    // https://github.com/FFmpeg/FFmpeg/blob/c810a9502cebe32e1dd08ee3d0d17053dde44aa9/libavdevice/avfoundation.m#L53-L80
+    switch (format) {
+        case AVBindFrameFormatI420:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8Planar;
+            break;
+        case AVBindFrameFormatNV21:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+            break;
+        case AVBindFrameFormatNV12:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+            break;
+        case AVBindFrameFormatUYVY:
+            *pFourCC = kCVPixelFormatType_422YpCbCr8;
+            break;
+        case AVBindFrameFormatYUY2:
+            *pFourCC = kCVPixelFormatType_422YpCbCr8_yuvs;
+            break;
+        // TODO: Add the rest of frame formats
+        default:
+            retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
+    }
+    return retStatus;
+}
+
+STATUS frameFormatFromFourCC(FourCharCode fourCC, AVBindFrameFormat *pFormat) {
+    STATUS retStatus = STATUS_OK;
+    switch (fourCC) {
+        case kCVPixelFormatType_420YpCbCr8Planar:
+            *pFormat = AVBindFrameFormatI420;
+            break;
+        case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+            *pFormat = AVBindFrameFormatNV21;
+            break;
+        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+            *pFormat = AVBindFrameFormatNV12;
+            break;
+        case kCVPixelFormatType_422YpCbCr8:
+            *pFormat = AVBindFrameFormatUYVY;
+            break;
+        case kCVPixelFormatType_422YpCbCr8_yuvs:
+            *pFormat = AVBindFrameFormatYUY2;
+            break;
+         // TODO: Add the rest of frame formats
+         default:
+             retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
+     }
+    return retStatus;
+}
+
+
+STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *pLen) {
+    static AVBindDevice devices[MAX_DEVICES];
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(mediaType == AVBindMediaTypeVideo || mediaType == AVBindMediaTypeAudio, STATUS_UNSUPPORTED_MEDIA_TYPE);
+    CHK(ppDevices != NULL && pLen != NULL, STATUS_NULL_ARG);
+
+    PAVBindDevice pDevice;
+    AVMediaType _mediaType = mediaType == AVBindMediaTypeVideo ? AVMediaTypeVideo : AVMediaTypeAudio;
+    NSArray *refAllTypes = @[
+        AVCaptureDeviceTypeBuiltInWideAngleCamera,
+        AVCaptureDeviceTypeBuiltInMicrophone,
+        AVCaptureDeviceTypeExternalUnknown
+    ];
+    AVCaptureDeviceDiscoverySession *refSession = [AVCaptureDeviceDiscoverySession
+        discoverySessionWithDeviceTypes: refAllTypes
+        mediaType: _mediaType
+        position: AVCaptureDevicePositionUnspecified];
+
+    int i = 0;
+    for (AVCaptureDevice *refDevice in refSession.devices) {
+        if (i >= MAX_DEVICES) {
+            break;
+        }
+
+        pDevice = devices + i;
+        strncpy(pDevice->uid, refDevice.uniqueID.UTF8String, MAX_DEVICE_UID_CHARS);
+        pDevice->uid[MAX_DEVICE_UID_CHARS] = '\0';
+        i++;
+    }
+
+    *ppDevices = devices;
+    *pLen = i;
+
+cleanup:
+    [refPool drain];
+    return retStatus;
+}
+
+struct AVBindSession {
+    AVBindDevice device;
+    AVCaptureSession *refCaptureSession;
+    AVBindMediaProperty properties[MAX_PROPERTIES];
+};
+
+
+STATUS AVBindSessionInit(AVBindDevice device, PAVBindSession *ppSessionResult) {
+    STATUS retStatus = STATUS_OK;
+    CHK(ppSessionResult != NULL, STATUS_NULL_ARG);
+    PAVBindSession pSession = malloc(sizeof(AVBindSession));
+    pSession->device = device;
+    pSession->refCaptureSession = NULL;
+    *ppSessionResult = pSession;
+
+cleanup:
+    return retStatus;
+}
+
+STATUS AVBindSessionFree(PAVBindSession *ppSession) {
+    STATUS retStatus = STATUS_OK;
+    CHK(ppSession != NULL, STATUS_NULL_ARG);
+    PAVBindSession pSession = *ppSession;
+    if (pSession->refCaptureSession != NULL) {
+        [pSession->refCaptureSession release];
+        pSession->refCaptureSession = NULL;
+    }
+    free(pSession);
+    *ppSession = NULL;
+
+cleanup:
+    return retStatus;
+}
+
+STATUS AVBindSessionOpen(PAVBindSession pSession,
+                  AVBindMediaProperty property,
+                  AVBindDataCallback dataCallback,
+                  void *pUserData) {
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(pSession != NULL && dataCallback != NULL, STATUS_NULL_ARG);
+
+    AVCaptureDeviceInput *refInput;
+    NSError *refErr = NULL;
+    NSString *refUID = [NSString stringWithUTF8String: pSession->device.uid];
+    AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refUID];
+
+    refInput = [[AVCaptureDeviceInput alloc] initWithDevice: refDevice error: &refErr];
+    CHK(refErr == NULL, STATUS_DEVICE_INIT_FAILED);
+
+    AVCaptureSession *refCaptureSession = [[AVCaptureSession alloc] init];
+    refCaptureSession.sessionPreset = AVCaptureSessionPresetMedium;
+    [refCaptureSession addInput: refInput];
+
+    if ([refDevice hasMediaType: AVMediaTypeVideo]) {
+        VideoDataDelegate *pDelegate = [[VideoDataDelegate alloc]
+                                        init: dataCallback
+                                        withUserData: pUserData];
+
+        AVCaptureVideoDataOutput *pOutput = [[AVCaptureVideoDataOutput alloc] init];
+        FourCharCode fourCC;
+        CHK_STATUS(frameFormatToFourCC(property.frameFormat, &fourCC));
+
+        pOutput.videoSettings = @{
+            (id)kCVPixelBufferWidthKey: @(property.width),
+            (id)kCVPixelBufferHeightKey: @(property.height),
+            (id)kCVPixelBufferPixelFormatTypeKey: @(fourCC),
+        };
+        pOutput.alwaysDiscardsLateVideoFrames = YES;
+        dispatch_queue_t queue =
+            dispatch_queue_create("captureQueue", DISPATCH_QUEUE_SERIAL);
+        [pOutput setSampleBufferDelegate:pDelegate queue:queue];
+        [refCaptureSession addOutput: pOutput];
+    } else {
+        // TODO: implement audio pipeline
+    }
+
+    pSession->refCaptureSession = [refCaptureSession retain];
+    [refCaptureSession startRunning];
+
+cleanup:
+    [refPool drain];
+    return retStatus;
+}
+
+
+STATUS AVBindSessionClose(PAVBindSession pSession) {
+    STATUS retStatus = STATUS_OK;
+    CHK(pSession != NULL, STATUS_NULL_ARG);
+    CHK(pSession->refCaptureSession != NULL, STATUS_OK);
+
+    [pSession->refCaptureSession stopRunning];
+    [pSession->refCaptureSession release];
+    pSession->refCaptureSession = NULL;
+
+cleanup:
+    return retStatus;
+}
+
+static NSString* FourCCString(FourCharCode code) {
+    NSString *result = [NSString stringWithFormat:@"%c%c%c%c",
+                        (code >> 24) & 0xff,
+                        (code >> 16) & 0xff,
+                        (code >> 8) & 0xff,
+                        code & 0xff];
+    NSCharacterSet *characterSet = [NSCharacterSet whitespaceCharacterSet];
+    return [result stringByTrimmingCharactersInSet:characterSet];
+}
+
+STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *ppProperties, int *pLen) {
+    STATUS retStatus = STATUS_OK;
+    NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
+    CHK(pSession != NULL && ppProperties != NULL && pLen != NULL, STATUS_NULL_ARG);
+
+    NSString *refDeviceUID = [NSString stringWithUTF8String: pSession->device.uid];
+    AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refDeviceUID];
+    FourCharCode fourCC;
+    CMVideoFormatDescriptionRef videoFormat;
+    CMVideoDimensions videoDimensions;
+
+    memset(pSession->properties, 0, sizeof(pSession->properties));
+    PAVBindMediaProperty pProperty = pSession->properties;
+    int len = 0;
+    for (AVCaptureDeviceFormat *refFormat in refDevice.formats) {
+        // TODO: Probably gives a warn to the user
+        if (len >= MAX_PROPERTIES) {
+            NSLog(@"[WARNING] skipping the rest of properties due to MAX_PROPERTIES");
+            break;
+        }
+
+        if ([refFormat.mediaType isEqual:AVMediaTypeVideo]) {
+            fourCC = CMFormatDescriptionGetMediaSubType(refFormat.formatDescription);
+            if (frameFormatFromFourCC(fourCC, &pProperty->frameFormat) != STATUS_OK) {
+                NSLog(@"[WARNING] skipping %@ %dx%d since it's not supported", FourCCString(fourCC), videoDimensions.width, videoDimensions.height);
+                continue;
+            }
+
+            videoFormat = (CMVideoFormatDescriptionRef) refFormat.formatDescription;
+            videoDimensions = CMVideoFormatDescriptionGetDimensions(videoFormat);
+            pProperty->height = videoDimensions.height;
+            pProperty->width = videoDimensions.width;
+        } else {
+            // TODO: Get audio properties
+        }
+
+        pProperty++;
+        len++;
+    }
+
+    *ppProperties = pSession->properties;
+    *pLen = len;
+
+cleanup:
+
+    [refPool drain];
+    return retStatus;
+}

pkg/avfoundation/avfoundation_callback_darwin.go

@@ -0,0 +1,54 @@
+package avfoundation
+
+import "C"
+import (
+	"sync"
+	"unsafe"
+)
+
+var mu sync.Mutex
+var nextID handleID
+
+type dataCb func(data []byte)
+
+var handles = make(map[handleID]dataCb)
+
+type handleID int
+
+//export onData
+func onData(userData unsafe.Pointer, buf unsafe.Pointer, length C.int) {
+	handleNum := (*C.int)(userData)
+	cb, ok := lookup(handleID(*handleNum))
+	if ok {
+		data := C.GoBytes(buf, length)
+		cb(data)
+	}
+}
+
+func register(fn dataCb) handleID {
+	mu.Lock()
+	defer mu.Unlock()
+
+	nextID++
+	for handles[nextID] != nil {
+		nextID++
+	}
+	handles[nextID] = fn
+
+	return nextID
+}
+
+func lookup(i handleID) (cb dataCb, ok bool) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	cb, ok = handles[i]
+	return
+}
+
+func unregister(i handleID) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	delete(handles, i)
+}

pkg/avfoundation/avfoundation_darwin.go

@@ -0,0 +1,221 @@
+// Package avfoundation provides AVFoundation binding for Go
+package avfoundation
+
+// #cgo CFLAGS: -x objective-c
+// #cgo LDFLAGS: -framework AVFoundation -framework Foundation -framework CoreMedia -framework CoreVideo
+// #include "AVFoundationBind/AVFoundationBind.h"
+// #include "AVFoundationBind/AVFoundationBind.m"
+// extern void onData(void*, void*, int);
+// void onDataBridge(void *userData, void *buf, int len) {
+// 	onData(userData, buf, len);
+// }
+import "C"
+import (
+	"fmt"
+	"io"
+	"unsafe"
+
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+type MediaType C.AVBindMediaType
+
+const (
+	Video = MediaType(C.AVBindMediaTypeVideo)
+	Audio = MediaType(C.AVBindMediaTypeAudio)
+)
+
+// Device represents a metadata that later can be used to retrieve back the
+// underlying device given by AVFoundation
+type Device struct {
+	// UID is a unique identifier for a device
+	UID     string
+	cDevice C.AVBindDevice
+}
+
+func frameFormatToAVBind(f frame.Format) (C.AVBindFrameFormat, bool) {
+	switch f {
+	case frame.FormatI420:
+		return C.AVBindFrameFormatI420, true
+	case frame.FormatNV21:
+		return C.AVBindFrameFormatNV21, true
+	case frame.FormatNV12:
+		return C.AVBindFrameFormatNV12, true
+	case frame.FormatYUY2:
+		return C.AVBindFrameFormatYUY2, true
+	case frame.FormatUYVY:
+		return C.AVBindFrameFormatUYVY, true
+	default:
+		return 0, false
+	}
+}
+
+func frameFormatFromAVBind(f C.AVBindFrameFormat) (frame.Format, bool) {
+	switch f {
+	case C.AVBindFrameFormatI420:
+		return frame.FormatI420, true
+	case C.AVBindFrameFormatNV21:
+		return frame.FormatNV21, true
+	case C.AVBindFrameFormatNV12:
+		return frame.FormatNV12, true
+	case C.AVBindFrameFormatYUY2:
+		return frame.FormatYUY2, true
+	case C.AVBindFrameFormatUYVY:
+		return frame.FormatUYVY, true
+	default:
+		return "", false
+	}
+}
+
+// Devices uses AVFoundation to query a list of devices based on the media type
+func Devices(mediaType MediaType) ([]Device, error) {
+	var cDevicesPtr C.PAVBindDevice
+	var cDevicesLen C.int
+
+	status := C.AVBindDevices(C.AVBindMediaType(mediaType), &cDevicesPtr, &cDevicesLen)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	cDevices := (*[1 << 28]C.AVBindDevice)(unsafe.Pointer(cDevicesPtr))[:cDevicesLen:cDevicesLen]
+	devices := make([]Device, cDevicesLen)
+
+	for i := range devices {
+		devices[i].UID = C.GoString(&cDevices[i].uid[0])
+		devices[i].cDevice = cDevices[i]
+	}
+
+	return devices, nil
+}
+
+// ReadCloser is a wrapper around the data callback from AVFoundation. The data received from the
+// the underlying callback can be retrieved by calling Read.
+type ReadCloser struct {
+	dataChan chan []byte
+	id       handleID
+	onClose  func()
+}
+
+func newReadCloser(onClose func()) *ReadCloser {
+	var rc ReadCloser
+	rc.dataChan = make(chan []byte, 1)
+	rc.onClose = onClose
+	rc.id = register(rc.dataCb)
+	return &rc
+}
+
+func (rc *ReadCloser) dataCb(data []byte) {
+	// TODO: add a policy for slow reader
+	rc.dataChan <- data
+}
+
+// Read reads raw data, the format is determined by the media type and property:
+//   - For video, each call will return a frame.
+//   - For audio, each call will return a chunk which its size configured by Latency
+func (rc *ReadCloser) Read() ([]byte, func(), error) {
+	data, ok := <-rc.dataChan
+	if !ok {
+		return nil, func() {}, io.EOF
+	}
+	return data, func() {}, nil
+}
+
+// Close closes the capturing session, and no data will flow anymore
+func (rc *ReadCloser) Close() {
+	if rc.onClose != nil {
+		rc.onClose()
+	}
+	close(rc.dataChan)
+	unregister(rc.id)
+}
+
+// Session represents a capturing session.
+type Session struct {
+	device   Device
+	cSession C.PAVBindSession
+}
+
+// NewSession creates a new capturing session
+func NewSession(device Device) (*Session, error) {
+	var session Session
+
+	status := C.AVBindSessionInit(device.cDevice, &session.cSession)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+
+	session.device = device
+	return &session, nil
+}
+
+// Close stops capturing session and frees up resources
+func (session *Session) Close() error {
+	if session.cSession == nil {
+		return nil
+	}
+
+	status := C.AVBindSessionFree(&session.cSession)
+	if status != nil {
+		return fmt.Errorf("%s", C.GoString(status))
+	}
+	return nil
+}
+
+// Open start capturing session. As soon as it returns successfully, the data will start
+// flowing. The raw data can be retrieved by using ReadCloser's Read method.
+func (session *Session) Open(property prop.Media) (*ReadCloser, error) {
+	frameFormat, ok := frameFormatToAVBind(property.FrameFormat)
+	if !ok {
+		return nil, fmt.Errorf("Unsupported frame format")
+	}
+
+	cProperty := C.AVBindMediaProperty{
+		width:       C.int(property.Width),
+		height:      C.int(property.Height),
+		frameFormat: frameFormat,
+	}
+
+	rc := newReadCloser(func() {
+		C.AVBindSessionClose(session.cSession)
+	})
+	status := C.AVBindSessionOpen(
+		session.cSession,
+		cProperty,
+		C.AVBindDataCallback(unsafe.Pointer(C.onDataBridge)),
+		unsafe.Pointer(&rc.id),
+	)
+	if status != nil {
+		return nil, fmt.Errorf("%s", C.GoString(status))
+	}
+	return rc, nil
+}
+
+// Properties queries a list of properties that device supports
+func (session *Session) Properties() []prop.Media {
+	var cPropertiesPtr C.PAVBindMediaProperty
+	var cPropertiesLen C.int
+
+	status := C.AVBindSessionProperties(session.cSession, &cPropertiesPtr, &cPropertiesLen)
+	if status != nil {
+		return nil
+	}
+
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	cProperties := (*[1 << 28]C.AVBindMediaProperty)(unsafe.Pointer(cPropertiesPtr))[:cPropertiesLen:cPropertiesLen]
+	var properties []prop.Media
+	for _, cProperty := range cProperties {
+		frameFormat, ok := frameFormatFromAVBind(cProperty.frameFormat)
+		if ok {
+			properties = append(properties, prop.Media{
+				Video: prop.Video{
+					Width:       int(cProperty.width),
+					Height:      int(cProperty.height),
+					FrameFormat: frameFormat,
+				},
+			})
+		}
+	}
+	return properties
+}

pkg/codec/codec.go

@@ -1,21 +1,96 @@
 package codec
 
 import (
-	"io"
-
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/rtp"
+	"github.com/pion/rtp/codecs"
+	"github.com/pion/webrtc/v3"
 )
 
+// RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.
+type RTPCodec struct {
+	webrtc.RTPCodecParameters
+	rtp.Payloader
+}
+
+// NewRTPH264Codec is a helper to create an H264 codec
+func NewRTPH264Codec(clockrate uint32) *RTPCodec {
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeH264,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 125,
+		},
+		Payloader: &codecs.H264Payloader{},
+	}
+}
+
+// NewRTPVP8Codec is a helper to create an VP8 codec
+func NewRTPVP8Codec(clockrate uint32) *RTPCodec {
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeVP8,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 96,
+		},
+		Payloader: &codecs.VP8Payloader{},
+	}
+}
+
+// NewRTPVP9Codec is a helper to create an VP9 codec
+func NewRTPVP9Codec(clockrate uint32) *RTPCodec {
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeVP9,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 98,
+		},
+		Payloader: &codecs.VP9Payloader{},
+	}
+}
+
+// NewRTPOpusCodec is a helper to create an Opus codec
+func NewRTPOpusCodec(clockrate uint32) *RTPCodec {
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeOpus,
+				ClockRate:    48000,
+				Channels:     2,
+				SDPFmtpLine:  "minptime=10;useinbandfec=1",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 111,
+		},
+		Payloader: &codecs.OpusPayloader{},
+	}
+}
+
 // AudioEncoderBuilder is the interface that wraps basic operations that are
 // necessary to build the audio encoder.
 //
 // This interface is for codec implementors to provide codec specific params,
 // but still giving generality for the users.
 type AudioEncoderBuilder interface {
-	// Name represents the codec name
-	Name() string
+	// RTPCodec represents the codec metadata
+	RTPCodec() *RTPCodec
 	// BuildAudioEncoder builds audio encoder by given media params and audio input
 	BuildAudioEncoder(r audio.Reader, p prop.Media) (ReadCloser, error)
 }
@@ -26,15 +101,16 @@ type AudioEncoderBuilder interface {
 // This interface is for codec implementors to provide codec specific params,
 // but still giving generality for the users.
 type VideoEncoderBuilder interface {
-	// Name represents the codec name
-	Name() string
+	// RTPCodec represents the codec metadata
+	RTPCodec() *RTPCodec
 	// BuildVideoEncoder builds video encoder by given media params and video input
 	BuildVideoEncoder(r video.Reader, p prop.Media) (ReadCloser, error)
 }
 
 // ReadCloser is an io.ReadCloser with methods for rate limiting: SetBitRate and ForceKeyFrame
 type ReadCloser interface {
-	io.ReadCloser
+	Read() (b []byte, release func(), err error)
+	Close() error
 	// SetBitRate sets current target bitrate, lower bitrate means smaller data will be transmitted
 	// but this also means that the quality will also be lower.
 	SetBitRate(int) error

pkg/codec/mmal/bridge.h

@@ -0,0 +1,196 @@
+#include <interface/mmal/mmal.h>
+#include <interface/mmal/util/mmal_default_components.h>
+#include <interface/mmal/util/mmal_util_params.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define CHK(__status, __msg)                                                                                           \
+  do {                                                                                                                 \
+    status.code = __status;                                                                                            \
+    if (status.code != MMAL_SUCCESS) {                                                                                 \
+      status.msg = __msg;                                                                                              \
+      goto CleanUp;                                                                                                    \
+    }                                                                                                                  \
+  } while (0)
+
+typedef struct Status {
+  MMAL_STATUS_T code;
+  const char *msg;
+} Status;
+
+typedef struct Slice {
+  uint8_t *data;
+  int len;
+} Slice;
+
+typedef struct Params {
+  int width, height;
+  uint32_t bitrate;
+  uint32_t key_frame_interval;
+} Params;
+
+typedef struct Encoder {
+  MMAL_COMPONENT_T *component;
+  MMAL_PORT_T *port_in, *port_out;
+  MMAL_QUEUE_T *queue_out;
+  MMAL_POOL_T *pool_in, *pool_out;
+} Encoder;
+
+Status enc_new(Params, Encoder *);
+Status enc_encode(Encoder *, Slice y, Slice cb, Slice cr, MMAL_BUFFER_HEADER_T **);
+Status enc_close(Encoder *);
+
+static void encoder_in_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) { mmal_buffer_header_release(buffer); }
+
+static void encoder_out_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) {
+  MMAL_QUEUE_T *queue = (MMAL_QUEUE_T *)port->userdata;
+  mmal_queue_put(queue, buffer);
+}
+
+Status enc_new(Params params, Encoder *encoder) {
+  Status status = {0};
+  bool created = false;
+
+  memset(encoder, 0, sizeof(Encoder));
+
+  CHK(mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder->component),
+      "Failed to create video encoder component");
+  created = true;
+
+  encoder->port_in = encoder->component->input[0];
+  encoder->port_in->format->type = MMAL_ES_TYPE_VIDEO;
+  encoder->port_in->format->encoding = MMAL_ENCODING_I420;
+  encoder->port_in->format->es->video.width = params.width;
+  encoder->port_in->format->es->video.height = params.height;
+  encoder->port_in->format->es->video.par.num = 1;
+  encoder->port_in->format->es->video.par.den = 1;
+  encoder->port_in->format->es->video.crop.x = 0;
+  encoder->port_in->format->es->video.crop.y = 0;
+  encoder->port_in->format->es->video.crop.width = params.width;
+  encoder->port_in->format->es->video.crop.height = params.height;
+  CHK(mmal_port_format_commit(encoder->port_in), "Failed to commit input port format");
+
+  encoder->port_out = encoder->component->output[0];
+  encoder->port_out->format->type = MMAL_ES_TYPE_VIDEO;
+  encoder->port_out->format->encoding = MMAL_ENCODING_H264;
+  encoder->port_out->format->bitrate = params.bitrate;
+  CHK(mmal_port_format_commit(encoder->port_out), "Failed to commit output port format");
+
+  MMAL_PARAMETER_VIDEO_PROFILE_T encoder_param_profile = {0};
+  encoder_param_profile.hdr.id = MMAL_PARAMETER_PROFILE;
+  encoder_param_profile.hdr.size = sizeof(encoder_param_profile);
+  encoder_param_profile.profile[0].profile = MMAL_VIDEO_PROFILE_H264_BASELINE;
+  encoder_param_profile.profile[0].level = MMAL_VIDEO_LEVEL_H264_42;
+  CHK(mmal_port_parameter_set(encoder->port_out, &encoder_param_profile.hdr), "Failed to set encoder profile param");
+
+  CHK(mmal_port_parameter_set_uint32(encoder->port_out, MMAL_PARAMETER_INTRAPERIOD, params.key_frame_interval),
+      "Failed to set intra period param");
+
+  MMAL_PARAMETER_VIDEO_RATECONTROL_T encoder_param_rate_control = {0};
+  encoder_param_rate_control.hdr.id = MMAL_PARAMETER_RATECONTROL;
+  encoder_param_rate_control.hdr.size = sizeof(encoder_param_rate_control);
+  encoder_param_rate_control.control = MMAL_VIDEO_RATECONTROL_VARIABLE;
+  CHK(mmal_port_parameter_set(encoder->port_out, &encoder_param_rate_control.hdr), "Failed to set rate control param");
+
+  // Some decoders expect SPS/PPS headers to be added to every frame
+  CHK(mmal_port_parameter_set_boolean(encoder->port_out, MMAL_PARAMETER_VIDEO_ENCODE_INLINE_HEADER, MMAL_TRUE),
+      "Failed to set inline header param");
+
+  CHK(mmal_port_parameter_set_boolean(encoder->port_out, MMAL_PARAMETER_VIDEO_ENCODE_HEADERS_WITH_FRAME, MMAL_TRUE),
+      "Failed to set headers with frame param");
+
+  /* FIXME: Somehow this flag is broken? When this flag is on, the encoder will get stuck.
+  // Since our use case is mainly for real time streaming, the encoder should optimized for low latency
+  CHK(mmal_port_parameter_set_boolean(encoder->port_out, MMAL_PARAMETER_VIDEO_ENCODE_H264_LOW_LATENCY, MMAL_TRUE),
+      "Failed to set low latency param");
+  */
+
+  // Now we know the format of both ports and the requirements of the encoder, we can create
+  // our buffer headers and their associated memory buffers. We use the buffer pool API for this.
+  encoder->port_in->buffer_num = encoder->port_in->buffer_num_min;
+  // mmal calculates recommended size that's big enough to store all of the pixels
+  encoder->port_in->buffer_size = encoder->port_in->buffer_size_recommended;
+  encoder->pool_in = mmal_pool_create(encoder->port_in->buffer_num, encoder->port_in->buffer_size);
+  encoder->port_out->buffer_num = encoder->port_out->buffer_num_min;
+  encoder->port_out->buffer_size = encoder->port_out->buffer_size_recommended;
+  encoder->pool_out = mmal_pool_create(encoder->port_out->buffer_num, encoder->port_out->buffer_size);
+
+  // Create a queue to store our encoded video frames. The callback we will get when
+  // a frame has been encoded will put the frame into this queue.
+  encoder->queue_out = mmal_queue_create();
+  encoder->port_out->userdata = (void *)encoder->queue_out;
+
+  // Enable all the input port and the output port.
+  // The callback specified here is the function which will be called when the buffer header
+  // we sent to the component has been processed.
+  CHK(mmal_port_enable(encoder->port_in, encoder_in_cb), "Failed to enable input port");
+  CHK(mmal_port_enable(encoder->port_out, encoder_out_cb), "Failed to enable output port");
+
+  // Enable the component. Components will only process data when they are enabled.
+  CHK(mmal_component_enable(encoder->component), "Failed to enable component");
+
+CleanUp:
+
+  if (status.code != MMAL_SUCCESS) {
+    if (created) {
+      enc_close(encoder);
+    }
+  }
+
+  return status;
+}
+
+// enc_encode encodes y, cb, cr. The encoded frame is going to be stored in encoded_buffer.
+// IMPORTANT: the caller is responsible to release the ownership of encoded_buffer
+Status enc_encode(Encoder *encoder, Slice y, Slice cb, Slice cr, MMAL_BUFFER_HEADER_T **encoded_buffer) {
+  Status status = {0};
+  MMAL_BUFFER_HEADER_T *buffer;
+  uint32_t required_size;
+
+  // buffer should always be available since the encoding process is blocking
+  buffer = mmal_queue_get(encoder->pool_in->queue);
+  assert(buffer != NULL);
+  // buffer->data should've been allocated with enough memory to contain a frame by pool_in
+  required_size = y.len + cb.len + cr.len;
+  assert(buffer->alloc_size >= required_size);
+  memcpy(buffer->data, y.data, y.len);
+  memcpy(buffer->data + y.len, cb.data, cb.len);
+  memcpy(buffer->data + y.len + cb.len, cr.data, cr.len);
+  buffer->length = required_size;
+  CHK(mmal_port_send_buffer(encoder->port_in, buffer), "Failed to send filled buffer to input port");
+
+  while (1) {
+    // Send empty buffers to the output port to allow the encoder to start
+    // producing frames as soon as it gets input data
+    while ((buffer = mmal_queue_get(encoder->pool_out->queue)) != NULL) {
+      CHK(mmal_port_send_buffer(encoder->port_out, buffer), "Failed to send empty buffers to output port");
+    }
+
+    while ((buffer = mmal_queue_wait(encoder->queue_out)) != NULL) {
+      if ((buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END) != 0) {
+        *encoded_buffer = buffer;
+        goto CleanUp;
+      }
+
+      mmal_buffer_header_release(buffer);
+    }
+  }
+
+CleanUp:
+
+  return status;
+}
+
+Status enc_close(Encoder *encoder) {
+  Status status = {0};
+
+  mmal_pool_destroy(encoder->pool_out);
+  mmal_pool_destroy(encoder->pool_in);
+  mmal_queue_destroy(encoder->queue_out);
+  mmal_component_destroy(encoder->component);
+
+CleanUp:
+
+  return status;
+}

pkg/codec/mmal/mmal.go

@@ -0,0 +1,113 @@
+// Package mmal implements a hardware accelerated H264 encoder for raspberry pi.
+// This package requires libmmal headers and libraries to be built.
+// Reference: https://github.com/raspberrypi/userland/tree/master/interface/mmal
+package mmal
+
+// #cgo CFLAGS: -I/opt/vc/include
+// #cgo LDFLAGS: -L/opt/vc/lib -lmmal -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host -lvcsm -lvcos
+// #include "bridge.h"
+import "C"
+import (
+	"fmt"
+	"image"
+	"io"
+	"sync"
+	"unsafe"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+type encoder struct {
+	engine C.Encoder
+	r      video.Reader
+	mu     sync.Mutex
+	closed bool
+	cntr   int
+}
+
+func statusToErr(status *C.Status) error {
+	return fmt.Errorf("(status = %d) %s", int(status.code), C.GoString(status.msg))
+}
+
+func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser, error) {
+	if params.KeyFrameInterval == 0 {
+		params.KeyFrameInterval = 60
+	}
+
+	if params.BitRate == 0 {
+		params.BitRate = 300000
+	}
+
+	e := encoder{
+		r: video.ToI420(r),
+	}
+	status := C.enc_new(C.Params{
+		width:              C.int(p.Width),
+		height:             C.int(p.Height),
+		bitrate:            C.uint(params.BitRate),
+		key_frame_interval: C.uint(params.KeyFrameInterval),
+	}, &e.engine)
+	if status.code != 0 {
+		return nil, statusToErr(&status)
+	}
+
+	return &e, nil
+}
+
+func (e *encoder) Read() ([]byte, func(), error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	if e.closed {
+		return nil, func() {}, io.EOF
+	}
+
+	img, _, err := e.r.Read()
+	if err != nil {
+		return nil, func() {}, err
+	}
+	imgReal := img.(*image.YCbCr)
+	var y, cb, cr C.Slice
+	y.data = (*C.uchar)(&imgReal.Y[0])
+	y.len = C.int(len(imgReal.Y))
+	cb.data = (*C.uchar)(&imgReal.Cb[0])
+	cb.len = C.int(len(imgReal.Cb))
+	cr.data = (*C.uchar)(&imgReal.Cr[0])
+	cr.len = C.int(len(imgReal.Cr))
+
+	var encodedBuffer *C.MMAL_BUFFER_HEADER_T
+	status := C.enc_encode(&e.engine, y, cb, cr, &encodedBuffer)
+	if status.code != 0 {
+		return nil, func() {}, statusToErr(&status)
+	}
+
+	// GoBytes copies the C array to Go slice. After this, it's safe to release the C array
+	encoded := C.GoBytes(unsafe.Pointer(encodedBuffer.data), C.int(encodedBuffer.length))
+	// Release the buffer so that mmal can reuse this memory
+	C.mmal_buffer_header_release(encodedBuffer)
+
+	return encoded, func() {}, err
+}
+
+func (e *encoder) SetBitRate(b int) error {
+	panic("SetBitRate is not implemented")
+}
+
+func (e *encoder) ForceKeyFrame() error {
+	panic("ForceKeyFrame is not implemented")
+}
+
+func (e *encoder) Close() error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	if e.closed {
+		return nil
+	}
+
+	e.closed = true
+	C.enc_close(&e.engine)
+	return nil
+}

pkg/codec/mmal/params.go

@@ -0,0 +1,31 @@
+package mmal
+
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+// Params stores libmmal specific encoding parameters.
+type Params struct {
+	codec.BaseParams
+}
+
+// NewParams returns default mmal codec specific parameters.
+func NewParams() (Params, error) {
+	return Params{
+		BaseParams: codec.BaseParams{
+			KeyFrameInterval: 60,
+		},
+	}, nil
+}
+
+// RTPCodec represents the codec metadata
+func (p *Params) RTPCodec() *codec.RTPCodec {
+	return codec.NewRTPH264Codec(90000)
+}
+
+// BuildVideoEncoder builds mmal encoder with given params
+func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (codec.ReadCloser, error) {
+	return newEncoder(r, property, *p)
+}

pkg/codec/openh264/.gitignore

@@ -0,0 +1 @@
+src

pkg/codec/openh264/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2013, Cisco Systems
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/codec/openh264/Makefile

@@ -0,0 +1,47 @@
+git_url := https://github.com/cisco/openh264.git
+version := v2.1.1
+src_root_dir := src
+lib_dir := lib
+include_dir := include/openh264
+lib_prefix := libopenh264
+src_dir := $(src_root_dir)/$(MEDIADEVICES_TARGET_PLATFORM)
+output_path := $(lib_dir)/$(lib_prefix)-$(MEDIADEVICES_TARGET_PLATFORM).a
+
+# OS and Arch mapping to OpenH264 parameters
+ifeq (windows,$(MEDIADEVICES_TARGET_OS))
+	os := mingw_nt
+else
+	os := $(MEDIADEVICES_TARGET_OS)
+endif
+
+ifneq (,$(findstring $(MEDIADEVICES_TARGET_ARCH),armv6 armv7 armv8))
+	arch := arm
+else ifeq (x64,$(MEDIADEVICES_TARGET_ARCH))
+	arch := x86_64
+else
+	arch := $(MEDIADEVICES_TARGET_ARCH)
+endif
+
+.PHONY: all
+all: guard-MEDIADEVICES_TARGET_PLATFORM guard-MEDIADEVICES_TARGET_PLATFORM \
+	guard-MEDIADEVICES_TARGET_OS guard-MEDIADEVICES_TARGET_ARCH \
+	$(output_path) headers
+
+headers: | $(src_dir) $(include_dir)
+	@cp $(src_dir)/codec/api/svc/*.h $(include_dir)
+
+$(output_path): $(src_dir)/$(lib_prefix).a | $(lib_dir)
+	@cp $< $@
+
+$(src_dir)/$(lib_prefix).a: | $(src_dir)
+	$(MEDIADEVICES_TOOLCHAIN_BIN) make --directory=$(src_dir) $(lib_prefix).a \
+		OS=$(os) ARCH=$(arch)
+
+$(src_dir): | $(src_root_dir)
+	git clone --depth=1 --branch=$(version) $(git_url) $@
+
+$(src_root_dir) $(lib_dir) $(include_dir):
+	@mkdir -p $@
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

pkg/codec/openh264/include/openh264/codec_app_def.h

@@ -167,8 +167,8 @@ typedef enum {
   DECODER_OPTION_LEVEL,                 ///< get current AU level info,only is used in GetOption
   DECODER_OPTION_STATISTICS_LOG_INTERVAL,///< set log output interval
   DECODER_OPTION_IS_REF_PIC,             ///< feedback current frame is ref pic or not
-  DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER  ///< number of frames remaining in decoder buffer when pictures are required to re-ordered into display-order.
-
+  DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER,  ///< number of frames remaining in decoder buffer when pictures are required to re-ordered into display-order.
+  DECODER_OPTION_NUM_OF_THREADS,         ///< number of decoding threads. The maximum thread count is equal or less than lesser of (cpu core counts and 16).
 } DECODER_OPTION;
 
 /**

pkg/codec/openh264/include/openh264/codec_def.h

@@ -201,6 +201,7 @@ typedef struct TagBufferInfo {
   union {
     SSysMEMBuffer sSystemBuffer; ///<  memory info for one picture
   } UsrData;                     ///<  output buffer info
+  unsigned char* pDst[3];  //point to picture YUV data
 } SBufferInfo;
 
 

pkg/codec/openh264/include/openh264/codec_ver.h

@@ -0,0 +1,15 @@
+//The current file is auto-generated by script: generate_codec_ver.sh
+#ifndef CODEC_VER_H
+#define CODEC_VER_H
+
+#include "codec_app_def.h"
+
+static const OpenH264Version g_stCodecVersion  = {2, 1, 1, 2005};
+static const char* const g_strCodecVer  = "OpenH264 version:2.1.1.2005";
+
+#define OPENH264_MAJOR (2)
+#define OPENH264_MINOR (1)
+#define OPENH264_REVISION (1)
+#define OPENH264_RESERVED (2005)
+
+#endif  // CODEC_VER_H

pkg/codec/openh264/openh264.go

@@ -1,7 +1,5 @@
 package openh264
 
-// #cgo CFLAGS: -I${SRCDIR}/../../../cvendor/include
-// #cgo CXXFLAGS: -I${SRCDIR}/../../../cvendor/include
 // #include <string.h>
 // #include <openh264/codec_api.h>
 // #include <errno.h>
@@ -16,7 +14,6 @@ import (
 	"unsafe"
 
 	"github.com/pion/mediadevices/pkg/codec"
-	mio "github.com/pion/mediadevices/pkg/io"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
@@ -24,7 +21,6 @@ import (
 type encoder struct {
 	engine *C.Encoder
 	r      video.Reader
-	buff   []byte
 
 	mu     sync.Mutex
 	closed bool
@@ -52,26 +48,17 @@ func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser,
 	}, nil
 }
 
-func (e *encoder) Read(p []byte) (n int, err error) {
+func (e *encoder) Read() ([]byte, func(), error) {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
 	if e.closed {
-		return 0, io.EOF
+		return nil, func() {}, io.EOF
 	}
 
-	if e.buff != nil {
-		n, err = mio.Copy(p, e.buff)
-		if err == nil {
-			e.buff = nil
-		}
-
-		return n, err
-	}
-
-	img, err := e.r.Read()
+	img, _, err := e.r.Read()
 	if err != nil {
-		return 0, err
+		return nil, func() {}, err
 	}
 
 	yuvImg := img.(*image.YCbCr)
@@ -85,16 +72,11 @@ func (e *encoder) Read(p []byte) (n int, err error) {
 		width:  C.int(bounds.Max.X - bounds.Min.X),
 	}, &rv)
 	if err := errResult(rv); err != nil {
-		return 0, fmt.Errorf("failed in encoding: %v", err)
+		return nil, func() {}, fmt.Errorf("failed in encoding: %v", err)
 	}
 
 	encoded := C.GoBytes(unsafe.Pointer(s.data), s.data_len)
-	n, err = mio.Copy(p, encoded)
-	if err != nil {
-		e.buff = encoded
-	}
-
-	return n, err
+	return encoded, func() {}, nil
 }
 
 func (e *encoder) SetBitRate(b int) error {

pkg/codec/openh264/openh264_darwin.go

@@ -1,6 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-darwin.a
-import "C"

pkg/codec/openh264/openh264_linux.go

@@ -1,6 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-linux.a
-import "C"

pkg/codec/openh264/openh264_static.go

@@ -0,0 +1,12 @@
+// +build !dynamic
+
+package openh264
+
+//#cgo CFLAGS: -I${SRCDIR}/include
+//#cgo CXXFLAGS: -I${SRCDIR}/include
+//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-armv7.a
+//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-arm64.a
+//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-x64.a
+//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-x64.a
+//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-windows-x64.a -lssp
+import "C"

pkg/codec/openh264/openh264_windows.go

@@ -1,14 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-windows.a -lssp
-// #include <stdio.h>
-// #include <crtdefs.h>
-// #if __MINGW64_VERSION_MAJOR < 6
-// // Workaround for the breaking ABI change between MinGW 5 and 6
-//   FILE *__cdecl __acrt_iob_func(unsigned index) { return &(__iob_func()[index]); }
-//   typedef FILE *__cdecl (*_f__acrt_iob_func)(unsigned index);
-//   _f__acrt_iob_func __MINGW_IMP_SYMBOL(__acrt_iob_func) = __acrt_iob_func;
-// #endif
-import "C"

pkg/codec/openh264/params.go

@@ -4,7 +4,6 @@ import (
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
 )
 
 // Params stores libopenh264 specific encoding parameters.
@@ -21,9 +20,9 @@ func NewParams() (Params, error) {
 	}, nil
 }
 
-// Name represents the codec name
-func (p *Params) Name() string {
-	return webrtc.H264
+// RTPCodec represents the codec metadata
+func (p *Params) RTPCodec() *codec.RTPCodec {
+	return codec.NewRTPH264Codec(90000)
 }
 
 // BuildVideoEncoder builds openh264 encoder with given params

pkg/codec/opus/.gitignore

@@ -0,0 +1 @@
+src

pkg/codec/opus/LICENSE

@@ -0,0 +1,44 @@
+Copyright 2001-2011 Xiph.Org, Skype Limited, Octasic,
+                    Jean-Marc Valin, Timothy B. Terriberry,
+                    CSIRO, Gregory Maxwell, Mark Borgerding,
+                    Erik de Castro Lopo
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+- Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+- Neither the name of Internet Society, IETF or IETF Trust, nor the
+names of specific contributors, may be used to endorse or promote
+products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Opus is subject to the royalty-free patent licenses which are
+specified at:
+
+Xiph.Org Foundation:
+https://datatracker.ietf.org/ipr/1524/
+
+Microsoft Corporation:
+https://datatracker.ietf.org/ipr/1914/
+
+Broadcom Corporation:
+https://datatracker.ietf.org/ipr/1526/

pkg/codec/opus/Makefile

@@ -0,0 +1,31 @@
+git_url := https://github.com/xiph/opus.git
+version := v1.3.1
+src_root_dir := src
+lib_dir := lib
+include_dir := include
+lib_prefix := libopus
+src_dir := $(src_root_dir)/$(MEDIADEVICES_TARGET_PLATFORM)
+output_path := $(lib_dir)/$(lib_prefix)-$(MEDIADEVICES_TARGET_PLATFORM).a
+
+.PHONY: all
+all: guard-MEDIADEVICES_TARGET_PLATFORM guard-MEDIADEVICES_TOOLCHAIN_BIN $(output_path) headers
+
+headers: | $(src_dir) $(include_dir)
+	@cp $(src_dir)/include/*.h $(include_dir)
+
+$(output_path): $(src_dir)/$(lib_prefix).a | $(lib_dir)
+	@cp $< $@
+
+$(src_dir)/$(lib_prefix).a: | $(src_dir)
+	cd $(src_dir) && \
+		$(MEDIADEVICES_TOOLCHAIN_BIN) cmake -DOPUS_STACK_PROTECTOR=OFF -DCMAKE_C_FLAGS="-fpic" && \
+		$(MEDIADEVICES_TOOLCHAIN_BIN) make VERBOSE=1
+
+$(src_dir): | $(src_root_dir)
+	git clone --depth=1 --branch=$(version) $(git_url) $@
+
+$(src_root_dir) $(lib_dir) $(include_dir):
+	@mkdir -p $@
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

pkg/codec/opus/include/opus.h

@@ -0,0 +1,981 @@
+/* Copyright (c) 2010-2011 Xiph.Org Foundation, Skype Limited
+   Written by Jean-Marc Valin and Koen Vos */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/**
+ * @file opus.h
+ * @brief Opus reference implementation API
+ */
+
+#ifndef OPUS_H
+#define OPUS_H
+
+#include "opus_types.h"
+#include "opus_defines.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @mainpage Opus
+ *
+ * The Opus codec is designed for interactive speech and audio transmission over the Internet.
+ * It is designed by the IETF Codec Working Group and incorporates technology from
+ * Skype's SILK codec and Xiph.Org's CELT codec.
+ *
+ * The Opus codec is designed to handle a wide range of interactive audio applications,
+ * including Voice over IP, videoconferencing, in-game chat, and even remote live music
+ * performances. It can scale from low bit-rate narrowband speech to very high quality
+ * stereo music. Its main features are:
+
+ * @li Sampling rates from 8 to 48 kHz
+ * @li Bit-rates from 6 kb/s to 510 kb/s
+ * @li Support for both constant bit-rate (CBR) and variable bit-rate (VBR)
+ * @li Audio bandwidth from narrowband to full-band
+ * @li Support for speech and music
+ * @li Support for mono and stereo
+ * @li Support for multichannel (up to 255 channels)
+ * @li Frame sizes from 2.5 ms to 60 ms
+ * @li Good loss robustness and packet loss concealment (PLC)
+ * @li Floating point and fixed-point implementation
+ *
+ * Documentation sections:
+ * @li @ref opus_encoder
+ * @li @ref opus_decoder
+ * @li @ref opus_repacketizer
+ * @li @ref opus_multistream
+ * @li @ref opus_libinfo
+ * @li @ref opus_custom
+ */
+
+/** @defgroup opus_encoder Opus Encoder
+  * @{
+  *
+  * @brief This page describes the process and functions used to encode Opus.
+  *
+  * Since Opus is a stateful codec, the encoding process starts with creating an encoder
+  * state. This can be done with:
+  *
+  * @code
+  * int          error;
+  * OpusEncoder *enc;
+  * enc = opus_encoder_create(Fs, channels, application, &error);
+  * @endcode
+  *
+  * From this point, @c enc can be used for encoding an audio stream. An encoder state
+  * @b must @b not be used for more than one stream at the same time. Similarly, the encoder
+  * state @b must @b not be re-initialized for each frame.
+  *
+  * While opus_encoder_create() allocates memory for the state, it's also possible
+  * to initialize pre-allocated memory:
+  *
+  * @code
+  * int          size;
+  * int          error;
+  * OpusEncoder *enc;
+  * size = opus_encoder_get_size(channels);
+  * enc = malloc(size);
+  * error = opus_encoder_init(enc, Fs, channels, application);
+  * @endcode
+  *
+  * where opus_encoder_get_size() returns the required size for the encoder state. Note that
+  * future versions of this code may change the size, so no assuptions should be made about it.
+  *
+  * The encoder state is always continuous in memory and only a shallow copy is sufficient
+  * to copy it (e.g. memcpy())
+  *
+  * It is possible to change some of the encoder's settings using the opus_encoder_ctl()
+  * interface. All these settings already default to the recommended value, so they should
+  * only be changed when necessary. The most common settings one may want to change are:
+  *
+  * @code
+  * opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate));
+  * opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(complexity));
+  * opus_encoder_ctl(enc, OPUS_SET_SIGNAL(signal_type));
+  * @endcode
+  *
+  * where
+  *
+  * @arg bitrate is in bits per second (b/s)
+  * @arg complexity is a value from 1 to 10, where 1 is the lowest complexity and 10 is the highest
+  * @arg signal_type is either OPUS_AUTO (default), OPUS_SIGNAL_VOICE, or OPUS_SIGNAL_MUSIC
+  *
+  * See @ref opus_encoderctls and @ref opus_genericctls for a complete list of parameters that can be set or queried. Most parameters can be set or changed at any time during a stream.
+  *
+  * To encode a frame, opus_encode() or opus_encode_float() must be called with exactly one frame (2.5, 5, 10, 20, 40 or 60 ms) of audio data:
+  * @code
+  * len = opus_encode(enc, audio_frame, frame_size, packet, max_packet);
+  * @endcode
+  *
+  * where
+  * <ul>
+  * <li>audio_frame is the audio data in opus_int16 (or float for opus_encode_float())</li>
+  * <li>frame_size is the duration of the frame in samples (per channel)</li>
+  * <li>packet is the byte array to which the compressed data is written</li>
+  * <li>max_packet is the maximum number of bytes that can be written in the packet (4000 bytes is recommended).
+  *     Do not use max_packet to control VBR target bitrate, instead use the #OPUS_SET_BITRATE CTL.</li>
+  * </ul>
+  *
+  * opus_encode() and opus_encode_float() return the number of bytes actually written to the packet.
+  * The return value <b>can be negative</b>, which indicates that an error has occurred. If the return value
+  * is 2 bytes or less, then the packet does not need to be transmitted (DTX).
+  *
+  * Once the encoder state if no longer needed, it can be destroyed with
+  *
+  * @code
+  * opus_encoder_destroy(enc);
+  * @endcode
+  *
+  * If the encoder was created with opus_encoder_init() rather than opus_encoder_create(),
+  * then no action is required aside from potentially freeing the memory that was manually
+  * allocated for it (calling free(enc) for the example above)
+  *
+  */
+
+/** Opus encoder state.
+  * This contains the complete state of an Opus encoder.
+  * It is position independent and can be freely copied.
+  * @see opus_encoder_create,opus_encoder_init
+  */
+typedef struct OpusEncoder OpusEncoder;
+
+/** Gets the size of an <code>OpusEncoder</code> structure.
+  * @param[in] channels <tt>int</tt>: Number of channels.
+  *                                   This must be 1 or 2.
+  * @returns The size in bytes.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_encoder_get_size(int channels);
+
+/**
+ */
+
+/** Allocates and initializes an encoder state.
+ * There are three coding modes:
+ *
+ * @ref OPUS_APPLICATION_VOIP gives best quality at a given bitrate for voice
+ *    signals. It enhances the  input signal by high-pass filtering and
+ *    emphasizing formants and harmonics. Optionally  it includes in-band
+ *    forward error correction to protect against packet loss. Use this
+ *    mode for typical VoIP applications. Because of the enhancement,
+ *    even at high bitrates the output may sound different from the input.
+ *
+ * @ref OPUS_APPLICATION_AUDIO gives best quality at a given bitrate for most
+ *    non-voice signals like music. Use this mode for music and mixed
+ *    (music/voice) content, broadcast, and applications requiring less
+ *    than 15 ms of coding delay.
+ *
+ * @ref OPUS_APPLICATION_RESTRICTED_LOWDELAY configures low-delay mode that
+ *    disables the speech-optimized mode in exchange for slightly reduced delay.
+ *    This mode can only be set on an newly initialized or freshly reset encoder
+ *    because it changes the codec delay.
+ *
+ * This is useful when the caller knows that the speech-optimized modes will not be needed (use with caution).
+ * @param [in] Fs <tt>opus_int32</tt>: Sampling rate of input signal (Hz)
+ *                                     This must be one of 8000, 12000, 16000,
+ *                                     24000, or 48000.
+ * @param [in] channels <tt>int</tt>: Number of channels (1 or 2) in input signal
+ * @param [in] application <tt>int</tt>: Coding mode (@ref OPUS_APPLICATION_VOIP/@ref OPUS_APPLICATION_AUDIO/@ref OPUS_APPLICATION_RESTRICTED_LOWDELAY)
+ * @param [out] error <tt>int*</tt>: @ref opus_errorcodes
+ * @note Regardless of the sampling rate and number channels selected, the Opus encoder
+ * can switch to a lower audio bandwidth or number of channels if the bitrate
+ * selected is too low. This also means that it is safe to always use 48 kHz stereo input
+ * and let the encoder optimize the encoding.
+ */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusEncoder *opus_encoder_create(
+    opus_int32 Fs,
+    int channels,
+    int application,
+    int *error
+);
+
+/** Initializes a previously allocated encoder state
+  * The memory pointed to by st must be at least the size returned by opus_encoder_get_size().
+  * This is intended for applications which use their own allocator instead of malloc.
+  * @see opus_encoder_create(),opus_encoder_get_size()
+  * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL.
+  * @param [in] st <tt>OpusEncoder*</tt>: Encoder state
+  * @param [in] Fs <tt>opus_int32</tt>: Sampling rate of input signal (Hz)
+ *                                      This must be one of 8000, 12000, 16000,
+ *                                      24000, or 48000.
+  * @param [in] channels <tt>int</tt>: Number of channels (1 or 2) in input signal
+  * @param [in] application <tt>int</tt>: Coding mode (OPUS_APPLICATION_VOIP/OPUS_APPLICATION_AUDIO/OPUS_APPLICATION_RESTRICTED_LOWDELAY)
+  * @retval #OPUS_OK Success or @ref opus_errorcodes
+  */
+OPUS_EXPORT int opus_encoder_init(
+    OpusEncoder *st,
+    opus_int32 Fs,
+    int channels,
+    int application
+) OPUS_ARG_NONNULL(1);
+
+/** Encodes an Opus frame.
+  * @param [in] st <tt>OpusEncoder*</tt>: Encoder state
+  * @param [in] pcm <tt>opus_int16*</tt>: Input signal (interleaved if 2 channels). length is frame_size*channels*sizeof(opus_int16)
+  * @param [in] frame_size <tt>int</tt>: Number of samples per channel in the
+  *                                      input signal.
+  *                                      This must be an Opus frame size for
+  *                                      the encoder's sampling rate.
+  *                                      For example, at 48 kHz the permitted
+  *                                      values are 120, 240, 480, 960, 1920,
+  *                                      and 2880.
+  *                                      Passing in a duration of less than
+  *                                      10 ms (480 samples at 48 kHz) will
+  *                                      prevent the encoder from using the LPC
+  *                                      or hybrid modes.
+  * @param [out] data <tt>unsigned char*</tt>: Output payload.
+  *                                            This must contain storage for at
+  *                                            least \a max_data_bytes.
+  * @param [in] max_data_bytes <tt>opus_int32</tt>: Size of the allocated
+  *                                                 memory for the output
+  *                                                 payload. This may be
+  *                                                 used to impose an upper limit on
+  *                                                 the instant bitrate, but should
+  *                                                 not be used as the only bitrate
+  *                                                 control. Use #OPUS_SET_BITRATE to
+  *                                                 control the bitrate.
+  * @returns The length of the encoded packet (in bytes) on success or a
+  *          negative error code (see @ref opus_errorcodes) on failure.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_encode(
+    OpusEncoder *st,
+    const opus_int16 *pcm,
+    int frame_size,
+    unsigned char *data,
+    opus_int32 max_data_bytes
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4);
+
+/** Encodes an Opus frame from floating point input.
+  * @param [in] st <tt>OpusEncoder*</tt>: Encoder state
+  * @param [in] pcm <tt>float*</tt>: Input in float format (interleaved if 2 channels), with a normal range of +/-1.0.
+  *          Samples with a range beyond +/-1.0 are supported but will
+  *          be clipped by decoders using the integer API and should
+  *          only be used if it is known that the far end supports
+  *          extended dynamic range.
+  *          length is frame_size*channels*sizeof(float)
+  * @param [in] frame_size <tt>int</tt>: Number of samples per channel in the
+  *                                      input signal.
+  *                                      This must be an Opus frame size for
+  *                                      the encoder's sampling rate.
+  *                                      For example, at 48 kHz the permitted
+  *                                      values are 120, 240, 480, 960, 1920,
+  *                                      and 2880.
+  *                                      Passing in a duration of less than
+  *                                      10 ms (480 samples at 48 kHz) will
+  *                                      prevent the encoder from using the LPC
+  *                                      or hybrid modes.
+  * @param [out] data <tt>unsigned char*</tt>: Output payload.
+  *                                            This must contain storage for at
+  *                                            least \a max_data_bytes.
+  * @param [in] max_data_bytes <tt>opus_int32</tt>: Size of the allocated
+  *                                                 memory for the output
+  *                                                 payload. This may be
+  *                                                 used to impose an upper limit on
+  *                                                 the instant bitrate, but should
+  *                                                 not be used as the only bitrate
+  *                                                 control. Use #OPUS_SET_BITRATE to
+  *                                                 control the bitrate.
+  * @returns The length of the encoded packet (in bytes) on success or a
+  *          negative error code (see @ref opus_errorcodes) on failure.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_encode_float(
+    OpusEncoder *st,
+    const float *pcm,
+    int frame_size,
+    unsigned char *data,
+    opus_int32 max_data_bytes
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4);
+
+/** Frees an <code>OpusEncoder</code> allocated by opus_encoder_create().
+  * @param[in] st <tt>OpusEncoder*</tt>: State to be freed.
+  */
+OPUS_EXPORT void opus_encoder_destroy(OpusEncoder *st);
+
+/** Perform a CTL function on an Opus encoder.
+  *
+  * Generally the request and subsequent arguments are generated
+  * by a convenience macro.
+  * @param st <tt>OpusEncoder*</tt>: Encoder state.
+  * @param request This and all remaining parameters should be replaced by one
+  *                of the convenience macros in @ref opus_genericctls or
+  *                @ref opus_encoderctls.
+  * @see opus_genericctls
+  * @see opus_encoderctls
+  */
+OPUS_EXPORT int opus_encoder_ctl(OpusEncoder *st, int request, ...) OPUS_ARG_NONNULL(1);
+/**@}*/
+
+/** @defgroup opus_decoder Opus Decoder
+  * @{
+  *
+  * @brief This page describes the process and functions used to decode Opus.
+  *
+  * The decoding process also starts with creating a decoder
+  * state. This can be done with:
+  * @code
+  * int          error;
+  * OpusDecoder *dec;
+  * dec = opus_decoder_create(Fs, channels, &error);
+  * @endcode
+  * where
+  * @li Fs is the sampling rate and must be 8000, 12000, 16000, 24000, or 48000
+  * @li channels is the number of channels (1 or 2)
+  * @li error will hold the error code in case of failure (or #OPUS_OK on success)
+  * @li the return value is a newly created decoder state to be used for decoding
+  *
+  * While opus_decoder_create() allocates memory for the state, it's also possible
+  * to initialize pre-allocated memory:
+  * @code
+  * int          size;
+  * int          error;
+  * OpusDecoder *dec;
+  * size = opus_decoder_get_size(channels);
+  * dec = malloc(size);
+  * error = opus_decoder_init(dec, Fs, channels);
+  * @endcode
+  * where opus_decoder_get_size() returns the required size for the decoder state. Note that
+  * future versions of this code may change the size, so no assuptions should be made about it.
+  *
+  * The decoder state is always continuous in memory and only a shallow copy is sufficient
+  * to copy it (e.g. memcpy())
+  *
+  * To decode a frame, opus_decode() or opus_decode_float() must be called with a packet of compressed audio data:
+  * @code
+  * frame_size = opus_decode(dec, packet, len, decoded, max_size, 0);
+  * @endcode
+  * where
+  *
+  * @li packet is the byte array containing the compressed data
+  * @li len is the exact number of bytes contained in the packet
+  * @li decoded is the decoded audio data in opus_int16 (or float for opus_decode_float())
+  * @li max_size is the max duration of the frame in samples (per channel) that can fit into the decoded_frame array
+  *
+  * opus_decode() and opus_decode_float() return the number of samples (per channel) decoded from the packet.
+  * If that value is negative, then an error has occurred. This can occur if the packet is corrupted or if the audio
+  * buffer is too small to hold the decoded audio.
+  *
+  * Opus is a stateful codec with overlapping blocks and as a result Opus
+  * packets are not coded independently of each other. Packets must be
+  * passed into the decoder serially and in the correct order for a correct
+  * decode. Lost packets can be replaced with loss concealment by calling
+  * the decoder with a null pointer and zero length for the missing packet.
+  *
+  * A single codec state may only be accessed from a single thread at
+  * a time and any required locking must be performed by the caller. Separate
+  * streams must be decoded with separate decoder states and can be decoded
+  * in parallel unless the library was compiled with NONTHREADSAFE_PSEUDOSTACK
+  * defined.
+  *
+  */
+
+/** Opus decoder state.
+  * This contains the complete state of an Opus decoder.
+  * It is position independent and can be freely copied.
+  * @see opus_decoder_create,opus_decoder_init
+  */
+typedef struct OpusDecoder OpusDecoder;
+
+/** Gets the size of an <code>OpusDecoder</code> structure.
+  * @param [in] channels <tt>int</tt>: Number of channels.
+  *                                    This must be 1 or 2.
+  * @returns The size in bytes.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decoder_get_size(int channels);
+
+/** Allocates and initializes a decoder state.
+  * @param [in] Fs <tt>opus_int32</tt>: Sample rate to decode at (Hz).
+  *                                     This must be one of 8000, 12000, 16000,
+  *                                     24000, or 48000.
+  * @param [in] channels <tt>int</tt>: Number of channels (1 or 2) to decode
+  * @param [out] error <tt>int*</tt>: #OPUS_OK Success or @ref opus_errorcodes
+  *
+  * Internally Opus stores data at 48000 Hz, so that should be the default
+  * value for Fs. However, the decoder can efficiently decode to buffers
+  * at 8, 12, 16, and 24 kHz so if for some reason the caller cannot use
+  * data at the full sample rate, or knows the compressed data doesn't
+  * use the full frequency range, it can request decoding at a reduced
+  * rate. Likewise, the decoder is capable of filling in either mono or
+  * interleaved stereo pcm buffers, at the caller's request.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusDecoder *opus_decoder_create(
+    opus_int32 Fs,
+    int channels,
+    int *error
+);
+
+/** Initializes a previously allocated decoder state.
+  * The state must be at least the size returned by opus_decoder_get_size().
+  * This is intended for applications which use their own allocator instead of malloc. @see opus_decoder_create,opus_decoder_get_size
+  * To reset a previously initialized state, use the #OPUS_RESET_STATE CTL.
+  * @param [in] st <tt>OpusDecoder*</tt>: Decoder state.
+  * @param [in] Fs <tt>opus_int32</tt>: Sampling rate to decode to (Hz).
+  *                                     This must be one of 8000, 12000, 16000,
+  *                                     24000, or 48000.
+  * @param [in] channels <tt>int</tt>: Number of channels (1 or 2) to decode
+  * @retval #OPUS_OK Success or @ref opus_errorcodes
+  */
+OPUS_EXPORT int opus_decoder_init(
+    OpusDecoder *st,
+    opus_int32 Fs,
+    int channels
+) OPUS_ARG_NONNULL(1);
+
+/** Decode an Opus packet.
+  * @param [in] st <tt>OpusDecoder*</tt>: Decoder state
+  * @param [in] data <tt>char*</tt>: Input payload. Use a NULL pointer to indicate packet loss
+  * @param [in] len <tt>opus_int32</tt>: Number of bytes in payload*
+  * @param [out] pcm <tt>opus_int16*</tt>: Output signal (interleaved if 2 channels). length
+  *  is frame_size*channels*sizeof(opus_int16)
+  * @param [in] frame_size Number of samples per channel of available space in \a pcm.
+  *  If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will
+  *  not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1),
+  *  then frame_size needs to be exactly the duration of audio that is missing, otherwise the
+  *  decoder will not be in the optimal state to decode the next incoming packet. For the PLC and
+  *  FEC cases, frame_size <b>must</b> be a multiple of 2.5 ms.
+  * @param [in] decode_fec <tt>int</tt>: Flag (0 or 1) to request that any in-band forward error correction data be
+  *  decoded. If no such data is available, the frame is decoded as if it were lost.
+  * @returns Number of decoded samples or @ref opus_errorcodes
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decode(
+    OpusDecoder *st,
+    const unsigned char *data,
+    opus_int32 len,
+    opus_int16 *pcm,
+    int frame_size,
+    int decode_fec
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4);
+
+/** Decode an Opus packet with floating point output.
+  * @param [in] st <tt>OpusDecoder*</tt>: Decoder state
+  * @param [in] data <tt>char*</tt>: Input payload. Use a NULL pointer to indicate packet loss
+  * @param [in] len <tt>opus_int32</tt>: Number of bytes in payload
+  * @param [out] pcm <tt>float*</tt>: Output signal (interleaved if 2 channels). length
+  *  is frame_size*channels*sizeof(float)
+  * @param [in] frame_size Number of samples per channel of available space in \a pcm.
+  *  If this is less than the maximum packet duration (120ms; 5760 for 48kHz), this function will
+  *  not be capable of decoding some packets. In the case of PLC (data==NULL) or FEC (decode_fec=1),
+  *  then frame_size needs to be exactly the duration of audio that is missing, otherwise the
+  *  decoder will not be in the optimal state to decode the next incoming packet. For the PLC and
+  *  FEC cases, frame_size <b>must</b> be a multiple of 2.5 ms.
+  * @param [in] decode_fec <tt>int</tt>: Flag (0 or 1) to request that any in-band forward error correction data be
+  *  decoded. If no such data is available the frame is decoded as if it were lost.
+  * @returns Number of decoded samples or @ref opus_errorcodes
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decode_float(
+    OpusDecoder *st,
+    const unsigned char *data,
+    opus_int32 len,
+    float *pcm,
+    int frame_size,
+    int decode_fec
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4);
+
+/** Perform a CTL function on an Opus decoder.
+  *
+  * Generally the request and subsequent arguments are generated
+  * by a convenience macro.
+  * @param st <tt>OpusDecoder*</tt>: Decoder state.
+  * @param request This and all remaining parameters should be replaced by one
+  *                of the convenience macros in @ref opus_genericctls or
+  *                @ref opus_decoderctls.
+  * @see opus_genericctls
+  * @see opus_decoderctls
+  */
+OPUS_EXPORT int opus_decoder_ctl(OpusDecoder *st, int request, ...) OPUS_ARG_NONNULL(1);
+
+/** Frees an <code>OpusDecoder</code> allocated by opus_decoder_create().
+  * @param[in] st <tt>OpusDecoder*</tt>: State to be freed.
+  */
+OPUS_EXPORT void opus_decoder_destroy(OpusDecoder *st);
+
+/** Parse an opus packet into one or more frames.
+  * Opus_decode will perform this operation internally so most applications do
+  * not need to use this function.
+  * This function does not copy the frames, the returned pointers are pointers into
+  * the input packet.
+  * @param [in] data <tt>char*</tt>: Opus packet to be parsed
+  * @param [in] len <tt>opus_int32</tt>: size of data
+  * @param [out] out_toc <tt>char*</tt>: TOC pointer
+  * @param [out] frames <tt>char*[48]</tt> encapsulated frames
+  * @param [out] size <tt>opus_int16[48]</tt> sizes of the encapsulated frames
+  * @param [out] payload_offset <tt>int*</tt>: returns the position of the payload within the packet (in bytes)
+  * @returns number of frames
+  */
+OPUS_EXPORT int opus_packet_parse(
+   const unsigned char *data,
+   opus_int32 len,
+   unsigned char *out_toc,
+   const unsigned char *frames[48],
+   opus_int16 size[48],
+   int *payload_offset
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(5);
+
+/** Gets the bandwidth of an Opus packet.
+  * @param [in] data <tt>char*</tt>: Opus packet
+  * @retval OPUS_BANDWIDTH_NARROWBAND Narrowband (4kHz bandpass)
+  * @retval OPUS_BANDWIDTH_MEDIUMBAND Mediumband (6kHz bandpass)
+  * @retval OPUS_BANDWIDTH_WIDEBAND Wideband (8kHz bandpass)
+  * @retval OPUS_BANDWIDTH_SUPERWIDEBAND Superwideband (12kHz bandpass)
+  * @retval OPUS_BANDWIDTH_FULLBAND Fullband (20kHz bandpass)
+  * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_bandwidth(const unsigned char *data) OPUS_ARG_NONNULL(1);
+
+/** Gets the number of samples per frame from an Opus packet.
+  * @param [in] data <tt>char*</tt>: Opus packet.
+  *                                  This must contain at least one byte of
+  *                                  data.
+  * @param [in] Fs <tt>opus_int32</tt>: Sampling rate in Hz.
+  *                                     This must be a multiple of 400, or
+  *                                     inaccurate results will be returned.
+  * @returns Number of samples per frame.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_samples_per_frame(const unsigned char *data, opus_int32 Fs) OPUS_ARG_NONNULL(1);
+
+/** Gets the number of channels from an Opus packet.
+  * @param [in] data <tt>char*</tt>: Opus packet
+  * @returns Number of channels
+  * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_channels(const unsigned char *data) OPUS_ARG_NONNULL(1);
+
+/** Gets the number of frames in an Opus packet.
+  * @param [in] packet <tt>char*</tt>: Opus packet
+  * @param [in] len <tt>opus_int32</tt>: Length of packet
+  * @returns Number of frames
+  * @retval OPUS_BAD_ARG Insufficient data was passed to the function
+  * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_frames(const unsigned char packet[], opus_int32 len) OPUS_ARG_NONNULL(1);
+
+/** Gets the number of samples of an Opus packet.
+  * @param [in] packet <tt>char*</tt>: Opus packet
+  * @param [in] len <tt>opus_int32</tt>: Length of packet
+  * @param [in] Fs <tt>opus_int32</tt>: Sampling rate in Hz.
+  *                                     This must be a multiple of 400, or
+  *                                     inaccurate results will be returned.
+  * @returns Number of samples
+  * @retval OPUS_BAD_ARG Insufficient data was passed to the function
+  * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_packet_get_nb_samples(const unsigned char packet[], opus_int32 len, opus_int32 Fs) OPUS_ARG_NONNULL(1);
+
+/** Gets the number of samples of an Opus packet.
+  * @param [in] dec <tt>OpusDecoder*</tt>: Decoder state
+  * @param [in] packet <tt>char*</tt>: Opus packet
+  * @param [in] len <tt>opus_int32</tt>: Length of packet
+  * @returns Number of samples
+  * @retval OPUS_BAD_ARG Insufficient data was passed to the function
+  * @retval OPUS_INVALID_PACKET The compressed data passed is corrupted or of an unsupported type
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_decoder_get_nb_samples(const OpusDecoder *dec, const unsigned char packet[], opus_int32 len) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2);
+
+/** Applies soft-clipping to bring a float signal within the [-1,1] range. If
+  * the signal is already in that range, nothing is done. If there are values
+  * outside of [-1,1], then the signal is clipped as smoothly as possible to
+  * both fit in the range and avoid creating excessive distortion in the
+  * process.
+  * @param [in,out] pcm <tt>float*</tt>: Input PCM and modified PCM
+  * @param [in] frame_size <tt>int</tt> Number of samples per channel to process
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @param [in,out] softclip_mem <tt>float*</tt>: State memory for the soft clipping process (one float per channel, initialized to zero)
+  */
+OPUS_EXPORT void opus_pcm_soft_clip(float *pcm, int frame_size, int channels, float *softclip_mem);
+
+
+/**@}*/
+
+/** @defgroup opus_repacketizer Repacketizer
+  * @{
+  *
+  * The repacketizer can be used to merge multiple Opus packets into a single
+  * packet or alternatively to split Opus packets that have previously been
+  * merged. Splitting valid Opus packets is always guaranteed to succeed,
+  * whereas merging valid packets only succeeds if all frames have the same
+  * mode, bandwidth, and frame size, and when the total duration of the merged
+  * packet is no more than 120 ms. The 120 ms limit comes from the
+  * specification and limits decoder memory requirements at a point where
+  * framing overhead becomes negligible.
+  *
+  * The repacketizer currently only operates on elementary Opus
+  * streams. It will not manipualte multistream packets successfully, except in
+  * the degenerate case where they consist of data from a single stream.
+  *
+  * The repacketizing process starts with creating a repacketizer state, either
+  * by calling opus_repacketizer_create() or by allocating the memory yourself,
+  * e.g.,
+  * @code
+  * OpusRepacketizer *rp;
+  * rp = (OpusRepacketizer*)malloc(opus_repacketizer_get_size());
+  * if (rp != NULL)
+  *     opus_repacketizer_init(rp);
+  * @endcode
+  *
+  * Then the application should submit packets with opus_repacketizer_cat(),
+  * extract new packets with opus_repacketizer_out() or
+  * opus_repacketizer_out_range(), and then reset the state for the next set of
+  * input packets via opus_repacketizer_init().
+  *
+  * For example, to split a sequence of packets into individual frames:
+  * @code
+  * unsigned char *data;
+  * int len;
+  * while (get_next_packet(&data, &len))
+  * {
+  *   unsigned char out[1276];
+  *   opus_int32 out_len;
+  *   int nb_frames;
+  *   int err;
+  *   int i;
+  *   err = opus_repacketizer_cat(rp, data, len);
+  *   if (err != OPUS_OK)
+  *   {
+  *     release_packet(data);
+  *     return err;
+  *   }
+  *   nb_frames = opus_repacketizer_get_nb_frames(rp);
+  *   for (i = 0; i < nb_frames; i++)
+  *   {
+  *     out_len = opus_repacketizer_out_range(rp, i, i+1, out, sizeof(out));
+  *     if (out_len < 0)
+  *     {
+  *        release_packet(data);
+  *        return (int)out_len;
+  *     }
+  *     output_next_packet(out, out_len);
+  *   }
+  *   opus_repacketizer_init(rp);
+  *   release_packet(data);
+  * }
+  * @endcode
+  *
+  * Alternatively, to combine a sequence of frames into packets that each
+  * contain up to <code>TARGET_DURATION_MS</code> milliseconds of data:
+  * @code
+  * // The maximum number of packets with duration TARGET_DURATION_MS occurs
+  * // when the frame size is 2.5 ms, for a total of (TARGET_DURATION_MS*2/5)
+  * // packets.
+  * unsigned char *data[(TARGET_DURATION_MS*2/5)+1];
+  * opus_int32 len[(TARGET_DURATION_MS*2/5)+1];
+  * int nb_packets;
+  * unsigned char out[1277*(TARGET_DURATION_MS*2/2)];
+  * opus_int32 out_len;
+  * int prev_toc;
+  * nb_packets = 0;
+  * while (get_next_packet(data+nb_packets, len+nb_packets))
+  * {
+  *   int nb_frames;
+  *   int err;
+  *   nb_frames = opus_packet_get_nb_frames(data[nb_packets], len[nb_packets]);
+  *   if (nb_frames < 1)
+  *   {
+  *     release_packets(data, nb_packets+1);
+  *     return nb_frames;
+  *   }
+  *   nb_frames += opus_repacketizer_get_nb_frames(rp);
+  *   // If adding the next packet would exceed our target, or it has an
+  *   // incompatible TOC sequence, output the packets we already have before
+  *   // submitting it.
+  *   // N.B., The nb_packets > 0 check ensures we've submitted at least one
+  *   // packet since the last call to opus_repacketizer_init(). Otherwise a
+  *   // single packet longer than TARGET_DURATION_MS would cause us to try to
+  *   // output an (invalid) empty packet. It also ensures that prev_toc has
+  *   // been set to a valid value. Additionally, len[nb_packets] > 0 is
+  *   // guaranteed by the call to opus_packet_get_nb_frames() above, so the
+  *   // reference to data[nb_packets][0] should be valid.
+  *   if (nb_packets > 0 && (
+  *       ((prev_toc & 0xFC) != (data[nb_packets][0] & 0xFC)) ||
+  *       opus_packet_get_samples_per_frame(data[nb_packets], 48000)*nb_frames >
+  *       TARGET_DURATION_MS*48))
+  *   {
+  *     out_len = opus_repacketizer_out(rp, out, sizeof(out));
+  *     if (out_len < 0)
+  *     {
+  *        release_packets(data, nb_packets+1);
+  *        return (int)out_len;
+  *     }
+  *     output_next_packet(out, out_len);
+  *     opus_repacketizer_init(rp);
+  *     release_packets(data, nb_packets);
+  *     data[0] = data[nb_packets];
+  *     len[0] = len[nb_packets];
+  *     nb_packets = 0;
+  *   }
+  *   err = opus_repacketizer_cat(rp, data[nb_packets], len[nb_packets]);
+  *   if (err != OPUS_OK)
+  *   {
+  *     release_packets(data, nb_packets+1);
+  *     return err;
+  *   }
+  *   prev_toc = data[nb_packets][0];
+  *   nb_packets++;
+  * }
+  * // Output the final, partial packet.
+  * if (nb_packets > 0)
+  * {
+  *   out_len = opus_repacketizer_out(rp, out, sizeof(out));
+  *   release_packets(data, nb_packets);
+  *   if (out_len < 0)
+  *     return (int)out_len;
+  *   output_next_packet(out, out_len);
+  * }
+  * @endcode
+  *
+  * An alternate way of merging packets is to simply call opus_repacketizer_cat()
+  * unconditionally until it fails. At that point, the merged packet can be
+  * obtained with opus_repacketizer_out() and the input packet for which
+  * opus_repacketizer_cat() needs to be re-added to a newly reinitialized
+  * repacketizer state.
+  */
+
+typedef struct OpusRepacketizer OpusRepacketizer;
+
+/** Gets the size of an <code>OpusRepacketizer</code> structure.
+  * @returns The size in bytes.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_repacketizer_get_size(void);
+
+/** (Re)initializes a previously allocated repacketizer state.
+  * The state must be at least the size returned by opus_repacketizer_get_size().
+  * This can be used for applications which use their own allocator instead of
+  * malloc().
+  * It must also be called to reset the queue of packets waiting to be
+  * repacketized, which is necessary if the maximum packet duration of 120 ms
+  * is reached or if you wish to submit packets with a different Opus
+  * configuration (coding mode, audio bandwidth, frame size, or channel count).
+  * Failure to do so will prevent a new packet from being added with
+  * opus_repacketizer_cat().
+  * @see opus_repacketizer_create
+  * @see opus_repacketizer_get_size
+  * @see opus_repacketizer_cat
+  * @param rp <tt>OpusRepacketizer*</tt>: The repacketizer state to
+  *                                       (re)initialize.
+  * @returns A pointer to the same repacketizer state that was passed in.
+  */
+OPUS_EXPORT OpusRepacketizer *opus_repacketizer_init(OpusRepacketizer *rp) OPUS_ARG_NONNULL(1);
+
+/** Allocates memory and initializes the new repacketizer with
+ * opus_repacketizer_init().
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT OpusRepacketizer *opus_repacketizer_create(void);
+
+/** Frees an <code>OpusRepacketizer</code> allocated by
+  * opus_repacketizer_create().
+  * @param[in] rp <tt>OpusRepacketizer*</tt>: State to be freed.
+  */
+OPUS_EXPORT void opus_repacketizer_destroy(OpusRepacketizer *rp);
+
+/** Add a packet to the current repacketizer state.
+  * This packet must match the configuration of any packets already submitted
+  * for repacketization since the last call to opus_repacketizer_init().
+  * This means that it must have the same coding mode, audio bandwidth, frame
+  * size, and channel count.
+  * This can be checked in advance by examining the top 6 bits of the first
+  * byte of the packet, and ensuring they match the top 6 bits of the first
+  * byte of any previously submitted packet.
+  * The total duration of audio in the repacketizer state also must not exceed
+  * 120 ms, the maximum duration of a single packet, after adding this packet.
+  *
+  * The contents of the current repacketizer state can be extracted into new
+  * packets using opus_repacketizer_out() or opus_repacketizer_out_range().
+  *
+  * In order to add a packet with a different configuration or to add more
+  * audio beyond 120 ms, you must clear the repacketizer state by calling
+  * opus_repacketizer_init().
+  * If a packet is too large to add to the current repacketizer state, no part
+  * of it is added, even if it contains multiple frames, some of which might
+  * fit.
+  * If you wish to be able to add parts of such packets, you should first use
+  * another repacketizer to split the packet into pieces and add them
+  * individually.
+  * @see opus_repacketizer_out_range
+  * @see opus_repacketizer_out
+  * @see opus_repacketizer_init
+  * @param rp <tt>OpusRepacketizer*</tt>: The repacketizer state to which to
+  *                                       add the packet.
+  * @param[in] data <tt>const unsigned char*</tt>: The packet data.
+  *                                                The application must ensure
+  *                                                this pointer remains valid
+  *                                                until the next call to
+  *                                                opus_repacketizer_init() or
+  *                                                opus_repacketizer_destroy().
+  * @param len <tt>opus_int32</tt>: The number of bytes in the packet data.
+  * @returns An error code indicating whether or not the operation succeeded.
+  * @retval #OPUS_OK The packet's contents have been added to the repacketizer
+  *                  state.
+  * @retval #OPUS_INVALID_PACKET The packet did not have a valid TOC sequence,
+  *                              the packet's TOC sequence was not compatible
+  *                              with previously submitted packets (because
+  *                              the coding mode, audio bandwidth, frame size,
+  *                              or channel count did not match), or adding
+  *                              this packet would increase the total amount of
+  *                              audio stored in the repacketizer state to more
+  *                              than 120 ms.
+  */
+OPUS_EXPORT int opus_repacketizer_cat(OpusRepacketizer *rp, const unsigned char *data, opus_int32 len) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2);
+
+
+/** Construct a new packet from data previously submitted to the repacketizer
+  * state via opus_repacketizer_cat().
+  * @param rp <tt>OpusRepacketizer*</tt>: The repacketizer state from which to
+  *                                       construct the new packet.
+  * @param begin <tt>int</tt>: The index of the first frame in the current
+  *                            repacketizer state to include in the output.
+  * @param end <tt>int</tt>: One past the index of the last frame in the
+  *                          current repacketizer state to include in the
+  *                          output.
+  * @param[out] data <tt>const unsigned char*</tt>: The buffer in which to
+  *                                                 store the output packet.
+  * @param maxlen <tt>opus_int32</tt>: The maximum number of bytes to store in
+  *                                    the output buffer. In order to guarantee
+  *                                    success, this should be at least
+  *                                    <code>1276</code> for a single frame,
+  *                                    or for multiple frames,
+  *                                    <code>1277*(end-begin)</code>.
+  *                                    However, <code>1*(end-begin)</code> plus
+  *                                    the size of all packet data submitted to
+  *                                    the repacketizer since the last call to
+  *                                    opus_repacketizer_init() or
+  *                                    opus_repacketizer_create() is also
+  *                                    sufficient, and possibly much smaller.
+  * @returns The total size of the output packet on success, or an error code
+  *          on failure.
+  * @retval #OPUS_BAD_ARG <code>[begin,end)</code> was an invalid range of
+  *                       frames (begin < 0, begin >= end, or end >
+  *                       opus_repacketizer_get_nb_frames()).
+  * @retval #OPUS_BUFFER_TOO_SMALL \a maxlen was insufficient to contain the
+  *                                complete output packet.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_repacketizer_out_range(OpusRepacketizer *rp, int begin, int end, unsigned char *data, opus_int32 maxlen) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4);
+
+/** Return the total number of frames contained in packet data submitted to
+  * the repacketizer state so far via opus_repacketizer_cat() since the last
+  * call to opus_repacketizer_init() or opus_repacketizer_create().
+  * This defines the valid range of packets that can be extracted with
+  * opus_repacketizer_out_range() or opus_repacketizer_out().
+  * @param rp <tt>OpusRepacketizer*</tt>: The repacketizer state containing the
+  *                                       frames.
+  * @returns The total number of frames contained in the packet data submitted
+  *          to the repacketizer state.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT int opus_repacketizer_get_nb_frames(OpusRepacketizer *rp) OPUS_ARG_NONNULL(1);
+
+/** Construct a new packet from data previously submitted to the repacketizer
+  * state via opus_repacketizer_cat().
+  * This is a convenience routine that returns all the data submitted so far
+  * in a single packet.
+  * It is equivalent to calling
+  * @code
+  * opus_repacketizer_out_range(rp, 0, opus_repacketizer_get_nb_frames(rp),
+  *                             data, maxlen)
+  * @endcode
+  * @param rp <tt>OpusRepacketizer*</tt>: The repacketizer state from which to
+  *                                       construct the new packet.
+  * @param[out] data <tt>const unsigned char*</tt>: The buffer in which to
+  *                                                 store the output packet.
+  * @param maxlen <tt>opus_int32</tt>: The maximum number of bytes to store in
+  *                                    the output buffer. In order to guarantee
+  *                                    success, this should be at least
+  *                                    <code>1277*opus_repacketizer_get_nb_frames(rp)</code>.
+  *                                    However,
+  *                                    <code>1*opus_repacketizer_get_nb_frames(rp)</code>
+  *                                    plus the size of all packet data
+  *                                    submitted to the repacketizer since the
+  *                                    last call to opus_repacketizer_init() or
+  *                                    opus_repacketizer_create() is also
+  *                                    sufficient, and possibly much smaller.
+  * @returns The total size of the output packet on success, or an error code
+  *          on failure.
+  * @retval #OPUS_BUFFER_TOO_SMALL \a maxlen was insufficient to contain the
+  *                                complete output packet.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_repacketizer_out(OpusRepacketizer *rp, unsigned char *data, opus_int32 maxlen) OPUS_ARG_NONNULL(1);
+
+/** Pads a given Opus packet to a larger size (possibly changing the TOC sequence).
+  * @param[in,out] data <tt>const unsigned char*</tt>: The buffer containing the
+  *                                                   packet to pad.
+  * @param len <tt>opus_int32</tt>: The size of the packet.
+  *                                 This must be at least 1.
+  * @param new_len <tt>opus_int32</tt>: The desired size of the packet after padding.
+  *                                 This must be at least as large as len.
+  * @returns an error code
+  * @retval #OPUS_OK \a on success.
+  * @retval #OPUS_BAD_ARG \a len was less than 1 or new_len was less than len.
+  * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet.
+  */
+OPUS_EXPORT int opus_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len);
+
+/** Remove all padding from a given Opus packet and rewrite the TOC sequence to
+  * minimize space usage.
+  * @param[in,out] data <tt>const unsigned char*</tt>: The buffer containing the
+  *                                                   packet to strip.
+  * @param len <tt>opus_int32</tt>: The size of the packet.
+  *                                 This must be at least 1.
+  * @returns The new size of the output packet on success, or an error code
+  *          on failure.
+  * @retval #OPUS_BAD_ARG \a len was less than 1.
+  * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_packet_unpad(unsigned char *data, opus_int32 len);
+
+/** Pads a given Opus multi-stream packet to a larger size (possibly changing the TOC sequence).
+  * @param[in,out] data <tt>const unsigned char*</tt>: The buffer containing the
+  *                                                   packet to pad.
+  * @param len <tt>opus_int32</tt>: The size of the packet.
+  *                                 This must be at least 1.
+  * @param new_len <tt>opus_int32</tt>: The desired size of the packet after padding.
+  *                                 This must be at least 1.
+  * @param nb_streams <tt>opus_int32</tt>: The number of streams (not channels) in the packet.
+  *                                 This must be at least as large as len.
+  * @returns an error code
+  * @retval #OPUS_OK \a on success.
+  * @retval #OPUS_BAD_ARG \a len was less than 1.
+  * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet.
+  */
+OPUS_EXPORT int opus_multistream_packet_pad(unsigned char *data, opus_int32 len, opus_int32 new_len, int nb_streams);
+
+/** Remove all padding from a given Opus multi-stream packet and rewrite the TOC sequence to
+  * minimize space usage.
+  * @param[in,out] data <tt>const unsigned char*</tt>: The buffer containing the
+  *                                                   packet to strip.
+  * @param len <tt>opus_int32</tt>: The size of the packet.
+  *                                 This must be at least 1.
+  * @param nb_streams <tt>opus_int32</tt>: The number of streams (not channels) in the packet.
+  *                                 This must be at least 1.
+  * @returns The new size of the output packet on success, or an error code
+  *          on failure.
+  * @retval #OPUS_BAD_ARG \a len was less than 1 or new_len was less than len.
+  * @retval #OPUS_INVALID_PACKET \a data did not contain a valid Opus packet.
+  */
+OPUS_EXPORT OPUS_WARN_UNUSED_RESULT opus_int32 opus_multistream_packet_unpad(unsigned char *data, opus_int32 len, int nb_streams);
+
+/**@}*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OPUS_H */

pkg/codec/opus/include/opus_custom.h

@@ -0,0 +1,342 @@
+/* Copyright (c) 2007-2008 CSIRO
+   Copyright (c) 2007-2009 Xiph.Org Foundation
+   Copyright (c) 2008-2012 Gregory Maxwell
+   Written by Jean-Marc Valin and Gregory Maxwell */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/**
+  @file opus_custom.h
+  @brief Opus-Custom reference implementation API
+ */
+
+#ifndef OPUS_CUSTOM_H
+#define OPUS_CUSTOM_H
+
+#include "opus_defines.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef CUSTOM_MODES
+# define OPUS_CUSTOM_EXPORT OPUS_EXPORT
+# define OPUS_CUSTOM_EXPORT_STATIC OPUS_EXPORT
+#else
+# define OPUS_CUSTOM_EXPORT
+# ifdef OPUS_BUILD
+#  define OPUS_CUSTOM_EXPORT_STATIC static OPUS_INLINE
+# else
+#  define OPUS_CUSTOM_EXPORT_STATIC
+# endif
+#endif
+
+/** @defgroup opus_custom Opus Custom
+  * @{
+  *  Opus Custom is an optional part of the Opus specification and
+  * reference implementation which uses a distinct API from the regular
+  * API and supports frame sizes that are not normally supported.\ Use
+  * of Opus Custom is discouraged for all but very special applications
+  * for which a frame size different from 2.5, 5, 10, or 20 ms is needed
+  * (for either complexity or latency reasons) and where interoperability
+  * is less important.
+  *
+  * In addition to the interoperability limitations the use of Opus custom
+  * disables a substantial chunk of the codec and generally lowers the
+  * quality available at a given bitrate. Normally when an application needs
+  * a different frame size from the codec it should buffer to match the
+  * sizes but this adds a small amount of delay which may be important
+  * in some very low latency applications. Some transports (especially
+  * constant rate RF transports) may also work best with frames of
+  * particular durations.
+  *
+  * Libopus only supports custom modes if they are enabled at compile time.
+  *
+  * The Opus Custom API is similar to the regular API but the
+  * @ref opus_encoder_create and @ref opus_decoder_create calls take
+  * an additional mode parameter which is a structure produced by
+  * a call to @ref opus_custom_mode_create. Both the encoder and decoder
+  * must create a mode using the same sample rate (fs) and frame size
+  * (frame size) so these parameters must either be signaled out of band
+  * or fixed in a particular implementation.
+  *
+  * Similar to regular Opus the custom modes support on the fly frame size
+  * switching, but the sizes available depend on the particular frame size in
+  * use. For some initial frame sizes on a single on the fly size is available.
+  */
+
+/** Contains the state of an encoder. One encoder state is needed
+    for each stream. It is initialized once at the beginning of the
+    stream. Do *not* re-initialize the state for every frame.
+   @brief Encoder state
+ */
+typedef struct OpusCustomEncoder OpusCustomEncoder;
+
+/** State of the decoder. One decoder state is needed for each stream.
+    It is initialized once at the beginning of the stream. Do *not*
+    re-initialize the state for every frame.
+   @brief Decoder state
+ */
+typedef struct OpusCustomDecoder OpusCustomDecoder;
+
+/** The mode contains all the information necessary to create an
+    encoder. Both the encoder and decoder need to be initialized
+    with exactly the same mode, otherwise the output will be
+    corrupted.
+   @brief Mode configuration
+ */
+typedef struct OpusCustomMode OpusCustomMode;
+
+/** Creates a new mode struct. This will be passed to an encoder or
+  * decoder. The mode MUST NOT BE DESTROYED until the encoders and
+  * decoders that use it are destroyed as well.
+  * @param [in] Fs <tt>int</tt>: Sampling rate (8000 to 96000 Hz)
+  * @param [in] frame_size <tt>int</tt>: Number of samples (per channel) to encode in each
+  *        packet (64 - 1024, prime factorization must contain zero or more 2s, 3s, or 5s and no other primes)
+  * @param [out] error <tt>int*</tt>: Returned error code (if NULL, no error will be returned)
+  * @return A newly created mode
+  */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomMode *opus_custom_mode_create(opus_int32 Fs, int frame_size, int *error);
+
+/** Destroys a mode struct. Only call this after all encoders and
+  * decoders using this mode are destroyed as well.
+  * @param [in] mode <tt>OpusCustomMode*</tt>: Mode to be freed.
+  */
+OPUS_CUSTOM_EXPORT void opus_custom_mode_destroy(OpusCustomMode *mode);
+
+
+#if !defined(OPUS_BUILD) || defined(CELT_ENCODER_C)
+
+/* Encoder */
+/** Gets the size of an OpusCustomEncoder structure.
+  * @param [in] mode <tt>OpusCustomMode *</tt>: Mode configuration
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @returns size
+  */
+OPUS_CUSTOM_EXPORT_STATIC OPUS_WARN_UNUSED_RESULT int opus_custom_encoder_get_size(
+    const OpusCustomMode *mode,
+    int channels
+) OPUS_ARG_NONNULL(1);
+
+# ifdef CUSTOM_MODES
+/** Initializes a previously allocated encoder state
+  * The memory pointed to by st must be the size returned by opus_custom_encoder_get_size.
+  * This is intended for applications which use their own allocator instead of malloc.
+  * @see opus_custom_encoder_create(),opus_custom_encoder_get_size()
+  * To reset a previously initialized state use the OPUS_RESET_STATE CTL.
+  * @param [in] st <tt>OpusCustomEncoder*</tt>: Encoder state
+  * @param [in] mode <tt>OpusCustomMode *</tt>: Contains all the information about the characteristics of
+  *  the stream (must be the same characteristics as used for the
+  *  decoder)
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @return OPUS_OK Success or @ref opus_errorcodes
+  */
+OPUS_CUSTOM_EXPORT int opus_custom_encoder_init(
+    OpusCustomEncoder *st,
+    const OpusCustomMode *mode,
+    int channels
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2);
+# endif
+#endif
+
+
+/** Creates a new encoder state. Each stream needs its own encoder
+  * state (can't be shared across simultaneous streams).
+  * @param [in] mode <tt>OpusCustomMode*</tt>: Contains all the information about the characteristics of
+  *  the stream (must be the same characteristics as used for the
+  *  decoder)
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @param [out] error <tt>int*</tt>: Returns an error code
+  * @return Newly created encoder state.
+*/
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomEncoder *opus_custom_encoder_create(
+    const OpusCustomMode *mode,
+    int channels,
+    int *error
+) OPUS_ARG_NONNULL(1);
+
+
+/** Destroys a an encoder state.
+  * @param[in] st <tt>OpusCustomEncoder*</tt>: State to be freed.
+  */
+OPUS_CUSTOM_EXPORT void opus_custom_encoder_destroy(OpusCustomEncoder *st);
+
+/** Encodes a frame of audio.
+  * @param [in] st <tt>OpusCustomEncoder*</tt>: Encoder state
+  * @param [in] pcm <tt>float*</tt>: PCM audio in float format, with a normal range of +/-1.0.
+  *          Samples with a range beyond +/-1.0 are supported but will
+  *          be clipped by decoders using the integer API and should
+  *          only be used if it is known that the far end supports
+  *          extended dynamic range. There must be exactly
+  *          frame_size samples per channel.
+  * @param [in] frame_size <tt>int</tt>: Number of samples per frame of input signal
+  * @param [out] compressed <tt>char *</tt>: The compressed data is written here. This may not alias pcm and must be at least maxCompressedBytes long.
+  * @param [in] maxCompressedBytes <tt>int</tt>: Maximum number of bytes to use for compressing the frame
+  *          (can change from one frame to another)
+  * @return Number of bytes written to "compressed".
+  *       If negative, an error has occurred (see error codes). It is IMPORTANT that
+  *       the length returned be somehow transmitted to the decoder. Otherwise, no
+  *       decoding is possible.
+  */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_encode_float(
+    OpusCustomEncoder *st,
+    const float *pcm,
+    int frame_size,
+    unsigned char *compressed,
+    int maxCompressedBytes
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4);
+
+/** Encodes a frame of audio.
+  * @param [in] st <tt>OpusCustomEncoder*</tt>: Encoder state
+  * @param [in] pcm <tt>opus_int16*</tt>: PCM audio in signed 16-bit format (native endian).
+  *          There must be exactly frame_size samples per channel.
+  * @param [in] frame_size <tt>int</tt>: Number of samples per frame of input signal
+  * @param [out] compressed <tt>char *</tt>: The compressed data is written here. This may not alias pcm and must be at least maxCompressedBytes long.
+  * @param [in] maxCompressedBytes <tt>int</tt>: Maximum number of bytes to use for compressing the frame
+  *          (can change from one frame to another)
+  * @return Number of bytes written to "compressed".
+  *       If negative, an error has occurred (see error codes). It is IMPORTANT that
+  *       the length returned be somehow transmitted to the decoder. Otherwise, no
+  *       decoding is possible.
+ */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_encode(
+    OpusCustomEncoder *st,
+    const opus_int16 *pcm,
+    int frame_size,
+    unsigned char *compressed,
+    int maxCompressedBytes
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2) OPUS_ARG_NONNULL(4);
+
+/** Perform a CTL function on an Opus custom encoder.
+  *
+  * Generally the request and subsequent arguments are generated
+  * by a convenience macro.
+  * @see opus_encoderctls
+  */
+OPUS_CUSTOM_EXPORT int opus_custom_encoder_ctl(OpusCustomEncoder * OPUS_RESTRICT st, int request, ...) OPUS_ARG_NONNULL(1);
+
+
+#if !defined(OPUS_BUILD) || defined(CELT_DECODER_C)
+/* Decoder */
+
+/** Gets the size of an OpusCustomDecoder structure.
+  * @param [in] mode <tt>OpusCustomMode *</tt>: Mode configuration
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @returns size
+  */
+OPUS_CUSTOM_EXPORT_STATIC OPUS_WARN_UNUSED_RESULT int opus_custom_decoder_get_size(
+    const OpusCustomMode *mode,
+    int channels
+) OPUS_ARG_NONNULL(1);
+
+/** Initializes a previously allocated decoder state
+  * The memory pointed to by st must be the size returned by opus_custom_decoder_get_size.
+  * This is intended for applications which use their own allocator instead of malloc.
+  * @see opus_custom_decoder_create(),opus_custom_decoder_get_size()
+  * To reset a previously initialized state use the OPUS_RESET_STATE CTL.
+  * @param [in] st <tt>OpusCustomDecoder*</tt>: Decoder state
+  * @param [in] mode <tt>OpusCustomMode *</tt>: Contains all the information about the characteristics of
+  *  the stream (must be the same characteristics as used for the
+  *  encoder)
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @return OPUS_OK Success or @ref opus_errorcodes
+  */
+OPUS_CUSTOM_EXPORT_STATIC int opus_custom_decoder_init(
+    OpusCustomDecoder *st,
+    const OpusCustomMode *mode,
+    int channels
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(2);
+
+#endif
+
+
+/** Creates a new decoder state. Each stream needs its own decoder state (can't
+  * be shared across simultaneous streams).
+  * @param [in] mode <tt>OpusCustomMode</tt>: Contains all the information about the characteristics of the
+  *          stream (must be the same characteristics as used for the encoder)
+  * @param [in] channels <tt>int</tt>: Number of channels
+  * @param [out] error <tt>int*</tt>: Returns an error code
+  * @return Newly created decoder state.
+  */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT OpusCustomDecoder *opus_custom_decoder_create(
+    const OpusCustomMode *mode,
+    int channels,
+    int *error
+) OPUS_ARG_NONNULL(1);
+
+/** Destroys a an decoder state.
+  * @param[in] st <tt>OpusCustomDecoder*</tt>: State to be freed.
+  */
+OPUS_CUSTOM_EXPORT void opus_custom_decoder_destroy(OpusCustomDecoder *st);
+
+/** Decode an opus custom frame with floating point output
+  * @param [in] st <tt>OpusCustomDecoder*</tt>: Decoder state
+  * @param [in] data <tt>char*</tt>: Input payload. Use a NULL pointer to indicate packet loss
+  * @param [in] len <tt>int</tt>: Number of bytes in payload
+  * @param [out] pcm <tt>float*</tt>: Output signal (interleaved if 2 channels). length
+  *  is frame_size*channels*sizeof(float)
+  * @param [in] frame_size Number of samples per channel of available space in *pcm.
+  * @returns Number of decoded samples or @ref opus_errorcodes
+  */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_decode_float(
+    OpusCustomDecoder *st,
+    const unsigned char *data,
+    int len,
+    float *pcm,
+    int frame_size
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4);
+
+/** Decode an opus custom frame
+  * @param [in] st <tt>OpusCustomDecoder*</tt>: Decoder state
+  * @param [in] data <tt>char*</tt>: Input payload. Use a NULL pointer to indicate packet loss
+  * @param [in] len <tt>int</tt>: Number of bytes in payload
+  * @param [out] pcm <tt>opus_int16*</tt>: Output signal (interleaved if 2 channels). length
+  *  is frame_size*channels*sizeof(opus_int16)
+  * @param [in] frame_size Number of samples per channel of available space in *pcm.
+  * @returns Number of decoded samples or @ref opus_errorcodes
+  */
+OPUS_CUSTOM_EXPORT OPUS_WARN_UNUSED_RESULT int opus_custom_decode(
+    OpusCustomDecoder *st,
+    const unsigned char *data,
+    int len,
+    opus_int16 *pcm,
+    int frame_size
+) OPUS_ARG_NONNULL(1) OPUS_ARG_NONNULL(4);
+
+/** Perform a CTL function on an Opus custom decoder.
+  *
+  * Generally the request and subsequent arguments are generated
+  * by a convenience macro.
+  * @see opus_genericctls
+  */
+OPUS_CUSTOM_EXPORT int opus_custom_decoder_ctl(OpusCustomDecoder * OPUS_RESTRICT st, int request, ...) OPUS_ARG_NONNULL(1);
+
+/**@}*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OPUS_CUSTOM_H */