Allow to enable frame copy on a video track (#405)

Also make the ReadCloser an Controllable allows to uncouple
the controller implementation from the encoder.
This is not needed for the 2 codec controller already implemented (openh264 and vpx)
but is more future proof in case it required for other codecs.

.github/workflows/ci.yaml

@@ -13,13 +13,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [ '1.15', '1.14' ]
+        go: [ '1.16', '1.15' ]
     name: Linux Go ${{ matrix.go }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -30,34 +30,22 @@ jobs:
             libva-dev \
             libvpx-dev \
             libx264-dev
-      - name: go vet
-        run: go vet $(go list ./... | grep -v mmal)
-      - name: go build
-        run: go build $(go list ./... | grep -v mmal)
-      - name: go build without CGO
-        run: go build . pkg/...
-        env:
-          CGO_ENABLED: 0
-      - name: go test
-        run: go test -v -race -coverprofile=coverage.txt -covermode=atomic $(go list ./... | grep -v mmal)
-      - uses: codecov/codecov-action@v1
-        if: matrix.go == '1.15'
-      - name: go test without CGO
-        run: go test . pkg/... -v
-        env:
-          CGO_ENABLED: 0
+      - name: Run Test Suite
+        run: make test
+      - uses: codecov/codecov-action@v3	
+        if: matrix.go == '1.16'
   build-darwin:
     runs-on: macos-latest
     strategy:
       fail-fast: false
       matrix:
-        go: [ '1.15', '1.14' ]
+        go: [ '1.16', '1.15' ]
     name: Darwin Go ${{ matrix.go }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -67,17 +55,19 @@ jobs:
             opus \
             libvpx \
             x264
-      - name: go vet
-        run: go vet $(go list ./... | grep -v mmal)
-      - name: go build
-        run: go build $(go list ./... | grep -v mmal)
-      - name: go build without CGO
-        run: go build . pkg/...
-        env:
-          CGO_ENABLED: 0
-      - name: go test
-        run: go test -v -race $(go list ./... | grep -v mmal)
-      - name: go test without CGO
-        run: go test . pkg/... -v
-        env:
-          CGO_ENABLED: 0
+      - name: Run Test Suite
+        run: make test
+  check-licenses:
+    runs-on: ubuntu-latest
+    name: Check Licenses
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.16'
+      - name: Installing go-licenses
+        run: go get github.com/google/go-licenses
+      - name: Checking licenses
+        run: go-licenses check ./...

.github/workflows/renovate-go-mod-fix.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
       - name: fix

.gitignore

@@ -12,3 +12,6 @@
 *.out
 
 scripts/cross
+coverage.txt
+
+.idea

Makefile

@@ -0,0 +1,83 @@
+docker_owner := lherman
+docker_prefix := cross
+toolchain_dockerfiles := dockerfiles
+script_path := $(realpath scripts)
+toolchain_path := $(script_path)/$(docker_prefix)
+os_list := \
+	linux \
+	windows \
+	darwin
+arch_list := \
+	armv7 \
+	arm64 \
+	x64
+supported_platforms := \
+  linux-armv7 \
+  linux-arm64 \
+  linux-x64 \
+  windows-x64 \
+  darwin-x64 \
+  darwin-arm64
+cmd_build := build
+cmd_test := test
+examples_dir := examples
+codec_dir := pkg/codec
+codec_list := $(shell ls $(codec_dir)/*/Makefile)
+codec_list := $(codec_list:$(codec_dir)/%/Makefile=%)
+targets := $(foreach codec, $(codec_list), $(addprefix $(cmd_build)-$(codec)-, $(supported_platforms)))
+pkgs_without_mmal := $(shell go list ./... | grep -v mmal)
+pkgs_without_cgo := $(shell go list ./... | grep -v pkg/codec | grep -v pkg/driver | grep -v pkg/avfoundation)
+
+define BUILD_TEMPLATE
+ifneq (,$$(findstring $(2)-$(3),$$(supported_platforms)))
+$$(cmd_build)-$(1)-$(2)-$(3): toolchain-$(2)-$(3)
+	$$(MAKE) --directory=$$(codec_dir)/$(1) \
+		MEDIADEVICES_TOOLCHAIN_BIN=$$(toolchain_path)/$(docker_prefix)-$(2)-$(3) \
+		MEDIADEVICES_TARGET_PLATFORM=$(2)-$(3) \
+		MEDIADEVICES_TARGET_OS=$(2) \
+		MEDIADEVICES_TARGET_ARCH=$(3)
+endif
+endef
+
+.PHONY: all
+all: $(cmd_test) $(cmd_build)
+
+# Subcommand:
+# 	make build[-<codec_name>-<os>-<arch>]
+#
+# Description:
+# 	Build codec dependencies to multiple platforms.
+#
+# Examples:
+#   * make build: build all codecs for all supported platforms
+#   * make build-opus-darwin-x64: only build opus for darwin-x64 platform
+$(cmd_build): $(targets)
+
+toolchain-%: $(toolchain_dockerfiles)
+	$(MAKE) --directory=$< "$*" \
+		MEDIADEVICES_DOCKER_OWNER=$(docker_owner) \
+		MEDIADEVICES_DOCKER_PREFIX=$(docker_prefix)
+	@mkdir -p $(toolchain_path)
+	@docker run $(docker_owner)/$(docker_prefix)-$* > \
+		$(toolchain_path)/$(docker_prefix)-$*
+	@chmod +x $(toolchain_path)/$(docker_prefix)-$*
+
+$(foreach codec, $(codec_list), \
+	$(foreach os, $(os_list), \
+		$(foreach arch, $(arch_list), \
+			$(eval $(call BUILD_TEMPLATE,$(codec),$(os),$(arch))))))
+
+# Subcommand:
+# 	make test
+#
+# Description:
+# 	Run a series of tests
+$(cmd_test):
+	go vet $(pkgs_without_mmal)
+	go build $(pkgs_without_mmal)
+	# go build without CGO
+	CGO_ENABLED=0 go build $(pkgs_without_cgo)
+	# go build with CGO
+	CGO_ENABLED=1 go build $(pkgs_without_mmal)
+	$(MAKE) --directory=$(examples_dir)
+	go test -v -race -coverprofile=coverage.txt -covermode=atomic $(pkgs_without_mmal)

README.md

@@ -87,7 +87,7 @@ func main() {
 | Microphone |  ✔️   | ✔️  |   ✔️    |
 |   Screen   |  ✔️   | ✔️  |   ✔️    |
 
-By default, there's no media input registered. This decision was made to allow you to pay what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
+By default, there's no media input registered. This decision was made to allow you to play only what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
 
 ```go
 import (

build.sh

@@ -1,70 +0,0 @@
-#!/bin/bash
-
-MEDIADEVICES_TOOLCHAIN_OWNER=lherman
-MEDIADEVICES_TOOLCHAIN_PREFIX=cross
-MEDIADEVICES_SCRIPT_PATH=$(realpath ./scripts)
-MEDIADEVICES_TOOLCHAIN_PATH=${MEDIADEVICES_SCRIPT_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}
-MEDIADEVICES_DOCKERFILES_PATH=dockerfiles
-
-# Reference: https://github.com/dockcross/dockcross#cross-compilers
-MEDIADEVICES_TARGET_PLATFORMS=(
-  linux-armv7
-  linux-arm64
-  linux-x64
-  windows-x64
-  darwin-x64
-)
-
-if [[ -z ${VERBOSE} ]]; then
-  MEDIADEVICES_OUTPUT=/dev/null
-else
-  MEDIADEVICES_OUTPUT=/dev/stdout
-fi
-
-install_toolchains() {
-  bash ${MEDIADEVICES_DOCKERFILES_PATH}/build.sh &> ${MEDIADEVICES_OUTPUT}
-  for platform in ${MEDIADEVICES_TARGET_PLATFORMS[@]}
-  do
-    mkdir -p ${MEDIADEVICES_TOOLCHAIN_PATH}
-    image=${MEDIADEVICES_TOOLCHAIN_OWNER}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
-    bin_path=${MEDIADEVICES_TOOLCHAIN_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
-    docker run ${image} > ${bin_path}
-    chmod +x ${bin_path}
-  done
-}
-
-build() {
-  sub_builds=$(find pkg -type f -name "build.sh")
-  for sub_build in ${sub_builds[@]}
-  do
-    sub_build=$(realpath ${sub_build})
-    sub_build_dir=$(dirname ${sub_build})
-    current_dir=${PWD}
-    cd $sub_build_dir
-    for platform in ${MEDIADEVICES_TARGET_PLATFORMS[@]}
-    do
-      export MEDIADEVICES_TOOLCHAIN_BIN=${MEDIADEVICES_TOOLCHAIN_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
-      # convert '-' to '_' since '_' is more common in library names
-      export MEDIADEVICES_TARGET_PLATFORM=${platform//-/_}
-      export MEDIADEVICES_TARGET_OS=$(echo $MEDIADEVICES_TARGET_PLATFORM | cut -d'_' -f1)
-      export MEDIADEVICES_TARGET_ARCH=${platform//${MEDIADEVICES_TARGET_OS}-/}
-
-      echo "Building ${sub_build_dir}:"
-      echo "  PLATFORM      : ${MEDIADEVICES_TARGET_PLATFORM}"
-      echo "  OS            : ${MEDIADEVICES_TARGET_OS}"
-      echo "  ARCH          : ${MEDIADEVICES_TARGET_ARCH}"
-      echo "  TOOLCHAIN_BIN : ${MEDIADEVICES_TOOLCHAIN_BIN}"
-      echo ""
-
-      ${sub_build} &> ${MEDIADEVICES_OUTPUT}
-    done
-    cd ${current_dir}
-  done
-}
-
-if [[ $# > 0 && $1 != "all" ]]; then
-  MEDIADEVICES_TARGET_PLATFORMS=($1)
-fi
-
-install_toolchains
-build

codec.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 // CodecSelector is a container of video and audio encoder builders, which later will be used
@@ -50,14 +50,15 @@ func NewCodecSelector(opts ...CodecSelectorOption) *CodecSelector {
 // Populate lets the webrtc engine be aware of supported codecs that are contained in CodecSelector
 func (selector *CodecSelector) Populate(setting *webrtc.MediaEngine) {
 	for _, encoder := range selector.videoEncoders {
-		setting.RegisterCodec(encoder.RTPCodec().RTPCodec)
+		setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeVideo)
 	}
 
 	for _, encoder := range selector.audioEncoders {
-		setting.RegisterCodec(encoder.RTPCodec().RTPCodec)
+		setting.RegisterCodec(encoder.RTPCodec().RTPCodecParameters, webrtc.RTPCodecTypeAudio)
 	}
 }
 
+// selectVideoCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "video/<codecName>" or "<codecName>"
 func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
 	var selectedEncoder codec.VideoEncoderBuilder
 	var encodedReader codec.ReadCloser
@@ -66,8 +67,10 @@ func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inpu
 
 outer:
 	for _, wantCodec := range codecNames {
+		wantCodecLower := strings.ToLower(wantCodec)
 		for _, encoder := range selector.videoEncoders {
-			if encoder.RTPCodec().Name == wantCodec {
+			// MimeType is formated as "video/<codecName>"
+			if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
 				encodedReader, err = encoder.BuildVideoEncoder(reader, inputProp)
 				if err == nil {
 					selectedEncoder = encoder
@@ -75,7 +78,7 @@ outer:
 				}
 			}
 
-			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().Name, err))
+			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
 		}
 	}
 
@@ -86,16 +89,17 @@ outer:
 	return encodedReader, selectedEncoder.RTPCodec(), nil
 }
 
-func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
+func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
 	var codecNames []string
 
 	for _, codec := range codecs {
-		codecNames = append(codecNames, codec.Name)
+		codecNames = append(codecNames, codec.MimeType)
 	}
 
 	return selector.selectVideoCodecByNames(reader, inputProp, codecNames...)
 }
 
+// selectAudioCodecByNames selects a single codec that can be built and matched. codecNames can be formatted as "audio/<codecName>" or "<codecName>"
 func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
 	var selectedEncoder codec.AudioEncoderBuilder
 	var encodedReader codec.ReadCloser
@@ -104,8 +108,10 @@ func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inpu
 
 outer:
 	for _, wantCodec := range codecNames {
+		wantCodecLower := strings.ToLower(wantCodec)
 		for _, encoder := range selector.audioEncoders {
-			if encoder.RTPCodec().Name == wantCodec {
+			// MimeType is formated as "audio/<codecName>"
+			if strings.HasSuffix(strings.ToLower(encoder.RTPCodec().MimeType), wantCodecLower) {
 				encodedReader, err = encoder.BuildAudioEncoder(reader, inputProp)
 				if err == nil {
 					selectedEncoder = encoder
@@ -113,7 +119,7 @@ outer:
 				}
 			}
 
-			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().Name, err))
+			errReasons = append(errReasons, fmt.Sprintf("%s: %s", encoder.RTPCodec().MimeType, err))
 		}
 	}
 
@@ -124,11 +130,11 @@ outer:
 	return encodedReader, selectedEncoder.RTPCodec(), nil
 }
 
-func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
+func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...webrtc.RTPCodecParameters) (codec.ReadCloser, *codec.RTPCodec, error) {
 	var codecNames []string
 
 	for _, codec := range codecs {
-		codecNames = append(codecNames, codec.Name)
+		codecNames = append(codecNames, codec.MimeType)
 	}
 
 	return selector.selectAudioCodecByNames(reader, inputProp, codecNames...)

dockerfiles/Makefile

@@ -0,0 +1,11 @@
+dockerfiles := $(wildcard *.Dockerfile)
+supported_platforms := $(dockerfiles:.Dockerfile=)
+
+.PHONY: all
+all: $(supported_platforms)
+
+%: %.Dockerfile guard-MEDIADEVICES_DOCKER_OWNER guard-MEDIADEVICES_DOCKER_PREFIX
+	docker build -t "$(MEDIADEVICES_DOCKER_OWNER)/$(MEDIADEVICES_DOCKER_PREFIX)-$@" -f "$<" . 
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

dockerfiles/build.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-cd $(dirname $0)
-
-OWNER=lherman
-PREFIX=cross
-IMAGES=$(ls *.Dockerfile)
-
-for image in ${IMAGES[@]}
-do
-  tag=${OWNER}/cross-${image//.Dockerfile/}
-  docker build -t "${tag}" -f "$image" .
-done

dockerfiles/darwin-arm64.Dockerfile

@@ -0,0 +1,47 @@
+FROM dockercore/golang-cross as m1cross
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -qq && apt-get install -y -q --no-install-recommends \
+    cmake \
+    git \
+    libssl-dev \
+    libxml2-dev \
+    libz-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV SDK_VERSION=11.3 \
+    TARGET_DIR=/osxcross/target \
+    UNATTENDED=1
+
+WORKDIR /work
+RUN git clone --depth=1 https://github.com/tpoechtrager/osxcross.git /work \
+  && cd /work/tarballs \
+  && wget -q https://github.com/phracker/MacOSX-SDKs/releases/download/${SDK_VERSION}/MacOSX${SDK_VERSION}.sdk.tar.xz
+
+# Build cross compile toolchain for Apple silicon
+RUN ./build.sh
+
+
+FROM dockcross/base
+
+ENV OSX_CROSS_PATH=/osxcross
+
+COPY --from=m1cross "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}/"
+ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+
+ENV CC=arm64-apple-darwin20.4-clang \
+    CXX=arm64-apple-darwin20.4-clang++ \
+    CPP=arm64-apple-darwin20.4-clang++ \
+    AR=arm64-apple-darwin20.4-ar \
+    AS=arm64-apple-darwin20.4-as \
+    LD=arm64-apple-darwin20.4-ld
+
+COPY darwin-arm64.cmake ${OSX_CROSS_PATH}/
+ENV CMAKE_TOOLCHAIN_FILE ${OSX_CROSS_PATH}/darwin-arm64.cmake
+
+ARG IMAGE=lherman/cross-darwin-arm64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}

dockerfiles/darwin-arm64.cmake

@@ -0,0 +1,8 @@
+set(CMAKE_SYSTEM_NAME Darwin)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR arm64)
+
+set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_CXX_COMPILER $ENV{CXX})
+set(CMAKE_AR $ENV{AR})
+set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})

examples/.gitignore

@@ -1 +0,0 @@
-go.sum

examples/Makefile

@@ -0,0 +1,8 @@
+examples := $(shell find * -maxdepth 0 -type d)
+examples := $(filter-out internal,$(examples))
+
+.PHONY: all $(examples)
+all: $(examples)
+
+$(examples):
+	cd $@ && go build -mod=mod

examples/archive/.gitignore

@@ -0,0 +1 @@
+archive

examples/archive/main.go

@@ -43,7 +43,7 @@ func main() {
 
 	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
 		},
@@ -68,7 +68,7 @@ func main() {
 		})
 	}))
 
-	reader, err := videoTrack.NewEncodedReader(x264Params.RTPCodec().Name)
+	reader, err := videoTrack.NewEncodedIOReader(x264Params.RTPCodec().MimeType)
 	must(err)
 	defer reader.Close()
 

examples/facedetection/.gitignore

@@ -0,0 +1 @@
+facedetection

examples/facedetection/main.go

@@ -8,8 +8,8 @@ import (
 
 	pigo "github.com/esimov/pigo/core"
 	"github.com/pion/mediadevices"
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
 	"github.com/pion/mediadevices/pkg/frame"
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -77,7 +77,7 @@ func main() {
 
 	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
+			c.FrameFormat = prop.FrameFormatOneOf{frame.FormatI420, frame.FormatYUY2}
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
 		},
@@ -97,7 +97,7 @@ func main() {
 		frame, release, err := videoReader.Read()
 		must(err)
 
-		// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
+		// Since we asked the frame format to be exactly I420/YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
 		if detectFace(frame.(*image.YCbCr)) {
 			log.Println("Detect a face")
 		}

examples/go.mod

@@ -2,8 +2,10 @@ module github.com/pion/mediadevices/examples
 
 go 1.14
 
-// Please don't commit require entries of examples.
-// `git checkout master examples/go.mod` to revert this file.
-require github.com/pion/mediadevices v0.0.0
+require (
+	github.com/esimov/pigo v1.4.3
+	github.com/pion/mediadevices v0.0.0
+	github.com/pion/webrtc/v3 v3.0.20
+)
 
 replace github.com/pion/mediadevices v0.0.0 => ../

examples/go.sum

@@ -0,0 +1,166 @@
+github.com/BurntSushi/xgb v0.0.0-20210121224620-deaf085860bc/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
+github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/disintegration/imaging v1.6.1/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=
+github.com/esimov/pigo v1.4.3 h1:xl098Z9CHmouywvyRZepuKx8aSWHBs/0lZtp7Yt5g28=
+github.com/esimov/pigo v1.4.3/go.mod h1:aOTYpOWsqniACzXKdSOGkqI6CnWQpP8tFjgtUOARoEs=
+github.com/fogleman/gg v1.0.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gen2brain/malgo v0.10.29 h1:bTYiUTUKJsEomNby+W0hgyLrOttUXIk4lTEnKA54iqM=
+github.com/gen2brain/malgo v0.10.29/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
+github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/kbinani/screenshot v0.0.0-20210326165202-b96eb3309bb0/go.mod h1:ZceVWGtzUZmxyN+/1I+oG31oOm1dOA2QUNbua9TLVdE=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
+github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
+github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
+github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8=
+github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
+github.com/pion/ice/v2 v2.1.7 h1:FjgDfUNrVYTxQabJrkBX6ld12tvYbgzHenqPh3PJF6E=
+github.com/pion/ice/v2 v2.1.7/go.mod h1:kV4EODVD5ux2z8XncbLHIOtcXKtYXVgLVCeVqnpoeP0=
+github.com/pion/interceptor v0.0.12 h1:eC1iVneBIAQJEfaNAfDqAncJWhMDAnaXPRCJsltdokE=
+github.com/pion/interceptor v0.0.12/go.mod h1:qzeuWuD/ZXvPqOnxNcnhWfkCZ2e1kwwslicyyPnhoK4=
+github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
+github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
+github.com/pion/mdns v0.0.5 h1:Q2oj/JB3NqfzY9xGZ1fPzZzK7sDSD8rZPOvcIQ10BCw=
+github.com/pion/mdns v0.0.5/go.mod h1:UgssrvdD3mxpi8tMxAXbsppL3vJ4Jipw1mTCW+al01g=
+github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
+github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
+github.com/pion/rtcp v1.2.6 h1:1zvwBbyd0TeEuuWftrd/4d++m+/kZSeiguxU61LFWpo=
+github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0=
+github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U=
+github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/rtp v1.6.5 h1:o2cZf8OascA5HF/b0PAbTxRKvOWxTQxWYt7SlToxFGI=
+github.com/pion/rtp v1.6.5/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
+github.com/pion/sctp v1.7.12 h1:GsatLufywVruXbZZT1CKg+Jr8ZTkwiPnmUC/oO9+uuY=
+github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
+github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
+github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk=
+github.com/pion/srtp/v2 v2.0.2 h1:664iGzVmaY7KYS5M0gleY0DscRo9ReDfTxQrq4UgGoU=
+github.com/pion/srtp/v2 v2.0.2/go.mod h1:VEyLv4CuxrwGY8cxM+Ng3bmVy8ckz/1t6A0q/msKOw0=
+github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
+github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
+github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
+github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
+github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw=
+github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A=
+github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
+github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
+github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
+github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk=
+golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210420210106-798c2154c571 h1:Q6Bg8xzKzpFPU4Oi1sBnBTHBwlMsLeEXpu4hYBY8rAg=
+golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

examples/http/.gitignore

@@ -0,0 +1 @@
+http

examples/rtp/.gitignore

@@ -0,0 +1 @@
+rtp

examples/rtp/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 
@@ -40,7 +41,7 @@ func main() {
 
 	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
 		},
@@ -51,7 +52,7 @@ func main() {
 	videoTrack := mediaStream.GetVideoTracks()[0]
 	defer videoTrack.Close()
 
-	rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().Name, mtu)
+	rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().MimeType, rand.Uint32(), mtu)
 	must(err)
 
 	addr, err := net.ResolveUDPAddr("udp", dest)

examples/vnc/README.md

@@ -0,0 +1,46 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download vnc example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+#### Compile vnc example
+
+```
+cd mediadevices/examples/vnc && go build
+```
+
+### Open example page
+
+[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
+
+### Run the webrtc example with your browsers SessionDescription as stdin
+
+In the jsfiddle the top textarea is your browser, copy that, and store the session description in an environment variable, `export SDP=<put_the_sdp_here>`
+
+Run `echo $SDP | ./vnc`
+
+In Windows
+
+```powershell
+type sdp.txt| .\vnc.exe
+```
+### Input webrtc's SessionDescription into your browser
+
+Copy the text that `./webrtc` just emitted and copy into second text area
+
+### Hit 'Start Session' in jsfiddle, enjoy your video!
+
+A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/vnc/main.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"fmt"
+	"github.com/pion/mediadevices/pkg/driver"
+	"github.com/pion/mediadevices/pkg/driver/vncdriver"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/examples/internal/signal"
+	"github.com/pion/webrtc/v3"
+
+	// If you don't like x264, you can also use vpx by importing as below
+	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
+	// or you can also use openh264 for alternative h264 implementation
+	// "github.com/pion/mediadevices/pkg/codec/openh264"
+	// or if you use a raspberry pi like, you can use mmal for using its hardware encoder
+	// "github.com/pion/mediadevices/pkg/codec/mmal"
+	"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
+
+	// Note: If you don't have a camera or microphone or your adapters are not supported,
+	//       you can always swap your adapters with our dummy adapters below.
+	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
+	// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
+)
+
+func main() {
+	config := webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{"stun:stun.l.google.com:19302"},
+			},
+		},
+	}
+	driver.GetManager().Register(
+		vncdriver.NewVnc("127.0.0.1:5900"),
+		driver.Info{Label: "VNC", DeviceType: driver.Camera, Priority: driver.PriorityLow},
+	)
+	// Wait for the offer to be pasted
+	offer := webrtc.SessionDescription{}
+	signal.Decode(signal.MustReadStdin(), &offer)
+
+	// Create a new RTCPeerConnection
+	x264Params, err := x264.NewParams()
+	if err != nil {
+		panic(err)
+	}
+	x264Params.BitRate = 500_000 // 500kbps
+
+	if err != nil {
+		panic(err)
+	}
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaEngine := webrtc.MediaEngine{}
+	codecSelector.Populate(&mediaEngine)
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(&mediaEngine))
+	peerConnection, err := api.NewPeerConnection(config)
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+	})
+
+	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+
+		},
+		Codec: codecSelector,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	for _, track := range s.GetTracks() {
+		track.OnEnded(func(err error) {
+			fmt.Printf("Track (ID: %s) ended with error: %v\n",
+				track.ID(), err)
+		})
+
+		_, err = peerConnection.AddTransceiverFromTrack(track,
+			webrtc.RtpTransceiverInit{
+				Direction: webrtc.RTPTransceiverDirectionSendonly,
+			},
+		)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Set the remote SessionDescription
+	err = peerConnection.SetRemoteDescription(offer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create an answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// Create channel that is blocked until ICE Gathering is complete
+	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+
+	// Sets the LocalDescription, and starts our UDP listeners
+	err = peerConnection.SetLocalDescription(answer)
+	if err != nil {
+		panic(err)
+	}
+
+	// Block until ICE Gathering is complete, disabling trickle ICE
+	// we do this because we only can exchange one signaling message
+	// in a production application you should exchange ICE Candidates via OnICECandidate
+	<-gatherComplete
+
+	// Output the answer in base64 so we can paste it in browser
+	fmt.Println(signal.Encode(*peerConnection.LocalDescription()))
+
+	// Block forever
+	select {}
+}

examples/webrtc/.gitignore

@@ -0,0 +1 @@
+webrtc

examples/webrtc/main.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pion/mediadevices/examples/internal/signal"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 
 	// If you don't like x264, you can also use vpx by importing as below
 	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
@@ -57,10 +57,7 @@ func main() {
 
 	mediaEngine := webrtc.MediaEngine{}
 	codecSelector.Populate(&mediaEngine)
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(&mediaEngine))
 	peerConnection, err := api.NewPeerConnection(config)
 	if err != nil {
 		panic(err)
@@ -74,7 +71,7 @@ func main() {
 
 	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.FrameFormat = prop.FrameFormat(frame.FormatI420)
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
 		},
@@ -86,19 +83,13 @@ func main() {
 		panic(err)
 	}
 
-	for _, tracker := range s.GetTracks() {
-		tracker.OnEnded(func(err error) {
+	for _, track := range s.GetTracks() {
+		track.OnEnded(func(err error) {
 			fmt.Printf("Track (ID: %s) ended with error: %v\n",
-				tracker.ID(), err)
+				track.ID(), err)
 		})
 
-		// In Pion/webrtc v3, bind will be called automatically after SDP negotiation
-		webrtcTrack, err := tracker.Bind(peerConnection)
-		if err != nil {
-			panic(err)
-		}
-
-		_, err = peerConnection.AddTransceiverFromTrack(webrtcTrack,
+		_, err = peerConnection.AddTransceiverFromTrack(track,
 			webrtc.RtpTransceiverInit{
 				Direction: webrtc.RTPTransceiverDirectionSendonly,
 			},
@@ -120,13 +111,23 @@ func main() {
 		panic(err)
 	}
 
+	// Create channel that is blocked until ICE Gathering is complete
+	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+
 	// Sets the LocalDescription, and starts our UDP listeners
 	err = peerConnection.SetLocalDescription(answer)
 	if err != nil {
 		panic(err)
 	}
 
+	// Block until ICE Gathering is complete, disabling trickle ICE
+	// we do this because we only can exchange one signaling message
+	// in a production application you should exchange ICE Candidates via OnICECandidate
+	<-gatherComplete
+
 	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
+	fmt.Println(signal.Encode(*peerConnection.LocalDescription()))
+
+	// Block forever
 	select {}
 }

go.mod

@@ -3,16 +3,15 @@ module github.com/pion/mediadevices
 go 1.13
 
 require (
-	github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd // indirect
-	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/gen2brain/malgo v0.10.27
-	github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
-	github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
-	github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 // indirect
+	github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165
+	github.com/gen2brain/malgo v0.10.35
+	github.com/google/uuid v1.3.0
+	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329
+	github.com/pion/interceptor v0.1.10
 	github.com/pion/logging v0.2.2
-	github.com/pion/rtp v1.6.2
-	github.com/pion/webrtc/v2 v2.2.26
-	github.com/satori/go.uuid v1.2.0
-	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
+	github.com/pion/rtcp v1.2.9
+	github.com/pion/rtp v1.7.13
+	github.com/pion/webrtc/v3 v3.1.34
+	golang.org/x/image v0.0.0-20220617043117-41969df76e82
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -1,138 +1,173 @@
-github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd h1:u7K2oMFMd8APDV3fM1j2rO3U/XJf1g1qC3DDTKou8iM=
-github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
-github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
-github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
-github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165 h1:QsIbRyO2tn5eSJZ/skuDqSTo0GWI5H4G1AT7Mm2H0Nw=
+github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gen2brain/malgo v0.10.27 h1:KlNitZIO8V4W2VnjtTM8AGMy/XBb2pN+fnIB5bEps8E=
-github.com/gen2brain/malgo v0.10.27/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gen2brain/malgo v0.10.35 h1:D6aNo/Q0SnzQLHomTydTXxj4AJFdGJcVoE7I8JxPoUo=
+github.com/gen2brain/malgo v0.10.35/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
 github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f h1:5hWo+DzJQSOBl6X+TDac0SPWffRonuRJ2///OYtYRT8=
-github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240 h1:dy+DS31tGEGCsZzB45HmJJNHjur8GDgtRNX9U7HnSX4=
+github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240/go.mod h1:3P4UH/k22rXyHIJD2w4h2XMqPX4Of/eySEZq9L6wqc4=
+github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o=
+github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
-github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 h1:4BxFx5XCtXc+nFtXDGDW+Uu5sPtsAbvPh6RObj3fG9o=
-github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
-github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e h1:H+t6A/QJMbhCSEH5rAuRxh+CtW96g0Or0Fxa9IKr4uc=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
-github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
-github.com/pion/dtls/v2 v2.0.1/go.mod h1:uMQkz2W0cSqY00xav7WByQ4Hb+18xeQh2oH2fRezr5U=
-github.com/pion/dtls/v2 v2.0.2 h1:FHCHTiM182Y8e15aFTiORroiATUI16ryHiQh8AIOJ1E=
-github.com/pion/dtls/v2 v2.0.2/go.mod h1:27PEO3MDdaCfo21heT59/vsdmZc0zMt9wQPcSlLu/1I=
-github.com/pion/ice v0.7.18 h1:KbAWlzWRUdX9SmehBh3gYpIFsirjhSQsCw6K2MjYMK0=
-github.com/pion/ice v0.7.18/go.mod h1:+Bvnm3nYC6Nnp7VV6glUkuOfToB/AtMRZpOU8ihuf4c=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E=
+github.com/pion/datachannel v1.5.2/go.mod h1:FTGQWaHrdCwIJ1rw6xBIfZVkslikjShim5yr05XFuCQ=
+github.com/pion/dtls/v2 v2.1.3 h1:3UF7udADqous+M2R5Uo2q/YaP4EzUoWKdfX2oscCUio=
+github.com/pion/dtls/v2 v2.1.3/go.mod h1:o6+WvyLDAlXF7YiPB/RlskRoeK+/JtuaZa5emwQcWus=
+github.com/pion/ice/v2 v2.2.6 h1:R/vaLlI1J2gCx141L5PEwtuGAGcyS6e7E0hDeJFq5Ig=
+github.com/pion/ice/v2 v2.2.6/go.mod h1:SWuHiOGP17lGromHTFadUe1EuPgFh/oCU6FCMZHooVE=
+github.com/pion/interceptor v0.1.10 h1:DJ2GjMGm4XGIQgMJxuEpdaExdY/6RdngT7Uh4oVmquU=
+github.com/pion/interceptor v0.1.10/go.mod h1:Lh3JSl/cbJ2wP8I3ccrjh1K/deRGRn3UlSPuOTiHb6U=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
-github.com/pion/mdns v0.0.4 h1:O4vvVqr4DGX63vzmO6Fw9vpy3lfztVWHGCQfyw0ZLSY=
-github.com/pion/mdns v0.0.4/go.mod h1:R1sL0p50l42S5lJs91oNdUL58nm0QHrhxnSegr++qC0=
-github.com/pion/quic v0.1.1 h1:D951FV+TOqI9A0rTF7tHx0Loooqz+nyzjEyj8o3PuMA=
-github.com/pion/quic v0.1.1/go.mod h1:zEU51v7ru8Mp4AUBJvj6psrSth5eEFNnVQK5K48oV3k=
-github.com/pion/randutil v0.0.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
+github.com/pion/mdns v0.0.5 h1:Q2oj/JB3NqfzY9xGZ1fPzZzK7sDSD8rZPOvcIQ10BCw=
+github.com/pion/mdns v0.0.5/go.mod h1:UgssrvdD3mxpi8tMxAXbsppL3vJ4Jipw1mTCW+al01g=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
-github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
-github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
-github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
-github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U=
-github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
-github.com/pion/sctp v1.7.10 h1:o3p3/hZB5Cx12RMGyWmItevJtZ6o2cpuxaw6GOS4x+8=
-github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
-github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=
-github.com/pion/sdp/v2 v2.4.0/go.mod h1:L2LxrOpSTJbAns244vfPChbciR/ReU1KWfG04OpkR7E=
-github.com/pion/srtp v1.5.1 h1:9Q3jAfslYZBt+C69SI/ZcONJh9049JUHZWYRRf5KEKw=
-github.com/pion/srtp v1.5.1/go.mod h1:B+QgX5xPeQTNc1CJStJPHzOlHK66ViMDWTT0HZTCkcA=
+github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0=
+github.com/pion/rtcp v1.2.9 h1:1ujStwg++IOLIEoOiIQ2s+qBuJ1VN81KW+9pMPsif+U=
+github.com/pion/rtcp v1.2.9/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo=
+github.com/pion/rtp v1.7.0/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/rtp v1.7.4/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA=
+github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/sctp v1.8.0/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
+github.com/pion/sctp v1.8.2 h1:yBBCIrUMJ4yFICL3RIvR4eh/H2BTTvlligmSTy+3kiA=
+github.com/pion/sctp v1.8.2/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
+github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
+github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk=
+github.com/pion/srtp/v2 v2.0.5 h1:ks3wcTvIUE/GHndO3FAvROQ9opy0uLELpwHJaQ1yqhQ=
+github.com/pion/srtp/v2 v2.0.5/go.mod h1:8k6AJlal740mrZ6WYxc4Dg6qDqqhxoRG2GSjlUhDF0A=
 github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
 github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
-github.com/pion/transport v0.6.0/go.mod h1:iWZ07doqOosSLMhZ+FXUTq+TamDoXSllxpbGcfkCmbE=
-github.com/pion/transport v0.8.10/go.mod h1:tBmha/UCjpum5hqTWhfAEs3CO4/tHSg0MYRhSzR+CZ8=
-github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE=
-github.com/pion/transport v0.10.1 h1:2W+yJT+0mOQ160ThZYUx5Zp2skzshiNgxrNE9GUfhJM=
-github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
-github.com/pion/turn/v2 v2.0.4 h1:oDguhEv2L/4rxwbL9clGLgtzQPjtuZwCdoM7Te8vQVk=
-github.com/pion/turn/v2 v2.0.4/go.mod h1:1812p4DcGVbYVBTiraUmP50XoKye++AMkbfp+N27mog=
-github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
-github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
-github.com/pion/webrtc/v2 v2.2.26 h1:01hWE26pL3LgqfxvQ1fr6O4ZtyRFFJmQEZK39pHWfFc=
-github.com/pion/webrtc/v2 v2.2.26/go.mod h1:XMZbZRNHyPDe1gzTIHFcQu02283YO45CbiwFgKvXnmc=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
+github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A=
+github.com/pion/transport v0.13.0 h1:KWTA5ZrQogizzYwPEciGtHPLwpAjE91FgXnyu+Hv2uY=
+github.com/pion/transport v0.13.0/go.mod h1:yxm9uXpK9bpBBWkITk13cLo1y5/ur5VQpG22ny6EP7g=
+github.com/pion/turn/v2 v2.0.8 h1:KEstL92OUN3k5k8qxsXHpr7WWfrdp7iJZHx99ud8muw=
+github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw=
+github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
+github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
+github.com/pion/webrtc/v3 v3.1.34 h1:GUfv2zxWge77x1FhZ6Fge8KQd3bTzvX1CMN1/LmTazM=
+github.com/pion/webrtc/v3 v3.1.34/go.mod h1:jClfnbJzt8wtmewGxhPzgE5wZ0U/gNB78XCtmc/uz3k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
-golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/image v0.0.0-20200927104501-e162460cd6b5 h1:QelT11PB4FXiDEXucrfNckHoFxwt8USGY1ajP1ZF5lM=
-golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd h1:zYlwaUHTmxuf6H7hwO2dgwqozQmH7zf4x+/qql4oVWc=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
-golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

ioreader.go

@@ -1,5 +1,7 @@
 package mediadevices
 
+import "github.com/pion/mediadevices/pkg/codec"
+
 type EncodedBuffer struct {
 	Data    []byte
 	Samples uint32
@@ -8,11 +10,13 @@ type EncodedBuffer struct {
 type EncodedReadCloser interface {
 	Read() (EncodedBuffer, func(), error)
 	Close() error
+	codec.Controllable
 }
 
 type encodedReadCloserImpl struct {
-	readFn  func() (EncodedBuffer, func(), error)
-	closeFn func() error
+	readFn       func() (EncodedBuffer, func(), error)
+	closeFn      func() error
+	controllerFn func() codec.EncoderController
 }
 
 func (r *encodedReadCloserImpl) Read() (EncodedBuffer, func(), error) {
@@ -23,9 +27,14 @@ func (r *encodedReadCloserImpl) Close() error {
 	return r.closeFn()
 }
 
+func (r *encodedReadCloserImpl) Controller() codec.EncoderController {
+	return r.controllerFn()
+}
+
 type encodedIOReadCloserImpl struct {
-	readFn  func([]byte) (int, error)
-	closeFn func() error
+	readFn     func([]byte) (int, error)
+	closeFn    func() error
+	controller func() codec.EncoderController
 }
 
 func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserImpl {
@@ -48,7 +57,8 @@ func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserIm
 			encoded.Data = encoded.Data[n:]
 			return n, nil
 		},
-		closeFn: reader.Close,
+		closeFn:    reader.Close,
+		controller: reader.Controller,
 	}
 }
 
@@ -59,3 +69,7 @@ func (r *encodedIOReadCloserImpl) Read(b []byte) (int, error) {
 func (r *encodedIOReadCloserImpl) Close() error {
 	return r.closeFn()
 }
+
+func (r *encodedIOReadCloserImpl) Controller() codec.EncoderController {
+	return r.controller()
+}

mediadevices_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pion/mediadevices/pkg/driver"
 	_ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/videotest"
+	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -97,37 +98,145 @@ func TestSelectBestDriverConstraintsResultIsSetProperly(t *testing.T) {
 		t.Fatal("expect to get at least 1 property")
 	}
 	expectedProp := driver.Properties()[0]
-	// Since this is a continuous value, bestConstraints should be set with the value that user specified
-	expectedProp.FrameRate = 30.0
 
-	wantConstraints := MediaTrackConstraints{
-		MediaConstraints: prop.MediaConstraints{
-			VideoConstraints: prop.VideoConstraints{
-				// By reducing the width from the driver by a tiny amount, this property should be chosen.
-				// At the same time, we'll be able to find out if the return constraints will be properly set
-				// to the best constraints.
-				Width:       prop.Int(expectedProp.Width - 1),
-				Height:      prop.Int(expectedProp.Width),
-				FrameFormat: prop.FrameFormat(expectedProp.FrameFormat),
-				FrameRate:   prop.Float(30.0),
-			},
+	// By reducing the value from the driver by a tiny amount, this property should be chosen.
+	// At the same time, we'll be able to find out if the return constraints will be properly set
+	// to the best constraints.
+	cases := map[string]struct {
+		width, height int
+		frameFormat   frame.Format
+		frameRate     float32
+	}{
+		"DifferentWidth": {
+			width:       expectedProp.Width - 1,
+			height:      expectedProp.Height,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentHeight": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height - 1,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentFrameFormat": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height,
+			frameFormat: frame.FormatI420,
+			frameRate:   expectedProp.FrameRate,
 		},
 	}
 
-	bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			var vc prop.VideoConstraints
+
+			if c.frameRate >= 0 {
+				vc = prop.VideoConstraints{
+					Width:       prop.Int(c.width),
+					Height:      prop.Int(c.height),
+					FrameFormat: prop.FrameFormat(c.frameFormat),
+					FrameRate:   prop.Float(c.frameRate),
+				}
+			} else {
+				// do not specify the framerate
+				vc = prop.VideoConstraints{
+					Width:       prop.Int(c.width),
+					Height:      prop.Int(c.height),
+					FrameFormat: prop.FrameFormat(c.frameFormat),
+				}
+			}
+			wantConstraints := MediaTrackConstraints{
+				MediaConstraints: prop.MediaConstraints{
+					VideoConstraints: vc,
+				},
+			}
+
+			bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if driver != bestDriver {
+				t.Fatal("best driver is not expected")
+			}
+
+			s := bestConstraints.selectedMedia
+			if s.Width != expectedProp.Width ||
+				s.Height != expectedProp.Height ||
+				s.FrameFormat != expectedProp.FrameFormat ||
+				s.FrameRate != expectedProp.FrameRate {
+				t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
+			}
+
+		})
+	}
+
+}
+
+func TestSelectBestDriverConstraintsNoFit(t *testing.T) {
+	filterFn := driver.FilterVideoRecorder()
+	drivers := driver.GetManager().Query(filterFn)
+	if len(drivers) == 0 {
+		t.Fatal("expect to get at least 1 driver")
+	}
+
+	driver := drivers[0]
+	err := driver.Open()
 	if err != nil {
-		t.Fatal(err)
+		t.Fatal("expect to open driver successfully")
+	}
+	defer driver.Close()
+
+	if len(driver.Properties()) == 0 {
+		t.Fatal("expect to get at least 1 property")
+	}
+	expectedProp := driver.Properties()[0]
+
+	cases := map[string]struct {
+		width, height int
+		frameFormat   frame.Format
+		frameRate     float32
+	}{
+		"DifferentWidth": {
+			width:       expectedProp.Width - 1,
+			height:      expectedProp.Height,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentHeight": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height - 1,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentFrameFormat": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height,
+			frameFormat: frame.FormatI420,
+			frameRate:   expectedProp.FrameRate,
+		},
 	}
 
-	if driver != bestDriver {
-		t.Fatal("best driver is not expected")
-	}
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			wantConstraints := MediaTrackConstraints{
+				MediaConstraints: prop.MediaConstraints{
+					VideoConstraints: prop.VideoConstraints{
+						Width:       prop.IntExact(c.width),
+						Height:      prop.IntExact(c.height),
+						FrameFormat: prop.FrameFormatExact(c.frameFormat),
+						FrameRate:   prop.FloatExact(c.frameRate),
+					},
+				},
+			}
 
-	s := bestConstraints.selectedMedia
-	if s.Width != expectedProp.Width ||
-		s.Height != expectedProp.Height ||
-		s.FrameFormat != expectedProp.FrameFormat ||
-		s.FrameRate != expectedProp.FrameRate {
-		t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
+			_, _, err := selectBestDriver(filterFn, wantConstraints)
+			if err == nil {
+				t.Fatal("expect to not find a driver that fits the constraints")
+			}
+		})
 	}
 }

mediastream.go

@@ -2,6 +2,8 @@ package mediadevices
 
 import (
 	"sync"
+
+	"github.com/pion/webrtc/v3"
 )
 
 // MediaStream is an interface that represents a collection of existing tracks.
@@ -23,7 +25,7 @@ type mediaStream struct {
 	l      sync.RWMutex
 }
 
-const trackTypeDefault MediaDeviceType = 0
+const trackTypeDefault webrtc.RTPCodecType = 0
 
 // NewMediaStream creates a MediaStream interface that's defined in
 // https://w3c.github.io/mediacapture-main/#dom-mediastream
@@ -40,11 +42,11 @@ func NewMediaStream(tracks ...Track) (MediaStream, error) {
 }
 
 func (m *mediaStream) GetAudioTracks() []Track {
-	return m.queryTracks(AudioInput)
+	return m.queryTracks(webrtc.RTPCodecTypeAudio)
 }
 
 func (m *mediaStream) GetVideoTracks() []Track {
-	return m.queryTracks(VideoInput)
+	return m.queryTracks(webrtc.RTPCodecTypeVideo)
 }
 
 func (m *mediaStream) GetTracks() []Track {
@@ -53,7 +55,7 @@ func (m *mediaStream) GetTracks() []Track {
 
 // queryTracks returns all tracks that are the same kind as t.
 // If t is 0, which is the default, queryTracks will return all the tracks.
-func (m *mediaStream) queryTracks(t MediaDeviceType) []Track {
+func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Track {
 	m.l.RLock()
 	defer m.l.RUnlock()
 

mediastream_test.go

@@ -4,7 +4,7 @@ import (
 	"io"
 	"testing"
 
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 type mockMediaStreamTrack struct {
@@ -15,26 +15,41 @@ func (track *mockMediaStreamTrack) ID() string {
 	return ""
 }
 
+func (track *mockMediaStreamTrack) StreamID() string {
+	return ""
+}
+
+func (track *mockMediaStreamTrack) RID() string {
+	return ""
+}
+
 func (track *mockMediaStreamTrack) Close() error {
 	return nil
 }
 
-func (track *mockMediaStreamTrack) Kind() MediaDeviceType {
-	return track.kind
+func (track *mockMediaStreamTrack) Kind() webrtc.RTPCodecType {
+	switch track.kind {
+	case AudioInput:
+		return webrtc.RTPCodecTypeAudio
+	case VideoInput:
+		return webrtc.RTPCodecTypeVideo
+	default:
+		panic("invalid track kind")
+	}
 }
 
 func (track *mockMediaStreamTrack) OnEnded(handler func(error)) {
 }
 
-func (track *mockMediaStreamTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error) {
-	return nil, nil
+func (track *mockMediaStreamTrack) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) {
+	return webrtc.RTPCodecParameters{}, nil
 }
 
-func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
+func (track *mockMediaStreamTrack) Unbind(ctx webrtc.TrackLocalContext) error {
 	return nil
 }
 
-func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+func (track *mockMediaStreamTrack) NewRTPReader(codecName string, ssrc uint32, mtu int) (RTPReadCloser, error) {
 	return nil, nil
 }
 

pkg/avfoundation/AVFoundationBind/AVFoundationBind.h

@@ -45,6 +45,7 @@ typedef enum AVBindMediaType {
 typedef enum AVBindFrameFormat {
     AVBindFrameFormatI420,
     AVBindFrameFormatNV21,
+    AVBindFrameFormatNV12,
     AVBindFrameFormatYUY2,
     AVBindFrameFormatUYVY,
 } AVBindFrameFormat;

pkg/avfoundation/AVFoundationBind/AVFoundationBind.m

@@ -1,17 +1,17 @@
 // MIT License
-// 
+//
 // Copyright (c) 2019-2020 Pion
-// 
+//
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
 // furnished to do so, subject to the following conditions:
-// 
+//
 // The above copyright notice and this permission notice shall be included in all
 // copies or substantial portions of the Software.
-// 
+//
 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -76,29 +76,23 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
         !CMSampleBufferDataIsReady(sampleBuffer)) {
       return;
     }
-    
+
     CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
     if (imageBuffer == NULL) {
       return;
     }
-    
+
     imageBuffer = CVBufferRetain(imageBuffer);
     CVReturn ret =
         CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
     if (ret != kCVReturnSuccess) {
       return;
     }
-    
-    size_t heightY = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
-    size_t bytesPerRowY = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
-    
-    size_t heightUV = CVPixelBufferGetHeightOfPlane(imageBuffer, 1);
-    size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 1);
-    
-    int len = (int)((heightY * bytesPerRowY) + (2 * heightUV * bytesPerRowUV));
+
     void *buf = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
-    _mCallback(_mPUserData, buf, len);
-    
+    size_t dataSize = CVPixelBufferGetDataSize(imageBuffer);
+    _mCallback(_mPUserData, buf, (int)dataSize);
+
     CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
     CVBufferRelease(imageBuffer);
 }
@@ -133,13 +127,21 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
 
 STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
     STATUS retStatus = STATUS_OK;
+    // Useful mapping reference from ffmpeg:
+    // https://github.com/FFmpeg/FFmpeg/blob/c810a9502cebe32e1dd08ee3d0d17053dde44aa9/libavdevice/avfoundation.m#L53-L80
     switch (format) {
-        case AVBindFrameFormatNV21:
-            *pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+        case AVBindFrameFormatI420:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8Planar;
+            break;
+        case AVBindFrameFormatNV12:
+            *pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
             break;
         case AVBindFrameFormatUYVY:
             *pFourCC = kCVPixelFormatType_422YpCbCr8;
             break;
+        case AVBindFrameFormatYUY2:
+            *pFourCC = kCVPixelFormatType_422YpCbCr8_yuvs;
+            break;
         // TODO: Add the rest of frame formats
         default:
             retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
@@ -150,15 +152,22 @@ STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
 STATUS frameFormatFromFourCC(FourCharCode fourCC, AVBindFrameFormat *pFormat) {
     STATUS retStatus = STATUS_OK;
     switch (fourCC) {
-         case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
-             *pFormat = AVBindFrameFormatNV21;
-             break;
-         case kCVPixelFormatType_422YpCbCr8:
-             *pFormat = AVBindFrameFormatUYVY;
-             break;
+        case kCVPixelFormatType_420YpCbCr8Planar:
+            *pFormat = AVBindFrameFormatI420;
+            break;
+        case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+            *pFormat = AVBindFrameFormatNV12;
+            break;
+        case kCVPixelFormatType_422YpCbCr8:
+            *pFormat = AVBindFrameFormatUYVY;
+            break;
+        case kCVPixelFormatType_422YpCbCr8_yuvs:
+            *pFormat = AVBindFrameFormatYUY2;
+            break;
          // TODO: Add the rest of frame formats
-         default:
-             retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
+        default:
+            retStatus = STATUS_UNSUPPORTED_FRAME_FORMAT;
      }
     return retStatus;
 }
@@ -170,7 +179,7 @@ STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *p
     NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
     CHK(mediaType == AVBindMediaTypeVideo || mediaType == AVBindMediaTypeAudio, STATUS_UNSUPPORTED_MEDIA_TYPE);
     CHK(ppDevices != NULL && pLen != NULL, STATUS_NULL_ARG);
-    
+
     PAVBindDevice pDevice;
     AVMediaType _mediaType = mediaType == AVBindMediaTypeVideo ? AVMediaTypeVideo : AVMediaTypeAudio;
     NSArray *refAllTypes = @[
@@ -182,22 +191,22 @@ STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *p
         discoverySessionWithDeviceTypes: refAllTypes
         mediaType: _mediaType
         position: AVCaptureDevicePositionUnspecified];
-    
+
     int i = 0;
     for (AVCaptureDevice *refDevice in refSession.devices) {
         if (i >= MAX_DEVICES) {
             break;
         }
-        
+
         pDevice = devices + i;
         strncpy(pDevice->uid, refDevice.uniqueID.UTF8String, MAX_DEVICE_UID_CHARS);
         pDevice->uid[MAX_DEVICE_UID_CHARS] = '\0';
         i++;
     }
-    
+
     *ppDevices = devices;
     *pLen = i;
-    
+
 cleanup:
     [refPool drain];
     return retStatus;
@@ -217,7 +226,7 @@ STATUS AVBindSessionInit(AVBindDevice device, PAVBindSession *ppSessionResult) {
     pSession->device = device;
     pSession->refCaptureSession = NULL;
     *ppSessionResult = pSession;
-    
+
 cleanup:
     return retStatus;
 }
@@ -244,15 +253,15 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
     STATUS retStatus = STATUS_OK;
     NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
     CHK(pSession != NULL && dataCallback != NULL, STATUS_NULL_ARG);
-    
+
     AVCaptureDeviceInput *refInput;
     NSError *refErr = NULL;
     NSString *refUID = [NSString stringWithUTF8String: pSession->device.uid];
     AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refUID];
-    
+
     refInput = [[AVCaptureDeviceInput alloc] initWithDevice: refDevice error: &refErr];
     CHK(refErr == NULL, STATUS_DEVICE_INIT_FAILED);
-    
+
     AVCaptureSession *refCaptureSession = [[AVCaptureSession alloc] init];
     refCaptureSession.sessionPreset = AVCaptureSessionPresetMedium;
     [refCaptureSession addInput: refInput];
@@ -261,7 +270,7 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
         VideoDataDelegate *pDelegate = [[VideoDataDelegate alloc]
                                         init: dataCallback
                                         withUserData: pUserData];
-        
+
         AVCaptureVideoDataOutput *pOutput = [[AVCaptureVideoDataOutput alloc] init];
         FourCharCode fourCC;
         CHK_STATUS(frameFormatToFourCC(property.frameFormat, &fourCC));
@@ -279,10 +288,10 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
     } else {
         // TODO: implement audio pipeline
     }
-    
+
     pSession->refCaptureSession = [refCaptureSession retain];
     [refCaptureSession startRunning];
-    
+
 cleanup:
     [refPool drain];
     return retStatus;
@@ -293,20 +302,30 @@ STATUS AVBindSessionClose(PAVBindSession pSession) {
     STATUS retStatus = STATUS_OK;
     CHK(pSession != NULL, STATUS_NULL_ARG);
     CHK(pSession->refCaptureSession != NULL, STATUS_OK);
-    
+
     [pSession->refCaptureSession stopRunning];
     [pSession->refCaptureSession release];
     pSession->refCaptureSession = NULL;
-        
+
 cleanup:
     return retStatus;
 }
 
+static NSString* FourCCString(FourCharCode code) {
+    NSString *result = [NSString stringWithFormat:@"%c%c%c%c",
+                        (code >> 24) & 0xff,
+                        (code >> 16) & 0xff,
+                        (code >> 8) & 0xff,
+                        code & 0xff];
+    NSCharacterSet *characterSet = [NSCharacterSet whitespaceCharacterSet];
+    return [result stringByTrimmingCharactersInSet:characterSet];
+}
+
 STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *ppProperties, int *pLen) {
     STATUS retStatus = STATUS_OK;
     NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
     CHK(pSession != NULL && ppProperties != NULL && pLen != NULL, STATUS_NULL_ARG);
-    
+
     NSString *refDeviceUID = [NSString stringWithUTF8String: pSession->device.uid];
     AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refDeviceUID];
     FourCharCode fourCC;
@@ -319,15 +338,17 @@ STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *pp
     for (AVCaptureDeviceFormat *refFormat in refDevice.formats) {
         // TODO: Probably gives a warn to the user
         if (len >= MAX_PROPERTIES) {
+            NSLog(@"[WARNING] skipping the rest of properties due to MAX_PROPERTIES");
             break;
         }
-        
+
         if ([refFormat.mediaType isEqual:AVMediaTypeVideo]) {
             fourCC = CMFormatDescriptionGetMediaSubType(refFormat.formatDescription);
             if (frameFormatFromFourCC(fourCC, &pProperty->frameFormat) != STATUS_OK) {
+                NSLog(@"[WARNING] skipping %@ %dx%d since it's not supported", FourCCString(fourCC), videoDimensions.width, videoDimensions.height);
                 continue;
             }
-            
+
             videoFormat = (CMVideoFormatDescriptionRef) refFormat.formatDescription;
             videoDimensions = CMVideoFormatDescriptionGetDimensions(videoFormat);
             pProperty->height = videoDimensions.height;
@@ -335,16 +356,16 @@ STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *pp
         } else {
             // TODO: Get audio properties
         }
-        
+
         pProperty++;
         len++;
     }
-    
+
     *ppProperties = pSession->properties;
     *pLen = len;
-    
+
 cleanup:
-    
+
     [refPool drain];
     return retStatus;
 }

pkg/avfoundation/avfoundation_callback_darwin.go

@@ -1,6 +1,5 @@
 package avfoundation
 
-// extern void onData(void*, void*, int);
 import "C"
 import (
 	"sync"
@@ -18,11 +17,10 @@ type handleID int
 
 //export onData
 func onData(userData unsafe.Pointer, buf unsafe.Pointer, length C.int) {
-	data := C.GoBytes(buf, length)
-
 	handleNum := (*C.int)(userData)
 	cb, ok := lookup(handleID(*handleNum))
 	if ok {
+		data := C.GoBytes(buf, length)
 		cb(data)
 	}
 }

pkg/avfoundation/avfoundation_darwin.go

@@ -11,6 +11,7 @@ package avfoundation
 // }
 import "C"
 import (
+	"context"
 	"fmt"
 	"io"
 	"unsafe"
@@ -40,6 +41,8 @@ func frameFormatToAVBind(f frame.Format) (C.AVBindFrameFormat, bool) {
 		return C.AVBindFrameFormatI420, true
 	case frame.FormatNV21:
 		return C.AVBindFrameFormatNV21, true
+	case frame.FormatNV12:
+		return C.AVBindFrameFormatNV12, true
 	case frame.FormatYUY2:
 		return C.AVBindFrameFormatYUY2, true
 	case frame.FormatUYVY:
@@ -55,6 +58,8 @@ func frameFormatFromAVBind(f C.AVBindFrameFormat) (frame.Format, bool) {
 		return frame.FormatI420, true
 	case C.AVBindFrameFormatNV21:
 		return frame.FormatNV21, true
+	case C.AVBindFrameFormatNV12:
+		return frame.FormatNV12, true
 	case C.AVBindFrameFormatYUY2:
 		return frame.FormatYUY2, true
 	case C.AVBindFrameFormatUYVY:
@@ -89,9 +94,11 @@ func Devices(mediaType MediaType) ([]Device, error) {
 // ReadCloser is a wrapper around the data callback from AVFoundation. The data received from the
 // the underlying callback can be retrieved by calling Read.
 type ReadCloser struct {
-	dataChan chan []byte
-	id       handleID
-	onClose  func()
+	dataChan   chan []byte
+	id         handleID
+	onClose    func()
+	cancelCtx  context.Context
+	cancelFunc func()
 }
 
 func newReadCloser(onClose func()) *ReadCloser {
@@ -99,12 +106,22 @@ func newReadCloser(onClose func()) *ReadCloser {
 	rc.dataChan = make(chan []byte, 1)
 	rc.onClose = onClose
 	rc.id = register(rc.dataCb)
+	cancelCtx, cancelFunc := context.WithCancel(context.Background())
+	rc.cancelCtx = cancelCtx
+	rc.cancelFunc = cancelFunc
 	return &rc
 }
 
 func (rc *ReadCloser) dataCb(data []byte) {
 	// TODO: add a policy for slow reader
-	rc.dataChan <- data
+	if rc.cancelCtx.Err() != nil {
+		return
+	}
+	select {
+	case <-rc.cancelCtx.Done():
+		close(rc.dataChan)
+	case rc.dataChan <- data:
+	}
 }
 
 // Read reads raw data, the format is determined by the media type and property:
@@ -123,7 +140,7 @@ func (rc *ReadCloser) Close() {
 	if rc.onClose != nil {
 		rc.onClose()
 	}
-	close(rc.dataChan)
+	rc.cancelFunc()
 	unregister(rc.id)
 }
 

pkg/codec/codec.go

@@ -1,35 +1,91 @@
 package codec
 
 import (
+	"time"
+
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/rtp"
+	"github.com/pion/rtp/codecs"
+	"github.com/pion/webrtc/v3"
 )
 
 // RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.
 type RTPCodec struct {
-	*webrtc.RTPCodec
+	webrtc.RTPCodecParameters
+	rtp.Payloader
+
+	// Latency of static frame size codec.
+	Latency time.Duration
 }
 
 // NewRTPH264Codec is a helper to create an H264 codec
 func NewRTPH264Codec(clockrate uint32) *RTPCodec {
-	return &RTPCodec{webrtc.NewRTPH264Codec(webrtc.DefaultPayloadTypeH264, clockrate)}
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeH264,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 125,
+		},
+		Payloader: &codecs.H264Payloader{},
+	}
 }
 
 // NewRTPVP8Codec is a helper to create an VP8 codec
 func NewRTPVP8Codec(clockrate uint32) *RTPCodec {
-	return &RTPCodec{webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, clockrate)}
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeVP8,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 96,
+		},
+		Payloader: &codecs.VP8Payloader{},
+	}
 }
 
 // NewRTPVP9Codec is a helper to create an VP9 codec
 func NewRTPVP9Codec(clockrate uint32) *RTPCodec {
-	return &RTPCodec{webrtc.NewRTPVP9Codec(webrtc.DefaultPayloadTypeVP9, clockrate)}
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeVP9,
+				ClockRate:    90000,
+				Channels:     0,
+				SDPFmtpLine:  "",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 98,
+		},
+		Payloader: &codecs.VP9Payloader{},
+	}
 }
 
 // NewRTPOpusCodec is a helper to create an Opus codec
 func NewRTPOpusCodec(clockrate uint32) *RTPCodec {
-	return &RTPCodec{webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, clockrate)}
+	return &RTPCodec{
+		RTPCodecParameters: webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeOpus,
+				ClockRate:    48000,
+				Channels:     2,
+				SDPFmtpLine:  "minptime=10;useinbandfec=1",
+				RTCPFeedback: nil,
+			},
+			PayloadType: 111,
+		},
+		Payloader: &codecs.OpusPayloader{},
+	}
 }
 
 // AudioEncoderBuilder is the interface that wraps basic operations that are
@@ -56,15 +112,37 @@ type VideoEncoderBuilder interface {
 	BuildVideoEncoder(r video.Reader, p prop.Media) (ReadCloser, error)
 }
 
-// ReadCloser is an io.ReadCloser with methods for rate limiting: SetBitRate and ForceKeyFrame
+// ReadCloser is an io.ReadCloser with a controller
 type ReadCloser interface {
 	Read() (b []byte, release func(), err error)
 	Close() error
+	Controllable
+}
+
+// EncoderController is the interface allowing to control the encoder behaviour after it's initialisation.
+// It will possibly have common control method in the future.
+// A controller can have optional methods represented by *Controller interfaces
+type EncoderController interface{}
+
+// Controllable is a interface representing a encoder which can be controlled
+// after it's initialisation with an EncoderController
+type Controllable interface {
+	Controller() EncoderController
+}
+
+// KeyFrameController is a interface representing an encoder that can be forced to produce key frame on demand
+type KeyFrameController interface {
+	EncoderController
+	// ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame.
+	ForceKeyFrame() error
+}
+
+// BitRateController is a interface representing an encoder which can have a variable bit rate
+type BitRateController interface {
+	EncoderController
 	// SetBitRate sets current target bitrate, lower bitrate means smaller data will be transmitted
 	// but this also means that the quality will also be lower.
 	SetBitRate(int) error
-	// ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame.
-	ForceKeyFrame() error
 }
 
 // BaseParams represents an codec's encoding properties

pkg/codec/internal/codectest/codectest.go

@@ -0,0 +1,121 @@
+// Package codectest provides shared test for codec implementations.
+package codectest
+
+import (
+	"image"
+	"io"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/io/audio"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+func assertNoPanic(t *testing.T, fn func() error, msg string) error {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("panic: %v: %s", r, msg)
+		}
+	}()
+	return fn()
+}
+
+func AudioEncoderSimpleReadTest(t *testing.T, c codec.AudioEncoderBuilder, p prop.Media, w wave.Audio) {
+	var eof bool
+	enc, err := c.BuildAudioEncoder(audio.ReaderFunc(func() (wave.Audio, func(), error) {
+		if eof {
+			return nil, nil, io.EOF
+		}
+		return w, nil, nil
+	}), p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 16; i++ {
+		b, release, err := enc.Read()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(b) == 0 {
+			t.Fatal("Encoded frame is empty")
+		}
+		release()
+	}
+
+	eof = true
+	if _, _, err := enc.Read(); err != io.EOF {
+		t.Fatalf("Expected EOF, got %v", err)
+	}
+
+	if err := enc.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func VideoEncoderSimpleReadTest(t *testing.T, c codec.VideoEncoderBuilder, p prop.Media, img image.Image) {
+	var eof bool
+	enc, err := c.BuildVideoEncoder(video.ReaderFunc(func() (image.Image, func(), error) {
+		if eof {
+			return nil, nil, io.EOF
+		}
+		return img, nil, nil
+	}), p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 16; i++ {
+		b, release, err := enc.Read()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(b) == 0 {
+			t.Errorf("Encoded frame is empty (%d)", i)
+		}
+		release()
+	}
+
+	eof = true
+	if _, _, err := enc.Read(); err != io.EOF {
+		t.Fatalf("Expected EOF, got %v", err)
+	}
+
+	if err := enc.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func AudioEncoderCloseTwiceTest(t *testing.T, c codec.AudioEncoderBuilder, p prop.Media) {
+	enc, err := c.BuildAudioEncoder(audio.ReaderFunc(func() (wave.Audio, func(), error) {
+		return nil, nil, io.EOF
+	}), p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := assertNoPanic(t, enc.Close, "on first Close()"); err != nil {
+		t.Fatal(err)
+	}
+	if err := assertNoPanic(t, enc.Close, "on second Close()"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func VideoEncoderCloseTwiceTest(t *testing.T, c codec.VideoEncoderBuilder, p prop.Media) {
+	enc, err := c.BuildVideoEncoder(video.ReaderFunc(func() (image.Image, func(), error) {
+		return nil, nil, io.EOF
+	}), p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := assertNoPanic(t, enc.Close, "on first Close()"); err != nil {
+		t.Fatal(err)
+	}
+	if err := assertNoPanic(t, enc.Close, "on second Close()"); err != nil {
+		t.Fatal(err)
+	}
+}

pkg/codec/mmal/mmal.go

@@ -91,12 +91,8 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
-
-func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {

pkg/codec/mmal/mmal_test.go

@@ -0,0 +1,65 @@
+package mmal
+
+import (
+	"image"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestEncoder(t *testing.T) {
+	t.Run("SimpleRead", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.VideoEncoderSimpleReadTest(t, &p,
+			prop.Media{
+				Video: prop.Video{
+					Width:       256,
+					Height:      144,
+					FrameFormat: frame.FormatI420,
+				},
+			},
+			image.NewYCbCr(
+				image.Rect(0, 0, 256, 144),
+				image.YCbCrSubsampleRatio420,
+			),
+		)
+	})
+	t.Run("CloseTwice", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.VideoEncoderCloseTwiceTest(t, &p, prop.Media{
+			Video: prop.Video{
+				Width:       640,
+				Height:      480,
+				FrameRate:   30,
+				FrameFormat: frame.FormatI420,
+			},
+		})
+	})
+}
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/openh264/Makefile

@@ -0,0 +1,47 @@
+git_url := https://github.com/cisco/openh264.git
+version := v2.1.1
+src_root_dir := src
+lib_dir := lib
+include_dir := include/openh264
+lib_prefix := libopenh264
+src_dir := $(src_root_dir)/$(MEDIADEVICES_TARGET_PLATFORM)
+output_path := $(lib_dir)/$(lib_prefix)-$(MEDIADEVICES_TARGET_PLATFORM).a
+
+# OS and Arch mapping to OpenH264 parameters
+ifeq (windows,$(MEDIADEVICES_TARGET_OS))
+	os := mingw_nt
+else
+	os := $(MEDIADEVICES_TARGET_OS)
+endif
+
+ifneq (,$(findstring $(MEDIADEVICES_TARGET_ARCH),armv6 armv7 armv8))
+	arch := arm
+else ifeq (x64,$(MEDIADEVICES_TARGET_ARCH))
+	arch := x86_64
+else
+	arch := $(MEDIADEVICES_TARGET_ARCH)
+endif
+
+.PHONY: all
+all: guard-MEDIADEVICES_TARGET_PLATFORM guard-MEDIADEVICES_TARGET_PLATFORM \
+	guard-MEDIADEVICES_TARGET_OS guard-MEDIADEVICES_TARGET_ARCH \
+	$(output_path) headers
+
+headers: | $(src_dir) $(include_dir)
+	@cp $(src_dir)/codec/api/svc/*.h $(include_dir)
+
+$(output_path): $(src_dir)/$(lib_prefix).a | $(lib_dir)
+	@cp $< $@
+
+$(src_dir)/$(lib_prefix).a: | $(src_dir)
+	$(MEDIADEVICES_TOOLCHAIN_BIN) make --directory=$(src_dir) $(lib_prefix).a \
+		OS=$(os) ARCH=$(arch)
+
+$(src_dir): | $(src_root_dir)
+	git clone --depth=1 --branch=$(version) $(git_url) $@
+
+$(src_root_dir) $(lib_dir) $(include_dir):
+	@mkdir -p $@
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

pkg/codec/openh264/bridge.cpp

@@ -21,29 +21,26 @@ Encoder *enc_new(const EncoderOptions opts, int *eresult) {
     return NULL;
   }
 
-  // TODO: Remove hardcoded values
-  params.iUsageType = CAMERA_VIDEO_REAL_TIME;
+  params.iUsageType = opts.usage_type;
   params.iPicWidth = opts.width;
   params.iPicHeight = opts.height;
   params.iTargetBitrate = opts.target_bitrate;
   params.iMaxBitrate = opts.target_bitrate;
-  params.iRCMode = RC_BITRATE_MODE;
+  params.iRCMode = opts.rc_mode;
   params.fMaxFrameRate = opts.max_fps;
-  params.bEnableFrameSkip = true;
-  params.uiMaxNalSize = 0;
-  params.uiIntraPeriod = 30;
-  // set to 0, so that it'll automatically use multi threads when needed
-  params.iMultipleThreadIdc = 0;
+  params.bEnableFrameSkip = opts.enable_frame_skip;
+  params.uiMaxNalSize = opts.max_nal_size;
+  params.uiIntraPeriod = opts.intra_period;
+  params.iMultipleThreadIdc = opts.multiple_thread_idc;
   // The base spatial layer 0 is the only one we use.
   params.sSpatialLayers[0].iVideoWidth = params.iPicWidth;
   params.sSpatialLayers[0].iVideoHeight = params.iPicHeight;
   params.sSpatialLayers[0].fFrameRate = params.fMaxFrameRate;
   params.sSpatialLayers[0].iSpatialBitrate = params.iTargetBitrate;
   params.sSpatialLayers[0].iMaxSpatialBitrate = params.iTargetBitrate;
-  // Single NAL unit mode
-  params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
-  params.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_SIZELIMITED_SLICE;
-  params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = 12800;
+  params.sSpatialLayers[0].sSliceArgument.uiSliceNum = opts.slice_num;
+  params.sSpatialLayers[0].sSliceArgument.uiSliceMode = opts.slice_mode;
+  params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = opts.slice_size_constraint;
 
   rv = engine->InitializeExt(&params);
   if (rv != 0) {
@@ -80,12 +77,16 @@ Slice enc_encode(Encoder *e, Frame f, int *eresult) {
   SFrameBSInfo info = {0};
   Slice payload = {0};
 
+  if(e->force_key_frame == 1) {
+    e->engine->ForceIntraFrame(true);
+    e->force_key_frame = 0;
+  }
+
   pic.iPicWidth = f.width;
   pic.iPicHeight = f.height;
   pic.iColorFormat = videoFormatI420;
-  // We always received I420 format
-  pic.iStride[0] = pic.iPicWidth;
-  pic.iStride[1] = pic.iStride[2] = pic.iPicWidth / 2;
+  pic.iStride[0] = f.ystride;
+  pic.iStride[1] = pic.iStride[2] = f.cstride;
   pic.pData[0] = (unsigned char *)f.y;
   pic.pData[1] = (unsigned char *)f.u;
   pic.pData[2] = (unsigned char *)f.v;

pkg/codec/openh264/bridge.hpp

@@ -12,6 +12,8 @@ typedef struct Slice {
 
 typedef struct Frame {
   void *y, *u, *v;
+  int ystride;
+  int cstride;
   int height;
   int width;
 } Frame;
@@ -20,6 +22,15 @@ typedef struct EncoderOptions {
   int width, height;
   int target_bitrate;
   float max_fps;
+  EUsageType usage_type;
+  RC_MODES rc_mode;
+  bool enable_frame_skip;
+  unsigned int max_nal_size;
+  unsigned int intra_period;
+  int multiple_thread_idc;
+  unsigned int slice_num;
+  SliceModeEnum slice_mode;
+  unsigned int slice_size_constraint;
 } EncoderOptions;
 
 typedef struct Encoder {
@@ -27,6 +38,7 @@ typedef struct Encoder {
   ISVCEncoder *engine;
   unsigned char *buff;
   int buff_size;
+  int force_key_frame;
 } Encoder;
 
 Encoder *enc_new(const EncoderOptions params, int *eresult);

pkg/codec/openh264/build.sh

@@ -1,36 +0,0 @@
-GIT_URL=https://github.com/cisco/openh264.git
-VERSION=v2.1.1
-SRC_DIR=src
-LIB_DIR=lib
-INCLUDE_DIR=include/openh264
-ROOT_DIR=${PWD}
-LIB_PREFIX=libopenh264
-
-OS=${MEDIADEVICES_TARGET_OS}
-ARCH=${MEDIADEVICES_TARGET_ARCH}
-
-case ${MEDIADEVICES_TARGET_OS} in
-  windows)
-    OS=mingw_nt
-    ;;
-esac
-
-case ${MEDIADEVICES_TARGET_ARCH} in
-  armv6 | armv7 | armv8)
-    ARCH=arm
-    ;;
-  x64)
-    ARCH=x86_64
-    ;;
-esac
-
-mkdir -p ${LIB_DIR} ${INCLUDE_DIR}
-
-git clone --depth=1 --branch=${VERSION} ${GIT_URL} ${SRC_DIR}
-cd ${SRC_DIR}
-${MEDIADEVICES_TOOLCHAIN_BIN} make -j2 ${LIB_PREFIX}.a OS=${OS} ARCH=${ARCH}
-${MEDIADEVICES_TOOLCHAIN_BIN} echo $PATH
-mv ${LIB_PREFIX}.a ${ROOT_DIR}/${LIB_DIR}/${LIB_PREFIX}_${MEDIADEVICES_TARGET_PLATFORM}.a
-mkdir -p ${ROOT_DIR}/${INCLUDE_DIR}
-cp codec/api/svc/*.h ${ROOT_DIR}/${INCLUDE_DIR}
-make clean

pkg/codec/openh264/include/openh264/codec_ver.h

@@ -4,12 +4,12 @@
 
 #include "codec_app_def.h"
 
-static const OpenH264Version g_stCodecVersion  = {2, 1, 0, 2002};
-static const char* const g_strCodecVer  = "OpenH264 version:2.1.0.2002";
+static const OpenH264Version g_stCodecVersion  = {2, 1, 1, 2005};
+static const char* const g_strCodecVer  = "OpenH264 version:2.1.1.2005";
 
 #define OPENH264_MAJOR (2)
 #define OPENH264_MINOR (1)
-#define OPENH264_REVISION (0)
-#define OPENH264_RESERVED (2002)
+#define OPENH264_REVISION (1)
+#define OPENH264_RESERVED (2005)
 
 #endif  // CODEC_VER_H

pkg/codec/openh264/openh264.go

@@ -33,10 +33,19 @@ func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser,
 
 	var rv C.int
 	cEncoder := C.enc_new(C.EncoderOptions{
-		width:          C.int(p.Width),
-		height:         C.int(p.Height),
-		target_bitrate: C.int(params.BitRate),
-		max_fps:        C.float(p.FrameRate),
+		width:                 C.int(p.Width),
+		height:                C.int(p.Height),
+		target_bitrate:        C.int(params.BitRate),
+		max_fps:               C.float(p.FrameRate),
+		usage_type:            C.EUsageType(params.UsageType),
+		rc_mode:               C.RC_MODES(params.RCMode),
+		enable_frame_skip:     C.bool(params.EnableFrameSkip),
+		max_nal_size:          C.uint(params.MaxNalSize),
+		intra_period:          C.uint(params.IntraPeriod),
+		multiple_thread_idc:   C.int(params.MultipleThreadIdc),
+		slice_num:             C.uint(params.SliceNum),
+		slice_mode:            C.SliceModeEnum(params.SliceMode),
+		slice_size_constraint: C.uint(params.SliceSizeConstraint),
 	}, &rv)
 	if err := errResult(rv); err != nil {
 		return nil, fmt.Errorf("failed in creating encoder: %v", err)
@@ -65,11 +74,13 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	bounds := yuvImg.Bounds()
 	var rv C.int
 	s := C.enc_encode(e.engine, C.Frame{
-		y:      unsafe.Pointer(&yuvImg.Y[0]),
-		u:      unsafe.Pointer(&yuvImg.Cb[0]),
-		v:      unsafe.Pointer(&yuvImg.Cr[0]),
-		height: C.int(bounds.Max.Y - bounds.Min.Y),
-		width:  C.int(bounds.Max.X - bounds.Min.X),
+		y:       unsafe.Pointer(&yuvImg.Y[0]),
+		u:       unsafe.Pointer(&yuvImg.Cb[0]),
+		v:       unsafe.Pointer(&yuvImg.Cr[0]),
+		ystride: C.int(yuvImg.YStride),
+		cstride: C.int(yuvImg.CStride),
+		height:  C.int(bounds.Max.Y - bounds.Min.Y),
+		width:   C.int(bounds.Max.X - bounds.Min.X),
 	}, &rv)
 	if err := errResult(rv); err != nil {
 		return nil, func() {}, fmt.Errorf("failed in encoding: %v", err)
@@ -79,18 +90,23 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, nil
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
+func (e *encoder) ForceKeyFrame() error {
+	e.engine.force_key_frame = C.int(1)
+	return nil
 }
 
-func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	if e.closed {
+		return nil
+	}
+
 	e.closed = true
 
 	var rv C.int

pkg/codec/openh264/openh264_static.go

@@ -4,9 +4,10 @@ package openh264
 
 //#cgo CFLAGS: -I${SRCDIR}/include
 //#cgo CXXFLAGS: -I${SRCDIR}/include
-//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_armv7.a
-//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_arm64.a
-//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_x64.a
-//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_darwin_x64.a
-//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_windows_x64.a -lssp
+//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-armv7.a
+//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-arm64.a
+//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-x64.a
+//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-arm64.a
+//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-windows-x64.a -lssp
 import "C"

pkg/codec/openh264/openh264_test.go

@@ -0,0 +1,63 @@
+package openh264
+
+import (
+	"image"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
+func TestEncoder(t *testing.T) {
+	t.Run("SimpleRead", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.VideoEncoderSimpleReadTest(t, &p,
+			prop.Media{
+				Video: prop.Video{
+					Width:       256,
+					Height:      144,
+					FrameFormat: frame.FormatI420,
+				},
+			},
+			image.NewYCbCr(
+				image.Rect(0, 0, 256, 144),
+				image.YCbCrSubsampleRatio420,
+			),
+		)
+	})
+	t.Run("CloseTwice", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.VideoEncoderCloseTwiceTest(t, &p, prop.Media{
+			Video: prop.Video{
+				Width:       640,
+				Height:      480,
+				FrameRate:   30,
+				FrameFormat: frame.FormatI420,
+			},
+		})
+	})
+}

pkg/codec/openh264/params.go

@@ -1,5 +1,8 @@
 package openh264
 
+// #include <openh264/codec_api.h>
+import "C"
+
 import (
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/video"
@@ -9,14 +12,62 @@ import (
 // Params stores libopenh264 specific encoding parameters.
 type Params struct {
 	codec.BaseParams
+	UsageType           UsageTypeEnum
+	RCMode              RCModeEnum
+	EnableFrameSkip     bool
+	MaxNalSize          uint
+	IntraPeriod         uint
+	MultipleThreadIdc   int
+	SliceNum            uint
+	SliceMode           SliceModeEnum
+	SliceSizeConstraint uint
 }
 
+type UsageTypeEnum int
+
+const (
+	CameraVideoRealTime      UsageTypeEnum = C.CAMERA_VIDEO_REAL_TIME   ///< camera video for real-time communication
+	ScreenContentRealTime    UsageTypeEnum = C.SCREEN_CONTENT_REAL_TIME ///< screen content signal
+	CameraVideoNonRealTime   UsageTypeEnum = C.CAMERA_VIDEO_NON_REAL_TIME
+	ScreenContentNonRealTime UsageTypeEnum = C.SCREEN_CONTENT_NON_REAL_TIME
+	InputContentTypeAll      UsageTypeEnum = C.INPUT_CONTENT_TYPE_ALL
+)
+
+type RCModeEnum int
+
+const (
+	RCQualityMode         RCModeEnum = C.RC_QUALITY_MODE           ///< quality mode
+	RCBitrateMode         RCModeEnum = C.RC_BITRATE_MODE           ///< bitrate mode
+	RCBufferbaseedMode    RCModeEnum = C.RC_BUFFERBASED_MODE       ///< no bitrate control,only using buffer status,adjust the video quality
+	RCTimestampMode       RCModeEnum = C.RC_TIMESTAMP_MODE         //rate control based timestamp
+	RCBitrateModePostSkip RCModeEnum = C.RC_BITRATE_MODE_POST_SKIP ///< this is in-building RC MODE, WILL BE DELETED after algorithm tuning!
+	RCOffMode             RCModeEnum = C.RC_OFF_MODE               ///< rate control off mode
+)
+
+type SliceModeEnum uint
+
+const (
+	SMSingleSlice      SliceModeEnum = C.SM_SINGLE_SLICE      ///< | SliceNum==1
+	SMFixedslcnumSlice SliceModeEnum = C.SM_FIXEDSLCNUM_SLICE ///< | according to SliceNum        | enabled dynamic slicing for multi-thread
+	SMRasterSlice      SliceModeEnum = C.SM_RASTER_SLICE      ///< | according to SlicesAssign    | need input of MB numbers each slice. In addition, if other constraint in SSliceArgument is presented, need to follow the constraints. Typically if MB num and slice size are both constrained, re-encoding may be involved.
+	SMSizelimitedSlice SliceModeEnum = C.SM_SIZELIMITED_SLICE ///< | according to SliceSize       | slicing according to size, the slicing will be dynamic(have no idea about slice_nums until encoding current frame)
+)
+
 // NewParams returns default openh264 codec specific parameters.
 func NewParams() (Params, error) {
 	return Params{
 		BaseParams: codec.BaseParams{
 			BitRate: 100000,
 		},
+		UsageType:           CameraVideoRealTime,
+		RCMode:              RCBitrateMode,
+		EnableFrameSkip:     true,
+		MaxNalSize:          0,
+		IntraPeriod:         30,
+		MultipleThreadIdc:   0, // Defaults to 0, so that it'll automatically use multi threads when needed
+		SliceNum:            1, // Defaults to single NAL unit mode
+		SliceMode:           SMSizelimitedSlice,
+		SliceSizeConstraint: 12800,
 	}, nil
 }
 

pkg/codec/opus/Makefile

@@ -0,0 +1,31 @@
+git_url := https://github.com/xiph/opus.git
+version := v1.3.1
+src_root_dir := src
+lib_dir := lib
+include_dir := include
+lib_prefix := libopus
+src_dir := $(src_root_dir)/$(MEDIADEVICES_TARGET_PLATFORM)
+output_path := $(lib_dir)/$(lib_prefix)-$(MEDIADEVICES_TARGET_PLATFORM).a
+
+.PHONY: all
+all: guard-MEDIADEVICES_TARGET_PLATFORM guard-MEDIADEVICES_TOOLCHAIN_BIN $(output_path) headers
+
+headers: | $(src_dir) $(include_dir)
+	@cp $(src_dir)/include/*.h $(include_dir)
+
+$(output_path): $(src_dir)/$(lib_prefix).a | $(lib_dir)
+	@cp $< $@
+
+$(src_dir)/$(lib_prefix).a: | $(src_dir)
+	cd $(src_dir) && \
+		$(MEDIADEVICES_TOOLCHAIN_BIN) cmake -DOPUS_STACK_PROTECTOR=OFF -DCMAKE_C_FLAGS="-fpic" && \
+		$(MEDIADEVICES_TOOLCHAIN_BIN) make VERBOSE=1
+
+$(src_dir): | $(src_root_dir)
+	git clone --depth=1 --branch=$(version) $(git_url) $@
+
+$(src_root_dir) $(lib_dir) $(include_dir):
+	@mkdir -p $@
+
+guard-%:
+	@if [ -z ${$*} ]; then echo "$* is a required environment variable"; exit 1; fi

pkg/codec/opus/build.sh

@@ -1,19 +0,0 @@
-GIT_URL=https://github.com/xiph/opus.git
-VERSION=v1.3.1
-SRC_DIR=src
-LIB_DIR=lib
-INCLUDE_DIR=include
-ROOT_DIR=${PWD}
-LIB_PREFIX=libopus
-
-mkdir -p ${LIB_DIR} ${INCLUDE_DIR}
-
-git clone --depth=1 --branch=${VERSION} ${GIT_URL} ${SRC_DIR}
-cd ${SRC_DIR}
-${MEDIADEVICES_TOOLCHAIN_BIN} cmake -DOPUS_STACK_PROTECTOR=OFF .
-${MEDIADEVICES_TOOLCHAIN_BIN} make -j2
-mv ${LIB_PREFIX}.a ${ROOT_DIR}/${LIB_DIR}/${LIB_PREFIX}_${MEDIADEVICES_TARGET_PLATFORM}.a
-mkdir -p ${ROOT_DIR}/${INCLUDE_DIR}
-cp include/*.h ${ROOT_DIR}/${INCLUDE_DIR}
-git clean -dfx
-git reset --hard

pkg/codec/opus/opus.go

@@ -3,7 +3,6 @@ package opus
 import (
 	"errors"
 	"fmt"
-	"math"
 
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/audio"
@@ -28,8 +27,6 @@ type encoder struct {
 	engine *C.OpusEncoder
 }
 
-var latencies = []float64{5, 10, 20, 40, 60}
-
 func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser, error) {
 	var cerror C.int
 
@@ -37,10 +34,6 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 		return nil, fmt.Errorf("opus: inProp.SampleRate is required")
 	}
 
-	if p.Latency == 0 {
-		p.Latency = 20
-	}
-
 	if params.BitRate == 0 {
 		params.BitRate = 32000
 	}
@@ -49,19 +42,8 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 		params.ChannelMixer = &mixer.MonoMixer{}
 	}
 
-	// Select the nearest supported latency
-	var targetLatency float64
-	// TODO: use p.Latency.Milliseconds() after Go 1.12 EOL
-	latencyInMS := float64(p.Latency.Nanoseconds() / 1000000)
-	nearestDist := math.Inf(+1)
-	for _, latency := range latencies {
-		dist := math.Abs(latency - latencyInMS)
-		if dist >= nearestDist {
-			break
-		}
-
-		nearestDist = dist
-		targetLatency = latency
+	if !params.Latency.Validate() {
+		return nil, fmt.Errorf("opus: unsupported latency %v", params.Latency)
 	}
 
 	channels := p.ChannelCount
@@ -77,7 +59,7 @@ func newEncoder(r audio.Reader, p prop.Media, params Params) (codec.ReadCloser,
 	}
 
 	rMix := audio.NewChannelMixer(channels, params.ChannelMixer)
-	rBuf := audio.NewBuffer(int(targetLatency * float64(p.SampleRate) / 1000))
+	rBuf := audio.NewBuffer(params.Latency.samples(p.SampleRate))
 	e := encoder{
 		engine: engine,
 		reader: rMix(rBuf(r)),
@@ -139,11 +121,14 @@ func (e *encoder) SetBitRate(bitRate int) error {
 	return nil
 }
 
-func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {
+	if e.engine == nil {
+		return nil
+	}
 	C.opus_encoder_destroy(e.engine)
 	e.engine = nil
 	return nil

pkg/codec/opus/opus_static.go

@@ -4,9 +4,10 @@ package opus
 
 //#cgo CFLAGS: -I${SRCDIR}/include
 //#cgo CXXFLAGS: -I${SRCDIR}/include
-//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopus_linux_armv7.a -lm
-//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopus_linux_arm64.a -lm
-//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopus_linux_x64.a -lm
-//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopus_darwin_x64.a
-//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopus_windows_x64.a
+//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopus-linux-armv7.a -lm
+//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-arm64.a -lm
+//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-x64.a -lm
+//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-arm64.a
+//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-windows-x64.a
 import "C"

pkg/codec/opus/opus_test.go

@@ -0,0 +1,60 @@
+package opus
+
+import (
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/mediadevices/pkg/wave"
+)
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
+func TestEncoder(t *testing.T) {
+	t.Run("SimpleRead", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.AudioEncoderSimpleReadTest(t, &p,
+			prop.Media{
+				Audio: prop.Audio{
+					SampleRate:   48000,
+					ChannelCount: 2,
+				},
+			},
+			wave.NewInt16Interleaved(wave.ChunkInfo{
+				Len:          960,
+				SamplingRate: 48000,
+				Channels:     2,
+			}),
+		)
+	})
+	t.Run("CloseTwice", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		codectest.AudioEncoderCloseTwiceTest(t, &p, prop.Media{
+			Audio: prop.Audio{
+				SampleRate:   48000,
+				ChannelCount: 2,
+			},
+		})
+	})
+}

pkg/codec/opus/params.go

@@ -1,27 +1,69 @@
 package opus
 
 import (
+	"time"
+
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/prop"
 	"github.com/pion/mediadevices/pkg/wave/mixer"
 )
 
+// Latency is a type of OPUS codec frame duration.
+type Latency time.Duration
+
+// Latency values available in OPUS codec.
+const (
+	Latency2500us Latency = Latency(2500 * time.Microsecond)
+	Latency5ms    Latency = Latency(5 * time.Millisecond)
+	Latency10ms   Latency = Latency(10 * time.Millisecond)
+	Latency20ms   Latency = Latency(20 * time.Millisecond)
+	Latency40ms   Latency = Latency(40 * time.Millisecond)
+	Latency60ms   Latency = Latency(60 * time.Millisecond)
+)
+
+// Validate that the Latency is allowed in OPUS.
+func (l Latency) Validate() bool {
+	switch l {
+	case Latency2500us, Latency5ms, Latency10ms, Latency20ms, Latency40ms, Latency60ms:
+		return true
+	default:
+		return false
+	}
+}
+
+// Duration returns latency in time.Duration.
+func (l Latency) Duration() time.Duration {
+	return time.Duration(l)
+}
+
+// samples returns number of samples for given sample rate.
+func (l Latency) samples(sampleRate int) int {
+	return int(l.Duration() * time.Duration(sampleRate) / time.Second)
+}
+
 // Params stores opus specific encoding parameters.
 type Params struct {
 	codec.BaseParams
 	// ChannelMixer is a mixer to be used if number of given and expected channels differ.
 	ChannelMixer mixer.ChannelMixer
+
+	// Expected latency of the codec.
+	Latency Latency
 }
 
 // NewParams returns default opus codec specific parameters.
 func NewParams() (Params, error) {
-	return Params{}, nil
+	return Params{
+		Latency: Latency20ms,
+	}, nil
 }
 
 // RTPCodec represents the codec metadata
 func (p *Params) RTPCodec() *codec.RTPCodec {
-	return codec.NewRTPOpusCodec(48000)
+	c := codec.NewRTPOpusCodec(48000)
+	c.Latency = time.Duration(p.Latency)
+	return c
 }
 
 // BuildAudioEncoder builds opus encoder with given params

pkg/codec/opus/params_test.go

@@ -0,0 +1,49 @@
+package opus
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestLatency_Validate(t *testing.T) {
+	t.Run("Valid", func(t *testing.T) {
+		for _, l := range []Latency{
+			Latency2500us, Latency5ms, Latency10ms, Latency20ms, Latency40ms, Latency60ms,
+		} {
+			if !l.Validate() {
+				t.Errorf("Defined Latency(%v) must be valid", l)
+			}
+		}
+	})
+	t.Run("Invalid", func(t *testing.T) {
+		for _, l := range []Latency{
+			0, Latency(time.Second),
+		} {
+			if l.Validate() {
+				t.Errorf("Latency(%v) must be valid", l)
+			}
+		}
+	})
+}
+
+func TestLatency_samples(t *testing.T) {
+	testCases := []struct {
+		latency    Latency
+		sampleRate int
+		samples    int
+	}{
+		{Latency5ms, 48000, 240},
+		{Latency20ms, 16000, 320},
+		{Latency20ms, 48000, 960},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(fmt.Sprintf("%v_%d", time.Duration(testCase.latency), testCase.sampleRate), func(t *testing.T) {
+			samples := testCase.latency.samples(testCase.sampleRate)
+			if samples != testCase.samples {
+				t.Errorf("Expected samples: %d, got: %d", testCase.samples, samples)
+			}
+		})
+	}
+}

pkg/codec/vaapi/vaapi.go

@@ -1,3 +1,4 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
 // +build dragonfly freebsd linux netbsd openbsd solaris
 
 // Package vaapi implements hardware accelerated codecs.

pkg/codec/vaapi/vaapi_test.go

@@ -0,0 +1,70 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package vaapi
+
+import (
+	"errors"
+	"image"
+	"os"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestEncoder(t *testing.T) {
+	if _, err := os.Stat("/dev/dri/card0"); errors.Is(err, os.ErrNotExist) {
+		t.Skip("/dev/dri/card0 not found")
+	}
+
+	for name, factory := range map[string]func() (codec.VideoEncoderBuilder, error){
+		"VP8": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP8Params()
+			return &p, err
+		},
+		"VP9": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP9Params()
+			return &p, err
+		},
+	} {
+		factory := factory
+		t.Run(name, func(t *testing.T) {
+			t.Run("SimpleRead", func(t *testing.T) {
+				p, err := factory()
+				if err != nil {
+					t.Fatal(err)
+				}
+				codectest.VideoEncoderSimpleReadTest(t, p,
+					prop.Media{
+						Video: prop.Video{
+							Width:       256,
+							Height:      144,
+							FrameFormat: frame.FormatI420,
+						},
+					},
+					image.NewYCbCr(
+						image.Rect(0, 0, 256, 144),
+						image.YCbCrSubsampleRatio420,
+					),
+				)
+			})
+			t.Run("CloseTwice", func(t *testing.T) {
+				p, err := factory()
+				if err != nil {
+					t.Fatal(err)
+				}
+				codectest.VideoEncoderCloseTwiceTest(t, p, prop.Media{
+					Video: prop.Video{
+						Width:       640,
+						Height:      480,
+						FrameRate:   30,
+						FrameFormat: frame.FormatI420,
+					},
+				})
+			})
+		})
+	}
+}

pkg/codec/vaapi/vp8.go

@@ -1,3 +1,4 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
 // +build dragonfly freebsd linux netbsd openbsd solaris
 
 package vaapi
@@ -540,18 +541,18 @@ func (e *encoderVP8) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoderVP8) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
-
-func (e *encoderVP8) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoderVP8) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoderVP8) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	if e.closed {
+		return nil
+	}
+
 	C.vaDestroySurfaces(e.display, &e.surfs[0], C.int(len(e.surfs)))
 	C.vaDestroyContext(e.display, e.ctxID)
 	C.vaDestroyConfig(e.display, e.confID)

pkg/codec/vaapi/vp8_test.go

@@ -0,0 +1,27 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package vaapi
+
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"testing"
+)
+
+func TestVP8ShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoderVP8{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestVP8ShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoderVP8{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/vaapi/vp9.go

@@ -1,3 +1,4 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
 // +build dragonfly freebsd linux netbsd openbsd solaris
 
 package vaapi
@@ -475,18 +476,18 @@ func (e *encoderVP9) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoderVP9) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
-
-func (e *encoderVP9) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoderVP9) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoderVP9) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	if e.closed {
+		return nil
+	}
+
 	C.vaDestroySurfaces(e.display, &e.surfs[0], C.int(len(e.surfs)))
 	C.vaDestroyContext(e.display, e.ctxID)
 	C.vaDestroyConfig(e.display, e.confID)

pkg/codec/vaapi/vp9_test.go

@@ -0,0 +1,27 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package vaapi
+
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"testing"
+)
+
+func TestVP9ShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoderVP9{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestVP9ShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoderVP9{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/vpx/params.go

@@ -16,6 +16,7 @@ type Params struct {
 	RateControlOvershootPercent  uint
 	RateControlMinQuantizer      uint
 	RateControlMaxQuantizer      uint
+	LagInFrames                  uint
 	ErrorResilient               ErrorResilientMode
 }
 

pkg/codec/vpx/vpx.go

@@ -23,6 +23,9 @@ package vpx
 // int pktSz(vpx_codec_cx_pkt_t *pkt) {
 //   return pkt->data.frame.sz;
 // }
+// vpx_codec_frame_flags_t pktFrameFlags(vpx_codec_cx_pkt_t *pkt) {
+//   return pkt->data.frame.flags;
+// }
 //
 // // Alloc helpers
 // vpx_codec_ctx_t *newCtx() {
@@ -61,15 +64,17 @@ import (
 )
 
 type encoder struct {
-	codec      *C.vpx_codec_ctx_t
-	raw        *C.vpx_image_t
-	cfg        *C.vpx_codec_enc_cfg_t
-	r          video.Reader
-	frameIndex int
-	tStart     int
-	tLastFrame int
-	frame      []byte
-	deadline   int
+	codec           *C.vpx_codec_ctx_t
+	raw             *C.vpx_image_t
+	cfg             *C.vpx_codec_enc_cfg_t
+	r               video.Reader
+	frameIndex      int
+	tStart          int
+	tLastFrame      int
+	frame           []byte
+	deadline        int
+	requireKeyFrame bool
+	isKeyFrame      bool
 
 	mu     sync.Mutex
 	closed bool
@@ -141,6 +146,7 @@ func newParams(codecIface *C.vpx_codec_iface_t) (Params, error) {
 		RateControlOvershootPercent:  uint(cfg.rc_overshoot_pct),
 		RateControlMinQuantizer:      uint(cfg.rc_min_quantizer),
 		RateControlMaxQuantizer:      uint(cfg.rc_max_quantizer),
+		LagInFrames:                  uint(cfg.g_lag_in_frames),
 		ErrorResilient:               ErrorResilientMode(cfg.g_error_resilient),
 	}, nil
 }
@@ -171,6 +177,7 @@ func newEncoder(r video.Reader, p prop.Media, params Params, codecIface *C.vpx_c
 	cfg.g_h = C.uint(p.Height)
 	cfg.g_timebase.num = 1
 	cfg.g_timebase.den = 1000
+	cfg.g_lag_in_frames = C.uint(params.LagInFrames)
 	cfg.rc_target_bitrate = C.uint(params.BitRate) / 1000
 	cfg.kf_max_dist = C.uint(params.KeyFrameInterval)
 
@@ -229,9 +236,16 @@ func (e *encoder) Read() ([]byte, func(), error) {
 
 	if e.cfg.g_w != C.uint(width) || e.cfg.g_h != C.uint(height) {
 		e.cfg.g_w, e.cfg.g_h = C.uint(width), C.uint(height)
-		if ec := C.vpx_codec_enc_config_set(e.codec, e.cfg); ec != C.VPX_CODEC_OK {
-			return nil, func() {}, fmt.Errorf("vpx_codec_enc_config_set failed (%d)", ec)
+
+		newCodec := C.newCtx()
+		if ec := C.vpx_codec_enc_init_ver(
+			newCodec, e.codec.iface, e.cfg, 0, C.VPX_ENCODER_ABI_VERSION,
+		); ec != 0 {
+			return nil, func() {}, fmt.Errorf("vpx_codec_enc_init failed (%d)", ec)
 		}
+		C.free(unsafe.Pointer(e.codec))
+		e.codec = newCodec
+
 		e.raw.w, e.raw.h = C.uint(width), C.uint(height)
 		e.raw.r_w, e.raw.r_h = C.uint(width), C.uint(height)
 		e.raw.d_w, e.raw.d_h = C.uint(width), C.uint(height)
@@ -247,6 +261,9 @@ func (e *encoder) Read() ([]byte, func(), error) {
 		duration = 1
 	}
 	var flags int
+	if e.requireKeyFrame {
+		flags = flags | C.VPX_EFLAG_FORCE_KF
+	}
 	if ec := C.encode_wrapper(
 		e.codec, e.raw,
 		C.long(t-e.tStart), C.ulong(duration), C.long(flags), C.ulong(e.deadline),
@@ -255,6 +272,7 @@ func (e *encoder) Read() ([]byte, func(), error) {
 		return nil, func() {}, fmt.Errorf("vpx_codec_encode failed (%d)", ec)
 	}
 
+	e.requireKeyFrame = false
 	e.frameIndex++
 	e.tLastFrame = t
 
@@ -266,6 +284,7 @@ func (e *encoder) Read() ([]byte, func(), error) {
 			break
 		}
 		if pkt.kind == C.VPX_CODEC_CX_FRAME_PKT {
+			e.isKeyFrame = C.pktFrameFlags(pkt)&C.VPX_FRAME_IS_KEY == C.VPX_FRAME_IS_KEY
 			encoded := C.GoBytes(unsafe.Pointer(C.pktBuf(pkt)), C.pktSz(pkt))
 			e.frame = append(e.frame, encoded...)
 		}
@@ -276,18 +295,25 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
+func (e *encoder) ForceKeyFrame() error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.requireKeyFrame = true
+	return nil
 }
 
-func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	if e.closed {
+		return nil
+	}
+
 	e.closed = true
 
 	C.free(unsafe.Pointer(e.raw))

pkg/codec/vpx/vpx_test.go

@@ -0,0 +1,228 @@
+package vpx
+
+import (
+	"image"
+	"io"
+	"sync/atomic"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestEncoder(t *testing.T) {
+	for name, factory := range map[string]func() (codec.VideoEncoderBuilder, error){
+		"VP8": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP8Params()
+			return &p, err
+		},
+		"VP9": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP9Params()
+			p.LagInFrames = 0
+			return &p, err
+		},
+	} {
+		factory := factory
+		t.Run(name, func(t *testing.T) {
+			t.Run("SimpleRead", func(t *testing.T) {
+				p, err := factory()
+				if err != nil {
+					t.Fatal(err)
+				}
+				codectest.VideoEncoderSimpleReadTest(t, p,
+					prop.Media{
+						Video: prop.Video{
+							Width:       256,
+							Height:      144,
+							FrameFormat: frame.FormatI420,
+						},
+					},
+					image.NewYCbCr(
+						image.Rect(0, 0, 256, 144),
+						image.YCbCrSubsampleRatio420,
+					),
+				)
+			})
+			t.Run("CloseTwice", func(t *testing.T) {
+				p, err := factory()
+				if err != nil {
+					t.Fatal(err)
+				}
+				codectest.VideoEncoderCloseTwiceTest(t, p, prop.Media{
+					Video: prop.Video{
+						Width:       640,
+						Height:      480,
+						FrameRate:   30,
+						FrameFormat: frame.FormatI420,
+					},
+				})
+			})
+		})
+	}
+}
+
+func TestImageSizeChange(t *testing.T) {
+	for name, factory := range map[string]func() (codec.VideoEncoderBuilder, error){
+		"VP8": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP8Params()
+			return &p, err
+		},
+		"VP9": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP9Params()
+			// Disable latency to ease test and begin to receive packets for each input frame
+			p.LagInFrames = 0
+			return &p, err
+		},
+	} {
+		factory := factory
+		t.Run(name, func(t *testing.T) {
+			param, err := factory()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			for name, testCase := range map[string]struct {
+				initialWidth, initialHeight int
+				width, height               int
+			}{
+				"NoChange": {
+					320, 240,
+					320, 240,
+				},
+				"Enlarge": {
+					320, 240,
+					640, 480,
+				},
+				"Shrink": {
+					640, 480,
+					320, 240,
+				},
+			} {
+				testCase := testCase
+				t.Run(name, func(t *testing.T) {
+					var cnt uint32
+					r, err := param.BuildVideoEncoder(
+						video.ReaderFunc(func() (image.Image, func(), error) {
+							i := atomic.AddUint32(&cnt, 1)
+							if i == 1 {
+								return image.NewYCbCr(
+									image.Rect(0, 0, testCase.width, testCase.height),
+									image.YCbCrSubsampleRatio420,
+								), func() {}, nil
+							}
+							return nil, nil, io.EOF
+						}),
+						prop.Media{
+							Video: prop.Video{
+								Width:       testCase.initialWidth,
+								Height:      testCase.initialHeight,
+								FrameRate:   1,
+								FrameFormat: frame.FormatI420,
+							},
+						},
+					)
+					if err != nil {
+						t.Fatal(err)
+					}
+					_, rel, err := r.Read()
+					if err != nil {
+						t.Fatal(err)
+					}
+					rel()
+					_, _, err = r.Read()
+					if err != io.EOF {
+						t.Fatal(err)
+					}
+				})
+			}
+		})
+	}
+}
+
+func TestRequestKeyFrame(t *testing.T) {
+	for name, factory := range map[string]func() (codec.VideoEncoderBuilder, error){
+		"VP8": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP8Params()
+			return &p, err
+		},
+		"VP9": func() (codec.VideoEncoderBuilder, error) {
+			p, err := NewVP9Params()
+			// Disable latency to ease test and begin to receive packets for each input frame
+			p.LagInFrames = 0
+			return &p, err
+		},
+	} {
+		factory := factory
+		t.Run(name, func(t *testing.T) {
+			param, err := factory()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var initialWidth, initialHeight, width, height int = 320, 240, 320, 240
+
+			var cnt uint32
+			r, err := param.BuildVideoEncoder(
+				video.ReaderFunc(func() (image.Image, func(), error) {
+					i := atomic.AddUint32(&cnt, 1)
+					if i == 3 {
+						return nil, nil, io.EOF
+					}
+					return image.NewYCbCr(
+						image.Rect(0, 0, width, height),
+						image.YCbCrSubsampleRatio420,
+					), func() {}, nil
+				}),
+				prop.Media{
+					Video: prop.Video{
+						Width:       initialWidth,
+						Height:      initialHeight,
+						FrameRate:   1,
+						FrameFormat: frame.FormatI420,
+					},
+				},
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+			_, rel, err := r.Read()
+			if err != nil {
+				t.Fatal(err)
+			}
+			rel()
+			r.Controller().(codec.KeyFrameController).ForceKeyFrame()
+			_, rel, err = r.Read()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !r.(*encoder).isKeyFrame {
+				t.Fatal("Not a key frame")
+			}
+			rel()
+			_, _, err = r.Read()
+			if err != io.EOF {
+				t.Fatal(err)
+			}
+		})
+
+	}
+}
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/x264/bridge.h

@@ -18,6 +18,7 @@ typedef struct Encoder {
   x264_t *h;
   x264_picture_t pic_in;
   x264_param_t param;
+  int force_key_frame;
 } Encoder;
 
 Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
@@ -85,8 +86,14 @@ Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
   e->pic_in.img.plane[0] = y;
   e->pic_in.img.plane[1] = cb;
   e->pic_in.img.plane[2] = cr;
+  if (e->force_key_frame) {
+    e->pic_in.i_type = X264_TYPE_IDR;
+  } else {
+    e->pic_in.i_type = X264_TYPE_AUTO;
+  }
 
   int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
+  e->force_key_frame = 0;
   Slice s = {.data_len = frame_size};
   if (frame_size <= 0) {
     *rc = ERR_ENCODE;

pkg/codec/x264/x264.go

@@ -124,12 +124,16 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
+// TODO: Implement bit rate control
+//var _ codec.BitRateController = (*encoder)(nil)
 
 func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+	e.engine.force_key_frame = C.int(1)
+	return nil
+}
+
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {

pkg/codec/x264/x264_test.go

@@ -0,0 +1,67 @@
+package x264
+
+import (
+	"image"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestEncoder(t *testing.T) {
+	t.Run("SimpleRead", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		p.BitRate = 200000
+		codectest.VideoEncoderSimpleReadTest(t, &p,
+			prop.Media{
+				Video: prop.Video{
+					Width:       256,
+					Height:      144,
+					FrameFormat: frame.FormatI420,
+				},
+			},
+			image.NewYCbCr(
+				image.Rect(0, 0, 256, 144),
+				image.YCbCrSubsampleRatio420,
+			),
+		)
+	})
+	t.Run("CloseTwice", func(t *testing.T) {
+		p, err := NewParams()
+		if err != nil {
+			t.Fatal(err)
+		}
+		p.BitRate = 200000
+		codectest.VideoEncoderCloseTwiceTest(t, &p, prop.Media{
+			Video: prop.Video{
+				Width:       640,
+				Height:      480,
+				FrameRate:   30,
+				FrameFormat: frame.FormatI420,
+			},
+		})
+	})
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}

pkg/driver/camera/camera.go

@@ -1 +1,16 @@
+/*
+Package camera provides a video camera driver.
+
+Device Label Generation Rules
+
+On Linux, the device label will be in the format of:
+	pci-0000:00:00.0-usb-0:0:0.0-video-index0;video0
+If /dev/v4l/by-path/* is not available (for example in a docker container without
+bindings in /dev/v4l/by-path/), it will be:
+	video0;video0
+*/
 package camera
+
+// LabelSeparator is used to separate labels for a driver that
+// is found from multiple locations on a host.
+const LabelSeparator = ";"

pkg/driver/camera/camera_bench_test.go

@@ -56,7 +56,7 @@ func BenchmarkRead(b *testing.B) {
 
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {
-				_, err := r.Read()
+				_, _, err := r.Read()
 				if err != nil {
 					b.Fatalf("Failed to read: %v", err)
 				}

pkg/driver/camera/camera_darwin.go

@@ -13,6 +13,7 @@ import (
 type camera struct {
 	device  avfoundation.Device
 	session *avfoundation.Session
+	rcClose func()
 }
 
 func init() {
@@ -43,6 +44,9 @@ func (cam *camera) Open() error {
 }
 
 func (cam *camera) Close() error {
+	if cam.rcClose != nil {
+		cam.rcClose()
+	}
 	return cam.session.Close()
 }
 
@@ -56,6 +60,7 @@ func (cam *camera) VideoRecord(property prop.Media) (video.Reader, error) {
 	if err != nil {
 		return nil, err
 	}
+	cam.rcClose = rc.Close
 	r := video.ReaderFunc(func() (image.Image, func(), error) {
 		frame, _, err := rc.Read()
 		if err != nil {

pkg/driver/camera/camera_darwin_test.go

@@ -0,0 +1,63 @@
+// +build darwin
+
+// $ go test -v . -tags darwin -run="^TestCameraFrameFormatSupport$"
+
+package camera
+
+import (
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/avfoundation"
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func TestCameraFrameFormatSupport(t *testing.T) {
+	devices, err := avfoundation.Devices(avfoundation.Video)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(devices) > 0 {
+		c := newCamera(devices[0])
+		if err := c.Open(); err != nil {
+			t.Fatal(err)
+		}
+		defer c.Close()
+
+		supportedFormats := make(map[frame.Format]struct{})
+		for _, p := range c.Properties() {
+			supportedFormats[p.FrameFormat] = struct{}{}
+		}
+
+		for _, format := range []frame.Format{
+			frame.FormatI420,
+			frame.FormatNV12,
+			frame.FormatNV21,
+			frame.FormatYUY2,
+			frame.FormatUYVY,
+		} {
+			if _, ok := supportedFormats[format]; !ok {
+				t.Logf("[%v] UNSUPPORTED", format)
+				continue
+			}
+			r, err := c.VideoRecord(prop.Media{
+				Video: prop.Video{
+					Width:       640,
+					Height:      480,
+					FrameFormat: format,
+				}})
+			if err != nil {
+				t.Logf("[%v] Failed to capture image: %v", format, err)
+				continue
+			}
+			for i := 0; i < 10; i++ {
+				_, _, err := r.Read()
+				if err != nil {
+					t.Logf("[%v] Failed to read: %v", format, err)
+					continue
+				}
+			}
+			t.Logf("[%v] OK", format)
+		}
+	}
+}

pkg/driver/camera/camera_linux.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/blackjack/webcam"
@@ -73,51 +74,51 @@ type camera struct {
 
 func init() {
 	discovered := make(map[string]struct{})
+	discover(discovered, "/dev/v4l/by-path/*")
+	discover(discovered, "/dev/video*")
+}
 
-	discover := func(pattern string) {
-		devices, err := filepath.Glob(pattern)
-		if err != nil {
-			// No v4l device.
-			return
-		}
-		for _, device := range devices {
-			label := filepath.Base(device)
-			reallink, err := os.Readlink(device)
-			if err != nil {
-				reallink = label
-			} else {
-				reallink = filepath.Base(reallink)
-			}
-
-			if _, ok := discovered[reallink]; ok {
-				continue
-			}
-
-			discovered[reallink] = struct{}{}
-			cam := newCamera(device)
-			priority := driver.PriorityNormal
-			if label == prioritizedDevice {
-				priority = driver.PriorityHigh
-			}
-			driver.GetManager().Register(cam, driver.Info{
-				Label:      label,
-				DeviceType: driver.Camera,
-				Priority:   priority,
-			})
-		}
+func discover(discovered map[string]struct{}, pattern string) {
+	devices, err := filepath.Glob(pattern)
+	if err != nil {
+		// No v4l device.
+		return
 	}
+	for _, device := range devices {
+		label := filepath.Base(device)
+		reallink, err := os.Readlink(device)
+		if err != nil {
+			reallink = label
+		} else {
+			reallink = filepath.Base(reallink)
+		}
+		if _, ok := discovered[reallink]; ok {
+			continue
+		}
 
-	discover("/dev/v4l/by-path/*")
-	discover("/dev/video*")
+		discovered[reallink] = struct{}{}
+		cam := newCamera(device)
+		priority := driver.PriorityNormal
+		if reallink == prioritizedDevice {
+			priority = driver.PriorityHigh
+		}
+		driver.GetManager().Register(cam, driver.Info{
+			Label:      label + LabelSeparator + reallink,
+			DeviceType: driver.Camera,
+			Priority:   priority,
+		})
+	}
 }
 
 func newCamera(path string) *camera {
 	formats := map[webcam.PixelFormat]frame.Format{
 		webcam.PixelFormat(C.V4L2_PIX_FMT_YUV420): frame.FormatI420,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_NV21):   frame.FormatNV21,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_NV12):   frame.FormatNV12,
 		webcam.PixelFormat(C.V4L2_PIX_FMT_YUYV):   frame.FormatYUYV,
 		webcam.PixelFormat(C.V4L2_PIX_FMT_UYVY):   frame.FormatUYVY,
-		webcam.PixelFormat(C.V4L2_PIX_FMT_NV12):   frame.FormatNV21,
 		webcam.PixelFormat(C.V4L2_PIX_FMT_MJPEG):  frame.FormatMJPEG,
+		webcam.PixelFormat(C.V4L2_PIX_FMT_Z16):    frame.FormatZ16,
 	}
 
 	reversedFormats := make(map[frame.Format]webcam.PixelFormat)
@@ -133,6 +134,19 @@ func newCamera(path string) *camera {
 	return c
 }
 
+func getCameraReadTimeout() uint32 {
+	// default to 5 seconds
+	var readTimeoutSec uint32 = 5
+	if val, ok := os.LookupEnv("PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"); ok {
+		if valInt, err := strconv.Atoi(val); err == nil {
+			if valInt > 0 {
+				readTimeoutSec = uint32(valInt)
+			}
+		}
+	}
+	return readTimeoutSec
+}
+
 func (c *camera) Open() error {
 	cam, err := webcam.Open(c.path)
 	if err != nil {
@@ -179,12 +193,21 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 		return nil, err
 	}
 
+	if p.FrameRate > 0 {
+		err = c.cam.SetFramerate(float32(p.FrameRate))
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if err := c.cam.StartStreaming(); err != nil {
 		return nil, err
 	}
 
 	cam := c.cam
 
+	readTimeoutSec := getCameraReadTimeout()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	c.cancel = cancel
 	var buf []byte
@@ -200,7 +223,7 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 				return nil, func() {}, io.EOF
 			}
 
-			err := cam.WaitForFrame(5) // 5 seconds
+			err := cam.WaitForFrame(readTimeoutSec)
 			switch err.(type) {
 			case nil:
 			case *webcam.Timeout:

pkg/driver/camera/camera_linux_test.go

@@ -0,0 +1,100 @@
+package camera
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/pion/mediadevices/pkg/driver"
+)
+
+func TestDiscover(t *testing.T) {
+	const (
+		shortName  = "unittest-video0"
+		shortName2 = "unittest-video1"
+		longName   = "unittest-long-device-name:0:1:2:3"
+	)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	byPathDir := filepath.Join(dir, "v4l", "by-path")
+	if err := os.MkdirAll(byPathDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(dir, shortName), []byte{}, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(dir, shortName2), []byte{}, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(
+		filepath.Join(dir, shortName),
+		filepath.Join(byPathDir, longName),
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	discovered := make(map[string]struct{})
+	discover(discovered, filepath.Join(byPathDir, "*"))
+	discover(discovered, filepath.Join(dir, "unittest-video*"))
+
+	drvs := driver.GetManager().Query(func(d driver.Driver) bool {
+		// Ignore real cameras.
+		return d.Info().DeviceType == driver.Camera && strings.Contains(d.Info().Label, "unittest")
+	})
+	if len(drvs) != 2 {
+		t.Fatalf("Expected 2 driver, got %d drivers", len(drvs))
+	}
+
+	labels := []string{
+		drvs[0].Info().Label,
+		drvs[1].Info().Label,
+	}
+	// Returned drivers are unordered. Sort to get static result.
+	sort.Sort(sort.StringSlice(labels))
+
+	expected := longName + LabelSeparator + shortName
+	if label := labels[0]; label != expected {
+		t.Errorf("Expected label: %s, got: %s", expected, label)
+	}
+
+	expectedNoLink := shortName2 + LabelSeparator + shortName2
+	if label := labels[1]; label != expectedNoLink {
+		t.Errorf("Expected label: %s, got: %s", expectedNoLink, label)
+	}
+}
+
+func TestGetCameraReadTimeout(t *testing.T) {
+	var expected uint32 = 5
+	value := getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	envVarName := "PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"
+	os.Setenv(envVarName, "text")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "-1")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "1")
+	expected = 1
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+}

pkg/driver/microphone/microphone.go

@@ -111,6 +111,8 @@ func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
 	config.PerformanceProfile = malgo.LowLatency
 	config.Capture.Channels = uint32(inputProp.ChannelCount)
 	config.SampleRate = uint32(inputProp.SampleRate)
+	//FIX: Turn on the microphone with the current device id
+	config.Capture.DeviceID = m.ID.Pointer()
 	if inputProp.SampleSize == 4 && inputProp.IsFloat {
 		config.Capture.Format = malgo.FormatF32
 	} else if inputProp.SampleSize == 2 && !inputProp.IsFloat {
@@ -155,9 +157,6 @@ func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
 		return decodedChunk, func() {}, err
 	})
 
-	// FIXME: The current audio detection and audio encoder can only work with a static latency. Since the latency from the driver
-	//        can fluctuate, we need to stabilize it. Maybe there's a better way for doing this?
-	reader = audio.NewBuffer(int(inputProp.Latency.Seconds() * float64(inputProp.SampleRate)))(reader)
 	return reader, nil
 }
 

pkg/driver/screen/x11capture_linux.go

@@ -2,6 +2,7 @@ package screen
 
 // #cgo pkg-config: x11 xext
 // #include <stdint.h>
+// #include <string.h>
 // #include <sys/shm.h>
 // #include <X11/Xlib.h>
 // #define XUTIL_DEFINE_FUNCTIONS
@@ -125,6 +126,14 @@ func (c colorFunc) RGBA() (r, g, b, a uint32) {
 
 func (s *shmImage) At(x, y int) color.Color {
 	switch s.pixFmt {
+	case pixFmtRGB24:
+		addr := (x + y*int(s.img.width)) * 4
+		r := uint32(s.b[addr]) * 0x100
+		g := uint32(s.b[addr+1]) * 0x100
+		b := uint32(s.b[addr+2]) * 0x100
+		return colorFunc(func() (_, _, _, _ uint32) {
+			return r, g, b, 0xFFFF
+		})
 	case pixFmtBGR24:
 		addr := (x + y*int(s.img.width)) * 4
 		b := uint32(s.b[addr]) * 0x100
@@ -133,6 +142,15 @@ func (s *shmImage) At(x, y int) color.Color {
 		return colorFunc(func() (_, _, _, _ uint32) {
 			return r, g, b, 0xFFFF
 		})
+	case pixFmtRGB16:
+		addr := (x + y*int(s.img.width)) * 2
+		b1, b2 := s.b[addr], s.b[addr+1]
+		r := uint32(b1>>3) * 0x100
+		g := uint32((b1&0x7)<<3|(b2&0xE0)>>5) * 0x100
+		b := uint32(b2&0x1F) * 0x100
+		return colorFunc(func() (_, _, _, _ uint32) {
+			return r, g, b, 0xFFFF
+		})
 	case pixFmtBGR16:
 		addr := (x + y*int(s.img.width)) * 2
 		b1, b2 := s.b[addr], s.b[addr+1]
@@ -149,12 +167,25 @@ func (s *shmImage) At(x, y int) color.Color {
 
 func (s *shmImage) RGBAAt(x, y int) color.RGBA {
 	switch s.pixFmt {
+	case pixFmtRGB24:
+		addr := (x + y*int(s.img.width)) * 4
+		r := s.b[addr]
+		g := s.b[addr+1]
+		b := s.b[addr+2]
+		return color.RGBA{R: r, G: g, B: b, A: 0xFF}
 	case pixFmtBGR24:
 		addr := (x + y*int(s.img.width)) * 4
 		b := s.b[addr]
 		g := s.b[addr+1]
 		r := s.b[addr+2]
 		return color.RGBA{R: r, G: g, B: b, A: 0xFF}
+	case pixFmtRGB16:
+		addr := (x + y*int(s.img.width)) * 2
+		b1, b2 := s.b[addr], s.b[addr+1]
+		r := b1 >> 3
+		g := (b1&0x7)<<3 | (b2&0xE0)>>5
+		b := b2 & 0x1F
+		return color.RGBA{R: r, G: g, B: b, A: 0xFF}
 	case pixFmtBGR16:
 		addr := (x + y*int(s.img.width)) * 2
 		b1, b2 := s.b[addr], s.b[addr+1]
@@ -178,11 +209,17 @@ func (s *shmImage) ToRGBA(dst *image.RGBA) *image.RGBA {
 		dst.Pix = dst.Pix[:l]
 	}
 	switch s.pixFmt {
+	case pixFmtRGB24:
+		C.memcpy(unsafe.Pointer(&dst.Pix[0]), unsafe.Pointer(s.img.data), C.size_t(len(dst.Pix)))
+		return dst
 	case pixFmtBGR24:
-		C.copyBGR24(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.ulong(len(dst.Pix)))
+		C.copyBGR24(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.size_t(len(dst.Pix)))
+		return dst
+	case pixFmtRGB16:
+		C.memcpy(unsafe.Pointer(&dst.Pix[0]), unsafe.Pointer(s.img.data), C.size_t(len(dst.Pix)))
 		return dst
 	case pixFmtBGR16:
-		C.copyBGR16(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.ulong(len(dst.Pix)))
+		C.copyBGR16(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.size_t(len(dst.Pix)))
 		return dst
 	default:
 		panic("unsupported pixel format")
@@ -199,8 +236,12 @@ func newShmImage(dp *C.Display, screen int) (*shmImage, error) {
 	s := &shmImage{dp: dp}
 
 	switch {
+	case v.red_mask == 0xFF && v.green_mask == 0xFF00 && v.blue_mask == 0xFF0000:
+		s.pixFmt = pixFmtRGB24
 	case v.red_mask == 0xFF0000 && v.green_mask == 0xFF00 && v.blue_mask == 0xFF:
 		s.pixFmt = pixFmtBGR24
+	case v.red_mask == 0x1F && v.green_mask == 0x7E0 && v.blue_mask == 0xF800:
+		s.pixFmt = pixFmtRGB16
 	case v.red_mask == 0xF800 && v.green_mask == 0x7E0 && v.blue_mask == 0x1F:
 		s.pixFmt = pixFmtBGR16
 	default:
@@ -209,7 +250,7 @@ func newShmImage(dp *C.Display, screen int) (*shmImage, error) {
 		return nil, errors.New("unsupported pixel format")
 	}
 
-	s.shm.shmid = C.shmget(C.IPC_PRIVATE, C.ulong(w*h*4+8), C.IPC_CREAT|0600)
+	s.shm.shmid = C.shmget(C.IPC_PRIVATE, C.size_t(w*h*4+8), C.IPC_CREAT|0600)
 	if s.shm.shmid == -1 {
 		return nil, errors.New("failed to get shared memory")
 	}
@@ -279,5 +320,5 @@ func (r *reader) Close() {
 
 // cAlign64 is fot testing
 func cAlign64(ptr uintptr) uintptr {
-	return uintptr(C.align64ForTest(C.ulong(uintptr(ptr))))
+	return uintptr(C.align64ForTest(C.size_t(uintptr(ptr))))
 }

pkg/driver/videotest/dummy.go

@@ -47,10 +47,6 @@ func (d *dummy) Close() error {
 }
 
 func (d *dummy) VideoRecord(p prop.Media) (video.Reader, error) {
-	if p.FrameRate == 0 {
-		p.FrameRate = 30
-	}
-
 	colors := [][3]byte{
 		{235, 128, 128},
 		{210, 16, 146},
@@ -143,6 +139,7 @@ func (d dummy) Properties() []prop.Media {
 				Width:       640,
 				Height:      480,
 				FrameFormat: frame.FormatYUYV,
+				FrameRate:   30,
 			},
 		},
 	}

pkg/driver/vncdriver/vnc/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

pkg/driver/vncdriver/vnc/README.md

@@ -0,0 +1,16 @@
+# VNC Library for Go
+
+go-vnc is a VNC library for Go, initially supporting VNC clients but
+with the goal of eventually implementing a VNC server.
+
+This library implements [RFC 6143](http://tools.ietf.org/html/rfc6143).
+
+## Usage & Installation
+
+The library is installable via standard `go get`. The package name is `vnc`.
+
+```
+$ go get github.com/mitchellh/go-vnc
+```
+
+Documentation is available on GoDoc: http://godoc.org/github.com/mitchellh/go-vnc

pkg/driver/vncdriver/vnc/client.go

@@ -0,0 +1,494 @@
+// Package vnc implements a VNC client.
+//
+// References:
+//   [PROTOCOL]: http://tools.ietf.org/html/rfc6143
+package vnc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"unicode"
+)
+
+type ClientConn struct {
+	c      net.Conn
+	config *ClientConfig
+
+	// If the pixel format uses a color map, then this is the color
+	// map that is used. This should not be modified directly, since
+	// the data comes from the server.
+	ColorMap [256]Color
+
+	// Encodings supported by the client. This should not be modified
+	// directly. Instead, SetEncodings should be used.
+	Encs []Encoding
+
+	// Width of the frame buffer in pixels, sent from the server.
+	FrameBufferWidth uint16
+
+	// Height of the frame buffer in pixels, sent from the server.
+	FrameBufferHeight uint16
+
+	// Name associated with the desktop, sent from the server.
+	DesktopName string
+
+	// The pixel format associated with the connection. This shouldn't
+	// be modified. If you wish to set a new pixel format, use the
+	// SetPixelFormat method.
+	PixelFormat PixelFormat
+}
+
+// A ClientConfig structure is used to configure a ClientConn. After
+// one has been passed to initialize a connection, it must not be modified.
+type ClientConfig struct {
+	// A slice of ClientAuth methods. Only the first instance that is
+	// suitable by the server will be used to authenticate.
+	Auth []ClientAuth
+
+	// Exclusive determines whether the connection is shared with other
+	// clients. If true, then all other clients connected will be
+	// disconnected when a connection is established to the VNC server.
+	Exclusive bool
+
+	// The channel that all messages received from the server will be
+	// sent on. If the channel blocks, then the goroutine reading data
+	// from the VNC server may block indefinitely. It is up to the user
+	// of the library to ensure that this channel is properly read.
+	// If this is not set, then all messages will be discarded.
+	ServerMessageCh chan<- ServerMessage
+
+	// A slice of supported messages that can be read from the server.
+	// This only needs to contain NEW server messages, and doesn't
+	// need to explicitly contain the RFC-required messages.
+	ServerMessages []ServerMessage
+}
+
+func Client(c net.Conn, cfg *ClientConfig) (*ClientConn, error) {
+	conn := &ClientConn{
+		c:      c,
+		config: cfg,
+	}
+
+	if err := conn.handshake(); err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	go conn.mainLoop()
+
+	return conn, nil
+}
+
+func (c *ClientConn) Close() error {
+	return c.c.Close()
+}
+
+// CutText tells the server that the client has new text in its cut buffer.
+// The text string MUST only contain Latin-1 characters. This encoding
+// is compatible with Go's native string format, but can only use up to
+// unicode.MaxLatin values.
+//
+// See RFC 6143 Section 7.5.6
+func (c *ClientConn) CutText(text string) error {
+	var buf bytes.Buffer
+
+	// This is the fixed size data we'll send
+	fixedData := []interface{}{
+		uint8(6),
+		uint8(0),
+		uint8(0),
+		uint8(0),
+		uint32(len(text)),
+	}
+
+	for _, val := range fixedData {
+		if err := binary.Write(&buf, binary.BigEndian, val); err != nil {
+			return err
+		}
+	}
+
+	for _, char := range text {
+		if char > unicode.MaxLatin1 {
+			return fmt.Errorf("Character '%d' is not valid Latin-1", char)
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, uint8(char)); err != nil {
+			return err
+		}
+	}
+
+	dataLength := 8 + len(text)
+	if _, err := c.c.Write(buf.Bytes()[0:dataLength]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Requests a framebuffer update from the server. There may be an indefinite
+// time between the request and the actual framebuffer update being
+// received.
+//
+// See RFC 6143 Section 7.5.3
+func (c *ClientConn) FramebufferUpdateRequest(incremental bool, x, y, width, height uint16) error {
+	var buf bytes.Buffer
+	var incrementalByte uint8 = 0
+
+	if incremental {
+		incrementalByte = 1
+	}
+
+	data := []interface{}{
+		uint8(3),
+		incrementalByte,
+		x, y, width, height,
+	}
+
+	for _, val := range data {
+		if err := binary.Write(&buf, binary.BigEndian, val); err != nil {
+			return err
+		}
+	}
+
+	if _, err := c.c.Write(buf.Bytes()[0:10]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// KeyEvent indiciates a key press or release and sends it to the server.
+// The key is indicated using the X Window System "keysym" value. Use
+// Google to find a reference of these values. To simulate a key press,
+// you must send a key with both a down event, and a non-down event.
+//
+// See 7.5.4.
+func (c *ClientConn) KeyEvent(keysym uint32, down bool) error {
+	var downFlag uint8 = 0
+	if down {
+		downFlag = 1
+	}
+
+	data := []interface{}{
+		uint8(4),
+		downFlag,
+		uint8(0),
+		uint8(0),
+		keysym,
+	}
+
+	for _, val := range data {
+		if err := binary.Write(c.c, binary.BigEndian, val); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PointerEvent indicates that pointer movement or a pointer button
+// press or release.
+//
+// The mask is a bitwise mask of various ButtonMask values. When a button
+// is set, it is pressed, when it is unset, it is released.
+//
+// See RFC 6143 Section 7.5.5
+func (c *ClientConn) PointerEvent(mask ButtonMask, x, y uint16) error {
+	var buf bytes.Buffer
+
+	data := []interface{}{
+		uint8(5),
+		uint8(mask),
+		x,
+		y,
+	}
+
+	for _, val := range data {
+		if err := binary.Write(&buf, binary.BigEndian, val); err != nil {
+			return err
+		}
+	}
+
+	if _, err := c.c.Write(buf.Bytes()[0:6]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetEncodings sets the encoding types in which the pixel data can
+// be sent from the server. After calling this method, the encs slice
+// given should not be modified.
+//
+// See RFC 6143 Section 7.5.2
+func (c *ClientConn) SetEncodings(encs []Encoding) error {
+	data := make([]interface{}, 3+len(encs))
+	data[0] = uint8(2)
+	data[1] = uint8(0)
+	data[2] = uint16(len(encs))
+
+	for i, enc := range encs {
+		data[3+i] = int32(enc.Type())
+	}
+
+	var buf bytes.Buffer
+	for _, val := range data {
+		if err := binary.Write(&buf, binary.BigEndian, val); err != nil {
+			return err
+		}
+	}
+
+	dataLength := 4 + (4 * len(encs))
+	if _, err := c.c.Write(buf.Bytes()[0:dataLength]); err != nil {
+		return err
+	}
+
+	c.Encs = encs
+
+	return nil
+}
+
+// SetPixelFormat sets the format in which pixel values should be sent
+// in FramebufferUpdate messages from the server.
+//
+// See RFC 6143 Section 7.5.1
+func (c *ClientConn) SetPixelFormat(format *PixelFormat) error {
+	var keyEvent [20]byte
+	keyEvent[0] = 0
+
+	pfBytes, err := writePixelFormat(format)
+	if err != nil {
+		return err
+	}
+
+	// Copy the pixel format bytes into the proper slice location
+	copy(keyEvent[4:], pfBytes)
+
+	// Send the data down the connection
+	if _, err := c.c.Write(keyEvent[:]); err != nil {
+		return err
+	}
+
+	// Reset the color map as according to RFC.
+	var newColorMap [256]Color
+	c.ColorMap = newColorMap
+
+	return nil
+}
+
+const pvLen = 12 // ProtocolVersion message length.
+
+func parseProtocolVersion(pv []byte) (uint, uint, error) {
+	var major, minor uint
+
+	if len(pv) < pvLen {
+		return 0, 0, fmt.Errorf("ProtocolVersion message too short (%v < %v)", len(pv), pvLen)
+	}
+
+	l, err := fmt.Sscanf(string(pv), "RFB %d.%d\n", &major, &minor)
+	if l != 2 {
+		return 0, 0, fmt.Errorf("error parsing ProtocolVersion.")
+	}
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return major, minor, nil
+}
+
+func (c *ClientConn) handshake() error {
+	var protocolVersion [pvLen]byte
+
+	// 7.1.1, read the ProtocolVersion message sent by the server.
+	if _, err := io.ReadFull(c.c, protocolVersion[:]); err != nil {
+		return err
+	}
+
+	maxMajor, maxMinor, err := parseProtocolVersion(protocolVersion[:])
+	if err != nil {
+		return err
+	}
+	if maxMajor < 3 {
+		return fmt.Errorf("unsupported major version, less than 3: %d", maxMajor)
+	}
+	if maxMinor < 3 {
+		return fmt.Errorf("unsupported minor version, less than 3: %d", maxMinor)
+	}
+
+	// Respond with the version we will support
+	if maxMinor<8 {
+		if _, err = c.c.Write([]byte("RFB 003.003\n")); err != nil {
+			return err
+		}
+		var numSecurityTypes uint32
+		if err = binary.Read(c.c, binary.BigEndian, &numSecurityTypes); err != nil {
+			return err
+		}
+
+		if numSecurityTypes == 0 {
+			return fmt.Errorf("no security types: %s", c.readErrorReason())
+		}
+	}else{
+		if _, err = c.c.Write([]byte("RFB 003.008\n")); err != nil {
+			return err
+		}
+		// 7.1.2 Security Handshake from server
+		var numSecurityTypes uint8
+		if err = binary.Read(c.c, binary.BigEndian, &numSecurityTypes); err != nil {
+			return err
+		}
+
+		if numSecurityTypes == 0 {
+			return fmt.Errorf("no security types: %s", c.readErrorReason())
+		}
+
+		securityTypes := make([]uint8, numSecurityTypes)
+		if err = binary.Read(c.c, binary.BigEndian, &securityTypes); err != nil {
+			return err
+		}
+
+		clientSecurityTypes := c.config.Auth
+		if clientSecurityTypes == nil {
+			clientSecurityTypes = []ClientAuth{new(ClientAuthNone)}
+		}
+
+		var auth ClientAuth
+	FindAuth:
+		for _, curAuth := range clientSecurityTypes {
+			for _, securityType := range securityTypes {
+				if curAuth.SecurityType() == securityType {
+					// We use the first matching supported authentication
+					auth = curAuth
+					break FindAuth
+				}
+			}
+		}
+
+		if auth == nil {
+			return fmt.Errorf("no suitable auth schemes found. server supported: %#v", securityTypes)
+		}
+
+		// Respond back with the security type we'll use
+		if err = binary.Write(c.c, binary.BigEndian, auth.SecurityType()); err != nil {
+			return err
+		}
+
+		if err = auth.Handshake(c.c); err != nil {
+			return err
+		}
+
+		// 7.1.3 SecurityResult Handshake
+		var securityResult uint32
+		if err = binary.Read(c.c, binary.BigEndian, &securityResult); err != nil {
+			return err
+		}
+
+		if securityResult == 1 {
+			return fmt.Errorf("security handshake failed: %s", c.readErrorReason())
+		}
+	}
+	// 7.3.1 ClientInit
+	var sharedFlag uint8 = 1
+	if c.config.Exclusive {
+		sharedFlag = 0
+	}
+
+	if err = binary.Write(c.c, binary.BigEndian, sharedFlag); err != nil {
+		return err
+	}
+
+	// 7.3.2 ServerInit
+	if err = binary.Read(c.c, binary.BigEndian, &c.FrameBufferWidth); err != nil {
+		return err
+	}
+
+	if err = binary.Read(c.c, binary.BigEndian, &c.FrameBufferHeight); err != nil {
+		return err
+	}
+
+	// Read the pixel format
+	if err = readPixelFormat(c.c, &c.PixelFormat); err != nil {
+		return err
+	}
+
+	var nameLength uint32
+	if err = binary.Read(c.c, binary.BigEndian, &nameLength); err != nil {
+		return err
+	}
+
+	nameBytes := make([]uint8, nameLength)
+	if err = binary.Read(c.c, binary.BigEndian, &nameBytes); err != nil {
+		return err
+	}
+
+	c.DesktopName = string(nameBytes)
+
+	return nil
+}
+
+// mainLoop reads messages sent from the server and routes them to the
+// proper channels for users of the client to read.
+func (c *ClientConn) mainLoop() {
+	defer c.Close()
+
+	// Build the map of available server messages
+	typeMap := make(map[uint8]ServerMessage)
+
+	defaultMessages := []ServerMessage{
+		new(FramebufferUpdateMessage),
+		new(SetColorMapEntriesMessage),
+		new(BellMessage),
+		new(ServerCutTextMessage),
+	}
+
+	for _, msg := range defaultMessages {
+		typeMap[msg.Type()] = msg
+	}
+
+	if c.config.ServerMessages != nil {
+		for _, msg := range c.config.ServerMessages {
+			typeMap[msg.Type()] = msg
+		}
+	}
+
+	for {
+		var messageType uint8
+		if err := binary.Read(c.c, binary.BigEndian, &messageType); err != nil {
+			break
+		}
+
+		msg, ok := typeMap[messageType]
+		if !ok {
+			// Unsupported message type! Bad!
+			break
+		}
+
+		parsedMsg, err := msg.Read(c, c.c)
+		if err != nil {
+			break
+		}
+
+		if c.config.ServerMessageCh == nil {
+			continue
+		}
+
+		c.config.ServerMessageCh <- parsedMsg
+	}
+}
+
+func (c *ClientConn) readErrorReason() string {
+	var reasonLen uint32
+	if err := binary.Read(c.c, binary.BigEndian, &reasonLen); err != nil {
+		return "<error>"
+	}
+
+	reason := make([]uint8, reasonLen)
+	if err := binary.Read(c.c, binary.BigEndian, &reason); err != nil {
+		return "<error>"
+	}
+
+	return string(reason)
+}

pkg/driver/vncdriver/vnc/client_auth.go

@@ -0,0 +1,124 @@
+package vnc
+
+import (
+	"net"
+
+	"crypto/des"
+	"encoding/binary"
+)
+
+// A ClientAuth implements a method of authenticating with a remote server.
+type ClientAuth interface {
+	// SecurityType returns the byte identifier sent by the server to
+	// identify this authentication scheme.
+	SecurityType() uint8
+
+	// Handshake is called when the authentication handshake should be
+	// performed, as part of the general RFB handshake. (see 7.2.1)
+	Handshake(net.Conn) error
+}
+
+// ClientAuthNone is the "none" authentication. See 7.2.1
+type ClientAuthNone byte
+
+func (*ClientAuthNone) SecurityType() uint8 {
+	return 1
+}
+
+func (*ClientAuthNone) Handshake(net.Conn) error {
+	return nil
+}
+
+// PasswordAuth is VNC authentication, 7.2.2
+type PasswordAuth struct {
+	Password string
+}
+
+func (p *PasswordAuth) SecurityType() uint8 {
+	return 2
+}
+
+func (p *PasswordAuth) Handshake(c net.Conn) error {
+	randomValue := make([]uint8, 16)
+	if err := binary.Read(c, binary.BigEndian, &randomValue); err != nil {
+		return err
+	}
+
+	crypted, err := p.encrypt(p.Password, randomValue)
+
+	if (err != nil) {
+		return err
+	}
+
+	if err := binary.Write(c, binary.BigEndian, &crypted); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *PasswordAuth) reverseBits(b byte) byte {
+	var reverse = [256]int{
+		0, 128, 64, 192, 32, 160, 96, 224,
+		16, 144, 80, 208, 48, 176, 112, 240,
+		8, 136, 72, 200, 40, 168, 104, 232,
+		24, 152, 88, 216, 56, 184, 120, 248,
+		4, 132, 68, 196, 36, 164, 100, 228,
+		20, 148, 84, 212, 52, 180, 116, 244,
+		12, 140, 76, 204, 44, 172, 108, 236,
+		28, 156, 92, 220, 60, 188, 124, 252,
+		2, 130, 66, 194, 34, 162, 98, 226,
+		18, 146, 82, 210, 50, 178, 114, 242,
+		10, 138, 74, 202, 42, 170, 106, 234,
+		26, 154, 90, 218, 58, 186, 122, 250,
+		6, 134, 70, 198, 38, 166, 102, 230,
+		22, 150, 86, 214, 54, 182, 118, 246,
+		14, 142, 78, 206, 46, 174, 110, 238,
+		30, 158, 94, 222, 62, 190, 126, 254,
+		1, 129, 65, 193, 33, 161, 97, 225,
+		17, 145, 81, 209, 49, 177, 113, 241,
+		9, 137, 73, 201, 41, 169, 105, 233,
+		25, 153, 89, 217, 57, 185, 121, 249,
+		5, 133, 69, 197, 37, 165, 101, 229,
+		21, 149, 85, 213, 53, 181, 117, 245,
+		13, 141, 77, 205, 45, 173, 109, 237,
+		29, 157, 93, 221, 61, 189, 125, 253,
+		3, 131, 67, 195, 35, 163, 99, 227,
+		19, 147, 83, 211, 51, 179, 115, 243,
+		11, 139, 75, 203, 43, 171, 107, 235,
+		27, 155, 91, 219, 59, 187, 123, 251,
+		7, 135, 71, 199, 39, 167, 103, 231,
+		23, 151, 87, 215, 55, 183, 119, 247,
+		15, 143, 79, 207, 47, 175, 111, 239,
+		31, 159, 95, 223, 63, 191, 127, 255,
+	}
+
+	return byte(reverse[int(b)])
+}
+
+func (p *PasswordAuth) encrypt(key string, bytes []byte) ([]byte, error) {
+	keyBytes := []byte{0,0,0,0,0,0,0,0}
+
+	if len(key) > 8 {
+		key = key[:8]
+	}
+
+	for i := 0; i < len(key); i++ {
+		keyBytes[i] = p.reverseBits(key[i])
+	}
+
+	block, err := des.NewCipher(keyBytes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	result1 := make([]byte, 8)
+	block.Encrypt(result1, bytes)
+	result2 := make([]byte, 8)
+	block.Encrypt(result2, bytes[8:])
+
+	crypted := append(result1, result2...)
+
+	return crypted, nil
+}

pkg/driver/vncdriver/vnc/color.go

@@ -0,0 +1,6 @@
+package vnc
+
+// Color represents a single color in a color map.
+type Color struct {
+	R, G, B uint16
+}

pkg/driver/vncdriver/vnc/encoding.go

@@ -0,0 +1,217 @@
+package vnc
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/binary"
+	"io"
+)
+
+// An Encoding implements a method for encoding pixel data that is
+// sent by the server to the client.
+type Encoding interface {
+	// The number that uniquely identifies this encoding type.
+	Type() int32
+
+	// Read reads the contents of the encoded pixel data from the reader.
+	// This should return a new Encoding implementation that contains
+	// the proper data.
+	Read(*ClientConn, *Rectangle, io.Reader) (Encoding, error)
+}
+type CursorEncoding struct {
+}
+
+func (*CursorEncoding) Type() int32 {
+	return -239
+}
+func (*CursorEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	size := int(rect.Height) * int(rect.Width) * int(c.PixelFormat.BPP) / 8
+	pixelBytes := make([]uint8, size)
+	if _, err := io.ReadFull(r, pixelBytes); err != nil {
+		return nil, err
+	}
+	mask := ((int(rect.Width) + 7) / 8) * int(rect.Height)
+	maskBytes := make([]uint8, mask)
+	if _, err := io.ReadFull(r, maskBytes); err != nil {
+		return nil, err
+	}
+	return &CursorEncoding{}, nil
+}
+
+// RawEncoding is raw pixel data sent by the server.
+//
+// See RFC 6143 Section 7.7.1
+type RawEncoding struct {
+	Colors   []Color
+	RawPixel []uint32 //RGBA
+}
+
+func (*RawEncoding) Type() int32 {
+	return 0
+}
+
+func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("RawEncoding")
+	bytesPerPixel := c.PixelFormat.BPP / 8
+	pixelBytes := make([]uint8, bytesPerPixel)
+
+	var byteOrder binary.ByteOrder = binary.LittleEndian
+	if c.PixelFormat.BigEndian {
+		byteOrder = binary.BigEndian
+	}
+
+	colors := make([]Color, int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
+	for y := uint16(0); y < rect.Height; y++ {
+		for x := uint16(0); x < rect.Width; x++ {
+			if _, err := io.ReadFull(r, pixelBytes); err != nil {
+				return nil, err
+			}
+
+			var rawPixel uint32
+			if c.PixelFormat.BPP == 8 {
+				rawPixel = uint32(pixelBytes[0])
+			} else if c.PixelFormat.BPP == 16 {
+				rawPixel = uint32(byteOrder.Uint16(pixelBytes))
+			} else if c.PixelFormat.BPP == 32 {
+				rawPixel = byteOrder.Uint32(pixelBytes)
+			}
+			//rawPixels[int(y)*int(rect.Width)+int(x)]=rawPixel
+			color := &colors[int(y)*int(rect.Width)+int(x)]
+			if c.PixelFormat.TrueColor {
+				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
+				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
+				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
+			} else {
+				*color = c.ColorMap[rawPixel]
+			}
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+			//fmt.Printf("%x %x",rawPixel,rawPixels[int(y)*int(rect.Width)+int(x)])
+		}
+	}
+
+	return &RawEncoding{colors, rawPixels}, nil
+}
+
+// ZlibEncoding is raw pixel data sent by the server compressed by Zlib.
+//
+// A single Zlib stream is created. There is only a single header for a framebuffer request response.
+type ZlibEncoding struct {
+	Colors   []Color
+	RawPixel []uint32
+	ZStream  *bytes.Buffer
+	ZReader  io.ReadCloser
+}
+
+func (*ZlibEncoding) Type() int32 {
+	return 6
+}
+
+func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("ZlibEncoding")
+	bytesPerPixel := c.PixelFormat.BPP / 8
+	pixelBytes := make([]uint8, bytesPerPixel)
+
+	var byteOrder binary.ByteOrder = binary.LittleEndian
+	if c.PixelFormat.BigEndian {
+		byteOrder = binary.BigEndian
+	}
+
+	// Format
+	// 4 bytes        | uint32 | length
+	// 'length' bytes | []byte | zlibData
+	// Read zlib length
+	var zipLength uint32
+	err := binary.Read(r, binary.BigEndian, &zipLength)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read all compressed data
+	zBytes := make([]byte, zipLength)
+	if _, err := io.ReadFull(r, zBytes); err != nil {
+		return nil, err
+	}
+
+	// Create new zlib stream if needed
+	if ze.ZStream == nil {
+		// Create and save the buffer
+		ze.ZStream = new(bytes.Buffer)
+		ze.ZStream.Write(zBytes)
+
+		// Create a reader for the buffer
+		ze.ZReader, err = zlib.NewReader(ze.ZStream)
+		if err != nil {
+			return nil, err
+		}
+
+		// This is needed to avoid 'zlib missing header'
+	} else {
+		// Just append if already created
+		ze.ZStream.Write(zBytes)
+	}
+
+	// Calculate zlib decompressed size
+	sizeToRead := int(rect.Height) * int(rect.Width) * int(bytesPerPixel)
+
+	// Create buffer for bytes
+	colorBytes := make([]byte, sizeToRead)
+
+	// Read all data from zlib stream
+	read, err := io.ReadFull(ze.ZReader, colorBytes)
+	if read != sizeToRead || err != nil {
+		return nil, err
+	}
+
+	// Create buffer for raw encoding
+	colorReader := bytes.NewReader(colorBytes)
+
+	colors := make([]Color, int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
+	for y := uint16(0); y < rect.Height; y++ {
+		for x := uint16(0); x < rect.Width; x++ {
+			if _, err := io.ReadFull(colorReader, pixelBytes); err != nil {
+				return nil, err
+			}
+
+			var rawPixel uint32
+			if c.PixelFormat.BPP == 8 {
+				rawPixel = uint32(pixelBytes[0])
+			} else if c.PixelFormat.BPP == 16 {
+				rawPixel = uint32(byteOrder.Uint16(pixelBytes))
+			} else if c.PixelFormat.BPP == 32 {
+				rawPixel = byteOrder.Uint32(pixelBytes)
+			}
+
+			color := &colors[int(y)*int(rect.Width)+int(x)]
+			if c.PixelFormat.TrueColor {
+				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
+				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
+				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
+			} else {
+				*color = c.ColorMap[rawPixel]
+			}
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+		}
+	}
+
+	return &ZlibEncoding{Colors: colors, RawPixel: rawPixels}, nil
+}
+
+func (ze *ZlibEncoding) Close() {
+	if ze.ZStream != nil {
+		ze.ZStream = nil
+		ze.ZReader.Close()
+		ze.ZReader = nil
+	}
+}

pkg/driver/vncdriver/vnc/pixel_format.go

@@ -0,0 +1,151 @@
+package vnc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+)
+
+// PixelFormat describes the way a pixel is formatted for a VNC connection.
+//
+// See RFC 6143 Section 7.4 for information on each of the fields.
+type PixelFormat struct {
+	BPP        uint8
+	Depth      uint8
+	BigEndian  bool
+	TrueColor  bool
+	RedMax     uint16
+	GreenMax   uint16
+	BlueMax    uint16
+	RedShift   uint8
+	GreenShift uint8
+	BlueShift  uint8
+}
+
+func readPixelFormat(r io.Reader, result *PixelFormat) error {
+	var rawPixelFormat [16]byte
+	if _, err := io.ReadFull(r, rawPixelFormat[:]); err != nil {
+		return err
+	}
+
+	var pfBoolByte uint8
+	brPF := bytes.NewReader(rawPixelFormat[:])
+	if err := binary.Read(brPF, binary.BigEndian, &result.BPP); err != nil {
+		return err
+	}
+
+	if err := binary.Read(brPF, binary.BigEndian, &result.Depth); err != nil {
+		return err
+	}
+
+	if err := binary.Read(brPF, binary.BigEndian, &pfBoolByte); err != nil {
+		return err
+	}
+
+	if pfBoolByte != 0 {
+		// Big endian is true
+		result.BigEndian = true
+	}
+
+	if err := binary.Read(brPF, binary.BigEndian, &pfBoolByte); err != nil {
+		return err
+	}
+
+	if pfBoolByte != 0 {
+		// True Color is true. So we also have to read all the color max & shifts.
+		result.TrueColor = true
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.RedMax); err != nil {
+			return err
+		}
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.GreenMax); err != nil {
+			return err
+		}
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.BlueMax); err != nil {
+			return err
+		}
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.RedShift); err != nil {
+			return err
+		}
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.GreenShift); err != nil {
+			return err
+		}
+
+		if err := binary.Read(brPF, binary.BigEndian, &result.BlueShift); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writePixelFormat(format *PixelFormat) ([]byte, error) {
+	var buf bytes.Buffer
+
+	// Byte 1
+	if err := binary.Write(&buf, binary.BigEndian, format.BPP); err != nil {
+		return nil, err
+	}
+
+	// Byte 2
+	if err := binary.Write(&buf, binary.BigEndian, format.Depth); err != nil {
+		return nil, err
+	}
+
+	var boolByte byte
+	if format.BigEndian {
+		boolByte = 1
+	} else {
+		boolByte = 0
+	}
+
+	// Byte 3 (BigEndian)
+	if err := binary.Write(&buf, binary.BigEndian, boolByte); err != nil {
+		return nil, err
+	}
+
+	if format.TrueColor {
+		boolByte = 1
+	} else {
+		boolByte = 0
+	}
+
+	// Byte 4 (TrueColor)
+	if err := binary.Write(&buf, binary.BigEndian, boolByte); err != nil {
+		return nil, err
+	}
+
+	// If we have true color enabled then we have to fill in the rest of the
+	// structure with the color values.
+	if format.TrueColor {
+		if err := binary.Write(&buf, binary.BigEndian, format.RedMax); err != nil {
+			return nil, err
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, format.GreenMax); err != nil {
+			return nil, err
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, format.BlueMax); err != nil {
+			return nil, err
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, format.RedShift); err != nil {
+			return nil, err
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, format.GreenShift); err != nil {
+			return nil, err
+		}
+
+		if err := binary.Write(&buf, binary.BigEndian, format.BlueShift); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes()[0:16], nil
+}

pkg/driver/vncdriver/vnc/pointer.go

@@ -0,0 +1,16 @@
+package vnc
+
+// ButtonMask represents a mask of pointer presses/releases.
+type ButtonMask uint8
+
+// All available button mask components.
+const (
+	ButtonLeft ButtonMask = 1 << iota
+	ButtonMiddle
+	ButtonRight
+	Button4
+	Button5
+	Button6
+	Button7
+	Button8
+)