Allow to enable frame copy on a video track (#405)

Also make the ReadCloser an Controllable allows to uncouple
the controller implementation from the encoder.
This is not needed for the 2 codec controller already implemented (openh264 and vpx)
but is more future proof in case it required for other codecs.

.github/workflows/ci.yaml

@@ -17,9 +17,9 @@ jobs:
     name: Linux Go ${{ matrix.go }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -32,7 +32,7 @@ jobs:
             libx264-dev
       - name: Run Test Suite
         run: make test
-      - uses: codecov/codecov-action@v2	
+      - uses: codecov/codecov-action@v3	
         if: matrix.go == '1.16'
   build-darwin:
     runs-on: macos-latest
@@ -43,9 +43,9 @@ jobs:
     name: Darwin Go ${{ matrix.go }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -62,9 +62,9 @@ jobs:
     name: Check Licenses
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: '1.16'
       - name: Installing go-licenses

.github/workflows/renovate-go-mod-fix.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 2
       - name: fix

.gitignore

@@ -13,3 +13,5 @@
 
 scripts/cross
 coverage.txt
+
+.idea

Makefile

@@ -16,7 +16,8 @@ supported_platforms := \
   linux-arm64 \
   linux-x64 \
   windows-x64 \
-  darwin-x64
+  darwin-x64 \
+  darwin-arm64
 cmd_build := build
 cmd_test := test
 examples_dir := examples

dockerfiles/darwin-arm64.Dockerfile

@@ -0,0 +1,47 @@
+FROM dockercore/golang-cross as m1cross
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -qq && apt-get install -y -q --no-install-recommends \
+    cmake \
+    git \
+    libssl-dev \
+    libxml2-dev \
+    libz-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV SDK_VERSION=11.3 \
+    TARGET_DIR=/osxcross/target \
+    UNATTENDED=1
+
+WORKDIR /work
+RUN git clone --depth=1 https://github.com/tpoechtrager/osxcross.git /work \
+  && cd /work/tarballs \
+  && wget -q https://github.com/phracker/MacOSX-SDKs/releases/download/${SDK_VERSION}/MacOSX${SDK_VERSION}.sdk.tar.xz
+
+# Build cross compile toolchain for Apple silicon
+RUN ./build.sh
+
+
+FROM dockcross/base
+
+ENV OSX_CROSS_PATH=/osxcross
+
+COPY --from=m1cross "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}/"
+ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+
+ENV CC=arm64-apple-darwin20.4-clang \
+    CXX=arm64-apple-darwin20.4-clang++ \
+    CPP=arm64-apple-darwin20.4-clang++ \
+    AR=arm64-apple-darwin20.4-ar \
+    AS=arm64-apple-darwin20.4-as \
+    LD=arm64-apple-darwin20.4-ld
+
+COPY darwin-arm64.cmake ${OSX_CROSS_PATH}/
+ENV CMAKE_TOOLCHAIN_FILE ${OSX_CROSS_PATH}/darwin-arm64.cmake
+
+ARG IMAGE=lherman/cross-darwin-arm64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}

dockerfiles/darwin-arm64.cmake

@@ -0,0 +1,8 @@
+set(CMAKE_SYSTEM_NAME Darwin)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR arm64)
+
+set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_CXX_COMPILER $ENV{CXX})
+set(CMAKE_AR $ENV{AR})
+set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})

examples/go.sum

@@ -13,6 +13,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/gen2brain/malgo v0.10.29 h1:bTYiUTUKJsEomNby+W0hgyLrOttUXIk4lTEnKA54iqM=
 github.com/gen2brain/malgo v0.10.29/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
@@ -21,6 +22,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -36,18 +38,19 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
 github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
-github.com/pion/dtls/v2 v2.0.8 h1:reGe8rNIMfO/UAeFLqO61tl64t154Qfkr4U3Gzu1tsg=
+github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8=
-github.com/pion/dtls/v2 v2.0.8/go.mod h1:QuDII+8FVvk9Dp5t5vYIMTo7hh7uBkra+8QIm7QGm10=
+github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
-github.com/pion/ice/v2 v2.0.16 h1:K6bzD8ef9vMKbGMTHaUweHXEyuNGnvr2zdqKoLKZPn0=
+github.com/pion/ice/v2 v2.1.7 h1:FjgDfUNrVYTxQabJrkBX6ld12tvYbgzHenqPh3PJF6E=
-github.com/pion/ice/v2 v2.0.16/go.mod h1:SJNJzC27gDZoOW0UoxIoC8Hf2PDxG28hQyNdSexDu38=
+github.com/pion/ice/v2 v2.1.7/go.mod h1:kV4EODVD5ux2z8XncbLHIOtcXKtYXVgLVCeVqnpoeP0=
 github.com/pion/interceptor v0.0.12 h1:eC1iVneBIAQJEfaNAfDqAncJWhMDAnaXPRCJsltdokE=
 github.com/pion/interceptor v0.0.12/go.mod h1:qzeuWuD/ZXvPqOnxNcnhWfkCZ2e1kwwslicyyPnhoK4=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -60,6 +63,8 @@ github.com/pion/rtcp v1.2.6 h1:1zvwBbyd0TeEuuWftrd/4d++m+/kZSeiguxU61LFWpo=
 github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0=
 github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U=
 github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/rtp v1.6.5 h1:o2cZf8OascA5HF/b0PAbTxRKvOWxTQxWYt7SlToxFGI=
+github.com/pion/rtp v1.6.5/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sctp v1.7.12 h1:GsatLufywVruXbZZT1CKg+Jr8ZTkwiPnmUC/oO9+uuY=
 github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
@@ -69,17 +74,14 @@ github.com/pion/srtp/v2 v2.0.2 h1:664iGzVmaY7KYS5M0gleY0DscRo9ReDfTxQrq4UgGoU=
 github.com/pion/srtp/v2 v2.0.2/go.mod h1:VEyLv4CuxrwGY8cxM+Ng3bmVy8ckz/1t6A0q/msKOw0=
 github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
 github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
-github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE=
 github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
 github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
 github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw=
 github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A=
 github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
 github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
-github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
+github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
-github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
+github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
-github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw=
-github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -90,24 +92,32 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk=
 golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210420210106-798c2154c571 h1:Q6Bg8xzKzpFPU4Oi1sBnBTHBwlMsLeEXpu4hYBY8rAg=
+golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -116,19 +126,23 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -147,5 +161,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go.mod

@@ -3,13 +3,15 @@ module github.com/pion/mediadevices
 go 1.13
 
 require (
-	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
+	github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165
 	github.com/gen2brain/malgo v0.10.35
 	github.com/google/uuid v1.3.0
 	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329
+	github.com/pion/interceptor v0.1.10
 	github.com/pion/logging v0.2.2
-	github.com/pion/rtp v1.7.4
+	github.com/pion/rtcp v1.2.9
-	github.com/pion/webrtc/v3 v3.1.10
+	github.com/pion/rtp v1.7.13
-	golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d
+	github.com/pion/webrtc/v3 v3.1.34
+	golang.org/x/image v0.0.0-20220617043117-41969df76e82
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
+github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165 h1:QsIbRyO2tn5eSJZ/skuDqSTo0GWI5H4G1AT7Mm2H0Nw=
-github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
+github.com/blackjack/webcam v0.0.0-20220329180758-ba064708e165/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -17,10 +17,12 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -40,19 +42,19 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E=
 github.com/pion/datachannel v1.5.2/go.mod h1:FTGQWaHrdCwIJ1rw6xBIfZVkslikjShim5yr05XFuCQ=
-github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
+github.com/pion/dtls/v2 v2.1.3 h1:3UF7udADqous+M2R5Uo2q/YaP4EzUoWKdfX2oscCUio=
-github.com/pion/dtls/v2 v2.0.10 h1:wgys7gPR1NMbWjmjJ3CW7lkUGaun8djgH8nahpNLnxI=
+github.com/pion/dtls/v2 v2.1.3/go.mod h1:o6+WvyLDAlXF7YiPB/RlskRoeK+/JtuaZa5emwQcWus=
-github.com/pion/dtls/v2 v2.0.10/go.mod h1:00OxfeCRWHShcqT9jx8pKKmBWuTt0NCZoVPCaC4VKvU=
+github.com/pion/ice/v2 v2.2.6 h1:R/vaLlI1J2gCx141L5PEwtuGAGcyS6e7E0hDeJFq5Ig=
-github.com/pion/ice/v2 v2.1.14 h1:nD9GZs3MiR1/dPa5EiMRMe8hLBG3/qqCdx/hTS2g8VE=
+github.com/pion/ice/v2 v2.2.6/go.mod h1:SWuHiOGP17lGromHTFadUe1EuPgFh/oCU6FCMZHooVE=
-github.com/pion/ice/v2 v2.1.14/go.mod h1:ovgYHUmwYLlRvcCLI67PnQ5YGe+upXZbGgllBDG/ktU=
+github.com/pion/interceptor v0.1.10 h1:DJ2GjMGm4XGIQgMJxuEpdaExdY/6RdngT7Uh4oVmquU=
-github.com/pion/interceptor v0.1.0 h1:SlXKaDlEvSl7cr4j8fJykzVz4UdH+7UDtcvx+u01wLU=
+github.com/pion/interceptor v0.1.10/go.mod h1:Lh3JSl/cbJ2wP8I3ccrjh1K/deRGRn3UlSPuOTiHb6U=
-github.com/pion/interceptor v0.1.0/go.mod h1:j5NIl3tJJPB3u8+Z2Xz8MZs/VV6rc+If9mXEKNuFmEM=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
 github.com/pion/mdns v0.0.5 h1:Q2oj/JB3NqfzY9xGZ1fPzZzK7sDSD8rZPOvcIQ10BCw=
@@ -60,64 +62,63 @@ github.com/pion/mdns v0.0.5/go.mod h1:UgssrvdD3mxpi8tMxAXbsppL3vJ4Jipw1mTCW+al01
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0=
-github.com/pion/rtcp v1.2.8/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo=
 github.com/pion/rtcp v1.2.9 h1:1ujStwg++IOLIEoOiIQ2s+qBuJ1VN81KW+9pMPsif+U=
 github.com/pion/rtcp v1.2.9/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo=
 github.com/pion/rtp v1.7.0/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
-github.com/pion/rtp v1.7.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
-github.com/pion/rtp v1.7.4 h1:4dMbjb1SuynU5OpA3kz1zHK+u+eOCQjW3MAeVHf1ODA=
 github.com/pion/rtp v1.7.4/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
-github.com/pion/sctp v1.8.0 h1:6erMF2qmQwXr+0iB1lm0AUSmDr9LdmpaBzgSVAEgehw=
+github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA=
+github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.8.0/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
+github.com/pion/sctp v1.8.2 h1:yBBCIrUMJ4yFICL3RIvR4eh/H2BTTvlligmSTy+3kiA=
+github.com/pion/sctp v1.8.2/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
 github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
 github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk=
 github.com/pion/srtp/v2 v2.0.5 h1:ks3wcTvIUE/GHndO3FAvROQ9opy0uLELpwHJaQ1yqhQ=
 github.com/pion/srtp/v2 v2.0.5/go.mod h1:8k6AJlal740mrZ6WYxc4Dg6qDqqhxoRG2GSjlUhDF0A=
 github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
 github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
-github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
 github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
-github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw=
 github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A=
-github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
+github.com/pion/transport v0.13.0 h1:KWTA5ZrQogizzYwPEciGtHPLwpAjE91FgXnyu+Hv2uY=
-github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
+github.com/pion/transport v0.13.0/go.mod h1:yxm9uXpK9bpBBWkITk13cLo1y5/ur5VQpG22ny6EP7g=
+github.com/pion/turn/v2 v2.0.8 h1:KEstL92OUN3k5k8qxsXHpr7WWfrdp7iJZHx99ud8muw=
+github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw=
 github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
 github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
-github.com/pion/webrtc/v3 v3.1.10 h1:DO99F6/X1HrQho3LxTWHPjI3c388btBf56lR5UNRNNk=
+github.com/pion/webrtc/v3 v3.1.34 h1:GUfv2zxWge77x1FhZ6Fge8KQd3bTzvX1CMN1/LmTazM=
-github.com/pion/webrtc/v3 v3.1.10/go.mod h1:eL2HHZOvX+W+Q+lenuidTrWfMD9gG3aobtGcCqJ5G48=
+github.com/pion/webrtc/v3 v3.1.34/go.mod h1:jClfnbJzt8wtmewGxhPzgE5wZ0U/gNB78XCtmc/uz3k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d h1:RNPAfi2nHY7C2srAV8A49jpsYr0ADedCk1wq6fTMTvs=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
-golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd h1:zYlwaUHTmxuf6H7hwO2dgwqozQmH7zf4x+/qql4oVWc=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -132,15 +133,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
@@ -155,6 +157,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

ioreader.go

@@ -1,5 +1,7 @@
 package mediadevices
 
+import "github.com/pion/mediadevices/pkg/codec"
+
 type EncodedBuffer struct {
 	Data    []byte
 	Samples uint32
@@ -8,11 +10,13 @@ type EncodedBuffer struct {
 type EncodedReadCloser interface {
 	Read() (EncodedBuffer, func(), error)
 	Close() error
+	codec.Controllable
 }
 
 type encodedReadCloserImpl struct {
-	readFn  func() (EncodedBuffer, func(), error)
+	readFn       func() (EncodedBuffer, func(), error)
-	closeFn func() error
+	closeFn      func() error
+	controllerFn func() codec.EncoderController
 }
 
 func (r *encodedReadCloserImpl) Read() (EncodedBuffer, func(), error) {
@@ -23,9 +27,14 @@ func (r *encodedReadCloserImpl) Close() error {
 	return r.closeFn()
 }
 
+func (r *encodedReadCloserImpl) Controller() codec.EncoderController {
+	return r.controllerFn()
+}
+
 type encodedIOReadCloserImpl struct {
-	readFn  func([]byte) (int, error)
+	readFn     func([]byte) (int, error)
-	closeFn func() error
+	closeFn    func() error
+	controller func() codec.EncoderController
 }
 
 func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserImpl {
@@ -48,7 +57,8 @@ func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserIm
 			encoded.Data = encoded.Data[n:]
 			return n, nil
 		},
-		closeFn: reader.Close,
+		closeFn:    reader.Close,
+		controller: reader.Controller,
 	}
 }
 
@@ -59,3 +69,7 @@ func (r *encodedIOReadCloserImpl) Read(b []byte) (int, error) {
 func (r *encodedIOReadCloserImpl) Close() error {
 	return r.closeFn()
 }
+
+func (r *encodedIOReadCloserImpl) Controller() codec.EncoderController {
+	return r.controller()
+}

mediadevices_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pion/mediadevices/pkg/driver"
 	_ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/videotest"
+	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -97,37 +98,145 @@ func TestSelectBestDriverConstraintsResultIsSetProperly(t *testing.T) {
 		t.Fatal("expect to get at least 1 property")
 	}
 	expectedProp := driver.Properties()[0]
-	// Since this is a continuous value, bestConstraints should be set with the value that user specified
-	expectedProp.FrameRate = 30.0
 
-	wantConstraints := MediaTrackConstraints{
+	// By reducing the value from the driver by a tiny amount, this property should be chosen.
-		MediaConstraints: prop.MediaConstraints{
+	// At the same time, we'll be able to find out if the return constraints will be properly set
-			VideoConstraints: prop.VideoConstraints{
+	// to the best constraints.
-				// By reducing the width from the driver by a tiny amount, this property should be chosen.
+	cases := map[string]struct {
-				// At the same time, we'll be able to find out if the return constraints will be properly set
+		width, height int
-				// to the best constraints.
+		frameFormat   frame.Format
-				Width:       prop.Int(expectedProp.Width - 1),
+		frameRate     float32
-				Height:      prop.Int(expectedProp.Width),
+	}{
-				FrameFormat: prop.FrameFormat(expectedProp.FrameFormat),
+		"DifferentWidth": {
-				FrameRate:   prop.Float(30.0),
+			width:       expectedProp.Width - 1,
-			},
+			height:      expectedProp.Height,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentHeight": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height - 1,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentFrameFormat": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height,
+			frameFormat: frame.FormatI420,
+			frameRate:   expectedProp.FrameRate,
 		},
 	}
 
-	bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			var vc prop.VideoConstraints
+
+			if c.frameRate >= 0 {
+				vc = prop.VideoConstraints{
+					Width:       prop.Int(c.width),
+					Height:      prop.Int(c.height),
+					FrameFormat: prop.FrameFormat(c.frameFormat),
+					FrameRate:   prop.Float(c.frameRate),
+				}
+			} else {
+				// do not specify the framerate
+				vc = prop.VideoConstraints{
+					Width:       prop.Int(c.width),
+					Height:      prop.Int(c.height),
+					FrameFormat: prop.FrameFormat(c.frameFormat),
+				}
+			}
+			wantConstraints := MediaTrackConstraints{
+				MediaConstraints: prop.MediaConstraints{
+					VideoConstraints: vc,
+				},
+			}
+
+			bestDriver, bestConstraints, err := selectBestDriver(filterFn, wantConstraints)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if driver != bestDriver {
+				t.Fatal("best driver is not expected")
+			}
+
+			s := bestConstraints.selectedMedia
+			if s.Width != expectedProp.Width ||
+				s.Height != expectedProp.Height ||
+				s.FrameFormat != expectedProp.FrameFormat ||
+				s.FrameRate != expectedProp.FrameRate {
+				t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
+			}
+
+		})
+	}
+
+}
+
+func TestSelectBestDriverConstraintsNoFit(t *testing.T) {
+	filterFn := driver.FilterVideoRecorder()
+	drivers := driver.GetManager().Query(filterFn)
+	if len(drivers) == 0 {
+		t.Fatal("expect to get at least 1 driver")
+	}
+
+	driver := drivers[0]
+	err := driver.Open()
 	if err != nil {
-		t.Fatal(err)
+		t.Fatal("expect to open driver successfully")
+	}
+	defer driver.Close()
+
+	if len(driver.Properties()) == 0 {
+		t.Fatal("expect to get at least 1 property")
+	}
+	expectedProp := driver.Properties()[0]
+
+	cases := map[string]struct {
+		width, height int
+		frameFormat   frame.Format
+		frameRate     float32
+	}{
+		"DifferentWidth": {
+			width:       expectedProp.Width - 1,
+			height:      expectedProp.Height,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentHeight": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height - 1,
+			frameFormat: expectedProp.FrameFormat,
+			frameRate:   expectedProp.FrameRate,
+		},
+		"DifferentFrameFormat": {
+			width:       expectedProp.Width,
+			height:      expectedProp.Height,
+			frameFormat: frame.FormatI420,
+			frameRate:   expectedProp.FrameRate,
+		},
 	}
 
-	if driver != bestDriver {
+	for name, c := range cases {
-		t.Fatal("best driver is not expected")
+		c := c
-	}
+		t.Run(name, func(t *testing.T) {
+			wantConstraints := MediaTrackConstraints{
+				MediaConstraints: prop.MediaConstraints{
+					VideoConstraints: prop.VideoConstraints{
+						Width:       prop.IntExact(c.width),
+						Height:      prop.IntExact(c.height),
+						FrameFormat: prop.FrameFormatExact(c.frameFormat),
+						FrameRate:   prop.FloatExact(c.frameRate),
+					},
+				},
+			}
 
-	s := bestConstraints.selectedMedia
+			_, _, err := selectBestDriver(filterFn, wantConstraints)
-	if s.Width != expectedProp.Width ||
+			if err == nil {
-		s.Height != expectedProp.Height ||
+				t.Fatal("expect to not find a driver that fits the constraints")
-		s.FrameFormat != expectedProp.FrameFormat ||
+			}
-		s.FrameRate != expectedProp.FrameRate {
+		})
-		t.Fatalf("failed to return best constraints\nexpected:\n%v\n\ngot:\n%v", expectedProp, bestConstraints.selectedMedia)
 	}
 }

pkg/avfoundation/avfoundation_darwin.go

@@ -11,6 +11,7 @@ package avfoundation
 // }
 import "C"
 import (
+	"context"
 	"fmt"
 	"io"
 	"unsafe"
@@ -93,9 +94,11 @@ func Devices(mediaType MediaType) ([]Device, error) {
 // ReadCloser is a wrapper around the data callback from AVFoundation. The data received from the
 // the underlying callback can be retrieved by calling Read.
 type ReadCloser struct {
-	dataChan chan []byte
+	dataChan   chan []byte
-	id       handleID
+	id         handleID
-	onClose  func()
+	onClose    func()
+	cancelCtx  context.Context
+	cancelFunc func()
 }
 
 func newReadCloser(onClose func()) *ReadCloser {
@@ -103,12 +106,22 @@ func newReadCloser(onClose func()) *ReadCloser {
 	rc.dataChan = make(chan []byte, 1)
 	rc.onClose = onClose
 	rc.id = register(rc.dataCb)
+	cancelCtx, cancelFunc := context.WithCancel(context.Background())
+	rc.cancelCtx = cancelCtx
+	rc.cancelFunc = cancelFunc
 	return &rc
 }
 
 func (rc *ReadCloser) dataCb(data []byte) {
 	// TODO: add a policy for slow reader
-	rc.dataChan <- data
+	if rc.cancelCtx.Err() != nil {
+		return
+	}
+	select {
+	case <-rc.cancelCtx.Done():
+		close(rc.dataChan)
+	case rc.dataChan <- data:
+	}
 }
 
 // Read reads raw data, the format is determined by the media type and property:
@@ -127,7 +140,7 @@ func (rc *ReadCloser) Close() {
 	if rc.onClose != nil {
 		rc.onClose()
 	}
-	close(rc.dataChan)
+	rc.cancelFunc()
 	unregister(rc.id)
 }
 

pkg/codec/codec.go

@@ -112,15 +112,37 @@ type VideoEncoderBuilder interface {
 	BuildVideoEncoder(r video.Reader, p prop.Media) (ReadCloser, error)
 }
 
-// ReadCloser is an io.ReadCloser with methods for rate limiting: SetBitRate and ForceKeyFrame
+// ReadCloser is an io.ReadCloser with a controller
 type ReadCloser interface {
 	Read() (b []byte, release func(), err error)
 	Close() error
+	Controllable
+}
+
+// EncoderController is the interface allowing to control the encoder behaviour after it's initialisation.
+// It will possibly have common control method in the future.
+// A controller can have optional methods represented by *Controller interfaces
+type EncoderController interface{}
+
+// Controllable is a interface representing a encoder which can be controlled
+// after it's initialisation with an EncoderController
+type Controllable interface {
+	Controller() EncoderController
+}
+
+// KeyFrameController is a interface representing an encoder that can be forced to produce key frame on demand
+type KeyFrameController interface {
+	EncoderController
+	// ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame.
+	ForceKeyFrame() error
+}
+
+// BitRateController is a interface representing an encoder which can have a variable bit rate
+type BitRateController interface {
+	EncoderController
 	// SetBitRate sets current target bitrate, lower bitrate means smaller data will be transmitted
 	// but this also means that the quality will also be lower.
 	SetBitRate(int) error
-	// ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame.
-	ForceKeyFrame() error
 }
 
 // BaseParams represents an codec's encoding properties

pkg/codec/mmal/mmal.go

@@ -91,12 +91,8 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
+func (e *encoder) Controller() codec.EncoderController {
-	panic("SetBitRate is not implemented")
+	return e
-}
-
-func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
 }
 
 func (e *encoder) Close() error {

pkg/codec/mmal/mmal_test.go

@@ -4,6 +4,7 @@ import (
 	"image"
 	"testing"
 
+	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -44,3 +45,21 @@ func TestEncoder(t *testing.T) {
 		})
 	})
 }
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/openh264/bridge.cpp

@@ -21,29 +21,26 @@ Encoder *enc_new(const EncoderOptions opts, int *eresult) {
     return NULL;
   }
 
-  // TODO: Remove hardcoded values
+  params.iUsageType = opts.usage_type;
-  params.iUsageType = CAMERA_VIDEO_REAL_TIME;
   params.iPicWidth = opts.width;
   params.iPicHeight = opts.height;
   params.iTargetBitrate = opts.target_bitrate;
   params.iMaxBitrate = opts.target_bitrate;
-  params.iRCMode = RC_BITRATE_MODE;
+  params.iRCMode = opts.rc_mode;
   params.fMaxFrameRate = opts.max_fps;
-  params.bEnableFrameSkip = true;
+  params.bEnableFrameSkip = opts.enable_frame_skip;
-  params.uiMaxNalSize = 0;
+  params.uiMaxNalSize = opts.max_nal_size;
-  params.uiIntraPeriod = 30;
+  params.uiIntraPeriod = opts.intra_period;
-  // set to 0, so that it'll automatically use multi threads when needed
+  params.iMultipleThreadIdc = opts.multiple_thread_idc;
-  params.iMultipleThreadIdc = 0;
   // The base spatial layer 0 is the only one we use.
   params.sSpatialLayers[0].iVideoWidth = params.iPicWidth;
   params.sSpatialLayers[0].iVideoHeight = params.iPicHeight;
   params.sSpatialLayers[0].fFrameRate = params.fMaxFrameRate;
   params.sSpatialLayers[0].iSpatialBitrate = params.iTargetBitrate;
   params.sSpatialLayers[0].iMaxSpatialBitrate = params.iTargetBitrate;
-  // Single NAL unit mode
+  params.sSpatialLayers[0].sSliceArgument.uiSliceNum = opts.slice_num;
-  params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
+  params.sSpatialLayers[0].sSliceArgument.uiSliceMode = opts.slice_mode;
-  params.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_SIZELIMITED_SLICE;
+  params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = opts.slice_size_constraint;
-  params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = 12800;
 
   rv = engine->InitializeExt(&params);
   if (rv != 0) {
@@ -81,16 +78,15 @@ Slice enc_encode(Encoder *e, Frame f, int *eresult) {
   Slice payload = {0};
 
   if(e->force_key_frame == 1) {
-    info.eFrameType = videoFrameTypeI;
+    e->engine->ForceIntraFrame(true);
     e->force_key_frame = 0;
   }
 
   pic.iPicWidth = f.width;
   pic.iPicHeight = f.height;
   pic.iColorFormat = videoFormatI420;
-  // We always received I420 format
+  pic.iStride[0] = f.ystride;
-  pic.iStride[0] = pic.iPicWidth;
+  pic.iStride[1] = pic.iStride[2] = f.cstride;
-  pic.iStride[1] = pic.iStride[2] = pic.iPicWidth / 2;
   pic.pData[0] = (unsigned char *)f.y;
   pic.pData[1] = (unsigned char *)f.u;
   pic.pData[2] = (unsigned char *)f.v;

pkg/codec/openh264/bridge.hpp

@@ -12,6 +12,8 @@ typedef struct Slice {
 
 typedef struct Frame {
   void *y, *u, *v;
+  int ystride;
+  int cstride;
   int height;
   int width;
 } Frame;
@@ -20,6 +22,15 @@ typedef struct EncoderOptions {
   int width, height;
   int target_bitrate;
   float max_fps;
+  EUsageType usage_type;
+  RC_MODES rc_mode;
+  bool enable_frame_skip;
+  unsigned int max_nal_size;
+  unsigned int intra_period;
+  int multiple_thread_idc;
+  unsigned int slice_num;
+  SliceModeEnum slice_mode;
+  unsigned int slice_size_constraint;
 } EncoderOptions;
 
 typedef struct Encoder {

pkg/codec/openh264/openh264.go

@@ -33,10 +33,19 @@ func newEncoder(r video.Reader, p prop.Media, params Params) (codec.ReadCloser,
 
 	var rv C.int
 	cEncoder := C.enc_new(C.EncoderOptions{
-		width:          C.int(p.Width),
+		width:                 C.int(p.Width),
-		height:         C.int(p.Height),
+		height:                C.int(p.Height),
-		target_bitrate: C.int(params.BitRate),
+		target_bitrate:        C.int(params.BitRate),
-		max_fps:        C.float(p.FrameRate),
+		max_fps:               C.float(p.FrameRate),
+		usage_type:            C.EUsageType(params.UsageType),
+		rc_mode:               C.RC_MODES(params.RCMode),
+		enable_frame_skip:     C.bool(params.EnableFrameSkip),
+		max_nal_size:          C.uint(params.MaxNalSize),
+		intra_period:          C.uint(params.IntraPeriod),
+		multiple_thread_idc:   C.int(params.MultipleThreadIdc),
+		slice_num:             C.uint(params.SliceNum),
+		slice_mode:            C.SliceModeEnum(params.SliceMode),
+		slice_size_constraint: C.uint(params.SliceSizeConstraint),
 	}, &rv)
 	if err := errResult(rv); err != nil {
 		return nil, fmt.Errorf("failed in creating encoder: %v", err)
@@ -65,11 +74,13 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	bounds := yuvImg.Bounds()
 	var rv C.int
 	s := C.enc_encode(e.engine, C.Frame{
-		y:      unsafe.Pointer(&yuvImg.Y[0]),
+		y:       unsafe.Pointer(&yuvImg.Y[0]),
-		u:      unsafe.Pointer(&yuvImg.Cb[0]),
+		u:       unsafe.Pointer(&yuvImg.Cb[0]),
-		v:      unsafe.Pointer(&yuvImg.Cr[0]),
+		v:       unsafe.Pointer(&yuvImg.Cr[0]),
-		height: C.int(bounds.Max.Y - bounds.Min.Y),
+		ystride: C.int(yuvImg.YStride),
-		width:  C.int(bounds.Max.X - bounds.Min.X),
+		cstride: C.int(yuvImg.CStride),
+		height:  C.int(bounds.Max.Y - bounds.Min.Y),
+		width:   C.int(bounds.Max.X - bounds.Min.X),
 	}, &rv)
 	if err := errResult(rv); err != nil {
 		return nil, func() {}, fmt.Errorf("failed in encoding: %v", err)
@@ -79,15 +90,15 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, nil
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
-
 func (e *encoder) ForceKeyFrame() error {
 	e.engine.force_key_frame = C.int(1)
 	return nil
 }
 
+func (e *encoder) Controller() codec.EncoderController {
+	return e
+}
+
 func (e *encoder) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()

pkg/codec/openh264/openh264_static.go

@@ -8,5 +8,6 @@ package openh264
 //#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-arm64.a
 //#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-x64.a
 //#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-arm64.a
 //#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-windows-x64.a -lssp
 import "C"

pkg/codec/openh264/openh264_test.go

@@ -4,11 +4,28 @@ import (
 	"image"
 	"testing"
 
+	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
 func TestEncoder(t *testing.T) {
 	t.Run("SimpleRead", func(t *testing.T) {
 		p, err := NewParams()

pkg/codec/openh264/params.go

@@ -1,5 +1,8 @@
 package openh264
 
+// #include <openh264/codec_api.h>
+import "C"
+
 import (
 	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/io/video"
@@ -9,14 +12,62 @@ import (
 // Params stores libopenh264 specific encoding parameters.
 type Params struct {
 	codec.BaseParams
+	UsageType           UsageTypeEnum
+	RCMode              RCModeEnum
+	EnableFrameSkip     bool
+	MaxNalSize          uint
+	IntraPeriod         uint
+	MultipleThreadIdc   int
+	SliceNum            uint
+	SliceMode           SliceModeEnum
+	SliceSizeConstraint uint
 }
 
+type UsageTypeEnum int
+
+const (
+	CameraVideoRealTime      UsageTypeEnum = C.CAMERA_VIDEO_REAL_TIME   ///< camera video for real-time communication
+	ScreenContentRealTime    UsageTypeEnum = C.SCREEN_CONTENT_REAL_TIME ///< screen content signal
+	CameraVideoNonRealTime   UsageTypeEnum = C.CAMERA_VIDEO_NON_REAL_TIME
+	ScreenContentNonRealTime UsageTypeEnum = C.SCREEN_CONTENT_NON_REAL_TIME
+	InputContentTypeAll      UsageTypeEnum = C.INPUT_CONTENT_TYPE_ALL
+)
+
+type RCModeEnum int
+
+const (
+	RCQualityMode         RCModeEnum = C.RC_QUALITY_MODE           ///< quality mode
+	RCBitrateMode         RCModeEnum = C.RC_BITRATE_MODE           ///< bitrate mode
+	RCBufferbaseedMode    RCModeEnum = C.RC_BUFFERBASED_MODE       ///< no bitrate control,only using buffer status,adjust the video quality
+	RCTimestampMode       RCModeEnum = C.RC_TIMESTAMP_MODE         //rate control based timestamp
+	RCBitrateModePostSkip RCModeEnum = C.RC_BITRATE_MODE_POST_SKIP ///< this is in-building RC MODE, WILL BE DELETED after algorithm tuning!
+	RCOffMode             RCModeEnum = C.RC_OFF_MODE               ///< rate control off mode
+)
+
+type SliceModeEnum uint
+
+const (
+	SMSingleSlice      SliceModeEnum = C.SM_SINGLE_SLICE      ///< | SliceNum==1
+	SMFixedslcnumSlice SliceModeEnum = C.SM_FIXEDSLCNUM_SLICE ///< | according to SliceNum        | enabled dynamic slicing for multi-thread
+	SMRasterSlice      SliceModeEnum = C.SM_RASTER_SLICE      ///< | according to SlicesAssign    | need input of MB numbers each slice. In addition, if other constraint in SSliceArgument is presented, need to follow the constraints. Typically if MB num and slice size are both constrained, re-encoding may be involved.
+	SMSizelimitedSlice SliceModeEnum = C.SM_SIZELIMITED_SLICE ///< | according to SliceSize       | slicing according to size, the slicing will be dynamic(have no idea about slice_nums until encoding current frame)
+)
+
 // NewParams returns default openh264 codec specific parameters.
 func NewParams() (Params, error) {
 	return Params{
 		BaseParams: codec.BaseParams{
 			BitRate: 100000,
 		},
+		UsageType:           CameraVideoRealTime,
+		RCMode:              RCBitrateMode,
+		EnableFrameSkip:     true,
+		MaxNalSize:          0,
+		IntraPeriod:         30,
+		MultipleThreadIdc:   0, // Defaults to 0, so that it'll automatically use multi threads when needed
+		SliceNum:            1, // Defaults to single NAL unit mode
+		SliceMode:           SMSizelimitedSlice,
+		SliceSizeConstraint: 12800,
 	}, nil
 }
 

pkg/codec/opus/opus.go

@@ -121,8 +121,8 @@ func (e *encoder) SetBitRate(bitRate int) error {
 	return nil
 }
 
-func (e *encoder) ForceKeyFrame() error {
+func (e *encoder) Controller() codec.EncoderController {
-	panic("ForceKeyFrame is not implemented")
+	return e
 }
 
 func (e *encoder) Close() error {

pkg/codec/opus/opus_static.go

@@ -8,5 +8,6 @@ package opus
 //#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-arm64.a -lm
 //#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-x64.a -lm
 //#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-arm64.a
 //#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-windows-x64.a
 import "C"

pkg/codec/opus/opus_test.go

@@ -3,11 +3,28 @@ package opus
 import (
 	"testing"
 
+	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
 	"github.com/pion/mediadevices/pkg/prop"
 	"github.com/pion/mediadevices/pkg/wave"
 )
 
+func TestShouldImplementBitRateControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
 func TestEncoder(t *testing.T) {
 	t.Run("SimpleRead", func(t *testing.T) {
 		p, err := NewParams()

pkg/codec/vaapi/vp8.go

@@ -541,12 +541,8 @@ func (e *encoderVP8) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoderVP8) SetBitRate(b int) error {
+func (e *encoderVP8) Controller() codec.EncoderController {
-	panic("SetBitRate is not implemented")
+	return e
-}
-
-func (e *encoderVP8) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
 }
 
 func (e *encoderVP8) Close() error {

pkg/codec/vaapi/vp8_test.go

@@ -0,0 +1,27 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package vaapi
+
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"testing"
+)
+
+func TestVP8ShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoderVP8{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestVP8ShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoderVP8{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/vaapi/vp9.go

@@ -476,12 +476,8 @@ func (e *encoderVP9) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoderVP9) SetBitRate(b int) error {
+func (e *encoderVP9) Controller() codec.EncoderController {
-	panic("SetBitRate is not implemented")
+	return e
-}
-
-func (e *encoderVP9) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
 }
 
 func (e *encoderVP9) Close() error {

pkg/codec/vaapi/vp9_test.go

@@ -0,0 +1,27 @@
+//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris
+// +build dragonfly freebsd linux netbsd openbsd solaris
+
+package vaapi
+
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"testing"
+)
+
+func TestVP9ShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoderVP9{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestVP9ShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoderVP9{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/vpx/vpx.go

@@ -295,10 +295,6 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
-	panic("SetBitRate is not implemented")
-}
-
 func (e *encoder) ForceKeyFrame() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()
@@ -306,6 +302,10 @@ func (e *encoder) ForceKeyFrame() error {
 	return nil
 }
 
+func (e *encoder) Controller() codec.EncoderController {
+	return e
+}
+
 func (e *encoder) Close() error {
 	e.mu.Lock()
 	defer e.mu.Unlock()

pkg/codec/vpx/vpx_test.go

@@ -193,7 +193,7 @@ func TestRequestKeyFrame(t *testing.T) {
 				t.Fatal(err)
 			}
 			rel()
-			r.ForceKeyFrame()
+			r.Controller().(codec.KeyFrameController).ForceKeyFrame()
 			_, rel, err = r.Read()
 			if err != nil {
 				t.Fatal(err)
@@ -210,3 +210,19 @@ func TestRequestKeyFrame(t *testing.T) {
 
 	}
 }
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}

pkg/codec/x264/bridge.h

@@ -18,6 +18,7 @@ typedef struct Encoder {
   x264_t *h;
   x264_picture_t pic_in;
   x264_param_t param;
+  int force_key_frame;
 } Encoder;
 
 Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
@@ -85,8 +86,14 @@ Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
   e->pic_in.img.plane[0] = y;
   e->pic_in.img.plane[1] = cb;
   e->pic_in.img.plane[2] = cr;
+  if (e->force_key_frame) {
+    e->pic_in.i_type = X264_TYPE_IDR;
+  } else {
+    e->pic_in.i_type = X264_TYPE_AUTO;
+  }
 
   int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
+  e->force_key_frame = 0;
   Slice s = {.data_len = frame_size};
   if (frame_size <= 0) {
     *rc = ERR_ENCODE;

pkg/codec/x264/x264.go

@@ -124,12 +124,16 @@ func (e *encoder) Read() ([]byte, func(), error) {
 	return encoded, func() {}, err
 }
 
-func (e *encoder) SetBitRate(b int) error {
+// TODO: Implement bit rate control
-	panic("SetBitRate is not implemented")
+//var _ codec.BitRateController = (*encoder)(nil)
-}
 
 func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+	e.engine.force_key_frame = C.int(1)
+	return nil
+}
+
+func (e *encoder) Controller() codec.EncoderController {
+	return e
 }
 
 func (e *encoder) Close() error {

pkg/codec/x264/x264_test.go

@@ -4,6 +4,7 @@ import (
 	"image"
 	"testing"
 
+	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/codec/internal/codectest"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -46,3 +47,21 @@ func TestEncoder(t *testing.T) {
 		})
 	})
 }
+
+func TestShouldImplementKeyFrameControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement key frame control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.KeyFrameController); !ok {
+		t.Error()
+	}
+}
+
+func TestShouldImplementBitRateControl(t *testing.T) {
+	t.SkipNow() // TODO: Implement bit rate control
+
+	e := &encoder{}
+	if _, ok := e.Controller().(codec.BitRateController); !ok {
+		t.Error()
+	}
+}

pkg/driver/camera/camera_darwin.go

@@ -13,6 +13,7 @@ import (
 type camera struct {
 	device  avfoundation.Device
 	session *avfoundation.Session
+	rcClose func()
 }
 
 func init() {
@@ -43,6 +44,9 @@ func (cam *camera) Open() error {
 }
 
 func (cam *camera) Close() error {
+	if cam.rcClose != nil {
+		cam.rcClose()
+	}
 	return cam.session.Close()
 }
 
@@ -56,6 +60,7 @@ func (cam *camera) VideoRecord(property prop.Media) (video.Reader, error) {
 	if err != nil {
 		return nil, err
 	}
+	cam.rcClose = rc.Close
 	r := video.ReaderFunc(func() (image.Image, func(), error) {
 		frame, _, err := rc.Read()
 		if err != nil {

pkg/driver/camera/camera_linux.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/blackjack/webcam"
@@ -133,6 +134,19 @@ func newCamera(path string) *camera {
 	return c
 }
 
+func getCameraReadTimeout() uint32 {
+	// default to 5 seconds
+	var readTimeoutSec uint32 = 5
+	if val, ok := os.LookupEnv("PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"); ok {
+		if valInt, err := strconv.Atoi(val); err == nil {
+			if valInt > 0 {
+				readTimeoutSec = uint32(valInt)
+			}
+		}
+	}
+	return readTimeoutSec
+}
+
 func (c *camera) Open() error {
 	cam, err := webcam.Open(c.path)
 	if err != nil {
@@ -179,12 +193,21 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 		return nil, err
 	}
 
+	if p.FrameRate > 0 {
+		err = c.cam.SetFramerate(float32(p.FrameRate))
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if err := c.cam.StartStreaming(); err != nil {
 		return nil, err
 	}
 
 	cam := c.cam
 
+	readTimeoutSec := getCameraReadTimeout()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	c.cancel = cancel
 	var buf []byte
@@ -200,7 +223,7 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 				return nil, func() {}, io.EOF
 			}
 
-			err := cam.WaitForFrame(5) // 5 seconds
+			err := cam.WaitForFrame(readTimeoutSec)
 			switch err.(type) {
 			case nil:
 			case *webcam.Timeout:

pkg/driver/camera/camera_linux_test.go

@@ -70,3 +70,31 @@ func TestDiscover(t *testing.T) {
 		t.Errorf("Expected label: %s, got: %s", expectedNoLink, label)
 	}
 }
+
+func TestGetCameraReadTimeout(t *testing.T) {
+	var expected uint32 = 5
+	value := getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	envVarName := "PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"
+	os.Setenv(envVarName, "text")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "-1")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "1")
+	expected = 1
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+}

pkg/driver/microphone/microphone.go

@@ -111,6 +111,8 @@ func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
 	config.PerformanceProfile = malgo.LowLatency
 	config.Capture.Channels = uint32(inputProp.ChannelCount)
 	config.SampleRate = uint32(inputProp.SampleRate)
+	//FIX: Turn on the microphone with the current device id
+	config.Capture.DeviceID = m.ID.Pointer()
 	if inputProp.SampleSize == 4 && inputProp.IsFloat {
 		config.Capture.Format = malgo.FormatF32
 	} else if inputProp.SampleSize == 2 && !inputProp.IsFloat {

pkg/driver/videotest/dummy.go

@@ -47,10 +47,6 @@ func (d *dummy) Close() error {
 }
 
 func (d *dummy) VideoRecord(p prop.Media) (video.Reader, error) {
-	if p.FrameRate == 0 {
-		p.FrameRate = 30
-	}
-
 	colors := [][3]byte{
 		{235, 128, 128},
 		{210, 16, 146},
@@ -143,6 +139,7 @@ func (d dummy) Properties() []prop.Media {
 				Width:       640,
 				Height:      480,
 				FrameFormat: frame.FormatYUYV,
+				FrameRate:   30,
 			},
 		},
 	}

pkg/driver/vncdriver/vnc/encoding.go

@@ -18,13 +18,32 @@ type Encoding interface {
 	// the proper data.
 	Read(*ClientConn, *Rectangle, io.Reader) (Encoding, error)
 }
+type CursorEncoding struct {
+}
+
+func (*CursorEncoding) Type() int32 {
+	return -239
+}
+func (*CursorEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	size := int(rect.Height) * int(rect.Width) * int(c.PixelFormat.BPP) / 8
+	pixelBytes := make([]uint8, size)
+	if _, err := io.ReadFull(r, pixelBytes); err != nil {
+		return nil, err
+	}
+	mask := ((int(rect.Width) + 7) / 8) * int(rect.Height)
+	maskBytes := make([]uint8, mask)
+	if _, err := io.ReadFull(r, maskBytes); err != nil {
+		return nil, err
+	}
+	return &CursorEncoding{}, nil
+}
 
 // RawEncoding is raw pixel data sent by the server.
 //
 // See RFC 6143 Section 7.7.1
 type RawEncoding struct {
-	Colors []Color
+	Colors   []Color
-	RawPixel []uint32	//RGBA
+	RawPixel []uint32 //RGBA
 }
 
 func (*RawEncoding) Type() int32 {
@@ -32,6 +51,7 @@ func (*RawEncoding) Type() int32 {
 }
 
 func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("RawEncoding")
 	bytesPerPixel := c.PixelFormat.BPP / 8
 	pixelBytes := make([]uint8, bytesPerPixel)
 
@@ -41,7 +61,7 @@ func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding,
 	}
 
 	colors := make([]Color, int(rect.Height)*int(rect.Width))
-	rawPixels:=make([]uint32,int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
 	for y := uint16(0); y < rect.Height; y++ {
 		for x := uint16(0); x < rect.Width; x++ {
 			if _, err := io.ReadFull(r, pixelBytes); err != nil {
@@ -62,24 +82,30 @@ func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding,
 				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
 				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
 				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
 			} else {
 				*color = c.ColorMap[rawPixel]
 			}
-			rawPixels[int(y)*int(rect.Width)+int(x)]=uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
 			//fmt.Printf("%x %x",rawPixel,rawPixels[int(y)*int(rect.Width)+int(x)])
 		}
 	}
 
-	return &RawEncoding{colors,rawPixels}, nil
+	return &RawEncoding{colors, rawPixels}, nil
 }
+
 // ZlibEncoding is raw pixel data sent by the server compressed by Zlib.
 //
 // A single Zlib stream is created. There is only a single header for a framebuffer request response.
 type ZlibEncoding struct {
-	Colors  []Color
+	Colors   []Color
-	RawPixel[] uint32
+	RawPixel []uint32
-	ZStream *bytes.Buffer
+	ZStream  *bytes.Buffer
-	ZReader io.ReadCloser
+	ZReader  io.ReadCloser
 }
 
 func (*ZlibEncoding) Type() int32 {
@@ -87,6 +113,7 @@ func (*ZlibEncoding) Type() int32 {
 }
 
 func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("ZlibEncoding")
 	bytesPerPixel := c.PixelFormat.BPP / 8
 	pixelBytes := make([]uint8, bytesPerPixel)
 
@@ -98,7 +125,6 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 	// Format
 	// 4 bytes        | uint32 | length
 	// 'length' bytes | []byte | zlibData
-
 	// Read zlib length
 	var zipLength uint32
 	err := binary.Read(r, binary.BigEndian, &zipLength)
@@ -146,7 +172,7 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 	colorReader := bytes.NewReader(colorBytes)
 
 	colors := make([]Color, int(rect.Height)*int(rect.Width))
-	rawPixels:=make([]uint32,int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
 	for y := uint16(0); y < rect.Height; y++ {
 		for x := uint16(0); x < rect.Width; x++ {
 			if _, err := io.ReadFull(colorReader, pixelBytes); err != nil {
@@ -167,14 +193,19 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
 				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
 				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
 			} else {
 				*color = c.ColorMap[rawPixel]
 			}
-			rawPixels[int(y)*int(rect.Width)+int(x)]=uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
 		}
 	}
 
-	return &ZlibEncoding{Colors: colors,RawPixel: rawPixels}, nil
+	return &ZlibEncoding{Colors: colors, RawPixel: rawPixels}, nil
 }
 
 func (ze *ZlibEncoding) Close() {
@@ -183,4 +214,4 @@ func (ze *ZlibEncoding) Close() {
 		ze.ZReader.Close()
 		ze.ZReader = nil
 	}
-}
+}

pkg/driver/vncdriver/vncdriver.go

@@ -5,13 +5,14 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
-	"github.com/pion/mediadevices/pkg/driver/vncdriver/vnc"
 	"image"
 	"io"
 	"net"
 	"sync"
 	"time"
 
+	"github.com/pion/mediadevices/pkg/driver/vncdriver/vnc"
+
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -32,12 +33,12 @@ func NewVnc(vncAddr string) *vncDevice {
 	return &vncDevice{vncAddr: vncAddr}
 }
 func (d *vncDevice) PointerEvent(mask uint8, x, y uint16) {
-	if d.vClient!=nil{
+	if d.vClient != nil {
 		d.vClient.PointerEvent(vnc.ButtonMask(mask), x, y)
 	}
 }
 func (d *vncDevice) KeyEvent(keysym uint32, down bool) {
-	if d.vClient!=nil {
+	if d.vClient != nil {
 		d.vClient.KeyEvent(keysym, down)
 	}
 }
@@ -49,7 +50,10 @@ func (d *vncDevice) Open() error {
 	d.closed = ctx.Done()
 	d.cancel = cancel
 	msg := make(chan vnc.ServerMessage, 1)
+	//auth:=new(vnc.PasswordAuth)
+	//auth.Password="####"
 	conf := vnc.ClientConfig{
+		//Auth: []vnc.ClientAuth{auth},
 		ServerMessageCh: msg,
 		Exclusive:       false,
 	}
@@ -66,6 +70,7 @@ func (d *vncDevice) Open() error {
 	d.vClient.SetEncodings([]vnc.Encoding{
 		&vnc.ZlibEncoding{},
 		&vnc.RawEncoding{},
+		&vnc.CursorEncoding{},
 	})
 	d.w = int(d.vClient.FrameBufferWidth)
 	d.h = int(d.vClient.FrameBufferHeight)
@@ -89,6 +94,9 @@ func (d *vncDevice) Open() error {
 					for _, rect := range t.Rectangles {
 						var pix []uint32
 						switch t := rect.Enc.(type) {
+						case *vnc.CursorEncoding:
+							//ignore remote cursor messages
+							continue
 						case *vnc.RawEncoding:
 							pix = t.RawPixel
 						case *vnc.ZlibEncoding:
@@ -97,19 +105,16 @@ func (d *vncDevice) Open() error {
 						for y := int(rect.Y); y < int(rect.Height+rect.Y); y++ {
 							for x := int(rect.X); x < int(rect.Width+rect.X); x++ {
 								binary.LittleEndian.PutUint32(d.rawPixel[(y*d.w+x)*4:], pix[(y-int(rect.Y))*int(rect.Width)+(x-int(rect.X))])
-								//BigEndian
 							}
 						}
 
 					}
-					//time.Sleep(33 * time.Millisecond)
 					d.vClient.FramebufferUpdateRequest(true, 0, 0, uint16(d.w), uint16(d.h))
 					break
 				default:
 
 				}
 			case <-time.After(10 * time.Second):
-				//fmt.Println("Timeout FramebufferUpdate")
 				if d.vClient.FramebufferUpdateRequest(true, 0, 0, uint16(d.w), uint16(d.h)) != nil {
 					d.cancel()
 					return
@@ -137,13 +142,12 @@ func (d *vncDevice) Close() error {
 
 func (d *vncDevice) VideoRecord(p prop.Media) (video.Reader, error) {
 	if p.FrameRate == 0 {
-		p.FrameRate = 15
+		p.FrameRate = 30
 	}
 
 	tick := time.NewTicker(time.Duration(float32(time.Second) / p.FrameRate))
 	d.tick = tick
 	closed := d.closed
-	pixs := make([]byte, d.h*d.w*4)
 	r := video.ReaderFunc(func() (image.Image, func(), error) {
 		select {
 		case <-closed:
@@ -153,9 +157,8 @@ func (d *vncDevice) VideoRecord(p prop.Media) (video.Reader, error) {
 		}
 
 		<-tick.C
-		copy(pixs, d.rawPixel)
 		return &image.RGBA{
-			Pix:    pixs,
+			Pix:    d.rawPixel,
 			Stride: 4,
 			Rect:   image.Rect(0, 0, d.w, d.h),
 		}, func() {}, nil

pkg/io/video/scale.go

@@ -109,7 +109,7 @@ func Scale(width, height int, scaler Scaler) TransformFunc {
 			yDy := rect.Dy()
 			cRect := fixedRect(rect, i1.SubsampleRatio)
 			cDx := cRect.Dx()
-			cDy := cRect.Dx()
+			cDy := cRect.Dy()
 			yLen := yDx * yDy
 			cLen := cDx * cDy
 			if len(imgDst.Y) < yLen {

pkg/io/video/scale_test.go

@@ -36,6 +36,32 @@ func TestScale(t *testing.T) {
 				Rect:   image.Rect(0, 0, 2, 2),
 			},
 		},
+		"RGBASameSize": {
+			src: &image.RGBA{
+				Pix: []uint8{
+					// R     G     B     A |   R     G     B     A |   R     G     B     A |   R     G     B     A
+					0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
+					0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
+					0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
+					0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
+				},
+				Stride: 16,
+				Rect:   image.Rect(0, 0, 4, 4),
+			},
+			width:  4,
+			height: 4,
+			expected: &image.RGBA{
+				Pix: []uint8{
+					// R     G     B     A |   R     G     B     A |   R     G     B     A |   R     G     B     A
+					0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
+					0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
+					0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
+					0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
+				},
+				Stride: 16,
+				Rect:   image.Rect(0, 0, 4, 4),
+			},
+		},
 		"I444": {
 			src: &image.YCbCr{
 				SubsampleRatio: image.YCbCrSubsampleRatio444,
@@ -91,6 +117,70 @@ func TestScale(t *testing.T) {
 				Rect:    image.Rect(0, 0, 3, 3),
 			},
 		},
+		"I444SameSize": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio444,
+				Y: []uint8{
+					0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
+					0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
+					0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
+					0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
+				},
+				YStride: 6,
+				CStride: 6,
+				Rect:    image.Rect(0, 0, 6, 6),
+			},
+			width:  6,
+			height: 6,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio444,
+				Y: []uint8{
+					0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
+					0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
+					0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
+					0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
+				},
+				YStride: 6,
+				CStride: 6,
+				Rect:    image.Rect(0, 0, 6, 6),
+			},
+		},
 		"I422": {
 			src: &image.YCbCr{
 				SubsampleRatio: image.YCbCrSubsampleRatio422,
@@ -155,6 +245,82 @@ func TestScale(t *testing.T) {
 				Rect:    image.Rect(0, 0, 4, 4),
 			},
 		},
+		"I422SameSize": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio422,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0xE0, 0xE0,
+					0x80, 0x80, 0xE0, 0xE0,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0xF0, 0xF0, 0x40, 0x40,
+					0xF0, 0xF0, 0x40, 0x40,
+				},
+				YStride: 8,
+				CStride: 4,
+				Rect:    image.Rect(0, 0, 8, 8),
+			},
+			width:  8,
+			height: 8,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio422,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0xE0, 0xE0,
+					0x80, 0x80, 0xE0, 0xE0,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+					0xF0, 0xF0, 0x40, 0x40,
+					0xF0, 0xF0, 0x40, 0x40,
+				},
+				YStride: 8,
+				CStride: 4,
+				Rect:    image.Rect(0, 0, 8, 8),
+			},
+		},
 		"I420": {
 			src: &image.YCbCr{
 				SubsampleRatio: image.YCbCrSubsampleRatio420,
@@ -207,6 +373,118 @@ func TestScale(t *testing.T) {
 				Rect:    image.Rect(0, 0, 4, 4),
 			},
 		},
+		"I420SameSize": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80,
+					0x80, 0x80, 0xE0, 0xE0,
+					0x80, 0x80, 0xE0, 0xE0,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80,
+					0xF0, 0xF0, 0x40, 0x40,
+					0xF0, 0xF0, 0x40, 0x40,
+				},
+				YStride: 8,
+				CStride: 4,
+				Rect:    image.Rect(0, 0, 8, 8),
+			},
+			width:  8,
+			height: 8,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80,
+					0x20, 0x20, 0x80, 0x80,
+					0x80, 0x80, 0xE0, 0xE0,
+					0x80, 0x80, 0xE0, 0xE0,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80,
+					0xE0, 0xE0, 0x80, 0x80,
+					0xF0, 0xF0, 0x40, 0x40,
+					0xF0, 0xF0, 0x40, 0x40,
+				},
+				YStride: 8,
+				CStride: 4,
+				Rect:    image.Rect(0, 0, 8, 8),
+			},
+		},
+		"I420NonSquareImage": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
+					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
+					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
+					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
+					0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
+					0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
+					0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
+				},
+				YStride: 12,
+				CStride: 6,
+				Rect:    image.Rect(0, 0, 12, 8),
+			},
+			width:  6,
+			height: 4,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x10, 0x00, 0x00, 0xF0, 0x10,
+					0x00, 0x00, 0x40, 0x00, 0x40, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x80, 0x30, 0x00, 0x30, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x80, 0x50,
+					0x80, 0xE0, 0x30,
+				},
+				Cr: []uint8{
+					0xE0, 0x80, 0xB0,
+					0xF0, 0x40, 0xC0,
+				},
+				YStride: 6,
+				CStride: 3,
+				Rect:    image.Rect(0, 0, 6, 4),
+			},
+		},
 	}
 	for name, algo := range scalerTestAlgos {
 		algo := algo
@@ -247,6 +525,100 @@ func TestScale(t *testing.T) {
 	}
 }
 
+func TestScaleFastBoxSampling(t *testing.T) {
+	cases := map[string]struct {
+		src           image.Image
+		width, height int
+		expected      image.Image
+	}{
+		"I420NonSquareImage": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
+					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
+					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
+					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
+					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
+					0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
+					0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
+					0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
+				},
+				YStride: 12,
+				CStride: 6,
+				Rect:    image.Rect(0, 0, 12, 8),
+			},
+			width:  6,
+			height: 4,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x80, 0x08, 0x00, 0x78, 0x80,
+					0x08, 0x00, 0x20, 0x20, 0x20, 0x20,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x40, 0x58, 0x18, 0x18, 0x18,
+				},
+				Cb: []uint8{
+					0x20, 0x50, 0x68,
+					0x68, 0xB0, 0x88,
+				},
+				Cr: []uint8{
+					0xE0, 0xB0, 0x98,
+					0xD0, 0x98, 0x80,
+				},
+				YStride: 6,
+				CStride: 3,
+				Rect:    image.Rect(0, 0, 6, 4),
+			},
+		},
+	}
+
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			trans := Scale(c.width, c.height, ScalerFastBoxSampling)
+			r := trans(ReaderFunc(func() (image.Image, func(), error) {
+				return c.src, func() {}, nil
+			}))
+			for i := 0; i < 4; i++ {
+				out, _, err := r.Read()
+				if err != nil {
+					t.Fatalf("Unexpected error: %v", err)
+				}
+				if !reflect.DeepEqual(c.expected, out) {
+					t.Errorf("Expected output image:\n%v\ngot:\n%v\nrepeat: %d", c.expected, out, i)
+				}
+				// Destroy output contents
+				switch v := out.(type) {
+				case *image.RGBA:
+					v.Stride = 10
+					v.Pix = v.Pix[:1]
+					v.Rect.Max.X = 1
+				case *image.YCbCr:
+					v.YStride = 10
+					v.CStride = 100
+					v.Y = v.Y[:1]
+					v.Cb = v.Cb[:2]
+					v.Cr = v.Cr[:1]
+					v.Rect.Max.X = 1
+				}
+			}
+		})
+	}
+}
+
 func BenchmarkScale(b *testing.B) {
 	for name, algo := range scalerBenchAlgos {
 		algo := algo

pkg/io/video/scaler_cgo.c

@@ -29,7 +29,7 @@ void fastBoxSampling(
     const int sw, const int sh, const int sstride,
     uint32_t* tmp)
 {
-  memset(tmp, 0, dw * dh * ch);
+  memset(tmp, 0, dw * dh * ch * sizeof(tmp[0]));
 
   for (int sy = 0; sy < sh; sy++)
   {

pkg/io/video/scaleycrcb.go

@@ -30,10 +30,18 @@ func (p *rgbLikeYCbCr) At(x, y int) color.Color {
 }
 
 func (p *rgbLikeYCbCr) Set(x, y int, c color.Color) {
-	rgb := c.(*color.RGBA64)
+	switch v := c.(type) {
-	p.y.SetGray(x, y, color.Gray{uint8(rgb.R / 0x100)})
+	case color.RGBA:
-	if (image.Point{x, y}.In(p.cb.Rect)) {
+		p.y.SetGray(x, y, color.Gray{v.R})
-		p.cb.SetGray(x, y, color.Gray{uint8(rgb.G / 0x100)})
+		if (image.Point{x, y}.In(p.cb.Rect)) {
-		p.cr.SetGray(x, y, color.Gray{uint8(rgb.B / 0x100)})
+			p.cb.SetGray(x, y, color.Gray{v.G})
+			p.cr.SetGray(x, y, color.Gray{v.B})
+		}
+	case *color.RGBA64:
+		p.y.SetGray(x, y, color.Gray{uint8(v.R / 0x100)})
+		if (image.Point{x, y}.In(p.cb.Rect)) {
+			p.cb.SetGray(x, y, color.Gray{uint8(v.G / 0x100)})
+			p.cr.SetGray(x, y, color.Gray{uint8(v.B / 0x100)})
+		}
 	}
 }

pkg/prop/prop.go

@@ -145,13 +145,17 @@ func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
 	cmps.add(p.Width, o.Width)
 	cmps.add(p.Height, o.Height)
 	cmps.add(p.FrameFormat, o.FrameFormat)
+	// The next line is comment out for now to not include framerate in the fitness function.
+	// As camera.Properties does not have access to the list of available framerate at the moment,
+	// no driver can be matched with a framerate constraint.
+	// Note this also affect screen caputre as screen.Properties does not fill in the Framerate field.
+	// cmps.add(p.FrameRate, o.FrameRate)
 	cmps.add(p.SampleRate, o.SampleRate)
 	cmps.add(p.Latency, o.Latency)
 	cmps.add(p.ChannelCount, o.ChannelCount)
 	cmps.add(p.IsBigEndian, o.IsBigEndian)
 	cmps.add(p.IsFloat, o.IsFloat)
 	cmps.add(p.IsInterleaved, o.IsInterleaved)
-
 	return cmps.fitnessDistance()
 }
 

rtpreader.go

@@ -1,15 +1,20 @@
 package mediadevices
 
-import "github.com/pion/rtp"
+import (
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/rtp"
+)
 
 type RTPReadCloser interface {
 	Read() (pkts []*rtp.Packet, release func(), err error)
 	Close() error
+	codec.Controllable
 }
 
 type rtpReadCloserImpl struct {
-	readFn  func() ([]*rtp.Packet, func(), error)
+	readFn       func() ([]*rtp.Packet, func(), error)
-	closeFn func() error
+	closeFn      func() error
+	controllerFn func() codec.EncoderController
 }
 
 func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
@@ -19,3 +24,7 @@ func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
 func (r *rtpReadCloserImpl) Close() error {
 	return r.closeFn()
 }
+
+func (r *rtpReadCloserImpl) Controller() codec.EncoderController {
+	return r.controllerFn()
+}

track.go

@@ -3,6 +3,8 @@ package mediadevices
 import (
 	"errors"
 	"fmt"
+	"github.com/pion/interceptor"
+	"github.com/pion/rtcp"
 	"image"
 	"io"
 	"strings"
@@ -157,6 +159,7 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac
 	defer track.mu.Unlock()
 
 	signalCh := make(chan chan<- struct{})
+	var stopRead chan struct{}
 	track.activePeerConnections[ctx.ID()] = signalCh
 
 	var encodedReader RTPReadCloser
@@ -182,6 +185,7 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac
 		var doneCh chan<- struct{}
 		writer := ctx.WriteStream()
 		defer func() {
+			close(stopRead)
 			encodedReader.Close()
 
 			// When there's another call to unbind, it won't block since we mark the signalCh to be closed
@@ -214,6 +218,40 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac
 		}
 	}()
 
+	keyFrameController, ok := encodedReader.Controller().(codec.KeyFrameController)
+	if ok {
+		stopRead = make(chan struct{})
+		go func() {
+			reader := ctx.RTCPReader()
+			for {
+				select {
+				case <-stopRead:
+					return
+				default:
+				}
+
+				var readerBuffer []byte
+				_, _, err := reader.Read(readerBuffer, interceptor.Attributes{})
+				if err != nil {
+					track.onError(err)
+					return
+				}
+
+				pkts, err := rtcp.Unmarshal(readerBuffer)
+
+				for _, pkt := range pkts {
+					switch pkt.(type) {
+					case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest:
+						if err := keyFrameController.ForceKeyFrame(); err != nil {
+							track.onError(err)
+							return
+						}
+					}
+				}
+			}
+		}()
+	}
+
 	return selectedCodec, nil
 }
 
@@ -261,6 +299,7 @@ func newTrackFromDriver(d driver.Driver, constraints MediaTrackConstraints, sele
 type VideoTrack struct {
 	*baseTrack
 	*video.Broadcaster
+	shouldCopyFrames bool
 }
 
 // NewVideoTrack constructs a new VideoTrack
@@ -268,6 +307,16 @@ func NewVideoTrack(source VideoSource, selector *CodecSelector) Track {
 	return newVideoTrackFromReader(source, source, selector)
 }
 
+// ShouldCopyFrames indicates if readers on this track should receive a clopy of the read buffer instead of sharing one.
+func (track *VideoTrack) ShouldCopyFrames() bool {
+	return track.shouldCopyFrames
+}
+
+// SetShouldCopyFrames enables frame copy for this track, sending each reader a different read buffer instead of sharing one.
+func (track *VideoTrack) SetShouldCopyFrames(shouldCopyFrames bool) {
+	track.shouldCopyFrames = shouldCopyFrames
+}
+
 func newVideoTrackFromReader(source Source, reader video.Reader, selector *CodecSelector) Track {
 	base := newBaseTrack(source, VideoInput, selector)
 	wrappedReader := video.ReaderFunc(func() (img image.Image, release func(), err error) {
@@ -312,7 +361,7 @@ func (track *VideoTrack) Unbind(ctx webrtc.TrackLocalContext) error {
 }
 
 func (track *VideoTrack) newEncodedReader(codecNames ...string) (EncodedReadCloser, *codec.RTPCodec, error) {
-	reader := track.NewReader(false)
+	reader := track.NewReader(track.shouldCopyFrames)
 	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
 	if err != nil {
 		return nil, nil, err
@@ -334,7 +383,8 @@ func (track *VideoTrack) newEncodedReader(codecNames ...string) (EncodedReadClos
 			}
 			return buffer, release, err
 		},
-		closeFn: encodedReader.Close,
+		closeFn:      encodedReader.Close,
+		controllerFn: encodedReader.Controller,
 	}, selectedCodec, nil
 }
 
@@ -372,7 +422,8 @@ func (track *VideoTrack) NewRTPReader(codecName string, ssrc uint32, mtu int) (R
 			pkts := packetizer.Packetize(encoded.Data, encoded.Samples)
 			return pkts, release, err
 		},
-		closeFn: encodedReader.Close,
+		closeFn:      encodedReader.Close,
+		controllerFn: encodedReader.Controller,
 	}, nil
 }