Camera timeout duration as parameter (#408)

Set PION_MEDIADEVICES_CAMERA_READ_TIMEOUT environment variable
to set the timeout duration.

.github/workflows/ci.yaml

@@ -32,7 +32,7 @@ jobs:
             libx264-dev
       - name: Run Test Suite
         run: make test
-      - uses: codecov/codecov-action@v2	
+      - uses: codecov/codecov-action@v3	
         if: matrix.go == '1.16'
   build-darwin:
     runs-on: macos-latest

Makefile

@@ -16,7 +16,8 @@ supported_platforms := \
   linux-arm64 \
   linux-x64 \
   windows-x64 \
-  darwin-x64
+  darwin-x64 \
+  darwin-arm64
 cmd_build := build
 cmd_test := test
 examples_dir := examples

dockerfiles/darwin-arm64.Dockerfile

@@ -0,0 +1,47 @@
+FROM dockercore/golang-cross as m1cross
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -qq && apt-get install -y -q --no-install-recommends \
+    cmake \
+    git \
+    libssl-dev \
+    libxml2-dev \
+    libz-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV SDK_VERSION=11.3 \
+    TARGET_DIR=/osxcross/target \
+    UNATTENDED=1
+
+WORKDIR /work
+RUN git clone --depth=1 https://github.com/tpoechtrager/osxcross.git /work \
+  && cd /work/tarballs \
+  && wget -q https://github.com/phracker/MacOSX-SDKs/releases/download/${SDK_VERSION}/MacOSX${SDK_VERSION}.sdk.tar.xz
+
+# Build cross compile toolchain for Apple silicon
+RUN ./build.sh
+
+
+FROM dockcross/base
+
+ENV OSX_CROSS_PATH=/osxcross
+
+COPY --from=m1cross "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}/"
+ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+
+ENV CC=arm64-apple-darwin20.4-clang \
+    CXX=arm64-apple-darwin20.4-clang++ \
+    CPP=arm64-apple-darwin20.4-clang++ \
+    AR=arm64-apple-darwin20.4-ar \
+    AS=arm64-apple-darwin20.4-as \
+    LD=arm64-apple-darwin20.4-ld
+
+COPY darwin-arm64.cmake ${OSX_CROSS_PATH}/
+ENV CMAKE_TOOLCHAIN_FILE ${OSX_CROSS_PATH}/darwin-arm64.cmake
+
+ARG IMAGE=lherman/cross-darwin-arm64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}

dockerfiles/darwin-arm64.cmake

@@ -0,0 +1,8 @@
+set(CMAKE_SYSTEM_NAME Darwin)
+set(CMAKE_SYSTEM_VERSION 1)
+set(CMAKE_SYSTEM_PROCESSOR arm64)
+
+set(CMAKE_C_COMPILER $ENV{CC})
+set(CMAKE_CXX_COMPILER $ENV{CXX})
+set(CMAKE_AR $ENV{AR})
+set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER})

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329
 	github.com/pion/logging v0.2.2
-	github.com/pion/rtp v1.7.9
-	github.com/pion/webrtc/v3 v3.1.27
-	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
+	github.com/pion/rtp v1.7.13
+	github.com/pion/webrtc/v3 v3.1.34
+	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -51,8 +51,8 @@ github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6
 github.com/pion/datachannel v1.5.2/go.mod h1:FTGQWaHrdCwIJ1rw6xBIfZVkslikjShim5yr05XFuCQ=
 github.com/pion/dtls/v2 v2.1.3 h1:3UF7udADqous+M2R5Uo2q/YaP4EzUoWKdfX2oscCUio=
 github.com/pion/dtls/v2 v2.1.3/go.mod h1:o6+WvyLDAlXF7YiPB/RlskRoeK+/JtuaZa5emwQcWus=
-github.com/pion/ice/v2 v2.2.2 h1:UfmAslxZ0u0itVjA4x7aw7WeQIv22FdF8VjW9cM+74g=
-github.com/pion/ice/v2 v2.2.2/go.mod h1:vLI7dFqxw8zMSb9J+ca74XU7JjLhddgfQB9+BbTydCo=
+github.com/pion/ice/v2 v2.2.6 h1:R/vaLlI1J2gCx141L5PEwtuGAGcyS6e7E0hDeJFq5Ig=
+github.com/pion/ice/v2 v2.2.6/go.mod h1:SWuHiOGP17lGromHTFadUe1EuPgFh/oCU6FCMZHooVE=
 github.com/pion/interceptor v0.1.10 h1:DJ2GjMGm4XGIQgMJxuEpdaExdY/6RdngT7Uh4oVmquU=
 github.com/pion/interceptor v0.1.10/go.mod h1:Lh3JSl/cbJ2wP8I3ccrjh1K/deRGRn3UlSPuOTiHb6U=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -66,8 +66,8 @@ github.com/pion/rtcp v1.2.9 h1:1ujStwg++IOLIEoOiIQ2s+qBuJ1VN81KW+9pMPsif+U=
 github.com/pion/rtcp v1.2.9/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo=
 github.com/pion/rtp v1.7.0/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/rtp v1.7.4/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
-github.com/pion/rtp v1.7.9 h1:17W5Mt2IM3MVfOh7yRfzXbbKXYzBZxV8eG4KKAy+0bg=
-github.com/pion/rtp v1.7.9/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
+github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA=
+github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.8.0/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
 github.com/pion/sctp v1.8.2 h1:yBBCIrUMJ4yFICL3RIvR4eh/H2BTTvlligmSTy+3kiA=
 github.com/pion/sctp v1.8.2/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
@@ -85,8 +85,8 @@ github.com/pion/turn/v2 v2.0.8 h1:KEstL92OUN3k5k8qxsXHpr7WWfrdp7iJZHx99ud8muw=
 github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw=
 github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
 github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
-github.com/pion/webrtc/v3 v3.1.27 h1:yQ6TuHKJR/vro3nLZMfPv+WGf7T1/4ItaQeuzIZLXs4=
-github.com/pion/webrtc/v3 v3.1.27/go.mod h1:hdduI+Rx0cpGvva18j0gKy/Iak611WPyhUIXs5W/FuI=
+github.com/pion/webrtc/v3 v3.1.34 h1:GUfv2zxWge77x1FhZ6Fge8KQd3bTzvX1CMN1/LmTazM=
+github.com/pion/webrtc/v3 v3.1.34/go.mod h1:jClfnbJzt8wtmewGxhPzgE5wZ0U/gNB78XCtmc/uz3k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
@@ -100,10 +100,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE=
 golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg=
-golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 h1:LRtI4W37N+KFebI/qV0OFiLUv4GLOWeEW5hn/KEJvxE=
+golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -116,8 +117,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd h1:zYlwaUHTmxuf6H7hwO2dgwqozQmH7zf4x+/qql4oVWc=
+golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

mediadevices_test.go

@@ -125,18 +125,6 @@ func TestSelectBestDriverConstraintsResultIsSetProperly(t *testing.T) {
 			frameFormat: frame.FormatI420,
 			frameRate:   expectedProp.FrameRate,
 		},
-		"DifferentFrameRate": {
-			width:       expectedProp.Width,
-			height:      expectedProp.Height,
-			frameFormat: expectedProp.FrameFormat,
-			frameRate:   expectedProp.FrameRate - 1,
-		},
-		"NoFrameRateConstraints": {
-			width:       expectedProp.Width,
-			height:      expectedProp.Height,
-			frameFormat: expectedProp.FrameFormat,
-			frameRate:   -1,
-		},
 	}
 
 	for name, c := range cases {
@@ -229,12 +217,6 @@ func TestSelectBestDriverConstraintsNoFit(t *testing.T) {
 			frameFormat: frame.FormatI420,
 			frameRate:   expectedProp.FrameRate,
 		},
-		"DifferentFrameRate": {
-			width:       expectedProp.Width,
-			height:      expectedProp.Height,
-			frameFormat: expectedProp.FrameFormat,
-			frameRate:   expectedProp.FrameRate - 1,
-		},
 	}
 
 	for name, c := range cases {

pkg/codec/openh264/bridge.cpp

@@ -81,7 +81,7 @@ Slice enc_encode(Encoder *e, Frame f, int *eresult) {
   Slice payload = {0};
 
   if(e->force_key_frame == 1) {
-    info.eFrameType = videoFrameTypeI;
+    e->engine->ForceIntraFrame(true);
     e->force_key_frame = 0;
   }
 

pkg/codec/openh264/openh264_static.go

@@ -8,5 +8,6 @@ package openh264
 //#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-arm64.a
 //#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-linux-x64.a
 //#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264-darwin-arm64.a
 //#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264-windows-x64.a -lssp
 import "C"

pkg/codec/opus/opus_static.go

@@ -8,5 +8,6 @@ package opus
 //#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-arm64.a -lm
 //#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-linux-x64.a -lm
 //#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-x64.a
+//#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/lib/libopus-darwin-arm64.a
 //#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopus-windows-x64.a
 import "C"

pkg/codec/x264/bridge.h

@@ -18,6 +18,7 @@ typedef struct Encoder {
   x264_t *h;
   x264_picture_t pic_in;
   x264_param_t param;
+  int force_key_frame;
 } Encoder;
 
 Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
@@ -85,8 +86,14 @@ Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
   e->pic_in.img.plane[0] = y;
   e->pic_in.img.plane[1] = cb;
   e->pic_in.img.plane[2] = cr;
+  if (e->force_key_frame) {
+    e->pic_in.i_type = X264_TYPE_IDR;
+  } else {
+    e->pic_in.i_type = X264_TYPE_AUTO;
+  }
 
   int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
+  e->force_key_frame = 0;
   Slice s = {.data_len = frame_size};
   if (frame_size <= 0) {
     *rc = ERR_ENCODE;

pkg/codec/x264/x264.go

@@ -129,7 +129,8 @@ func (e *encoder) SetBitRate(b int) error {
 }
 
 func (e *encoder) ForceKeyFrame() error {
-	panic("ForceKeyFrame is not implemented")
+	e.engine.force_key_frame = C.int(1)
+	return nil
 }
 
 func (e *encoder) Close() error {

pkg/driver/camera/camera_linux.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/blackjack/webcam"
@@ -133,6 +134,19 @@ func newCamera(path string) *camera {
 	return c
 }
 
+func getCameraReadTimeout() uint32 {
+	// default to 5 seconds
+	var readTimeoutSec uint32 = 5
+	if val, ok := os.LookupEnv("PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"); ok {
+		if valInt, err := strconv.Atoi(val); err == nil {
+			if valInt > 0 {
+				readTimeoutSec = uint32(valInt)
+			}
+		}
+	}
+	return readTimeoutSec
+}
+
 func (c *camera) Open() error {
 	cam, err := webcam.Open(c.path)
 	if err != nil {
@@ -192,6 +206,8 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 
 	cam := c.cam
 
+	readTimeoutSec := getCameraReadTimeout()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	c.cancel = cancel
 	var buf []byte
@@ -207,7 +223,7 @@ func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) {
 				return nil, func() {}, io.EOF
 			}
 
-			err := cam.WaitForFrame(5) // 5 seconds
+			err := cam.WaitForFrame(readTimeoutSec)
 			switch err.(type) {
 			case nil:
 			case *webcam.Timeout:

pkg/driver/camera/camera_linux_test.go

@@ -70,3 +70,31 @@ func TestDiscover(t *testing.T) {
 		t.Errorf("Expected label: %s, got: %s", expectedNoLink, label)
 	}
 }
+
+func TestGetCameraReadTimeout(t *testing.T) {
+	var expected uint32 = 5
+	value := getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	envVarName := "PION_MEDIADEVICES_CAMERA_READ_TIMEOUT"
+	os.Setenv(envVarName, "text")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "-1")
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+
+	os.Setenv(envVarName, "1")
+	expected = 1
+	value = getCameraReadTimeout()
+	if value != expected {
+		t.Errorf("Expected: %d, got: %d", expected, value)
+	}
+}

pkg/driver/microphone/microphone.go

@@ -111,6 +111,8 @@ func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
 	config.PerformanceProfile = malgo.LowLatency
 	config.Capture.Channels = uint32(inputProp.ChannelCount)
 	config.SampleRate = uint32(inputProp.SampleRate)
+	//FIX: Turn on the microphone with the current device id
+	config.Capture.DeviceID = m.ID.Pointer()
 	if inputProp.SampleSize == 4 && inputProp.IsFloat {
 		config.Capture.Format = malgo.FormatF32
 	} else if inputProp.SampleSize == 2 && !inputProp.IsFloat {

pkg/driver/vncdriver/vnc/encoding.go

@@ -18,13 +18,32 @@ type Encoding interface {
 	// the proper data.
 	Read(*ClientConn, *Rectangle, io.Reader) (Encoding, error)
 }
+type CursorEncoding struct {
+}
+
+func (*CursorEncoding) Type() int32 {
+	return -239
+}
+func (*CursorEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	size := int(rect.Height) * int(rect.Width) * int(c.PixelFormat.BPP) / 8
+	pixelBytes := make([]uint8, size)
+	if _, err := io.ReadFull(r, pixelBytes); err != nil {
+		return nil, err
+	}
+	mask := ((int(rect.Width) + 7) / 8) * int(rect.Height)
+	maskBytes := make([]uint8, mask)
+	if _, err := io.ReadFull(r, maskBytes); err != nil {
+		return nil, err
+	}
+	return &CursorEncoding{}, nil
+}
 
 // RawEncoding is raw pixel data sent by the server.
 //
 // See RFC 6143 Section 7.7.1
 type RawEncoding struct {
-	Colors []Color
-	RawPixel []uint32	//RGBA
+	Colors   []Color
+	RawPixel []uint32 //RGBA
 }
 
 func (*RawEncoding) Type() int32 {
@@ -32,6 +51,7 @@ func (*RawEncoding) Type() int32 {
 }
 
 func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("RawEncoding")
 	bytesPerPixel := c.PixelFormat.BPP / 8
 	pixelBytes := make([]uint8, bytesPerPixel)
 
@@ -41,7 +61,7 @@ func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding,
 	}
 
 	colors := make([]Color, int(rect.Height)*int(rect.Width))
-	rawPixels:=make([]uint32,int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
 	for y := uint16(0); y < rect.Height; y++ {
 		for x := uint16(0); x < rect.Width; x++ {
 			if _, err := io.ReadFull(r, pixelBytes); err != nil {
@@ -62,24 +82,30 @@ func (*RawEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding,
 				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
 				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
 				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
 			} else {
 				*color = c.ColorMap[rawPixel]
 			}
-			rawPixels[int(y)*int(rect.Width)+int(x)]=uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
 			//fmt.Printf("%x %x",rawPixel,rawPixels[int(y)*int(rect.Width)+int(x)])
 		}
 	}
 
-	return &RawEncoding{colors,rawPixels}, nil
+	return &RawEncoding{colors, rawPixels}, nil
 }
+
 // ZlibEncoding is raw pixel data sent by the server compressed by Zlib.
 //
 // A single Zlib stream is created. There is only a single header for a framebuffer request response.
 type ZlibEncoding struct {
-	Colors  []Color
-	RawPixel[] uint32
-	ZStream *bytes.Buffer
-	ZReader io.ReadCloser
+	Colors   []Color
+	RawPixel []uint32
+	ZStream  *bytes.Buffer
+	ZReader  io.ReadCloser
 }
 
 func (*ZlibEncoding) Type() int32 {
@@ -87,6 +113,7 @@ func (*ZlibEncoding) Type() int32 {
 }
 
 func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encoding, error) {
+	//fmt.Println("ZlibEncoding")
 	bytesPerPixel := c.PixelFormat.BPP / 8
 	pixelBytes := make([]uint8, bytesPerPixel)
 
@@ -98,7 +125,6 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 	// Format
 	// 4 bytes        | uint32 | length
 	// 'length' bytes | []byte | zlibData
-
 	// Read zlib length
 	var zipLength uint32
 	err := binary.Read(r, binary.BigEndian, &zipLength)
@@ -146,7 +172,7 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 	colorReader := bytes.NewReader(colorBytes)
 
 	colors := make([]Color, int(rect.Height)*int(rect.Width))
-	rawPixels:=make([]uint32,int(rect.Height)*int(rect.Width))
+	rawPixels := make([]uint32, int(rect.Height)*int(rect.Width))
 	for y := uint16(0); y < rect.Height; y++ {
 		for x := uint16(0); x < rect.Width; x++ {
 			if _, err := io.ReadFull(colorReader, pixelBytes); err != nil {
@@ -167,14 +193,19 @@ func (ze *ZlibEncoding) Read(c *ClientConn, rect *Rectangle, r io.Reader) (Encod
 				color.R = uint16((rawPixel >> c.PixelFormat.RedShift) & uint32(c.PixelFormat.RedMax))
 				color.G = uint16((rawPixel >> c.PixelFormat.GreenShift) & uint32(c.PixelFormat.GreenMax))
 				color.B = uint16((rawPixel >> c.PixelFormat.BlueShift) & uint32(c.PixelFormat.BlueMax))
+				if c.PixelFormat.BPP == 16 {
+					color.B = color.B<<3 | color.B>>2
+					color.G = color.G<<2 | color.G>>2
+					color.R = color.R<<3 | color.R>>2
+				}
 			} else {
 				*color = c.ColorMap[rawPixel]
 			}
-			rawPixels[int(y)*int(rect.Width)+int(x)]=uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
+			rawPixels[int(y)*int(rect.Width)+int(x)] = uint32(0xff)<<24 | uint32(color.B)<<16 | uint32(color.G)<<8 | uint32(color.R)
 		}
 	}
 
-	return &ZlibEncoding{Colors: colors,RawPixel: rawPixels}, nil
+	return &ZlibEncoding{Colors: colors, RawPixel: rawPixels}, nil
 }
 
 func (ze *ZlibEncoding) Close() {
@@ -183,4 +214,4 @@ func (ze *ZlibEncoding) Close() {
 		ze.ZReader.Close()
 		ze.ZReader = nil
 	}
-}
+}

pkg/driver/vncdriver/vncdriver.go

@@ -5,13 +5,14 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
-	"github.com/pion/mediadevices/pkg/driver/vncdriver/vnc"
 	"image"
 	"io"
 	"net"
 	"sync"
 	"time"
 
+	"github.com/pion/mediadevices/pkg/driver/vncdriver/vnc"
+
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -32,12 +33,12 @@ func NewVnc(vncAddr string) *vncDevice {
 	return &vncDevice{vncAddr: vncAddr}
 }
 func (d *vncDevice) PointerEvent(mask uint8, x, y uint16) {
-	if d.vClient!=nil{
+	if d.vClient != nil {
 		d.vClient.PointerEvent(vnc.ButtonMask(mask), x, y)
 	}
 }
 func (d *vncDevice) KeyEvent(keysym uint32, down bool) {
-	if d.vClient!=nil {
+	if d.vClient != nil {
 		d.vClient.KeyEvent(keysym, down)
 	}
 }
@@ -49,7 +50,10 @@ func (d *vncDevice) Open() error {
 	d.closed = ctx.Done()
 	d.cancel = cancel
 	msg := make(chan vnc.ServerMessage, 1)
+	//auth:=new(vnc.PasswordAuth)
+	//auth.Password="####"
 	conf := vnc.ClientConfig{
+		//Auth: []vnc.ClientAuth{auth},
 		ServerMessageCh: msg,
 		Exclusive:       false,
 	}
@@ -66,6 +70,7 @@ func (d *vncDevice) Open() error {
 	d.vClient.SetEncodings([]vnc.Encoding{
 		&vnc.ZlibEncoding{},
 		&vnc.RawEncoding{},
+		&vnc.CursorEncoding{},
 	})
 	d.w = int(d.vClient.FrameBufferWidth)
 	d.h = int(d.vClient.FrameBufferHeight)
@@ -89,6 +94,9 @@ func (d *vncDevice) Open() error {
 					for _, rect := range t.Rectangles {
 						var pix []uint32
 						switch t := rect.Enc.(type) {
+						case *vnc.CursorEncoding:
+							//ignore remote cursor messages
+							continue
 						case *vnc.RawEncoding:
 							pix = t.RawPixel
 						case *vnc.ZlibEncoding:
@@ -97,19 +105,16 @@ func (d *vncDevice) Open() error {
 						for y := int(rect.Y); y < int(rect.Height+rect.Y); y++ {
 							for x := int(rect.X); x < int(rect.Width+rect.X); x++ {
 								binary.LittleEndian.PutUint32(d.rawPixel[(y*d.w+x)*4:], pix[(y-int(rect.Y))*int(rect.Width)+(x-int(rect.X))])
-								//BigEndian
 							}
 						}
 
 					}
-					//time.Sleep(33 * time.Millisecond)
 					d.vClient.FramebufferUpdateRequest(true, 0, 0, uint16(d.w), uint16(d.h))
 					break
 				default:
 
 				}
 			case <-time.After(10 * time.Second):
-				//fmt.Println("Timeout FramebufferUpdate")
 				if d.vClient.FramebufferUpdateRequest(true, 0, 0, uint16(d.w), uint16(d.h)) != nil {
 					d.cancel()
 					return
@@ -137,13 +142,12 @@ func (d *vncDevice) Close() error {
 
 func (d *vncDevice) VideoRecord(p prop.Media) (video.Reader, error) {
 	if p.FrameRate == 0 {
-		p.FrameRate = 15
+		p.FrameRate = 30
 	}
 
 	tick := time.NewTicker(time.Duration(float32(time.Second) / p.FrameRate))
 	d.tick = tick
 	closed := d.closed
-	pixs := make([]byte, d.h*d.w*4)
 	r := video.ReaderFunc(func() (image.Image, func(), error) {
 		select {
 		case <-closed:
@@ -153,9 +157,8 @@ func (d *vncDevice) VideoRecord(p prop.Media) (video.Reader, error) {
 		}
 
 		<-tick.C
-		copy(pixs, d.rawPixel)
 		return &image.RGBA{
-			Pix:    pixs,
+			Pix:    d.rawPixel,
 			Stride: 4,
 			Rect:   image.Rect(0, 0, d.w, d.h),
 		}, func() {}, nil

pkg/prop/prop.go

@@ -145,14 +145,17 @@ func (p *MediaConstraints) FitnessDistance(o Media) (float64, bool) {
 	cmps.add(p.Width, o.Width)
 	cmps.add(p.Height, o.Height)
 	cmps.add(p.FrameFormat, o.FrameFormat)
-	cmps.add(p.FrameRate, o.FrameRate)
+	// The next line is comment out for now to not include framerate in the fitness function.
+	// As camera.Properties does not have access to the list of available framerate at the moment,
+	// no driver can be matched with a framerate constraint.
+	// Note this also affect screen caputre as screen.Properties does not fill in the Framerate field.
+	// cmps.add(p.FrameRate, o.FrameRate)
 	cmps.add(p.SampleRate, o.SampleRate)
 	cmps.add(p.Latency, o.Latency)
 	cmps.add(p.ChannelCount, o.ChannelCount)
 	cmps.add(p.IsBigEndian, o.IsBigEndian)
 	cmps.add(p.IsFloat, o.IsFloat)
 	cmps.add(p.IsInterleaved, o.IsInterleaved)
-
 	return cmps.fitnessDistance()
 }