Add go2rtc and add restream role / live source (#4082)

* Pull go2rtc dependency

* Add go2rtc to local services and add to s6

* Add relay controller for go2rtc

* Add restream role

* Add restream role

* Add restream to nginx

* Add camera live source config

* Disable RTMP by default and use restream

* Use go2rtc for camera config

* Fix go2rtc move

* Start restream on frigate start

* Send restream to camera level

* Fix restream

* Make sure jsmpeg works as expected

* Make view rspect live size config

* Tweak player options to fit live view

* Adjust VideoPlayer to accept live option which disables irrelevant controls

* Add multiple options from restream live view

* Add base for webrtc option

* Setup specific restream modules

* Make mp4 the default streaming for now

* Expose 8554 for rtsp relay from go2rtc

* Formatting

* Update docs to suggest new restream method.

* Update docs to reflect restream role

* Update docs to reflect restream role

* Add webrtc player

* Improvements to webRTC

* Support webrtc

* Cleanup

* Adjust rtmp test and add restream test

* Fix tests

* Add restream tests

* Add live view docs and show different options

* Small docs tweak

* Support all stream types

* Update to beta 9 of go2rtc

* Formatting

* Make jsmpeg the default

* Support wss if made from https

* Support wss if made from https

* Use onEffect

* Set url outside onEffect

* Fix passed deps

* Update docs about required host mode

* Try memo instead

* Close websocket on changing camera

* Formatting

* Close pc connection

* Set video source to null on cleanup

* Use full path since go2rtc can't see PATH var

* Adjust audio codec to enable browser audio by default

* Cleanup stream creation

* Add restream tests

* Format tests

* Mock requests

* Adjust paths

* Move stream configs to restream

* Remove live source

* Remove live config

* Use live persistence for which view to use on each camera

* Fix live sizes

* Only use jsmpeg sizes for jsmpeg live

* Set max live size

* Remove access of live config

* Add selector for live view source in web view

* Remove RTMP from default list of roles

* Update docs

* Fix tests

* Fix docs for live view modes

* make default undefined to avoid race condition

* Wait until camera source is loaded to avoid race condition

* Fix tests

* Add config to go2rtc

* Work with config

* Set full path for config

* Set to use stun

* Check for mounted file

* Look for frigate-go2rtc

* Update docs to reflect webRTC configuration.

* Add link to go2rtc config

* Update docs to be more clear

* Update docs to be more clear

* Update format

Co-authored-by: Felipe Santos <felipecassiors@gmail.com>

* Update live docs

* Improve bash startup script

* Add option to force audio compatibility

* Formatting

* Fix mapping

* Fix broken link

* Update go2rtc version

* Get go2rtc webui working

* Add support for mse

* Remove mp4 option

* Undo changes to video player

* Update docs for new live view options

* Make separate path for mse

* Remove unused

* Remove mp4 path

* Try to get go2rtc proxy working

* Try to get go2rtc proxy working

* Remove unused callback

* Allow websocket on restrea dashboard

* Make mse default stream option

* Fix mse sizing

* don't assume roles is defined

* Remove nginx mapping to go2rtc ui

Co-authored-by: Felipe Santos <felipecassiors@gmail.com>
Co-authored-by: Blake Blackshear <blakeb@blakeshome.com>

web/config/handlers.js

@@ -19,7 +19,7 @@ export const handlers = [
             record: { enabled: true },
             detect: { width: 1280, height: 720 },
             snapshots: {},
-            live: { height: 720 },
+            restream: { enabled: true, jsmpeg: { height: 720 } },
             ui: { dashboard: true, order: 0 },
           },
           side: {
@@ -28,7 +28,7 @@ export const handlers = [
             record: { enabled: false },
             detect: { width: 1280, height: 720 },
             snapshots: {},
-            live: { height: 720 },
+            restream: { enabled: true, jsmpeg: { height: 720 } },
             ui: { dashboard: true, order: 1 },
           },
         },

web/src/components/JSMpegPlayer.jsx

@@ -5,7 +5,7 @@ import JSMpeg from '@cycjimmy/jsmpeg-player';
 
 export default function JSMpegPlayer({ camera, width, height }) {
   const playerRef = useRef();
-  const url = `${baseUrl.replace(/^http/, 'ws')}live/${camera}`;
+  const url = `${baseUrl.replace(/^http/, 'ws')}live/jsmpeg/${camera}`;
 
   useEffect(() => {
     const video = new JSMpeg.VideoElement(

web/src/components/MsePlayer.jsx

@@ -0,0 +1,93 @@
+import { h } from 'preact';
+import { baseUrl } from '../api/baseUrl';
+import { useEffect } from 'preact/hooks';
+
+export default function MsePlayer({ camera, width, height }) {
+  const url = `${baseUrl.replace(/^http/, 'ws')}live/mse/api/ws?src=${camera}`;
+
+  useEffect(() => {
+    const video = document.querySelector('#video');
+
+    // support api_path
+    const ws = new WebSocket(url);
+    ws.binaryType = 'arraybuffer';
+
+    let mediaSource;
+
+    ws.onopen = () => {
+      // https://web.dev/i18n/en/fast-playback-with-preload/#manual_buffering
+      // https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API
+      mediaSource = new MediaSource();
+      video.src = URL.createObjectURL(mediaSource);
+      mediaSource.onsourceopen = () => {
+        mediaSource.onsourceopen = null;
+        URL.revokeObjectURL(video.src);
+        ws.send(JSON.stringify({ type: 'mse' }));
+      };
+    };
+
+    let sourceBuffer,
+      queueBuffer = [];
+
+    ws.onmessage = (ev) => {
+      if (typeof ev.data === 'string') {
+        const data = JSON.parse(ev.data);
+
+        if (data.type === 'mse') {
+          sourceBuffer = mediaSource.addSourceBuffer(data.value);
+          // important: segments supports TrackFragDecodeTime
+          // sequence supports only TrackFragRunEntry Duration
+          sourceBuffer.mode = 'segments';
+          sourceBuffer.onupdateend = () => {
+            if (!sourceBuffer.updating && queueBuffer.length > 0) {
+              sourceBuffer.appendBuffer(queueBuffer.shift());
+            }
+          };
+        }
+      } else if (sourceBuffer.updating) {
+        queueBuffer.push(ev.data);
+      } else {
+        sourceBuffer.appendBuffer(ev.data);
+      }
+    };
+
+    let offsetTime = 1,
+      noWaiting = 0;
+
+    setInterval(() => {
+      if (video.paused || video.seekable.length === 0) return;
+
+      if (noWaiting < 0) {
+        offsetTime = Math.min(offsetTime * 1.1, 5);
+      } else if (noWaiting >= 30) {
+        noWaiting = 0;
+        offsetTime = Math.max(offsetTime * 0.9, 0.5);
+      }
+      noWaiting += 1;
+
+      const endTime = video.seekable.end(video.seekable.length - 1);
+      let playbackRate = (endTime - video.currentTime) / offsetTime;
+      if (playbackRate < 0.1) {
+        // video.currentTime = endTime - offsetTime;
+        playbackRate = 0.1;
+      } else if (playbackRate > 10) {
+        // video.currentTime = endTime - offsetTime;
+        playbackRate = 10;
+      }
+      // https://github.com/GoogleChrome/developer.chrome.com/issues/135
+      video.playbackRate = playbackRate;
+    }, 1000);
+
+    video.onwaiting = () => {
+      const endTime = video.seekable.end(video.seekable.length - 1);
+      video.currentTime = endTime - offsetTime;
+      noWaiting = -1;
+    };
+  }, [url]);
+
+  return (
+    <div>
+      <video id="video" autoplay playsinline controls muted width={width} height={height} />
+    </div>
+  );
+}

web/src/components/VideoPlayer.jsx

@@ -98,4 +98,4 @@ export default function VideoPlayer({ children, options, seekOptions = {}, onRea
       {children}
     </div>
   );
-}
+}

web/src/components/WebRtcPlayer.jsx

@@ -0,0 +1,72 @@
+import { h } from 'preact';
+import { baseUrl } from '../api/baseUrl';
+import { useEffect } from 'preact/hooks';
+
+export default function WebRtcPlayer({ camera, width, height }) {
+  const url = `${baseUrl.replace(/^http/, 'ws')}live/webrtc/api/ws?src=${camera}`;
+
+  useEffect(() => {
+    const ws = new WebSocket(url);
+    ws.onopen = () => {
+      pc.createOffer().then((offer) => {
+        pc.setLocalDescription(offer).then(() => {
+          const msg = { type: 'webrtc/offer', value: pc.localDescription.sdp };
+          ws.send(JSON.stringify(msg));
+        });
+      });
+    };
+    ws.onmessage = (ev) => {
+      const msg = JSON.parse(ev.data);
+
+      if (msg.type === 'webrtc/candidate') {
+        pc.addIceCandidate({ candidate: msg.value, sdpMid: '' });
+      } else if (msg.type === 'webrtc/answer') {
+        pc.setRemoteDescription({ type: 'answer', sdp: msg.value });
+      }
+    };
+
+    const pc = new RTCPeerConnection({
+      iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
+    });
+    pc.onicecandidate = (ev) => {
+      if (ev.candidate !== null) {
+        ws.send(
+          JSON.stringify({
+            type: 'webrtc/candidate',
+            value: ev.candidate.toJSON().candidate,
+          })
+        );
+      }
+    };
+    pc.ontrack = (ev) => {
+      const video = document.getElementById('video');
+
+      // when audio track not exist in Chrome
+      if (ev.streams.length === 0) return;
+      // when audio track not exist in Firefox
+      if (ev.streams[0].id[0] === '{') return;
+      // when stream already init
+      if (video.srcObject !== null) return;
+
+      video.srcObject = ev.streams[0];
+    };
+
+    // Safari don't support "offerToReceiveVideo"
+    // so need to create transeivers manually
+    pc.addTransceiver('video', { direction: 'recvonly' });
+    pc.addTransceiver('audio', { direction: 'recvonly' });
+
+    return () => {
+      const video = document.getElementById('video');
+      video.srcObject = null;
+      pc.close();
+      ws.close();
+    };
+  }, [url]);
+
+  return (
+    <div>
+      <video id="video" autoplay playsinline controls muted width={width} height={height} />
+    </div>
+  );
+}

web/src/routes/Camera.jsx

@@ -13,6 +13,8 @@ import { usePersistence } from '../context';
 import { useCallback, useMemo, useState } from 'preact/hooks';
 import { useApiHost } from '../api';
 import useSWR from 'swr';
+import WebRtcPlayer from '../components/WebRtcPlayer';
+import MsePlayer from '../components/MsePlayer';
 
 const emptyObject = Object.freeze({});
 
@@ -23,9 +25,11 @@ export default function Camera({ camera }) {
   const [viewMode, setViewMode] = useState('live');
 
   const cameraConfig = config?.cameras[camera];
-  const liveWidth = cameraConfig
-    ? Math.round(cameraConfig.live.height * (cameraConfig.detect.width / cameraConfig.detect.height))
+  const jsmpegWidth = cameraConfig
+    ? Math.round(cameraConfig.restream.jsmpeg.height * (cameraConfig.detect.width / cameraConfig.detect.height))
     : 0;
+  const [viewSource, setViewSource, sourceIsLoaded] = usePersistence(`${camera}-source`, 'mse');
+  const sourceValues = cameraConfig && cameraConfig.restream.enabled ? ['mse', 'webrtc', 'jsmpeg'] : ['mse'];
   const [options, setOptions] = usePersistence(`${camera}-feed`, emptyObject);
 
   const handleSetOption = useCallback(
@@ -51,7 +55,7 @@ export default function Camera({ camera }) {
     setShowSettings(!showSettings);
   }, [showSettings, setShowSettings]);
 
-  if (!cameraConfig) {
+  if (!cameraConfig || !sourceIsLoaded) {
     return <ActivityIndicator />;
   }
 
@@ -93,13 +97,31 @@ export default function Camera({ camera }) {
 
   let player;
   if (viewMode === 'live') {
-    player = (
-      <Fragment>
-        <div>
-          <JSMpegPlayer camera={camera} width={liveWidth} height={cameraConfig.live.height} />
-        </div>
-      </Fragment>
-    );
+    if (viewSource == 'mse') {
+      player = (
+        <Fragment>
+          <div className="max-w-5xl">
+            <MsePlayer camera={camera} />
+          </div>
+        </Fragment>
+      );
+    } else if (viewSource == 'webrtc') {
+      player = (
+        <Fragment>
+          <div className="max-w-5xl">
+            <WebRtcPlayer camera={camera} />
+          </div>
+        </Fragment>
+      );
+    } else {
+      player = (
+        <Fragment>
+          <div>
+            <JSMpegPlayer camera={camera} width={jsmpegWidth} height={cameraConfig.restream.jsmpeg.height} />
+          </div>
+        </Fragment>
+      );
+    }
   } else if (viewMode === 'debug') {
     player = (
       <Fragment>
@@ -120,7 +142,23 @@ export default function Camera({ camera }) {
 
   return (
     <div className="space-y-4 p-2 px-4">
-      <Heading size="2xl">{camera.replaceAll('_', ' ')}</Heading>
+      <div className="flex justify-between">
+        <Heading className="p-2" size="2xl">
+          {camera.replaceAll('_', ' ')}
+        </Heading>
+        <select
+          className="basis-1/8 cursor-pointer rounded dark:bg-slate-800"
+          value={viewSource}
+          onChange={(e) => setViewSource(e.target.value)}
+        >
+          {sourceValues.map((item) => (
+            <option key={item} value={item}>
+              {item}
+            </option>
+          ))}
+        </select>
+      </div>
+
       <ButtonsTabbed viewModes={['live', 'debug']} setViewMode={setViewMode} />
 
       {player}

web/src/routes/__tests__/Camera.test.jsx

@@ -11,7 +11,7 @@ describe('Camera Route', () => {
 
   beforeEach(() => {
     mockSetOptions = jest.fn();
-    mockUsePersistence = jest.spyOn(Context, 'usePersistence').mockImplementation(() => [{}, mockSetOptions]);
+    mockUsePersistence = jest.spyOn(Context, 'usePersistence').mockImplementation(() => [{}, mockSetOptions, true]);
     jest.spyOn(AutoUpdatingCameraImage, 'default').mockImplementation(({ searchParams }) => {
       return <div data-testid="mock-image">{searchParams.toString()}</div>;
     });
@@ -32,11 +32,12 @@ describe('Camera Route', () => {
         regions: false,
       },
       mockSetOptions,
+      true,
     ]);
 
     render(<Camera camera="front" />);
 
-    await waitForElementToBeRemoved(() => screen.queryByLabelText('Loading…'));
+    await waitForElementToBeRemoved(() => screen.queryByLabelText('Loading…'), { timeout: 10 });
 
     fireEvent.click(screen.queryByText('Debug'));
     fireEvent.click(screen.queryByText('Show Options'));
@@ -47,15 +48,20 @@ describe('Camera Route', () => {
 
   test('updates camera feed options to persistence', async () => {
     mockUsePersistence
-      .mockReturnValueOnce([{}, mockSetOptions])
-      .mockReturnValueOnce([{}, mockSetOptions])
-      .mockReturnValueOnce([{}, mockSetOptions])
-      .mockReturnValueOnce([{ bbox: true }, mockSetOptions])
-      .mockReturnValueOnce([{ bbox: true, timestamp: true }, mockSetOptions]);
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{}, mockSetOptions, true])
+      .mockReturnValueOnce([{ bbox: true }, mockSetOptions, true])
+      .mockReturnValueOnce([{ bbox: true, timestamp: true }, mockSetOptions, true]);
 
     render(<Camera camera="front" />);
 
-    await waitForElementToBeRemoved(() => screen.queryByLabelText('Loading…'));
+    await waitForElementToBeRemoved(() => screen.queryByLabelText('Loading…'), { timeout: 10 });
 
     fireEvent.click(screen.queryByText('Debug'));
     fireEvent.click(screen.queryByText('Show Options'));
@@ -63,9 +69,8 @@ describe('Camera Route', () => {
     fireEvent.change(screen.queryByTestId('timestamp-input'), { target: { checked: true } });
     fireEvent.click(screen.queryByText('Hide Options'));
 
-    expect(mockUsePersistence).toHaveBeenCalledTimes(5);
+    expect(mockUsePersistence).toHaveBeenCalledTimes(10);
     expect(mockSetOptions).toHaveBeenCalledTimes(2);
-    expect(mockSetOptions).toHaveBeenCalledWith({ bbox: true, timestamp: true });
     expect(screen.queryByTestId('mock-image')).toHaveTextContent('bbox=1&timestamp=1');
   });
 });