Audio events (#6848)

* Initial audio classification model implementation

* fix mypy

* Keep audio labelmap local

* Cleanup

* Start adding config for audio

* Add the detector

* Add audio detection process keypoints

* Build out base config

* Load labelmap correctly

* Fix config bugs

* Start audio process

* Fix startup issues

* Try to cleanup restarting

* Add ffmpeg input args

* Get audio detection working

* Save event to db

* End events if not heard for 30 seconds

* Use not heard config

* Stop ffmpeg when shutting down

* Fixes

* End events correctly

* Use api instead of event queue to save audio events

* Get events working

* Close threads when stop event is sent

* remove unused

* Only start audio process if at least one camera is enabled

* Add const for float

* Cleanup labelmap

* Add audio icon in frontend

* Add ability to toggle audio with mqtt

* Set initial audio value

* Fix audio enabling

* Close logpipe

* Isort

* Formatting

* Fix web tests

* Fix web tests

* Handle cases where args are a string

* Remove log

* Cleanup process close

* Use correct field

* Simplify if statement

* Use var for localhost

* Add audio detectors docs

* Add restream docs to mention audio detection

* Add full config docs

* Fix links to other docs

---------

Co-authored-by: Jason Hunter <hunterjm@gmail.com>

web/src/api/__tests__/ws.test.jsx

@@ -113,8 +113,8 @@ describe('WsProvider', () => {
     vi.spyOn(Date, 'now').mockReturnValue(123456);
     const config = {
       cameras: {
-        front: { name: 'front', detect: { enabled: true }, record: { enabled: false }, snapshots: { enabled: true } },
-        side: { name: 'side', detect: { enabled: false }, record: { enabled: false }, snapshots: { enabled: false } },
+        front: { name: 'front', detect: { enabled: true }, record: { enabled: false }, snapshots: { enabled: true }, audio: { enabled: false } },
+        side: { name: 'side', detect: { enabled: false }, record: { enabled: false }, snapshots: { enabled: false }, audio: { enabled: false } },
       },
     };
     render(

web/src/api/ws.jsx

@@ -41,10 +41,11 @@ export function WsProvider({
 
   useEffect(() => {
     Object.keys(config.cameras).forEach((camera) => {
-      const { name, record, detect, snapshots } = config.cameras[camera];
+      const { name, record, detect, snapshots, audio } = config.cameras[camera];
       dispatch({ topic: `${name}/recordings/state`, payload: record.enabled ? 'ON' : 'OFF', retain: false });
       dispatch({ topic: `${name}/detect/state`, payload: detect.enabled ? 'ON' : 'OFF', retain: false });
       dispatch({ topic: `${name}/snapshots/state`, payload: snapshots.enabled ? 'ON' : 'OFF', retain: false });
+      dispatch({ topic: `${name}/audio/state`, payload: audio.enabled ? 'ON' : 'OFF', retain: false });
     });
   }, [config]);
 
@@ -120,6 +121,15 @@ export function useSnapshotsState(camera) {
   return { payload, send, connected };
 }
 
+export function useAudioState(camera) {
+  const {
+    value: { payload },
+    send,
+    connected,
+  } = useWs(`${camera}/audio/state`, `${camera}/audio/set`);
+  return { payload, send, connected };
+}
+
 export function usePtzCommand(camera) {
   const {
     value: { payload },

web/src/icons/Audio.jsx

@@ -0,0 +1,36 @@
+import { h } from 'preact';
+import { memo } from 'preact/compat';
+
+export function Snapshot({ className = 'h-6 w-6', stroke = 'currentColor', onClick = () => {} }) {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      className={className}
+      fill="none"
+      viewBox="0 0 32 32"
+      stroke={stroke}
+      onClick={onClick}
+    >
+      <path
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        stroke-width="2"
+        d="M18 30v-2a10.011 10.011 0 0010-10h2a12.013 12.013 0 01-12 12z"
+      />
+      <path
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        stroke-width="2"
+        d="M18 22v-2a2.002 2.002 0 002-2h2a4.004 4.004 0 01-4 4zM10 2a9.01 9.01 0 00-9 9h2a7 7 0 0114 0 7.09 7.09 0 01-3.501 6.135l-.499.288v3.073a2.935 2.935 0 01-.9 2.151 4.182 4.182 0 01-4.633 1.03A4.092 4.092 0 015 20H3a6.116 6.116 0 003.67 5.512 5.782 5.782 0 002.314.486 6.585 6.585 0 004.478-1.888A4.94 4.94 0 0015 20.496v-1.942A9.108 9.108 0 0019 11a9.01 9.01 0 00-9-9z"
+      />
+      <path
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        stroke-width="2"
+        d="M9.28 8.082A3.006 3.006 0 0113 11h2a4.979 4.979 0 00-1.884-3.911 5.041 5.041 0 00-4.281-.957 4.95 4.95 0 00-3.703 3.703 5.032 5.032 0 002.304 5.458A3.078 3.078 0 019 17.924V20h2v-2.077a5.06 5.06 0 00-2.537-4.346 3.002 3.002 0 01.817-5.494z"
+      />
+    </svg>
+  );
+}
+
+export default memo(Snapshot);

web/src/routes/Cameras.jsx

@@ -2,10 +2,11 @@ import { h, Fragment } from 'preact';
 import ActivityIndicator from '../components/ActivityIndicator';
 import Card from '../components/Card';
 import CameraImage from '../components/CameraImage';
+import AudioIcon from '../icons/Audio';
 import ClipIcon from '../icons/Clip';
 import MotionIcon from '../icons/Motion';
 import SnapshotIcon from '../icons/Snapshot';
-import { useDetectState, useRecordingsState, useSnapshotsState } from '../api/ws';
+import { useAudioState, useDetectState, useRecordingsState, useSnapshotsState } from '../api/ws';
 import { useMemo } from 'preact/hooks';
 import useSWR from 'swr';
 
@@ -43,6 +44,7 @@ function Camera({ name, config }) {
   const { payload: detectValue, send: sendDetect } = useDetectState(name);
   const { payload: recordValue, send: sendRecordings } = useRecordingsState(name);
   const { payload: snapshotValue, send: sendSnapshots } = useSnapshotsState(name);
+  const { payload: audioValue, send: sendAudio } = useAudioState(name);
   const href = `/cameras/${name}`;
   const buttons = useMemo(() => {
     return [
@@ -50,10 +52,9 @@ function Camera({ name, config }) {
       { name: 'Recordings', href: `/recording/${name}` },
     ];
   }, [name]);
-  const cleanName = useMemo(
-    () => { return `${name.replaceAll('_', ' ')}` },
-    [name]
-  );
+  const cleanName = useMemo(() => {
+    return `${name.replaceAll('_', ' ')}`;
+  }, [name]);
   const icons = useMemo(
     () => [
       {
@@ -65,7 +66,9 @@ function Camera({ name, config }) {
         },
       },
       {
-        name: config.record.enabled_in_config ? `Toggle recordings ${recordValue === 'ON' ? 'off' : 'on'}` : 'Recordings must be enabled in the config to be turned on in the UI.',
+        name: config.record.enabled_in_config
+          ? `Toggle recordings ${recordValue === 'ON' ? 'off' : 'on'}`
+          : 'Recordings must be enabled in the config to be turned on in the UI.',
         icon: ClipIcon,
         color: config.record.enabled_in_config ? (recordValue === 'ON' ? 'blue' : 'gray') : 'red',
         onClick: () => {
@@ -82,11 +85,27 @@ function Camera({ name, config }) {
           sendSnapshots(snapshotValue === 'ON' ? 'OFF' : 'ON', true);
         },
       },
-    ],
-    [config, detectValue, sendDetect, recordValue, sendRecordings, snapshotValue, sendSnapshots]
+      config.audio.enabled_in_config
+        ? {
+          name: `Toggle audio detection ${audioValue === 'ON' ? 'off' : 'on'}`,
+          icon: AudioIcon,
+          color: audioValue === 'ON' ? 'blue' : 'gray',
+          onClick: () => {
+            sendAudio(audioValue === 'ON' ? 'OFF' : 'ON', true);
+          },
+        }
+        : null,
+    ].filter((button) => button != null),
+    [config, audioValue, sendAudio, detectValue, sendDetect, recordValue, sendRecordings, snapshotValue, sendSnapshots]
   );
 
   return (
-    <Card buttons={buttons} href={href} header={cleanName} icons={icons} media={<CameraImage camera={name} stretch />} />
+    <Card
+      buttons={buttons}
+      href={href}
+      header={cleanName}
+      icons={icons}
+      media={<CameraImage camera={name} stretch />}
+    />
   );
 }