Improve audio detection debugging (#19753)

* create audio activity manager

move publishing logic out of audio detector

* dispatcher changes

* correctly publish full array of audio detections in onConnect

* frontend websocket hooks

* line graph

* debug tab and i18n

* docs

* clean up

* fix i18n key

web/public/locales/en/views/settings.json

@@ -411,6 +411,13 @@
     "debugging": "Debugging",
     "objectList": "Object List",
     "noObjects": "No objects",
+    "audio": {
+      "title": "Audio",
+      "noAudioDetections": "No audio detections",
+      "score": "score",
+      "currentRMS": "Current RMS",
+      "currentdbFS": "Current dbFS"
+    },
     "boundingBoxes": {
       "title": "Bounding boxes",
       "desc": "Show bounding boxes around tracked objects",

web/src/api/ws.tsx

@@ -10,6 +10,7 @@ import {
   ToggleableSetting,
   TrackedObjectUpdateReturnType,
   TriggerStatus,
+  FrigateAudioDetections,
 } from "@/types/ws";
 import { FrigateStats } from "@/types/stats";
 import { createContainer } from "react-tracked";
@@ -341,6 +342,13 @@ export function useFrigateEvents(): { payload: FrigateEvent } {
   return { payload: JSON.parse(payload as string) };
 }
 
+export function useAudioDetections(): { payload: FrigateAudioDetections } {
+  const {
+    value: { payload },
+  } = useWs("audio_detections", "");
+  return { payload: JSON.parse(payload as string) };
+}
+
 export function useFrigateReviews(): FrigateReview {
   const {
     value: { payload },

web/src/components/audio/AudioLevelGraph.tsx

@@ -0,0 +1,165 @@
+import { useEffect, useMemo, useState, useCallback } from "react";
+import { MdCircle } from "react-icons/md";
+import Chart from "react-apexcharts";
+import { useTheme } from "@/context/theme-provider";
+import { useWs } from "@/api/ws";
+import { useDateLocale } from "@/hooks/use-date-locale";
+import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { useTranslation } from "react-i18next";
+
+const GRAPH_COLORS = ["#3b82f6", "#ef4444"]; // RMS, dBFS
+
+interface AudioLevelGraphProps {
+  cameraName: string;
+}
+
+export function AudioLevelGraph({ cameraName }: AudioLevelGraphProps) {
+  const [audioData, setAudioData] = useState<
+    { timestamp: number; rms: number; dBFS: number }[]
+  >([]);
+  const [maxDataPoints] = useState(50);
+
+  // config for time formatting
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+  const locale = useDateLocale();
+  const { t } = useTranslation(["common"]);
+
+  const {
+    value: { payload: audioRms },
+  } = useWs(`${cameraName}/audio/rms`, "");
+  const {
+    value: { payload: audioDBFS },
+  } = useWs(`${cameraName}/audio/dBFS`, "");
+
+  useEffect(() => {
+    if (typeof audioRms === "number") {
+      const now = Date.now();
+      setAudioData((prev) => {
+        const next = [
+          ...prev,
+          {
+            timestamp: now,
+            rms: audioRms,
+            dBFS: typeof audioDBFS === "number" ? audioDBFS : 0,
+          },
+        ];
+        return next.slice(-maxDataPoints);
+      });
+    }
+  }, [audioRms, audioDBFS, maxDataPoints]);
+
+  const series = useMemo(
+    () => [
+      {
+        name: "RMS",
+        data: audioData.map((p) => ({ x: p.timestamp, y: p.rms })),
+      },
+      {
+        name: "dBFS",
+        data: audioData.map((p) => ({ x: p.timestamp, y: p.dBFS })),
+      },
+    ],
+    [audioData],
+  );
+
+  const lastValues = useMemo(() => {
+    if (!audioData.length) return undefined;
+    const last = audioData[audioData.length - 1];
+    return [last.rms, last.dBFS];
+  }, [audioData]);
+
+  const timeFormat = config?.ui.time_format === "24hour" ? "24hour" : "12hour";
+  const formatString = useMemo(
+    () =>
+      t(`time.formattedTimestampHourMinuteSecond.${timeFormat}`, {
+        ns: "common",
+      }),
+    [t, timeFormat],
+  );
+
+  const formatTime = useCallback(
+    (val: unknown) => {
+      const seconds = Math.round(Number(val) / 1000);
+      return formatUnixTimestampToDateTime(seconds, {
+        timezone: config?.ui.timezone,
+        date_format: formatString,
+        locale,
+      });
+    },
+    [config?.ui.timezone, formatString, locale],
+  );
+
+  const { theme, systemTheme } = useTheme();
+
+  const options = useMemo(() => {
+    return {
+      chart: {
+        id: `${cameraName}-audio`,
+        selection: { enabled: false },
+        toolbar: { show: false },
+        zoom: { enabled: false },
+        animations: { enabled: false },
+      },
+      colors: GRAPH_COLORS,
+      grid: {
+        show: true,
+        borderColor: "#374151",
+        strokeDashArray: 3,
+        xaxis: { lines: { show: true } },
+        yaxis: { lines: { show: true } },
+      },
+      legend: { show: false },
+      dataLabels: { enabled: false },
+      stroke: { width: 1 },
+      markers: { size: 0 },
+      tooltip: {
+        theme: systemTheme || theme,
+        x: { formatter: (val: number) => formatTime(val) },
+        y: { formatter: (v: number) => v.toFixed(1) },
+      },
+      xaxis: {
+        type: "datetime",
+        labels: {
+          rotate: 0,
+          formatter: formatTime,
+          style: { colors: "#6B6B6B", fontSize: "10px" },
+        },
+        axisBorder: { show: false },
+        axisTicks: { show: false },
+      },
+      yaxis: {
+        show: true,
+        labels: {
+          formatter: (val: number) => Math.round(val).toString(),
+          style: { colors: "#6B6B6B", fontSize: "10px" },
+        },
+      },
+    } as ApexCharts.ApexOptions;
+  }, [cameraName, theme, systemTheme, formatTime]);
+
+  return (
+    <div className="my-4 flex flex-col">
+      {lastValues && (
+        <div className="mb-2 flex flex-wrap items-center gap-2.5">
+          {["RMS", "dBFS"].map((label, idx) => (
+            <div key={label} className="flex items-center gap-1">
+              <MdCircle
+                className="size-2"
+                style={{ color: GRAPH_COLORS[idx] }}
+              />
+              <div className="text-xs text-secondary-foreground">{label}</div>
+              <div className="text-xs text-primary">
+                {lastValues[idx].toFixed(1)}
+              </div>
+            </div>
+          ))}
+        </div>
+      )}
+      <Chart type="line" options={options} series={series} />
+    </div>
+  );
+}

web/src/hooks/use-camera-activity.ts

@@ -1,4 +1,5 @@
 import {
+  useAudioDetections,
   useEnabledState,
   useFrigateEvents,
   useInitialCameraState,
@@ -8,7 +9,7 @@ import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
 import { MotionData, ReviewSegment } from "@/types/review";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import { useTimelineUtils } from "./use-timeline-utils";
-import { ObjectType } from "@/types/ws";
+import { AudioDetection, ObjectType } from "@/types/ws";
 import useDeepMemo from "./use-deep-memo";
 import { isEqual } from "lodash";
 import { useAutoFrigateStats } from "./use-stats";
@@ -20,6 +21,7 @@ type useCameraActivityReturn = {
   activeTracking: boolean;
   activeMotion: boolean;
   objects: ObjectType[];
+  audio_detections: AudioDetection[];
   offline: boolean;
 };
 
@@ -38,6 +40,9 @@ export function useCameraActivity(
     return getAttributeLabels(config);
   }, [config]);
   const [objects, setObjects] = useState<ObjectType[] | undefined>([]);
+  const [audioDetections, setAudioDetections] = useState<
+    AudioDetection[] | undefined
+  >([]);
 
   // init camera activity
 
@@ -51,6 +56,15 @@ export function useCameraActivity(
     }
   }, [updatedCameraState, camera]);
 
+  const { payload: updatedAudioState } = useAudioDetections();
+  const memoizedAudioState = useDeepMemo(updatedAudioState);
+
+  useEffect(() => {
+    if (memoizedAudioState) {
+      setAudioDetections(memoizedAudioState[camera.name]);
+    }
+  }, [memoizedAudioState, camera]);
+
   // handle camera activity
 
   const hasActiveObjects = useMemo(
@@ -160,6 +174,7 @@ export function useCameraActivity(
         : updatedCameraState?.motion === true
       : false,
     objects: isCameraEnabled ? (objects ?? []) : [],
+    audio_detections: isCameraEnabled ? (audioDetections ?? []) : [],
     offline,
   };
 }

web/src/types/ws.ts

@@ -51,6 +51,12 @@ export type ObjectType = {
   sub_label: string;
 };
 
+export type AudioDetection = {
+  id: string;
+  label: string;
+  score: number;
+};
+
 export interface FrigateCameraState {
   config: {
     enabled: boolean;
@@ -69,6 +75,10 @@ export interface FrigateCameraState {
   };
   motion: boolean;
   objects: ObjectType[];
+  audio_detections: AudioDetection[];
+}
+export interface FrigateAudioDetections {
+  [camera: string]: AudioDetection[];
 }
 
 export type ModelState =

web/src/views/settings/ObjectSettingsView.tsx

@@ -16,7 +16,7 @@ import {
   PopoverContent,
   PopoverTrigger,
 } from "@/components/ui/popover";
-import { ObjectType } from "@/types/ws";
+import { AudioDetection, ObjectType } from "@/types/ws";
 import useDeepMemo from "@/hooks/use-deep-memo";
 import { Card } from "@/components/ui/card";
 import { getIconForLabel } from "@/utils/iconUtil";
@@ -30,6 +30,8 @@ import { isDesktop } from "react-device-detect";
 import { Trans, useTranslation } from "react-i18next";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { getTranslatedLabel } from "@/utils/i18n";
+import { AudioLevelGraph } from "@/components/audio/AudioLevelGraph";
+import { useWs } from "@/api/ws";
 
 type ObjectSettingsViewProps = {
   selectedCamera?: string;
@@ -126,9 +128,12 @@ export default function ObjectSettingsView({
     }
   }, [config, selectedCamera]);
 
-  const { objects } = useCameraActivity(cameraConfig ?? ({} as CameraConfig));
+  const { objects, audio_detections } = useCameraActivity(
+    cameraConfig ?? ({} as CameraConfig),
+  );
 
   const memoizedObjects = useDeepMemo(objects);
+  const memoizedAudio = useDeepMemo(audio_detections);
 
   const searchParams = useMemo(() => {
     if (!optionsLoaded) {
@@ -189,11 +194,18 @@ export default function ObjectSettingsView({
         )}
 
         <Tabs defaultValue="debug" className="w-full">
-          <TabsList className="grid w-full grid-cols-2">
+          <TabsList
+            className={`grid w-full ${cameraConfig.ffmpeg.inputs.some((input) => input.roles.includes("audio")) ? "grid-cols-3" : "grid-cols-2"}`}
+          >
             <TabsTrigger value="debug">{t("debug.debugging")}</TabsTrigger>
             <TabsTrigger value="objectlist">
               {t("debug.objectList")}
             </TabsTrigger>
+            {cameraConfig.ffmpeg.inputs.some((input) =>
+              input.roles.includes("audio"),
+            ) && (
+              <TabsTrigger value="audio">{t("debug.audio.title")}</TabsTrigger>
+            )}
           </TabsList>
           <TabsContent value="debug">
             <div className="flex w-full flex-col space-y-6">
@@ -304,6 +316,16 @@ export default function ObjectSettingsView({
           <TabsContent value="objectlist">
             <ObjectList cameraConfig={cameraConfig} objects={memoizedObjects} />
           </TabsContent>
+          {cameraConfig.ffmpeg.inputs.some((input) =>
+            input.roles.includes("audio"),
+          ) && (
+            <TabsContent value="audio">
+              <AudioList
+                cameraConfig={cameraConfig}
+                audioDetections={memoizedAudio}
+              />
+            </TabsContent>
+          )}
         </Tabs>
       </div>
 
@@ -362,7 +384,7 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) {
   return (
     <div className="scrollbar-container flex w-full flex-col overflow-y-auto">
       {objects && objects.length > 0 ? (
-        objects.map((obj) => {
+        objects.map((obj: ObjectType) => {
           return (
             <Card className="mb-1 p-2 text-sm" key={obj.id}>
               <div className="flex flex-row items-center gap-3 pb-1">
@@ -438,3 +460,61 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) {
     </div>
   );
 }
+
+type AudioListProps = {
+  cameraConfig: CameraConfig;
+  audioDetections?: AudioDetection[];
+};
+
+function AudioList({ cameraConfig, audioDetections }: AudioListProps) {
+  const { t } = useTranslation(["views/settings"]);
+
+  // Get audio levels directly from ws hooks
+  const {
+    value: { payload: audioRms },
+  } = useWs(`${cameraConfig.name}/audio/rms`, "");
+  const {
+    value: { payload: audioDBFS },
+  } = useWs(`${cameraConfig.name}/audio/dBFS`, "");
+
+  return (
+    <div className="scrollbar-container flex w-full flex-col overflow-y-auto">
+      {audioDetections && Object.keys(audioDetections).length > 0 ? (
+        Object.entries(audioDetections).map(([key, obj]) => (
+          <Card className="mb-1 p-2 text-sm" key={obj.id ?? key}>
+            <div className="flex flex-row items-center gap-3 pb-1">
+              <div className="flex flex-1 flex-row items-center justify-start p-3 pl-1">
+                <div className="rounded-lg bg-selected p-2">
+                  {getIconForLabel(key, "size-5 text-white")}
+                </div>
+                <div className="ml-3 text-lg">{getTranslatedLabel(key)}</div>
+              </div>
+              <div className="flex w-8/12 flex-row items-center justify-end">
+                <div className="text-md mr-2 w-1/3">
+                  <div className="flex flex-col items-end justify-end">
+                    <p className="mb-1.5 text-sm text-primary-variant">
+                      {t("debug.audio.score")}
+                    </p>
+                    {obj.score ? (obj.score * 100).toFixed(1).toString() : "-"}%
+                  </div>
+                </div>
+              </div>
+            </div>
+          </Card>
+        ))
+      ) : (
+        <div className="p-3 text-center">
+          <p className="mb-2">{t("debug.audio.noAudioDetections")}</p>
+          <p className="text-xs text-muted-foreground">
+            {t("debug.audio.currentRMS")}{" "}
+            {(typeof audioRms === "number" ? audioRms : 0).toFixed(1)} |{" "}
+            {t("debug.audio.currentdbFS")}{" "}
+            {(typeof audioDBFS === "number" ? audioDBFS : 0).toFixed(1)}
+          </p>
+        </div>
+      )}
+
+      <AudioLevelGraph cameraName={cameraConfig.name} />
+    </div>
+  );
+}