Review genai updates (#19448)

* Include extra level for normal activity

* Add dynamic toggling

* Update docs

* Add different threshold for genai

* Adjust webUI for object and review description feature

* Adjust config

* Send on startup

* Cleanup config setting

* Set config

* Fix config name

web/public/locales/en/views/settings.json

@@ -150,9 +150,13 @@
       "title": "Streams",
       "desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.<br /> <em>Note: This does not disable go2rtc restreams.</em>"
     },
-    "genai": {
-      "title": "Generative AI",
-      "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    "object_descriptions": {
+      "title": "Generative AI Object Descriptions",
+      "desc": "Temporarily enable/disable Generative AI object descriptions for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    },
+    "review_descriptions": {
+      "title": "Generative AI Review Descriptions",
+      "desc": "Temporarily enable/disable Generative AI review descriptions for this camera. When disabled, AI generated descriptions will not be requested for review items on this camera."
     },
     "review": {
       "title": "Review",

web/src/api/ws.tsx

@@ -68,7 +68,8 @@ function useValue(): useValueReturn {
         autotracking,
         alerts,
         detections,
-        genai,
+        object_descriptions,
+        review_descriptions,
       } = state["config"];
       cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF";
       cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF";
@@ -90,7 +91,12 @@ function useValue(): useValueReturn {
       cameraStates[`${name}/review_detections/state`] = detections
         ? "ON"
         : "OFF";
-      cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF";
+      cameraStates[`${name}/object_descriptions/state`] = object_descriptions
+        ? "ON"
+        : "OFF";
+      cameraStates[`${name}/review_descriptions/state`] = review_descriptions
+        ? "ON"
+        : "OFF";
     });
 
     setWsState((prevState) => ({
@@ -278,14 +284,31 @@ export function useDetectionsState(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useGenAIState(camera: string): {
+export function useObjectDescriptionState(camera: string): {
   payload: ToggleableSetting;
   send: (payload: ToggleableSetting, retain?: boolean) => void;
 } {
   const {
     value: { payload },
     send,
-  } = useWs(`${camera}/genai/state`, `${camera}/genai/set`);
+  } = useWs(
+    `${camera}/object_descriptions/state`,
+    `${camera}/object_descriptions/set`,
+  );
+  return { payload: payload as ToggleableSetting, send };
+}
+
+export function useReviewDescriptionState(camera: string): {
+  payload: ToggleableSetting;
+  send: (payload: ToggleableSetting, retain?: boolean) => void;
+} {
+  const {
+    value: { payload },
+    send,
+  } = useWs(
+    `${camera}/review_descriptions/state`,
+    `${camera}/review_descriptions/set`,
+  );
   return { payload: payload as ToggleableSetting, send };
 }
 

web/src/types/frigateConfig.ts

@@ -218,6 +218,12 @@ export interface CameraConfig {
         mode: string;
       };
     };
+    genai?: {
+      enabled: boolean;
+      enabled_in_config: boolean;
+      alerts: boolean;
+      detections: boolean;
+    };
   };
   rtmp: {
     enabled: boolean;

web/src/types/graph.ts

@@ -23,6 +23,11 @@ export const EmbeddingThreshold = {
   error: 1000,
 } as Threshold;
 
+export const GenAIThreshold = {
+  warning: 30000,
+  error: 60000,
+} as Threshold;
+
 export const DetectorTempThreshold = {
   warning: 72,
   error: 80,

web/src/types/ws.ts

@@ -64,7 +64,8 @@ export interface FrigateCameraState {
     autotracking: boolean;
     alerts: boolean;
     detections: boolean;
-    genai: boolean;
+    object_descriptions: boolean;
+    review_descriptions: boolean;
   };
   motion: boolean;
   objects: ObjectType[];

web/src/views/settings/CameraSettingsView.tsx

@@ -35,7 +35,8 @@ import {
   useAlertsState,
   useDetectionsState,
   useEnabledState,
-  useGenAIState,
+  useObjectDescriptionState,
+  useReviewDescriptionState,
 } from "@/api/ws";
 import CameraEditForm from "@/components/settings/CameraEditForm";
 import { LuPlus } from "react-icons/lu";
@@ -150,8 +151,10 @@ export default function CameraSettingsView({
   const { payload: detectionsState, send: sendDetections } =
     useDetectionsState(selectedCamera);
 
-  const { payload: genAIState, send: sendGenAI } =
-    useGenAIState(selectedCamera);
+  const { payload: objDescState, send: sendObjDesc } =
+    useObjectDescriptionState(selectedCamera);
+  const { payload: revDescState, send: sendRevDesc } =
+    useReviewDescriptionState(selectedCamera);
 
   const handleCheckedChange = useCallback(
     (isChecked: boolean) => {
@@ -418,7 +421,9 @@ export default function CameraSettingsView({
                   <Separator className="my-2 flex bg-secondary" />
 
                   <Heading as="h4" className="my-2">
-                    <Trans ns="views/settings">camera.genai.title</Trans>
+                    <Trans ns="views/settings">
+                      camera.object_descriptions.title
+                    </Trans>
                   </Heading>
 
                   <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
@@ -426,9 +431,9 @@ export default function CameraSettingsView({
                       <Switch
                         id="alerts-enabled"
                         className="mr-3"
-                        checked={genAIState == "ON"}
+                        checked={objDescState == "ON"}
                         onCheckedChange={(isChecked) => {
-                          sendGenAI(isChecked ? "ON" : "OFF");
+                          sendObjDesc(isChecked ? "ON" : "OFF");
                         }}
                       />
                       <div className="space-y-0.5">
@@ -438,7 +443,44 @@ export default function CameraSettingsView({
                       </div>
                     </div>
                     <div className="mt-3 text-sm text-muted-foreground">
-                      <Trans ns="views/settings">camera.genai.desc</Trans>
+                      <Trans ns="views/settings">
+                        camera.object_descriptions.desc
+                      </Trans>
+                    </div>
+                  </div>
+                </>
+              )}
+
+              {cameraConfig?.review?.genai?.enabled_in_config && (
+                <>
+                  <Separator className="my-2 flex bg-secondary" />
+
+                  <Heading as="h4" className="my-2">
+                    <Trans ns="views/settings">
+                      camera.review_descriptions.title
+                    </Trans>
+                  </Heading>
+
+                  <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
+                    <div className="flex flex-row items-center">
+                      <Switch
+                        id="alerts-enabled"
+                        className="mr-3"
+                        checked={revDescState == "ON"}
+                        onCheckedChange={(isChecked) => {
+                          sendRevDesc(isChecked ? "ON" : "OFF");
+                        }}
+                      />
+                      <div className="space-y-0.5">
+                        <Label htmlFor="genai-enabled">
+                          <Trans>button.enabled</Trans>
+                        </Label>
+                      </div>
+                    </div>
+                    <div className="mt-3 text-sm text-muted-foreground">
+                      <Trans ns="views/settings">
+                        camera.review_descriptions.desc
+                      </Trans>
                     </div>
                   </div>
                 </>

web/src/views/system/EnrichmentMetrics.tsx

@@ -1,8 +1,8 @@
 import useSWR from "swr";
 import { FrigateStats } from "@/types/stats";
-import { useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { useFrigateStats } from "@/api/ws";
-import { EmbeddingThreshold } from "@/types/graph";
+import { EmbeddingThreshold, GenAIThreshold, Threshold } from "@/types/graph";
 import { Skeleton } from "@/components/ui/skeleton";
 import { ThresholdBarGraph } from "@/components/graph/SystemGraph";
 import { cn } from "@/lib/utils";
@@ -50,6 +50,14 @@ export default function EnrichmentMetrics({
     }
   }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]);
 
+  const getThreshold = useCallback((key: string) => {
+    if (key.includes("description")) {
+      return GenAIThreshold;
+    }
+
+    return EmbeddingThreshold;
+  }, []);
+
   // timestamps
 
   const updateTimes = useMemo(
@@ -65,7 +73,11 @@ export default function EnrichmentMetrics({
     }
 
     const series: {
-      [key: string]: { name: string; data: { x: number; y: number }[] };
+      [key: string]: {
+        name: string;
+        metrics: Threshold;
+        data: { x: number; y: number }[];
+      };
     } = {};
 
     statsHistory.forEach((stats, statsIdx) => {
@@ -79,6 +91,7 @@ export default function EnrichmentMetrics({
         if (!(key in series)) {
           series[key] = {
             name: t("enrichments.embeddings." + rawKey),
+            metrics: getThreshold(rawKey),
             data: [],
           };
         }
@@ -87,7 +100,7 @@ export default function EnrichmentMetrics({
       });
     });
     return Object.values(series);
-  }, [statsHistory, t]);
+  }, [statsHistory, t, getThreshold]);
 
   return (
     <>
@@ -112,7 +125,7 @@ export default function EnrichmentMetrics({
                       graphId={`${series.name}-inference`}
                       name={series.name}
                       unit="ms"
-                      threshold={EmbeddingThreshold}
+                      threshold={series.metrics}
                       updateTimes={updateTimes}
                       data={[series]}
                     />