Review Item GenAI metadata (#19442)

* Rename existing function

* Keep track of thumbnial updates

* Tinkering with genai prompt

* Adjust input format

* Create model for review description output

* testing prompt changes

* Prompt improvements and image saving

* Add config for review items genai

* Use genai review config

* Actual config usage

* Adjust debug image saving

* Fix

* Fix review creation

* Adjust prompt

* Prompt adjustment

* Run genai in thread

* Fix detections block

* Adjust prompt

* Prompt changes

* Save genai response to metadata model

* Handle metadata

* Send review update to dispatcher

* Save review metadata to DB

* Send review notification updates

* Quick fix

* Fix name

* Fix update type

* Correctly dump model

* Add card

* Add card

* Remove message

* Cleanup typing and UI

* Adjust prompt

* Formatting

* Add log

* Formatting

* Add inference speed and keep alive

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -11,7 +11,11 @@ import { FrigateConfig } from "@/types/frigateConfig";
 import { useFormattedTimestamp } from "@/hooks/use-date-utils";
 import { getIconForLabel } from "@/utils/iconUtil";
 import { useApiHost } from "@/api";
-import { ReviewDetailPaneType, ReviewSegment } from "@/types/review";
+import {
+  ReviewDetailPaneType,
+  ReviewSegment,
+  ThreatLevel,
+} from "@/types/review";
 import { Event } from "@/types/event";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { cn } from "@/lib/utils";
@@ -69,6 +73,25 @@ export default function ReviewDetailDialog({
     review ? ["event_ids", { ids: review.data.detections.join(",") }] : null,
   );
 
+  const aiAnalysis = useMemo(() => review?.data?.metadata, [review]);
+
+  const aiThreatLevel = useMemo(() => {
+    if (!aiAnalysis?.potential_threat_level) {
+      return "None";
+    }
+
+    switch (aiAnalysis.potential_threat_level) {
+      case ThreatLevel.UNUSUAL:
+        return "Unusual Activity";
+      case ThreatLevel.SUSPICIOUS:
+        return "Suspicious Activity";
+      case ThreatLevel.DANGER:
+        return "Danger";
+    }
+
+    return "Unknown";
+  }, [aiAnalysis]);
+
   const hasMismatch = useMemo(() => {
     if (!review || !events) {
       return false;
@@ -232,6 +255,22 @@ export default function ReviewDetailDialog({
           )}
           {pane == "overview" && (
             <div className="flex flex-col gap-5 md:mt-3">
+              {aiAnalysis != undefined && (
+                <div
+                  className={cn(
+                    "m-2 flex h-full w-full flex-col gap-2 rounded-md bg-card p-2",
+                    isDesktop && "w-[90%]",
+                  )}
+                >
+                  AI Analysis
+                  <div className="text-sm text-primary/40">Description</div>
+                  <div className="text-sm">{aiAnalysis.scene}</div>
+                  <div className="text-sm text-primary/40">Score</div>
+                  <div className="text-sm">{aiAnalysis.confidence * 100}%</div>
+                  <div className="text-sm text-primary/40">Threat Level</div>
+                  <div className="text-sm">{aiThreatLevel}</div>
+                </div>
+              )}
               <div className="flex w-full flex-row">
                 <div className="flex w-full flex-col gap-3">
                   <div className="flex flex-col gap-1.5">

web/src/types/review.ts

@@ -18,6 +18,11 @@ export type ReviewData = {
   sub_labels?: string[];
   significant_motion_areas: number[];
   zones: string[];
+  metadata?: {
+    scene: string;
+    confidence: number;
+    potential_threat_level?: number;
+  };
 };
 
 export type SegmentedReviewData =
@@ -73,3 +78,9 @@ export type ConsolidatedSegmentData = {
 };
 
 export type TimelineZoomDirection = "in" | "out" | null;
+
+export enum ThreatLevel {
+  UNUSUAL = 1,
+  SUSPICIOUS = 2,
+  DANGER = 3,
+}