Metadata Timeline (#6194)

* Create timeline table

* Fix indexes

* Add other fields

* Adjust schema to be less descriptive

* Handle timeline queue from tracked object data

* Setup timeline queue in events

* Add source id for index

* Add other fields

* Fixes

* Formatting

* Store better data

* Add api with filtering

* Setup basic UI for timeline in events

* Cleanups

* Add recordings snapshot url

* Start working on timeline ui

* Add tooltip with info

* Improve icons

* Fix start time with clip

* Move player logic back to clips

* Make box in timeline relative coordinates

* Make region relative

* Get box overlay working

* Remove overlay when playing again

* Add disclaimer when selecting overlay points

* Add docs for new apis

* Fix mobile

* Fix docs

* Change color of bottom center box

* Fix vscode formatting

web/src/components/TimelineSummary.jsx

@@ -0,0 +1,95 @@
+import { h } from 'preact';
+import useSWR from 'swr';
+import ActivityIndicator from './ActivityIndicator';
+import { formatUnixTimestampToDateTime } from '../utils/dateUtil';
+import PlayIcon from '../icons/Play';
+import ExitIcon from '../icons/Exit';
+import { Zone } from '../icons/Zone';
+import { useState } from 'preact/hooks';
+import Button from './Button';
+
+export default function TimelineSummary({ event, onFrameSelected }) {
+  const { data: eventTimeline } = useSWR([
+    'timeline',
+    {
+      source_id: event.id,
+    },
+  ]);
+
+  const { data: config } = useSWR('config');
+
+  const [timeIndex, setTimeIndex] = useState(-1);
+
+  const onSelectMoment = async (index) => {
+    setTimeIndex(index);
+    onFrameSelected(eventTimeline[index]);
+  };
+
+  if (!eventTimeline || !config) {
+    return <ActivityIndicator />;
+  }
+
+  return (
+    <div className="flex flex-col">
+      <div className="h-14 flex justify-center">
+        <div className="sm:w-1 md:w-1/4 flex flex-row flex-nowrap justify-between overflow-auto">
+          {eventTimeline.map((item, index) =>
+            item.class_type == 'visible' || item.class_type == 'gone' ? (
+              <Button
+                key={index}
+                className="rounded-full"
+                type="text"
+                color={index == timeIndex ? 'blue' : 'gray'}
+                aria-label={window.innerWidth > 640 ? getTimelineItemDescription(config, item, event) : ''}
+                onClick={() => onSelectMoment(index)}
+              >
+                {item.class_type == 'visible' ? <PlayIcon className="w-8" /> : <ExitIcon className="w-8" />}
+              </Button>
+            ) : (
+              <Button
+                key={index}
+                className="rounded-full"
+                type="text"
+                color={index == timeIndex ? 'blue' : 'gray'}
+                aria-label={window.innerWidth > 640 ? getTimelineItemDescription(config, item, event) : ''}
+                onClick={() => onSelectMoment(index)}
+              >
+                <Zone className="w-8" />
+              </Button>
+            )
+          )}
+        </div>
+      </div>
+      {timeIndex >= 0 ? (
+        <div className="bg-gray-500 p-4 m-2 max-w-md self-center">
+          Disclaimer: This data comes from the detect feed but is shown on the recordings, it is unlikely that the
+          streams are perfectly in sync so the bounding box and the footage will not line up perfectly.
+        </div>
+      ) : null}
+    </div>
+  );
+}
+
+function getTimelineItemDescription(config, timelineItem, event) {
+  if (timelineItem.class_type == 'visible') {
+    return `${event.label} detected at ${formatUnixTimestampToDateTime(timelineItem.timestamp, {
+      date_style: 'short',
+      time_style: 'medium',
+      time_format: config.ui.time_format,
+    })}`;
+  } else if (timelineItem.class_type == 'entered_zone') {
+    return `${event.label.replaceAll('_', ' ')} entered ${timelineItem.data.zones
+      .join(' and ')
+      .replaceAll('_', ' ')} at ${formatUnixTimestampToDateTime(timelineItem.timestamp, {
+      date_style: 'short',
+      time_style: 'medium',
+      time_format: config.ui.time_format,
+    })}`;
+  }
+
+  return `${event.label} left at ${formatUnixTimestampToDateTime(timelineItem.timestamp, {
+    date_style: 'short',
+    time_style: 'medium',
+    time_format: config.ui.time_format,
+  })}`;
+}

web/src/components/Tooltip.jsx

@@ -49,7 +49,7 @@ export default function Tooltip({ relativeTo, text }) {
   const tooltip = (
     <div
       role="tooltip"
-      className={`shadow max-w-lg absolute pointer-events-none bg-gray-900 dark:bg-gray-200 bg-opacity-80 rounded px-2 py-1 transition-transform transition-opacity duration-75 transform scale-90 opacity-0 text-gray-100 dark:text-gray-900 text-sm ${
+      className={`shadow max-w-lg absolute pointer-events-none bg-gray-900 dark:bg-gray-200 bg-opacity-80 rounded px-2 py-1 transition-transform transition-opacity duration-75 transform scale-90 opacity-0 text-gray-100 dark:text-gray-900 text-sm capitalize ${
         position.top >= 0 ? 'opacity-100 scale-100' : ''
       }`}
       ref={ref}

web/src/icons/Exit.jsx

@@ -0,0 +1,12 @@
+import { h } from 'preact';
+import { memo } from 'preact/compat';
+
+function Exit({ className = '' }) {
+  return (
+    <svg className={`fill-current ${className}`} viewBox="0 0 24 24">
+      <path d="M22 12l-4-4v3h-8v2h8v3m2 2a10 10 0 110-12h-2.73a8 8 0 100 12z" />
+    </svg>
+  );
+}
+
+export default memo(Exit);

web/src/icons/Play.jsx

@@ -1,9 +1,9 @@
 import { h } from 'preact';
 import { memo } from 'preact/compat';
 
-export function Play() {
+export function Play({ className = '' }) {
   return (
-    <svg style="width:24px;height:24px" viewBox="0 0 24 24">
+    <svg className={`fill-current ${className}`} viewBox="0 0 24 24">
       <path fill="currentColor" d="M8,5.14V19.14L19,12.14L8,5.14Z" />
     </svg>
   );

web/src/routes/Events.jsx

@@ -26,6 +26,7 @@ import Dialog from '../components/Dialog';
 import MultiSelect from '../components/MultiSelect';
 import { formatUnixTimestampToDateTime, getDurationFromTimestamps } from '../utils/dateUtil';
 import TimeAgo from '../components/TimeAgo';
+import TimelineSummary from '../components/TimelineSummary';
 
 const API_LIMIT = 25;
 
@@ -60,6 +61,7 @@ export default function Events({ path, ...props }) {
   });
   const [uploading, setUploading] = useState([]);
   const [viewEvent, setViewEvent] = useState();
+  const [eventOverlay, setEventOverlay] = useState();
   const [eventDetailType, setEventDetailType] = useState('clip');
   const [downloadEvent, setDownloadEvent] = useState({
     id: null,
@@ -180,6 +182,18 @@ export default function Events({ path, ...props }) {
     onFilter(name, items);
   };
 
+  const onEventFrameSelected = (event, frame) => {
+    const eventDuration = event.end_time - event.start_time;
+
+    if (this.player) {
+      this.player.pause();
+      const videoOffset = this.player.duration() - eventDuration;
+      const startTime = videoOffset + (frame.timestamp - event.start_time);
+      this.player.currentTime(startTime);
+      setEventOverlay(frame);
+    }
+  };
+
   const datePicker = useRef();
 
   const downloadButton = useRef();
@@ -526,7 +540,7 @@ export default function Events({ path, ...props }) {
                         </div>
                       </div>
                       <div class="hidden sm:flex flex-col justify-end mr-2">
-                        {(event.end_time && event.has_snapshot) && (
+                        {event.end_time && event.has_snapshot && (
                           <Fragment>
                             {event.plus_id ? (
                               <div className="uppercase text-xs">Sent to Frigate+</div>
@@ -573,20 +587,52 @@ export default function Events({ path, ...props }) {
 
                         <div>
                           {eventDetailType == 'clip' && event.has_clip ? (
-                            <VideoPlayer
-                              options={{
-                                preload: 'auto',
-                                autoplay: true,
-                                sources: [
-                                  {
-                                    src: `${apiHost}vod/event/${event.id}/master.m3u8`,
-                                    type: 'application/vnd.apple.mpegurl',
-                                  },
-                                ],
-                              }}
-                              seekOptions={{ forward: 10, backward: 5 }}
-                              onReady={() => {}}
-                            />
+                            <div>
+                              <TimelineSummary
+                                event={event}
+                                onFrameSelected={(frame) => onEventFrameSelected(event, frame)}
+                              />
+                              <div>
+                                <VideoPlayer
+                                  options={{
+                                    preload: 'auto',
+                                    autoplay: true,
+                                    sources: [
+                                      {
+                                        src: `${apiHost}vod/event/${event.id}/master.m3u8`,
+                                        type: 'application/vnd.apple.mpegurl',
+                                      },
+                                    ],
+                                  }}
+                                  seekOptions={{ forward: 10, backward: 5 }}
+                                  onReady={(player) => {
+                                    this.player = player;
+                                    this.player.on('playing', () => {
+                                      setEventOverlay(undefined);
+                                    });
+                                  }}
+                                  onDispose={() => {
+                                    this.player = null;
+                                  }}
+                                >
+                                  {eventOverlay ? (
+                                    <div
+                                      className="absolute border-4 border-red-600"
+                                      style={{
+                                        left: `${Math.round(eventOverlay.data.box[0] * 100)}%`,
+                                        top: `${Math.round(eventOverlay.data.box[1] * 100)}%`,
+                                        right: `${Math.round((1 - eventOverlay.data.box[2]) * 100)}%`,
+                                        bottom: `${Math.round((1 - eventOverlay.data.box[3]) * 100)}%`,
+                                      }}
+                                    >
+                                      {eventOverlay.class_type == 'entered_zone' ? (
+                                        <div className="absolute w-2 h-2 bg-yellow-500 left-[50%] bottom-0" />
+                                      ) : null}
+                                    </div>
+                                  ) : null}
+                                </VideoPlayer>
+                              </div>
+                            </div>
                           ) : null}
 
                           {eventDetailType == 'image' || !event.has_clip ? (