From c97aac6c94cc8888864b3e6c45b24969f45a78c7 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 13 Nov 2022 11:48:14 -0700 Subject: [PATCH] Revamped debug UI and add camera / process info, ffprobe copying (#4349) * Move each camera to a separate card and show per process info * Install top * Add support for cpu usage stats * Use cpu usage stats in debug * Increase number of runs to ensure good results * Add ffprobe endpoint * Get ffprobe for multiple inputs * Copy ffprobe in output * Add fps to camera metrics * Fix lint errors * Update stats config * Add ffmpeg pid * Use grid display so more cameras can take less vertical space * Fix hanging characters * Only show the current detector * Fix bad if statement * Return full output of ffprobe process * Return full output of ffprobe process * Don't specify rtsp_transport * Make ffprobe button show dialog with output and option to copy * Adjust ffprobe api to take paths directly * Add docs for ffprobe api --- docker/Dockerfile | 1 + docs/docs/integrations/api.md | 8 ++ frigate/http.py | 40 +++++++- frigate/stats.py | 7 ++ frigate/util.py | 48 +++++++++ web/__test__/handlers.js | 4 +- web/src/routes/Debug.jsx | 179 +++++++++++++++++++++++++--------- 7 files changed, 235 insertions(+), 52 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0b7998f2b..5e3c592c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -63,6 +63,7 @@ RUN apt-get -qq update \ apt-transport-https \ gnupg \ wget \ + procps \ unzip tzdata libxml2 xz-utils \ python3-pip \ # add raspberry pi repo diff --git a/docs/docs/integrations/api.md b/docs/docs/integrations/api.md index c7a963ca6..d79b4b3cc 100644 --- a/docs/docs/integrations/api.md +++ b/docs/docs/integrations/api.md @@ -264,3 +264,11 @@ Get recording segment details for the given timestamp range. | -------- | ---- | ------------------------------------- | | `after` | int | Unix timestamp for beginning of range | | `before` | int | Unix timestamp for end of range | + +### `GET /api/ffprobe` + +Get ffprobe output for camera feed paths. + +| param | Type | Description | +| ------- | ------ | ---------------------------------- | +| `paths` | string | `,` separated list of camera paths | diff --git a/frigate/http.py b/frigate/http.py index 5d8b8db2f..5b67575e1 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -1,8 +1,8 @@ import base64 -from collections import OrderedDict from datetime import datetime, timedelta import copy import logging +import json import os import subprocess as sp import time @@ -28,9 +28,9 @@ from playhouse.shortcuts import model_to_dict from frigate.const import CLIPS_DIR from frigate.models import Event, Recordings -from frigate.object_processing import TrackedObject, TrackedObjectProcessor +from frigate.object_processing import TrackedObject from frigate.stats import stats_snapshot -from frigate.util import clean_camera_user_pass +from frigate.util import clean_camera_user_pass, ffprobe_stream from frigate.version import VERSION logger = logging.getLogger(__name__) @@ -957,3 +957,37 @@ def imagestream(detected_frames_processor, camera_name, fps, height, draw_option b"--frame\r\n" b"Content-Type: image/jpeg\r\n\r\n" + jpg.tobytes() + b"\r\n\r\n" ) + + +@bp.route("/ffprobe", methods=["GET"]) +def ffprobe(): + path_param = request.args.get("paths", "") + + if not path_param: + return jsonify( + {"success": False, "message": f"Path needs to be provided."}, "404" + ) + + if "," in clean_camera_user_pass(path_param): + paths = path_param.split(",") + else: + paths = [path_param] + + # user has multiple streams + output = [] + + for path in paths: + ffprobe = ffprobe_stream(path) + output.append( + { + "return_code": ffprobe.returncode, + "stderr": json.loads(ffprobe.stderr.decode("unicode_escape").strip()) + if ffprobe.stderr.decode() + else {}, + "stdout": json.loads(ffprobe.stdout.decode("unicode_escape").strip()) + if ffprobe.stdout.decode() + else {}, + } + ) + + return jsonify(output) diff --git a/frigate/stats.py b/frigate/stats.py index 7605ea1bc..d4899bb6d 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -14,6 +14,7 @@ from frigate.config import FrigateConfig from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR from frigate.types import StatsTrackingTypes, CameraMetricsTypes from frigate.version import VERSION +from frigate.util import get_cpu_stats from frigate.object_detection import ObjectDetectProcess logger = logging.getLogger(__name__) @@ -90,6 +91,9 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]: for name, camera_stats in camera_metrics.items(): total_detection_fps += camera_stats["detection_fps"].value pid = camera_stats["process"].pid if camera_stats["process"] else None + ffmpeg_pid = ( + camera_stats["ffmpeg_pid"].value if camera_stats["ffmpeg_pid"] else None + ) cpid = ( camera_stats["capture_process"].pid if camera_stats["capture_process"] @@ -102,6 +106,7 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]: "detection_fps": round(camera_stats["detection_fps"].value, 2), "pid": pid, "capture_pid": cpid, + "ffmpeg_pid": ffmpeg_pid, } stats["detectors"] = {} @@ -114,6 +119,8 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]: } stats["detection_fps"] = round(total_detection_fps, 2) + stats["cpu_usages"] = get_cpu_stats() + stats["service"] = { "uptime": (int(time.time()) - stats_tracking["started"]), "version": VERSION, diff --git a/frigate/util.py b/frigate/util.py index 8b3703891..05e5fec29 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -1,6 +1,8 @@ import copy import datetime import logging +import subprocess as sp +import json import re import signal import traceback @@ -679,6 +681,52 @@ def escape_special_characters(path: str) -> str: return path +def get_cpu_stats() -> dict[str, dict]: + """Get cpu usages for each process id""" + usages = {} + # -n=2 runs to ensure extraneous values are not included + top_command = ["top", "-b", "-n", "2"] + + p = sp.run( + top_command, + encoding="ascii", + capture_output=True, + ) + + if p.returncode != 0: + logger.error(p.stderr) + return usages + else: + lines = p.stdout.split("\n") + + for line in lines: + stats = list(filter(lambda a: a != "", line.strip().split(" "))) + try: + usages[stats[0]] = { + "cpu": stats[8], + "mem": stats[9], + } + except: + continue + + return usages + + +def ffprobe_stream(path: str) -> sp.CompletedProcess: + """Run ffprobe on stream.""" + ffprobe_cmd = [ + "ffprobe", + "-print_format", + "json", + "-show_entries", + "stream=codec_long_name,width,height,bit_rate,duration,display_aspect_ratio,avg_frame_rate", + "-loglevel", + "quiet", + path, + ] + return sp.run(ffprobe_cmd, capture_output=True) + + class FrameManager(ABC): @abstractmethod def create(self, name, size) -> AnyStr: diff --git a/web/__test__/handlers.js b/web/__test__/handlers.js index c24fcd8d2..23433d840 100644 --- a/web/__test__/handlers.js +++ b/web/__test__/handlers.js @@ -39,9 +39,10 @@ export const handlers = [ return res( ctx.status(200), ctx.json({ + cpu_usages: { 74: {cpu: 6, mem: 6}, 64: { cpu: 5, mem: 5 }, 54: { cpu: 4, mem: 4 }, 71: { cpu: 3, mem: 3}, 60: {cpu: 2, mem: 2}, 72: {cpu: 1, mem: 1} }, detection_fps: 0.0, detectors: { coral: { detection_start: 0.0, inference_speed: 8.94, pid: 52 } }, - front: { camera_fps: 5.0, capture_pid: 64, detection_fps: 0.0, pid: 54, process_fps: 0.0, skipped_fps: 0.0 }, + front: { camera_fps: 5.0, capture_pid: 64, detection_fps: 0.0, pid: 54, process_fps: 0.0, skipped_fps: 0.0, ffmpeg_pid: 72 }, side: { camera_fps: 6.9, capture_pid: 71, @@ -49,6 +50,7 @@ export const handlers = [ pid: 60, process_fps: 0.0, skipped_fps: 0.0, + ffmpeg_pid: 74, }, service: { uptime: 34812, version: '0.8.1-d376f6b' }, }) diff --git a/web/src/routes/Debug.jsx b/web/src/routes/Debug.jsx index b404c61a9..3c8c8a8ff 100644 --- a/web/src/routes/Debug.jsx +++ b/web/src/routes/Debug.jsx @@ -5,12 +5,15 @@ import Heading from '../components/Heading'; import Link from '../components/Link'; import { useMqtt } from '../api/mqtt'; import useSWR from 'swr'; +import axios from 'axios'; import { Table, Tbody, Thead, Tr, Th, Td } from '../components/Table'; -import { useCallback } from 'preact/hooks'; +import { useCallback, useState } from 'preact/hooks'; +import Dialog from '../components/Dialog'; const emptyObject = Object.freeze({}); export default function Debug() { + const [state, setState] = useState({ showFfprobe: false, ffprobe: '' }); const { data: config } = useSWR('config'); const { @@ -18,12 +21,10 @@ export default function Debug() { } = useMqtt('stats'); const { data: initialStats } = useSWR('stats'); - const { detectors, service = {}, detection_fps: _, ...cameras } = stats || initialStats || emptyObject; + const { cpu_usages, detectors, service = {}, detection_fps: _, ...cameras } = stats || initialStats || emptyObject; const detectorNames = Object.keys(detectors || emptyObject); - const detectorDataKeys = Object.keys(detectors ? detectors[detectorNames[0]] : emptyObject); const cameraNames = Object.keys(cameras || emptyObject); - const cameraDataKeys = Object.keys(cameras[cameraNames[0]] || emptyObject); const handleCopyConfig = useCallback(() => { async function copy() { @@ -32,11 +33,64 @@ export default function Debug() { copy(); }, [config]); + const onHandleFfprobe = async (camera, e) => { + if (e) { + e.stopPropagation(); + } + + setState({ ...state, showFfprobe: true }); + let paths = ''; + config.cameras[camera].ffmpeg.inputs.forEach((input) => { + if (paths) { + paths += ','; + paths += input.path; + } else { + paths = input.path; + } + }); + const response = await axios.get('ffprobe', { + params: { + paths, + }, + }); + + if (response.status === 200) { + setState({ showFfprobe: true, ffprobe: JSON.stringify(response.data, null, 2) }); + } else { + setState({ ...state, ffprobe: 'There was an error getting the ffprobe output.' }); + } + }; + + const onCopyFfprobe = async () => { + await window.navigator.clipboard.writeText(JSON.stringify(state.ffprobe, null, 2)); + setState({ ...state, ffprobe: '', showFfprobe: false }); + }; + return (
Debug {service.version} + {state.showFfprobe && ( + +
+ Ffprobe Output + {state.ffprobe != '' ?

{state.ffprobe}

: } +
+
+ + +
+ + )} {!detectors ? (
@@ -44,52 +98,81 @@ export default function Debug() {
) : ( -
- - - - - {detectorDataKeys.map((name) => ( - - ))} - - - - {detectorNames.map((detector, i) => ( - - - {detectorDataKeys.map((name) => ( - - ))} - - ))} - -
detector{name.replace('_', ' ')}
{detector}{detectors[detector][name]}
+ Detectors +
+ {detectorNames.map((detector) => ( +
+
{detector}
+
+ + + + + + + + + + + + + + + +
P-IDDetection StartInference Speed
{detectors[detector]['pid']}{detectors[detector]['detection_start']}{detectors[detector]['inference_speed']}
+
+
+ ))}
-
- - - - - {cameraDataKeys.map((name) => ( - - ))} - - - - {cameraNames.map((camera, i) => ( - - - {cameraDataKeys.map((name) => ( - - ))} - - ))} - -
camera{name.replace('_', ' ')}
- {camera.replaceAll('_', ' ')} - {cameras[camera][name]}
+ Cameras +
+ {cameraNames.map((camera) => ( +
+
+ {camera.replaceAll('_', ' ')} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ProcessP-IDfpsCpu %Memory %
Capture{cameras[camera]['capture_pid']}{cameras[camera]['process_fps']}{cpu_usages[cameras[camera]['capture_pid']]['cpu']}%{cpu_usages[cameras[camera]['capture_pid']]['mem']}%
Detect{cameras[camera]['pid']} + {cameras[camera]['detection_fps']} ({cameras[camera]['skipped_fps']} skipped) + {cpu_usages[cameras[camera]['pid']]['cpu']}%{cpu_usages[cameras[camera]['pid']]['cpu']}%
ffmpeg{cameras[camera]['ffmpeg_pid']}{cameras[camera]['camera_fps']}{cpu_usages[cameras[camera]['ffmpeg_pid']]['cpu']}%{cpu_usages[cameras[camera]['ffmpeg_pid']]['cpu']}%
+
+
+ ))}

Debug stats update automatically every {config.mqtt.stats_interval} seconds.