prevent the camera process from hanging

Frigate provides an HTTP server that can be used to detect if frigate is running or not. Using the docker-compose "healthcheck" feature we can set automations to restart the service if it stops working.

Dockerfile

@@ -1,63 +0,0 @@
-FROM ubuntu:18.04
-LABEL maintainer "blakeb@blakeshome.com"
-
-ENV DEBIAN_FRONTEND=noninteractive
-# Install packages for apt repo
-RUN apt -qq update && apt -qq install --no-install-recommends -y \
-    software-properties-common \
-    # apt-transport-https ca-certificates \
-    build-essential \
-    gnupg wget unzip tzdata \
-    # libcap-dev \
-    && add-apt-repository ppa:deadsnakes/ppa -y \
-    && apt -qq install --no-install-recommends -y \
-        python3.7 \
-        python3.7-dev \
-        python3-pip \
-        ffmpeg \
-        # VAAPI drivers for Intel hardware accel
-        libva-drm2 libva2 i965-va-driver vainfo \
-    && python3.7 -m pip install -U pip \
-    && python3.7 -m pip install -U wheel setuptools \
-    && python3.7 -m pip install -U \
-        opencv-python-headless \
-        # python-prctl \
-        numpy \
-        imutils \
-        scipy \
-        psutil \
-    && python3.7 -m pip install -U \
-        Flask \
-        paho-mqtt \
-        PyYAML \
-        matplotlib \
-        pyarrow \
-        click \
-    && echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
-    && wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
-    && apt -qq update \
-    && echo "libedgetpu1-max libedgetpu/accepted-eula boolean true" | debconf-set-selections \
-    && apt -qq install --no-install-recommends -y \
-        libedgetpu1-max \
-    ## Tensorflow lite (python 3.7 only)
-    && wget -q https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp37-cp37m-linux_x86_64.whl \
-    && python3.7 -m pip install tflite_runtime-2.1.0.post1-cp37-cp37m-linux_x86_64.whl \
-    && rm tflite_runtime-2.1.0.post1-cp37-cp37m-linux_x86_64.whl \
-    && rm -rf /var/lib/apt/lists/* \
-    && (apt-get autoremove -y; apt-get autoclean -y)
-
-# get model and labels
-RUN wget -q https://github.com/google-coral/edgetpu/raw/master/test_data/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite -O /edgetpu_model.tflite --trust-server-names
-COPY labelmap.txt /labelmap.txt
-RUN wget -q https://github.com/google-coral/edgetpu/raw/master/test_data/ssd_mobilenet_v2_coco_quant_postprocess.tflite -O /cpu_model.tflite 
-
-
-RUN mkdir /cache /clips
-
-WORKDIR /opt/frigate/
-ADD frigate frigate/
-COPY detect_objects.py .
-COPY benchmark.py .
-COPY process_clip.py .
-
-CMD ["python3.7", "-u", "detect_objects.py"]

Makefile

@@ -0,0 +1,37 @@
+default_target: amd64_frigate
+
+amd64_wheels:
+	docker build --tag blakeblackshear/frigate-wheels:amd64 --file docker/Dockerfile.wheels .
+
+amd64_ffmpeg:
+	docker build --tag blakeblackshear/frigate-ffmpeg:amd64 --file docker/Dockerfile.ffmpeg.amd64 .
+
+amd64_frigate:
+	docker build --tag frigate-base --build-arg ARCH=amd64 --file docker/Dockerfile.base .
+	docker build --tag frigate --file docker/Dockerfile.amd64 .
+
+amd64_all: amd64_wheels amd64_ffmpeg amd64_frigate
+
+arm64_wheels:
+	docker build --tag blakeblackshear/frigate-wheels:arm64 --file docker/Dockerfile.wheels.arm64 .
+
+arm64_ffmpeg:
+	docker build --tag blakeblackshear/frigate-ffmpeg:arm64 --file docker/Dockerfile.ffmpeg.arm64 .
+
+arm64_frigate:
+	docker build --tag frigate-base --build-arg ARCH=arm64 --file docker/Dockerfile.base .
+	docker build --tag frigate --file docker/Dockerfile.arm64 .
+
+armv7hf_all: arm64_wheels arm64_ffmpeg arm64_frigate
+
+armv7hf_wheels:
+	docker build --tag blakeblackshear/frigate-wheels:armv7hf --file docker/Dockerfile.wheels .
+
+armv7hf_ffmpeg:
+	docker build --tag blakeblackshear/frigate-ffmpeg:armv7hf --file docker/Dockerfile.ffmpeg.armv7hf .
+
+armv7hf_frigate:
+	docker build --tag frigate-base --build-arg ARCH=armv7hf --file docker/Dockerfile.base .
+	docker build --tag frigate --file docker/Dockerfile.armv7hf .
+
+armv7hf_all: armv7hf_wheels armv7hf_ffmpeg armv7hf_frigate

README.md

@@ -15,13 +15,17 @@ Use of a [Google Coral Accelerator](https://coral.ai/products/) is optional, but
 You see multiple bounding boxes because it draws bounding boxes from all frames in the past 1 second where a person was detected. Not all of the bounding boxes were from the current frame.
 [![](http://img.youtube.com/vi/nqHbCtyo4dY/0.jpg)](http://www.youtube.com/watch?v=nqHbCtyo4dY "Frigate")
 
+## Documentation
+- [Camera Specific Docs](docs/CAMERAS.md)
+- [Hardware Acceleration](docs/HWACCEL.md)
+
 ## Getting Started
 Run the container with
 ```bash
 docker run --rm \
--name blakeblackshear/frigate:stable \
+--name frigate \
 --privileged \
---shm-size=512m \ # should work for a 2-3 cameras
+--shm-size=100m \ # only needed with large numbers of high res cameras
 -v /dev/bus/usb:/dev/bus/usb \
 -v <path_to_config_dir>:/config:ro \
 -v /etc/localtime:/etc/localtime:ro \
@@ -36,7 +40,7 @@ Example docker-compose:
     container_name: frigate
     restart: unless-stopped
     privileged: true
-    shm_size: '1g' # should work for 5-7 cameras
+    shm_size: '100m' # only needed with large numbers of high res cameras
     image: blakeblackshear/frigate:stable
     volumes:
       - /dev/bus/usb:/dev/bus/usb
@@ -47,9 +51,23 @@ Example docker-compose:
       - "5000:5000"
     environment:
       FRIGATE_RTSP_PASSWORD: "password"
+    healthcheck:
+      test: ["CMD", "wget" , "-q", "-O-", "http://localhost:5000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 3m
 ```
 
-A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and device specific info can be found [here](docs/DEVICES.md).
+A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and camera specific info can be found [here](docs/CAMERAS.md).
+
+### Calculating shm-size
+The default shm-size of 64m should be fine for most setups. If you start seeing segfault errors, it could be because you have too many high resolution cameras and you need to specify a higher shm size.
+
+You can calculate the necessary shm-size for each camera with the following formula:
+```
+(width * height * 3 + 270480)/1048576 = <shm size in mb>
+```
 
 ## Recommended Hardware
 |Name|Inference Speed|Notes|
@@ -58,8 +76,9 @@ A `config.yml` file must exist in the `config` directory. See example [here](con
 |Intel NUC NUC7i3BNK|8-10ms|Best possible performance. Can handle 7+ cameras at 5fps depending on typical amounts of motion.|
 |BMAX B2 Plus|10-12ms|Good balance of performance and cost. Also capable of running many other services at the same time as frigate.|
 |Minisforum GK41|9-10ms|Great alternative to a NUC. Easily handiles 4 1080p cameras.|
-
-ARM boards are not officially supported at the moment due to some python dependencies that require modification to work on ARM devices. The Raspberry Pi4 gets about 16ms inference speeds, but the hardware acceleration for ffmpeg does not work for converting yuv420 to rgb24. The Atomic Pi is x86 and much more efficient.
+|Raspberry Pi 3B (32bit)|60ms|Can handle a small number of cameras, but the detection speeds are slow|
+|Raspberry Pi 4 (32bit)|15-20ms|Can handle a small number of cameras. The 2GB version runs fine.|
+|Raspberry Pi 4 (64bit)|10-15ms|Can handle a small number of cameras. The 2GB version runs fine.|
 
 Users have reported varying success in getting frigate to run in a VM. In some cases, the virtualization layer introduces a significant delay in communication with the Coral. If running virtualized in Proxmox, pass the USB card/interface to the virtual machine not the USB ID for faster inference speed.
 
@@ -90,7 +109,8 @@ sensor:
     scan_interval: 5
     json_attributes:
       - <camera_name>
-      - coral
+      - detection_fps
+      - detectors
     value_template: 'OK'  
   - platform: template
     sensors:
@@ -103,11 +123,11 @@ sensor:
       <camera_name>_detection_fps: 
         value_template: '{{ states.sensor.frigate_debug.attributes["<camera_name>"]["detection_fps"] }}'
         unit_of_measurement: 'FPS'
-      frigate_coral_fps: 
-        value_template: '{{ states.sensor.frigate_debug.attributes["coral"]["fps"] }}'
+      frigate_detection_fps: 
+        value_template: '{{ states.sensor.frigate_debug.attributes["detection_fps"] }}'
         unit_of_measurement: 'FPS'
       frigate_coral_inference:
-        value_template: '{{ states.sensor.frigate_debug.attributes["coral"]["inference_speed"] }}' 
+        value_template: '{{ states.sensor.frigate_debug.attributes["detectors"]["coral"]["inference_speed"] }}' 
         unit_of_measurement: 'ms'
         
 automation:
@@ -213,7 +233,7 @@ Same as `frigate/<camera_name>/events/start`, but with an `end_time` property as
 ### frigate/<zone_name>/<object_name>
 Publishes `ON` or `OFF` and is designed to be used a as a binary sensor in HomeAssistant for whether or not that object type is detected in the zone.
 
-## Understanding min_score and threshold
+## Understanding min_score and threshold filters
 `min_score` defines the minimum score for Frigate to begin tracking a detected object. Any single detection below `min_score` will be ignored as a false positive. `threshold` is based on the median of the history of scores for a tracked object. Consider the following frames when `min_score` is set to 0.6 and threshold is set to 0.85:
 
 | Frame | Current Score | Score History | Computed Score | Detected Object |
@@ -255,8 +275,8 @@ Frigate can save video clips without any CPU overhead for encoding by simply cop
 - `pre_capture`: Defines how much time should be included in the clip prior to the beginning of the event. Defaults to 30 seconds.
 - `objects`: List of object types to save clips for. Object types here must be listed for tracking at the camera or global configuration. Defaults to all tracked objects.
 
-## Google Coral Configuration
-Frigate attempts to detect your Coral device automatically. If you have multiple Coral devices or a version that is not detected automatically, you can specify using the `tensorflow_device` config option.
+## Detectors Configuration
+Frigate attempts to detect your Coral device automatically. If you have multiple Coral devices or a version that is not detected automatically, you can specify using the `detectors` config option as shown in the example config.
 
 ## Masks and limiting detection to a certain area
 The mask works by looking at the bottom center of any bounding box (first image, red dot below) and comparing that to your mask. If that red dot falls on an area of your mask that is black, the detection (and motion) will be ignored. The mask in the second image would limit detection on this camera to only objects that are in the front yard and not the street. 
@@ -336,7 +356,13 @@ During testing, `draw_zones` can be set in the config to tell frigate to draw th
         ***************/
         "skipped_fps": 0.0
     },
-    /* Coral Stats */
+    /***************
+    * Sum of detection_fps across all cameras and detectors. 
+    * This should be the sum of all detection_fps values from cameras.
+    ***************/
+    "detection_fps": 5.0,
+    /* Detectors Stats */
+    "detectors": {
       "coral": {
           /***************
           * Timestamp when object detection started. If this value stays non-zero and constant
@@ -344,10 +370,6 @@ During testing, `draw_zones` can be set in the config to tell frigate to draw th
           ***************/
           "detection_start": 0.0,
           /***************
-        * Frames per second of the Coral. This should be the sum of all detection_fps values from cameras.
-        ***************/
-        "fps": 6.9,
-        /***************
           * Time spent running object detection in milliseconds.
           ***************/
           "inference_speed": 10.48,
@@ -355,8 +377,8 @@ During testing, `draw_zones` can be set in the config to tell frigate to draw th
           * PID for the shared process that runs object detection on the Coral.
           ***************/
           "pid": 25321
-    },
-    "plasma_store_rc": null // Return code for the plasma store. This should be null normally.
+      }
+    }
 }
 ```
 

benchmark.py

@@ -11,7 +11,7 @@ labels = load_labels('/labelmap.txt')
 ######
 # Minimal same process runner
 ######
-# object_detector = ObjectDetector()
+# object_detector = LocalObjectDetector()
 # tensor_input = np.expand_dims(np.full((300,300,3), 0, np.uint8), axis=0)
 
 # start = datetime.datetime.now().timestamp()
@@ -37,11 +37,9 @@ labels = load_labels('/labelmap.txt')
 # print(f"Processed for {duration:.2f} seconds.")
 # print(f"Average frame processing time: {mean(frame_times)*1000:.2f}ms")
 
-######
-# Separate process runner
-######
-def start(id, num_detections, detection_queue):
-  object_detector = RemoteObjectDetector(str(id), '/labelmap.txt', detection_queue)
+
+def start(id, num_detections, detection_queue, event):
+  object_detector = RemoteObjectDetector(str(id), '/labelmap.txt', detection_queue, event)
   start = datetime.datetime.now().timestamp()
 
   frame_times = []
@@ -51,23 +49,39 @@ def start(id, num_detections, detection_queue):
     frame_times.append(datetime.datetime.now().timestamp()-start_frame)
 
   duration = datetime.datetime.now().timestamp()-start
+  object_detector.cleanup()
   print(f"{id} - Processed for {duration:.2f} seconds.")
+  print(f"{id} - FPS: {object_detector.fps.eps():.2f}")
   print(f"{id} - Average frame processing time: {mean(frame_times)*1000:.2f}ms")
 
-edgetpu_process = EdgeTPUProcess()
+######
+# Separate process runner
+######
+# event = mp.Event()
+# detection_queue = mp.Queue()
+# edgetpu_process = EdgeTPUProcess(detection_queue, {'1': event}, 'usb:0')
 
-# start(1, 1000, edgetpu_process.detect_lock, edgetpu_process.detect_ready, edgetpu_process.frame_ready)
+# start(1, 1000, edgetpu_process.detection_queue, event)
+# print(f"Average raw inference speed: {edgetpu_process.avg_inference_speed.value*1000:.2f}ms")
 
 ####
 # Multiple camera processes
 ####
 camera_processes = []
+
+events = {}
 for x in range(0, 10):
-  camera_process = mp.Process(target=start, args=(x, 100, edgetpu_process.detection_queue))
+  events[str(x)] = mp.Event()
+detection_queue = mp.Queue()
+edgetpu_process_1 = EdgeTPUProcess(detection_queue, events, 'usb:0')
+edgetpu_process_2 = EdgeTPUProcess(detection_queue, events, 'usb:1')
+
+for x in range(0, 10):
+  camera_process = mp.Process(target=start, args=(x, 300, detection_queue, events[str(x)]))
   camera_process.daemon = True
   camera_processes.append(camera_process)
 
-start = datetime.datetime.now().timestamp()
+start_time = datetime.datetime.now().timestamp()
 
 for p in camera_processes:
   p.start()
@@ -75,5 +89,5 @@ for p in camera_processes:
 for p in camera_processes:
   p.join()
 
-duration = datetime.datetime.now().timestamp()-start
+duration = datetime.datetime.now().timestamp()-start_time
 print(f"Total - Processed for {duration:.2f} seconds.")

config/config.example.yml

@@ -1,10 +1,14 @@
 web_port: 5000
 
 ################
-## Tell frigate to look for a specific EdgeTPU device. Useful if you want to run multiple instances of frigate
-## on the same machine with multiple EdgeTPUs. https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api
+## List of detectors.
+## Currently supported types: cpu, edgetpu
+## EdgeTPU requires device as defined here: https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api
 ################
-tensorflow_device: usb
+detectors:
+  coral:
+    type: edgetpu
+    device: usb
 
 mqtt:
   host: mqtt.server.com
@@ -63,7 +67,7 @@ save_clips:
 #     - -f
 #     - rawvideo
 #     - -pix_fmt
-#     - rgb24
+#     - yuv420p
 
 ####################
 # Global object configuration. Applies to all cameras

detect_objects.py

@@ -1,3 +1,4 @@
+import faulthandler; faulthandler.enable()
 import os
 import signal
 import sys
@@ -54,32 +55,22 @@ FFMPEG_DEFAULT_CONFIG = {
          '-use_wallclock_as_timestamps', '1']),
     'output_args': FFMPEG_CONFIG.get('output_args',
         ['-f', 'rawvideo',
-         '-pix_fmt', 'rgb24'])
+         '-pix_fmt', 'yuv420p'])
 }
 
 GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
 
 WEB_PORT = CONFIG.get('web_port', 5000)
-DEBUG = (CONFIG.get('debug', '0') == '1')
-TENSORFLOW_DEVICE = CONFIG.get('tensorflow_device')
-
-def start_plasma_store():
-    plasma_cmd = ['plasma_store', '-m', '400000000', '-s', '/tmp/plasma']
-    plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
-    time.sleep(1)
-    rc = plasma_process.poll()
-    if rc is not None:
-        return None
-    return plasma_process
+DETECTORS = CONFIG.get('detectors', {'coral': {'type': 'edgetpu', 'device': 'usb'}})
 
 class CameraWatchdog(threading.Thread):
-    def __init__(self, camera_processes, config, tflite_process, tracked_objects_queue, plasma_process, stop_event):
+    def __init__(self, camera_processes, config, detectors, detection_queue, tracked_objects_queue, stop_event):
         threading.Thread.__init__(self)
         self.camera_processes = camera_processes
         self.config = config
-        self.tflite_process = tflite_process
+        self.detectors = detectors
+        self.detection_queue = detection_queue
         self.tracked_objects_queue = tracked_objects_queue
-        self.plasma_process = plasma_process
         self.stop_event = stop_event
 
     def run(self):
@@ -94,21 +85,16 @@ class CameraWatchdog(threading.Thread):
 
             now = datetime.datetime.now().timestamp()
 
-            # check the plasma process
-            rc = self.plasma_process.poll()
-            if rc != None:
-                print(f"plasma_process exited unexpectedly with {rc}")
-                self.plasma_process = start_plasma_store()
-
-            # check the detection process
-            detection_start = self.tflite_process.detection_start.value
+            # check the detection processes
+            for detector in self.detectors.values():
+                detection_start = detector.detection_start.value
                 if (detection_start > 0.0 and 
                     now - detection_start > 10):
                     print("Detection appears to be stuck. Restarting detection process")
-                self.tflite_process.start_or_restart()
-            elif not self.tflite_process.detect_process.is_alive():
+                    detector.start_or_restart()
+                elif not detector.detect_process.is_alive():
                     print("Detection appears to have stopped. Restarting detection process")
-                self.tflite_process.start_or_restart()
+                    detector.start_or_restart()
 
             # check the camera processes
             for name, camera_process in self.camera_processes.items():
@@ -119,9 +105,9 @@ class CameraWatchdog(threading.Thread):
                     camera_process['detection_fps'].value = 0.0
                     camera_process['read_start'].value = 0.0
                     process = mp.Process(target=track_camera, args=(name, self.config[name], camera_process['frame_queue'],
-                        camera_process['frame_shape'], self.tflite_process.detection_queue, self.tracked_objects_queue, 
+                        camera_process['frame_shape'], self.detection_queue, self.tracked_objects_queue, 
                         camera_process['process_fps'], camera_process['detection_fps'],
-                        camera_process['read_start'], camera_process['detection_frame'], self.stop_event))
+                        camera_process['read_start'], self.stop_event))
                     process.daemon = True
                     camera_process['process'] = process
                     process.start()
@@ -132,7 +118,7 @@ class CameraWatchdog(threading.Thread):
                     frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
                     ffmpeg_process = start_or_restart_ffmpeg(camera_process['ffmpeg_cmd'], frame_size)
                     camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, camera_process['frame_queue'], 
-                        camera_process['take_frame'], camera_process['camera_fps'], camera_process['detection_frame'], self.stop_event)
+                        camera_process['take_frame'], camera_process['camera_fps'], self.stop_event)
                     camera_capture.start()
                     camera_process['ffmpeg_process'] = ffmpeg_process
                     camera_process['capture_thread'] = camera_capture
@@ -172,8 +158,6 @@ def main():
     client.connect(MQTT_HOST, MQTT_PORT, 60)
     client.loop_start()
 
-    plasma_process = start_plasma_store()
-
     ##
     # Setup config defaults for cameras
     ##
@@ -190,10 +174,26 @@ def main():
     # Queue for clip processing
     event_queue = mp.Queue()
 
-    # Start the shared tflite process
-    tflite_process = EdgeTPUProcess(TENSORFLOW_DEVICE)
+    # create the detection pipes and shms
+    out_events = {}
+    camera_shms = []
+    for name in CONFIG['cameras'].keys():
+        out_events[name] = mp.Event()
+        shm_in = mp.shared_memory.SharedMemory(name=name, create=True, size=300*300*3)
+        shm_out = mp.shared_memory.SharedMemory(name=f"out-{name}", create=True, size=20*6*4)
+        camera_shms.append(shm_in)
+        camera_shms.append(shm_out)
 
-    # start the camera processes
+    detection_queue = mp.Queue()
+
+    detectors = {}
+    for name, detector in DETECTORS.items():
+        if detector['type'] == 'cpu':
+            detectors[name] = EdgeTPUProcess(detection_queue, out_events=out_events, tf_device='cpu')
+        if detector['type'] == 'edgetpu':
+            detectors[name] = EdgeTPUProcess(detection_queue, out_events=out_events, tf_device=detector['device'])
+
+    # create the camera processes
     camera_processes = {}
     for name, config in CONFIG['cameras'].items():
         # Merge the ffmpeg config with the global config
@@ -237,16 +237,18 @@ def main():
         else:
             frame_shape = get_frame_shape(ffmpeg_input)
         
+        config['frame_shape'] = frame_shape
+
         frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
         take_frame = config.get('take_frame', 1)
 
         detection_frame = mp.Value('d', 0.0)
 
         ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_size)
-        frame_queue = mp.Queue()
+        frame_queue = mp.Queue(maxsize=2)
         camera_fps = EventsPerSecond()
         camera_fps.start()
-        camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps, detection_frame, stop_event)
+        camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps, stop_event)
         camera_capture.start()
 
         camera_processes[name] = {
@@ -275,12 +277,13 @@ def main():
         }
 
         camera_process = mp.Process(target=track_camera, args=(name, config, frame_queue, frame_shape,
-            tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['process_fps'], 
+            detection_queue, out_events[name], tracked_objects_queue, camera_processes[name]['process_fps'], 
             camera_processes[name]['detection_fps'], 
             camera_processes[name]['read_start'], camera_processes[name]['detection_frame'], stop_event))
         camera_process.daemon = True
         camera_processes[name]['process'] = camera_process
 
+    # start the camera_processes
     for name, camera_process in camera_processes.items():
         camera_process['process'].start()
         print(f"Camera_process started for {name}: {camera_process['process'].pid}")
@@ -291,7 +294,7 @@ def main():
     object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue, event_queue, stop_event)
     object_processor.start()
     
-    camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue, plasma_process, stop_event)
+    camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], detectors, detection_queue, tracked_objects_queue, stop_event)
     camera_watchdog.start()
 
     def receiveSignal(signalNumber, frame):
@@ -300,11 +303,20 @@ def main():
         event_processor.join()
         object_processor.join()
         camera_watchdog.join()
-        for name, camera_process in camera_processes.items():
+        for camera_name, camera_process in camera_processes.items():
             camera_process['capture_thread'].join()
-        rc = camera_watchdog.plasma_process.poll()
-        if rc == None:
-            camera_watchdog.plasma_process.terminate()
+            # cleanup the frame queue
+            while not camera_process['frame_queue'].empty():
+                frame_time = camera_process['frame_queue'].get()
+                shm = mp.shared_memory.SharedMemory(name=f"{camera_name}{frame_time}")
+                shm.close()
+                shm.unlink()
+
+        for detector in detectors:
+            detector.stop()
+        for shm in camera_shms:
+            shm.close()
+            shm.unlink()
         sys.exit()
     
     signal.signal(signal.SIGTERM, receiveSignal)
@@ -361,15 +373,14 @@ def main():
                 }
             }
         
-        stats['coral'] = {
-            'fps': round(total_detection_fps, 2),
-            'inference_speed': round(tflite_process.avg_inference_speed.value*1000, 2),
-            'detection_start': tflite_process.detection_start.value,
-            'pid': tflite_process.detect_process.pid
+        stats['detectors'] = {}
+        for name, detector in detectors.items():
+            stats['detectors'][name] = {
+                'inference_speed': round(detector.avg_inference_speed.value*1000, 2),
+                'detection_start': detector.detection_start.value,
+                'pid': detector.detect_process.pid
             }
-
-        rc = camera_watchdog.plasma_process.poll()
-        stats['plasma_store_rc'] = rc
+        stats['detection_fps'] = round(total_detection_fps, 2)
 
         return jsonify(stats)
 
@@ -379,6 +390,8 @@ def main():
             best_object = object_processor.get_best(camera_name, label)
             best_frame = best_object.get('frame', np.zeros((720,1280,3), np.uint8))
 
+            best_frame = cv2.cvtColor(best_frame, cv2.COLOR_YUV2BGR_I420)
+            
             crop = bool(request.args.get('crop', 0, type=int))
             if crop:
                 region = best_object.get('region', [0,0,300,300])
@@ -388,7 +401,6 @@ def main():
             width = int(height*best_frame.shape[1]/best_frame.shape[0])
 
             best_frame = cv2.resize(best_frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
-            best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
             ret, jpg = cv2.imencode('.jpg', best_frame)
             response = make_response(jpg.tobytes())
             response.headers['Content-Type'] = 'image/jpg'
@@ -419,7 +431,6 @@ def main():
             width = int(height*frame.shape[1]/frame.shape[0])
 
             frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
-            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
             ret, jpg = cv2.imencode('.jpg', frame)
             response = make_response(jpg.tobytes())
@@ -432,14 +443,12 @@ def main():
         while True:
             # max out at specified FPS
             time.sleep(1/fps)
-            frame = object_processor.get_current_frame(camera_name)
+            frame = object_processor.get_current_frame(camera_name, draw=True)
             if frame is None:
                 frame = np.zeros((height,int(height*16/9),3), np.uint8)
 
             width = int(height*frame.shape[1]/frame.shape[0])
-
             frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_LINEAR)
-            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
             ret, jpg = cv2.imencode('.jpg', frame)
             yield (b'--frame\r\n'
@@ -449,7 +458,5 @@ def main():
 
     object_processor.join()
 
-    plasma_process.terminate()
-
 if __name__ == '__main__':
     main()

docker/Dockerfile.amd64

@@ -0,0 +1,16 @@
+FROM frigate-base
+LABEL maintainer "blakeb@blakeshome.com"
+
+# Install packages for apt repo
+RUN apt-get -qq update \
+    && apt-get -qq install --no-install-recommends -y \
+        # ffmpeg dependencies
+        libgomp1 \
+        # VAAPI drivers for Intel hardware accel
+        libva-drm2 libva2 i965-va-driver vainfo \
+        ## Tensorflow lite
+    && wget -q https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp38-cp38-linux_x86_64.whl \
+    && python3.8 -m pip install tflite_runtime-2.1.0.post1-cp38-cp38-linux_x86_64.whl \
+    && rm tflite_runtime-2.1.0.post1-cp38-cp38-linux_x86_64.whl \
+    && rm -rf /var/lib/apt/lists/* \
+    && (apt-get autoremove -y; apt-get autoclean -y)

docker/Dockerfile.arm64

@@ -0,0 +1,22 @@
+FROM frigate-base
+LABEL maintainer "blakeb@blakeshome.com"
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Install packages for apt repo
+RUN apt-get -qq update \
+    && apt-get -qq install --no-install-recommends -y \
+        # ffmpeg runtime dependencies
+        libgomp1 \
+        # runtime dependencies
+        libopenexr24 \
+        libgstreamer1.0-0 \
+        libgstreamer-plugins-base1.0-0 \
+        libopenblas-base \
+        libjpeg-turbo8 \
+        libpng16-16 \
+        libtiff5 \
+        libdc1394-22 \
+    ## Tensorflow lite
+    && pip3 install https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp38-cp38-linux_aarch64.whl \
+    && rm -rf /var/lib/apt/lists/* \
+    && (apt-get autoremove -y; apt-get autoclean -y)

docker/Dockerfile.armv7hf

@@ -0,0 +1,24 @@
+FROM frigate-base
+LABEL maintainer "blakeb@blakeshome.com"
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Install packages for apt repo
+RUN apt-get -qq update \
+    && apt-get -qq install --no-install-recommends -y \
+        # ffmpeg runtime dependencies
+        libgomp1 \
+        # runtime dependencies
+        libopenexr24 \
+        libgstreamer1.0-0 \
+        libgstreamer-plugins-base1.0-0 \
+        libopenblas-base \
+        libjpeg-turbo8 \
+        libpng16-16 \
+        libtiff5 \
+        libdc1394-22 \
+        libaom0 \
+        libx265-179 \
+    ## Tensorflow lite
+    && pip3 install https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp38-cp38-linux_armv7l.whl \
+    && rm -rf /var/lib/apt/lists/* \
+    && (apt-get autoremove -y; apt-get autoclean -y)

docker/Dockerfile.base

@@ -0,0 +1,41 @@
+ARG ARCH=amd64
+FROM blakeblackshear/frigate-wheels:${ARCH} as wheels
+FROM blakeblackshear/frigate-ffmpeg:${ARCH} as ffmpeg
+
+FROM ubuntu:20.04
+LABEL maintainer "blakeb@blakeshome.com"
+
+COPY --from=ffmpeg /usr/local /usr/local/
+
+COPY --from=wheels /wheels/. /wheels/
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Install packages for apt repo
+RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
+    gnupg wget unzip tzdata \
+    && apt-get -qq install --no-install-recommends -y \
+        python3-pip \
+    && pip3 install -U /wheels/*.whl \
+    && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn apt-key adv --fetch-keys https://packages.cloud.google.com/apt/doc/apt-key.gpg \
+    && echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
+    && echo "libedgetpu1-max libedgetpu/accepted-eula select true" | debconf-set-selections \
+    && apt-get -qq update && apt-get -qq install --no-install-recommends -y \
+        libedgetpu1-max \
+    && rm -rf /var/lib/apt/lists/* /wheels \
+    && (apt-get autoremove -y; apt-get autoclean -y)
+
+# get model and labels
+ARG MODEL_REFS=7064b94dd5b996189242320359dbab8b52c94a84
+COPY labelmap.txt /labelmap.txt
+RUN wget -q https://github.com/google-coral/edgetpu/raw/$MODEL_REFS/test_data/ssd_mobilenet_v2_coco_quant_postprocess_edgetpu.tflite -O /edgetpu_model.tflite
+RUN wget -q https://github.com/google-coral/edgetpu/raw/$MODEL_REFS/test_data/ssd_mobilenet_v2_coco_quant_postprocess.tflite -O /cpu_model.tflite
+
+RUN mkdir /cache /clips
+
+WORKDIR /opt/frigate/
+ADD frigate frigate/
+COPY detect_objects.py .
+COPY benchmark.py .
+COPY process_clip.py .
+
+CMD ["python3", "-u", "detect_objects.py"]

docker/Dockerfile.ffmpeg.amd64

@@ -0,0 +1,526 @@
+# inspired by:
+# https://github.com/collelog/ffmpeg/blob/master/4.3.1-alpine-rpi4-arm64v8.Dockerfile
+# https://github.com/jrottenberg/ffmpeg/pull/158/files
+# https://github.com/jrottenberg/ffmpeg/pull/239
+FROM        ubuntu:20.04 AS base
+
+WORKDIR     /tmp/workdir
+
+ENV     DEBIAN_FRONTEND=noninteractive
+RUN     apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ca-certificates expat libgomp1 && \
+        apt-get autoremove -y && \
+        apt-get clean -y
+
+FROM base as build
+
+ENV         FFMPEG_VERSION=4.3.1 \
+            AOM_VERSION=v1.0.0 \
+            FDKAAC_VERSION=0.1.5 \
+            FONTCONFIG_VERSION=2.12.4 \
+            FREETYPE_VERSION=2.5.5 \
+            FRIBIDI_VERSION=0.19.7 \
+            KVAZAAR_VERSION=1.2.0 \
+            LAME_VERSION=3.100 \
+            LIBASS_VERSION=0.13.7 \
+            LIBPTHREAD_STUBS_VERSION=0.4 \
+            LIBVIDSTAB_VERSION=1.1.0 \
+            LIBXCB_VERSION=1.13.1 \
+            XCBPROTO_VERSION=1.13 \
+            OGG_VERSION=1.3.2 \
+            OPENCOREAMR_VERSION=0.1.5 \
+            OPUS_VERSION=1.2 \
+            OPENJPEG_VERSION=2.1.2 \
+            THEORA_VERSION=1.1.1 \
+            VORBIS_VERSION=1.3.5 \
+            VPX_VERSION=1.8.0 \
+            WEBP_VERSION=1.0.2 \
+            X264_VERSION=20170226-2245-stable \
+            X265_VERSION=3.1.1 \
+            XAU_VERSION=1.0.9 \
+            XORG_MACROS_VERSION=1.19.2 \
+            XPROTO_VERSION=7.0.31 \
+            XVID_VERSION=1.3.4 \
+            LIBXML2_VERSION=2.9.10 \
+            LIBBLURAY_VERSION=1.1.2 \
+            LIBZMQ_VERSION=4.3.2 \
+            SRC=/usr/local
+
+ARG         FREETYPE_SHA256SUM="5d03dd76c2171a7601e9ce10551d52d4471cf92cd205948e60289251daddffa8 freetype-2.5.5.tar.gz"
+ARG         FRIBIDI_SHA256SUM="3fc96fa9473bd31dcb5500bdf1aa78b337ba13eb8c301e7c28923fea982453a8 0.19.7.tar.gz"
+ARG         LIBASS_SHA256SUM="8fadf294bf701300d4605e6f1d92929304187fca4b8d8a47889315526adbafd7 0.13.7.tar.gz"
+ARG         LIBVIDSTAB_SHA256SUM="14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb v1.1.0.tar.gz"
+ARG         OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz"
+ARG         OPUS_SHA256SUM="77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz"
+ARG         THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz"
+ARG         VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz"
+ARG         XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz"
+ARG         LIBXML2_SHA256SUM="f07dab13bf42d2b8db80620cce7419b3b87827cc937c8bb20fe13b8571ee9501  libxml2-v2.9.10.tar.gz"
+ARG         LIBBLURAY_SHA256SUM="a3dd452239b100dc9da0d01b30e1692693e2a332a7d29917bf84bb10ea7c0b42 libbluray-1.1.2.tar.bz2"
+ARG         LIBZMQ_SHA256SUM="02ecc88466ae38cf2c8d79f09cfd2675ba299a439680b64ade733e26a349edeb v4.3.2.tar.gz"
+
+
+ARG         LD_LIBRARY_PATH=/opt/ffmpeg/lib
+ARG         MAKEFLAGS="-j2"
+ARG         PKG_CONFIG_PATH="/opt/ffmpeg/share/pkgconfig:/opt/ffmpeg/lib/pkgconfig:/opt/ffmpeg/lib64/pkgconfig"
+ARG         PREFIX=/opt/ffmpeg
+ARG         LD_LIBRARY_PATH="/opt/ffmpeg/lib:/opt/ffmpeg/lib64:/usr/lib64:/usr/lib:/lib64:/lib"
+
+
+RUN      buildDeps="autoconf \
+                    automake \
+                    cmake \
+                    curl \
+                    bzip2 \
+                    libexpat1-dev \
+                    g++ \
+                    gcc \
+                    git \
+                    gperf \
+                    libtool \
+                    make \
+                    nasm \
+                    perl \
+                    pkg-config \
+                    python \
+                    libssl-dev \
+                    yasm \
+                    libva-dev \
+                    zlib1g-dev" && \
+        apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ${buildDeps}
+## opencore-amr https://sourceforge.net/projects/opencore-amr/
+RUN \
+        DIR=/tmp/opencore-amr && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## x264 http://www.videolan.org/developers/x264.html
+RUN \
+        DIR=/tmp/x264 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://download.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-${X264_VERSION}.tar.bz2 | \
+        tar -jx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared --enable-pic --disable-cli && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### x265 http://x265.org/
+RUN \
+        DIR=/tmp/x265 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://download.videolan.org/pub/videolan/x265/x265_${X265_VERSION}.tar.gz  | \
+        tar -zx && \
+        cd x265_${X265_VERSION}/build/linux && \
+        sed -i "/-DEXTRA_LIB/ s/$/ -DCMAKE_INSTALL_PREFIX=\${PREFIX}/" multilib.sh && \
+        sed -i "/^cmake/ s/$/ -DENABLE_CLI=OFF/" multilib.sh && \
+        ./multilib.sh && \
+        make -C 8bit install && \
+        rm -rf ${DIR}
+### libogg https://www.xiph.org/ogg/
+RUN \
+        DIR=/tmp/ogg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \
+        echo ${OGG_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libopus https://www.opus-codec.org/
+RUN \
+        DIR=/tmp/opus && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \
+        echo ${OPUS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libvorbis https://xiph.org/vorbis/
+RUN \
+        DIR=/tmp/vorbis && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \
+        echo ${VORBIS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libtheora http://www.theora.org/
+RUN \
+        DIR=/tmp/theora && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz && \
+        echo ${THEORA_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libtheora-${THEORA_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libvpx https://www.webmproject.org/code/
+RUN \
+        DIR=/tmp/vpx && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://codeload.github.com/webmproject/libvpx/tar.gz/v${VPX_VERSION} | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-vp8 --enable-vp9 --enable-vp9-highbitdepth --enable-pic --enable-shared \
+        --disable-debug --disable-examples --disable-docs --disable-install-bins  && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libwebp https://developers.google.com/speed/webp/
+RUN \
+        DIR=/tmp/vebp && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### libmp3lame http://lame.sourceforge.net/
+RUN \
+        DIR=/tmp/lame && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/lame/lame/$(echo ${LAME_VERSION} | sed -e 's/[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)/\1.\2/')/lame-${LAME_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" --enable-shared --enable-nasm --disable-frontend && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### xvid https://www.xvid.com/
+RUN \
+        DIR=/tmp/xvid && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xvid.org/downloads/xvidcore-${XVID_VERSION}.tar.gz && \
+        echo ${XVID_SHA256SUM} | sha256sum --check && \
+        tar -zx -f xvidcore-${XVID_VERSION}.tar.gz && \
+        cd xvidcore/build/generic && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+### fdk-aac https://github.com/mstorsjo/fdk-aac
+RUN \
+        DIR=/tmp/fdk-aac && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/mstorsjo/fdk-aac/archive/v${FDKAAC_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared --datadir="${DIR}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## openjpeg https://github.com/uclouvain/openjpeg
+RUN \
+        DIR=/tmp/openjpeg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        cmake -DBUILD_THIRDPARTY:BOOL=ON -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## freetype https://www.freetype.org/
+RUN  \
+        DIR=/tmp/freetype && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.savannah.gnu.org/releases/freetype/freetype-${FREETYPE_VERSION}.tar.gz && \
+        echo ${FREETYPE_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f freetype-${FREETYPE_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## libvstab https://github.com/georgmartius/vid.stab
+RUN  \
+        DIR=/tmp/vid.stab && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/georgmartius/vid.stab/archive/v${LIBVIDSTAB_VERSION}.tar.gz && \
+        echo ${LIBVIDSTAB_SHA256SUM} | sha256sum --check &&  \
+        tar -zx --strip-components=1 -f v${LIBVIDSTAB_VERSION}.tar.gz && \
+        cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## fridibi https://www.fribidi.org/
+RUN  \
+        DIR=/tmp/fribidi && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/fribidi/fribidi/archive/${FRIBIDI_VERSION}.tar.gz && \
+        echo ${FRIBIDI_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${FRIBIDI_VERSION}.tar.gz && \
+        sed -i 's/^SUBDIRS =.*/SUBDIRS=gen.tab charset lib bin/' Makefile.am && \
+        ./bootstrap --no-config --auto && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j1 && \
+        make install && \
+        rm -rf ${DIR}
+## fontconfig https://www.freedesktop.org/wiki/Software/fontconfig/
+RUN  \
+        DIR=/tmp/fontconfig && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.freedesktop.org/software/fontconfig/release/fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## libass https://github.com/libass/libass
+RUN  \
+        DIR=/tmp/libass && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/libass/libass/archive/${LIBASS_VERSION}.tar.gz && \
+        echo ${LIBASS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${LIBASS_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+## kvazaar https://github.com/ultravideo/kvazaar
+RUN \
+        DIR=/tmp/kvazaar && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/ultravideo/kvazaar/archive/v${KVAZAAR_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f v${KVAZAAR_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/aom && \
+        git clone --branch ${AOM_VERSION} --depth 1 https://aomedia.googlesource.com/aom ${DIR} ; \
+        cd ${DIR} ; \
+        rm -rf CMakeCache.txt CMakeFiles ; \
+        mkdir -p ./aom_build ; \
+        cd ./aom_build ; \
+        cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=1 ..; \
+        make ; \
+        make install ; \
+        rm -rf ${DIR}
+
+## libxcb (and supporting libraries) for screen capture https://xcb.freedesktop.org/
+RUN \
+        DIR=/tmp/xorg-macros && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive//individual/util/util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/xproto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/proto/xproto-${XPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xproto-${XPROTO_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libXau && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/lib/libXau-${XAU_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libXau-${XAU_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libpthread-stubs && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb-proto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libxcb-${LIBXCB_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libxcb-${LIBXCB_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+## libxml2 - for libbluray
+RUN \
+        DIR=/tmp/libxml2 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://gitlab.gnome.org/GNOME/libxml2/-/archive/v${LIBXML2_VERSION}/libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        echo ${LIBXML2_SHA256SUM} | sha256sum --check && \
+        tar -xz --strip-components=1 -f libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        ./autogen.sh --prefix="${PREFIX}" --with-ftp=no --with-http=no --with-python=no && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+## libbluray - Requires libxml, freetype, and fontconfig
+RUN \
+        DIR=/tmp/libbluray && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.videolan.org/pub/videolan/libbluray/${LIBBLURAY_VERSION}/libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        echo ${LIBBLURAY_SHA256SUM} | sha256sum --check && \
+        tar -jx --strip-components=1 -f libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-examples --disable-bdjava-jar --disable-static --enable-shared && \
+        make && \
+        make install && \
+        rm -rf ${DIR}
+
+## libzmq https://github.com/zeromq/libzmq/
+RUN \
+        DIR=/tmp/libzmq && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/zeromq/libzmq/archive/v${LIBZMQ_VERSION}.tar.gz && \
+        echo ${LIBZMQ_SHA256SUM} | sha256sum --check && \
+        tar -xz --strip-components=1 -f v${LIBZMQ_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make && \
+        make check && \
+        make install && \
+        rm -rf ${DIR}
+
+## ffmpeg https://ffmpeg.org/
+RUN  \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2
+
+
+
+RUN \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        ./configure \
+        --disable-debug \
+        --disable-doc \
+        --disable-ffplay \
+        --enable-shared \
+        --enable-avresample \
+        --enable-libopencore-amrnb \
+        --enable-libopencore-amrwb \
+        --enable-gpl \
+        --enable-libass \
+        --enable-fontconfig \
+        --enable-libfreetype \
+        --enable-libvidstab \
+        --enable-libmp3lame \
+        --enable-libopus \
+        --enable-libtheora \
+        --enable-libvorbis \
+        --enable-libvpx \
+        --enable-libwebp \
+        --enable-libxcb \
+        --enable-libx265 \
+        --enable-libxvid \
+        --enable-libx264 \
+        --enable-nonfree \
+        --enable-openssl \
+        --enable-libfdk_aac \
+        --enable-postproc \
+        --enable-small \
+        --enable-version3 \
+        --enable-libbluray \
+        --enable-libzmq \
+        --extra-libs=-ldl \
+        --prefix="${PREFIX}" \
+        --enable-libopenjpeg \
+        --enable-libkvazaar \
+        --enable-libaom \
+        --extra-libs=-lpthread \
+        --enable-vaapi \
+        --extra-cflags="-I${PREFIX}/include" \
+        --extra-ldflags="-L${PREFIX}/lib" && \
+        make && \
+        make install && \
+        make tools/zmqsend && cp tools/zmqsend ${PREFIX}/bin/ && \
+        make distclean && \
+        hash -r && \
+        cd tools && \
+        make qt-faststart && cp qt-faststart ${PREFIX}/bin/
+
+## cleanup
+RUN \
+        ldd ${PREFIX}/bin/ffmpeg | grep opt/ffmpeg | cut -d ' ' -f 3 | xargs -i cp {} /usr/local/lib/ && \
+        for lib in /usr/local/lib/*.so.*; do ln -s "${lib##*/}" "${lib%%.so.*}".so; done && \
+        cp ${PREFIX}/bin/* /usr/local/bin/ && \
+        cp -r ${PREFIX}/share/ffmpeg /usr/local/share/ && \
+        LD_LIBRARY_PATH=/usr/local/lib ffmpeg -buildconf && \
+        cp -r ${PREFIX}/include/libav* ${PREFIX}/include/libpostproc ${PREFIX}/include/libsw* /usr/local/include && \
+        mkdir -p /usr/local/lib/pkgconfig && \
+        for pc in ${PREFIX}/lib/pkgconfig/libav*.pc ${PREFIX}/lib/pkgconfig/libpostproc.pc ${PREFIX}/lib/pkgconfig/libsw*.pc; do \
+          sed "s:${PREFIX}:/usr/local:g" <"$pc" >/usr/local/lib/pkgconfig/"${pc##*/}"; \
+        done
+
+FROM        base AS release
+
+ENV         LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/lib:/usr/lib64:/lib:/lib64
+
+CMD         ["--help"]
+ENTRYPOINT  ["ffmpeg"]
+
+COPY --from=build /usr/local /usr/local/
+
+RUN \
+	apt-get update -y && \
+	apt-get install -y --no-install-recommends libva-drm2 libva2 i965-va-driver && \
+	rm -rf /var/lib/apt/lists/*

docker/Dockerfile.ffmpeg.arm64

@@ -0,0 +1,533 @@
+# inspired by:
+# https://github.com/collelog/ffmpeg/blob/master/4.3.1-alpine-rpi4-arm64v8.Dockerfile
+# https://github.com/mmastrac/ffmpeg-omx-rpi-docker/blob/master/Dockerfile
+# https://github.com/jrottenberg/ffmpeg/pull/158/files
+# https://github.com/jrottenberg/ffmpeg/pull/239
+FROM        ubuntu:20.04 AS base
+
+WORKDIR     /tmp/workdir
+
+ENV     DEBIAN_FRONTEND=noninteractive
+RUN     apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ca-certificates expat libgomp1 && \
+        apt-get autoremove -y && \
+        apt-get clean -y
+
+FROM base as build
+
+ENV         FFMPEG_VERSION=4.3.1 \
+            AOM_VERSION=v1.0.0 \
+            FDKAAC_VERSION=0.1.5 \
+            FONTCONFIG_VERSION=2.12.4 \
+            FREETYPE_VERSION=2.5.5 \
+            FRIBIDI_VERSION=0.19.7 \
+            KVAZAAR_VERSION=1.2.0 \
+            LAME_VERSION=3.100 \
+            LIBASS_VERSION=0.13.7 \
+            LIBPTHREAD_STUBS_VERSION=0.4 \
+            LIBVIDSTAB_VERSION=1.1.0 \
+            LIBXCB_VERSION=1.13.1 \
+            XCBPROTO_VERSION=1.13 \
+            OGG_VERSION=1.3.2 \
+            OPENCOREAMR_VERSION=0.1.5 \
+            OPUS_VERSION=1.2 \
+            OPENJPEG_VERSION=2.1.2 \
+            THEORA_VERSION=1.1.1 \
+            VORBIS_VERSION=1.3.5 \
+            VPX_VERSION=1.8.0 \
+            WEBP_VERSION=1.0.2 \
+            X264_VERSION=20170226-2245-stable \
+            X265_VERSION=3.1.1 \
+            XAU_VERSION=1.0.9 \
+            XORG_MACROS_VERSION=1.19.2 \
+            XPROTO_VERSION=7.0.31 \
+            XVID_VERSION=1.3.4 \
+            LIBXML2_VERSION=2.9.10 \
+            LIBBLURAY_VERSION=1.1.2 \
+            LIBZMQ_VERSION=4.3.2 \
+            SRC=/usr/local
+
+ARG         FREETYPE_SHA256SUM="5d03dd76c2171a7601e9ce10551d52d4471cf92cd205948e60289251daddffa8 freetype-2.5.5.tar.gz"
+ARG         FRIBIDI_SHA256SUM="3fc96fa9473bd31dcb5500bdf1aa78b337ba13eb8c301e7c28923fea982453a8 0.19.7.tar.gz"
+ARG         LIBASS_SHA256SUM="8fadf294bf701300d4605e6f1d92929304187fca4b8d8a47889315526adbafd7 0.13.7.tar.gz"
+ARG         LIBVIDSTAB_SHA256SUM="14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb v1.1.0.tar.gz"
+ARG         OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz"
+ARG         OPUS_SHA256SUM="77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz"
+ARG         THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz"
+ARG         VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz"
+ARG         XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz"
+ARG         LIBXML2_SHA256SUM="f07dab13bf42d2b8db80620cce7419b3b87827cc937c8bb20fe13b8571ee9501  libxml2-v2.9.10.tar.gz"
+ARG         LIBBLURAY_SHA256SUM="a3dd452239b100dc9da0d01b30e1692693e2a332a7d29917bf84bb10ea7c0b42 libbluray-1.1.2.tar.bz2"
+ARG         LIBZMQ_SHA256SUM="02ecc88466ae38cf2c8d79f09cfd2675ba299a439680b64ade733e26a349edeb v4.3.2.tar.gz"
+
+
+ARG         LD_LIBRARY_PATH=/opt/ffmpeg/lib
+ARG         MAKEFLAGS="-j2"
+ARG         PKG_CONFIG_PATH="/opt/ffmpeg/share/pkgconfig:/opt/ffmpeg/lib/pkgconfig:/opt/ffmpeg/lib64/pkgconfig"
+ARG         PREFIX=/opt/ffmpeg
+ARG         LD_LIBRARY_PATH="/opt/ffmpeg/lib:/opt/ffmpeg/lib64:/usr/lib64:/usr/lib:/lib64:/lib"
+
+
+RUN      buildDeps="autoconf \
+                    automake \
+                    cmake \
+                    curl \
+                    bzip2 \
+                    libexpat1-dev \
+                    g++ \
+                    gcc \
+                    git \
+                    gperf \
+                    libtool \
+                    make \
+                    nasm \
+                    perl \
+                    pkg-config \
+                    python \
+                    libssl-dev \
+                    yasm \
+                    linux-headers-raspi2 \
+                    libomxil-bellagio-dev \
+                    zlib1g-dev" && \
+        apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ${buildDeps}
+## opencore-amr https://sourceforge.net/projects/opencore-amr/
+RUN \
+        DIR=/tmp/opencore-amr && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## x264 http://www.videolan.org/developers/x264.html
+RUN \
+        DIR=/tmp/x264 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://download.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-${X264_VERSION}.tar.bz2 | \
+        tar -jx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared --enable-pic --disable-cli && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### x265 http://x265.org/
+RUN \
+        DIR=/tmp/x265 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://download.videolan.org/pub/videolan/x265/x265_${X265_VERSION}.tar.gz  | \
+        tar -zx && \
+        cd x265_${X265_VERSION}/build/linux && \
+        sed -i "/-DEXTRA_LIB/ s/$/ -DCMAKE_INSTALL_PREFIX=\${PREFIX}/" multilib.sh && \
+        sed -i "/^cmake/ s/$/ -DENABLE_CLI=OFF/" multilib.sh && \
+        export CXXFLAGS="${CXXFLAGS} -fPIC" && \
+        ./multilib.sh && \
+        make -C 8bit install && \
+        rm -rf ${DIR}
+### libogg https://www.xiph.org/ogg/
+RUN \
+        DIR=/tmp/ogg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \
+        echo ${OGG_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libopus https://www.opus-codec.org/
+RUN \
+        DIR=/tmp/opus && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \
+        echo ${OPUS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libvorbis https://xiph.org/vorbis/
+RUN \
+        DIR=/tmp/vorbis && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \
+        echo ${VORBIS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libtheora http://www.theora.org/
+RUN \
+        DIR=/tmp/theora && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz && \
+        echo ${THEORA_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libtheora-${THEORA_VERSION}.tar.gz && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libvpx https://www.webmproject.org/code/
+RUN \
+        DIR=/tmp/vpx && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://codeload.github.com/webmproject/libvpx/tar.gz/v${VPX_VERSION} | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-vp8 --enable-vp9 --enable-vp9-highbitdepth --enable-pic --enable-shared \
+        --disable-debug --disable-examples --disable-docs --disable-install-bins  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libwebp https://developers.google.com/speed/webp/
+RUN \
+        DIR=/tmp/vebp && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libmp3lame http://lame.sourceforge.net/
+RUN \
+        DIR=/tmp/lame && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/lame/lame/$(echo ${LAME_VERSION} | sed -e 's/[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)/\1.\2/')/lame-${LAME_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" --enable-shared --enable-nasm --disable-frontend && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### xvid https://www.xvid.com/
+RUN \
+        DIR=/tmp/xvid && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xvid.org/downloads/xvidcore-${XVID_VERSION}.tar.gz && \
+        echo ${XVID_SHA256SUM} | sha256sum --check && \
+        tar -zx -f xvidcore-${XVID_VERSION}.tar.gz && \
+        cd xvidcore/build/generic && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### fdk-aac https://github.com/mstorsjo/fdk-aac
+RUN \
+        DIR=/tmp/fdk-aac && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/mstorsjo/fdk-aac/archive/v${FDKAAC_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared --datadir="${DIR}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## openjpeg https://github.com/uclouvain/openjpeg
+RUN \
+        DIR=/tmp/openjpeg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        export CFLAGS="${CFLAGS} -DPNG_ARM_NEON_OPT=0" && \
+        cmake -DBUILD_THIRDPARTY:BOOL=ON -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## freetype https://www.freetype.org/
+RUN  \
+        DIR=/tmp/freetype && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.savannah.gnu.org/releases/freetype/freetype-${FREETYPE_VERSION}.tar.gz && \
+        echo ${FREETYPE_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f freetype-${FREETYPE_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## libvstab https://github.com/georgmartius/vid.stab
+RUN  \
+        DIR=/tmp/vid.stab && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/georgmartius/vid.stab/archive/v${LIBVIDSTAB_VERSION}.tar.gz && \
+        echo ${LIBVIDSTAB_SHA256SUM} | sha256sum --check &&  \
+        tar -zx --strip-components=1 -f v${LIBVIDSTAB_VERSION}.tar.gz && \
+        cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## fridibi https://www.fribidi.org/
+RUN  \
+        DIR=/tmp/fribidi && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/fribidi/fribidi/archive/${FRIBIDI_VERSION}.tar.gz && \
+        echo ${FRIBIDI_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${FRIBIDI_VERSION}.tar.gz && \
+        sed -i 's/^SUBDIRS =.*/SUBDIRS=gen.tab charset lib bin/' Makefile.am && \
+        ./bootstrap --no-config --auto && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j1 && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## fontconfig https://www.freedesktop.org/wiki/Software/fontconfig/
+RUN  \
+        DIR=/tmp/fontconfig && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.freedesktop.org/software/fontconfig/release/fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## libass https://github.com/libass/libass
+RUN  \
+        DIR=/tmp/libass && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/libass/libass/archive/${LIBASS_VERSION}.tar.gz && \
+        echo ${LIBASS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${LIBASS_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## kvazaar https://github.com/ultravideo/kvazaar
+RUN \
+        DIR=/tmp/kvazaar && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/ultravideo/kvazaar/archive/v${KVAZAAR_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f v${KVAZAAR_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/aom && \
+        git clone --branch ${AOM_VERSION} --depth 1 https://aomedia.googlesource.com/aom ${DIR} ; \
+        cd ${DIR} ; \
+        rm -rf CMakeCache.txt CMakeFiles ; \
+        mkdir -p ./aom_build ; \
+        cd ./aom_build ; \
+        cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=1 ..; \
+        make ; \
+        make install ; \
+        rm -rf ${DIR}
+
+## libxcb (and supporting libraries) for screen capture https://xcb.freedesktop.org/
+RUN \
+        DIR=/tmp/xorg-macros && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive//individual/util/util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/xproto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/proto/xproto-${XPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xproto-${XPROTO_VERSION}.tar.gz && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libXau && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/lib/libXau-${XAU_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libXau-${XAU_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libpthread-stubs && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb-proto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libxcb-${LIBXCB_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libxcb-${LIBXCB_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libxml2 - for libbluray
+RUN \
+        DIR=/tmp/libxml2 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://gitlab.gnome.org/GNOME/libxml2/-/archive/v${LIBXML2_VERSION}/libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        echo ${LIBXML2_SHA256SUM} | sha256sum --check && \
+        tar -xz --strip-components=1 -f libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        ./autogen.sh --prefix="${PREFIX}" --with-ftp=no --with-http=no --with-python=no && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libbluray - Requires libxml, freetype, and fontconfig
+RUN \
+        DIR=/tmp/libbluray && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.videolan.org/pub/videolan/libbluray/${LIBBLURAY_VERSION}/libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        echo ${LIBBLURAY_SHA256SUM} | sha256sum --check && \
+        tar -jx --strip-components=1 -f libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-examples --disable-bdjava-jar --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libzmq https://github.com/zeromq/libzmq/
+RUN \
+        DIR=/tmp/libzmq && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/zeromq/libzmq/archive/v${LIBZMQ_VERSION}.tar.gz && \
+        echo ${LIBZMQ_SHA256SUM} | sha256sum --check && \
+        tar -xz --strip-components=1 -f v${LIBZMQ_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make check && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## ffmpeg https://ffmpeg.org/
+RUN  \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2
+
+RUN \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        ./configure \
+        --disable-debug \
+        --disable-doc \
+        --disable-ffplay \
+        --enable-shared \
+        --enable-avresample \
+        --enable-libopencore-amrnb \
+        --enable-libopencore-amrwb \
+        --enable-gpl \
+        --enable-libass \
+        --enable-fontconfig \
+        --enable-libfreetype \
+        --enable-libvidstab \
+        --enable-libmp3lame \
+        --enable-libopus \
+        --enable-libtheora \
+        --enable-libvorbis \
+        --enable-libvpx \
+        --enable-libwebp \
+        --enable-libxcb \
+        --enable-libx265 \
+        --enable-libxvid \
+        --enable-libx264 \
+        --enable-nonfree \
+        --enable-openssl \
+        --enable-libfdk_aac \
+        --enable-postproc \
+        --enable-small \
+        --enable-version3 \
+        --enable-libbluray \
+        --enable-libzmq \
+        --extra-libs=-ldl \
+        --prefix="${PREFIX}" \
+        --enable-libopenjpeg \
+        --enable-libkvazaar \
+        --enable-libaom \
+        --extra-libs=-lpthread \
+        # --enable-omx \
+        # --enable-omx-rpi \
+        # --enable-mmal \
+        --enable-v4l2_m2m \
+        --enable-neon \
+        --extra-cflags="-I${PREFIX}/include" \
+        --extra-ldflags="-L${PREFIX}/lib" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        make tools/zmqsend && cp tools/zmqsend ${PREFIX}/bin/ && \
+        make distclean && \
+        hash -r && \
+        cd tools && \
+        make qt-faststart && cp qt-faststart ${PREFIX}/bin/
+
+## cleanup
+RUN \
+        ldd ${PREFIX}/bin/ffmpeg | grep opt/ffmpeg | cut -d ' ' -f 3 | xargs -i cp {} /usr/local/lib/ && \
+        for lib in /usr/local/lib/*.so.*; do ln -s "${lib##*/}" "${lib%%.so.*}".so; done && \
+        cp ${PREFIX}/bin/* /usr/local/bin/ && \
+        cp -r ${PREFIX}/share/ffmpeg /usr/local/share/ && \
+        LD_LIBRARY_PATH=/usr/local/lib ffmpeg -buildconf && \
+        cp -r ${PREFIX}/include/libav* ${PREFIX}/include/libpostproc ${PREFIX}/include/libsw* /usr/local/include && \
+        mkdir -p /usr/local/lib/pkgconfig && \
+        for pc in ${PREFIX}/lib/pkgconfig/libav*.pc ${PREFIX}/lib/pkgconfig/libpostproc.pc ${PREFIX}/lib/pkgconfig/libsw*.pc; do \
+          sed "s:${PREFIX}:/usr/local:g" <"$pc" >/usr/local/lib/pkgconfig/"${pc##*/}"; \
+        done
+
+FROM        base AS release
+
+ENV         LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/lib:/usr/lib64:/lib:/lib64
+
+CMD         ["--help"]
+ENTRYPOINT  ["ffmpeg"]
+
+COPY --from=build /usr/local /usr/local/
+
+# Run ffmpeg with -c:v h264_v4l2m2m to enable HW accell for decoding on raspberry pi4 64-bit

docker/Dockerfile.ffmpeg.armv7hf

@@ -0,0 +1,549 @@
+# inspired by:
+# https://github.com/collelog/ffmpeg/blob/master/4.3.1-alpine-rpi4-arm64v8.Dockerfile
+# https://github.com/mmastrac/ffmpeg-omx-rpi-docker/blob/master/Dockerfile
+# https://github.com/jrottenberg/ffmpeg/pull/158/files
+# https://github.com/jrottenberg/ffmpeg/pull/239
+FROM        ubuntu:20.04 AS base
+
+WORKDIR     /tmp/workdir
+
+ENV     DEBIAN_FRONTEND=noninteractive
+RUN     apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ca-certificates expat libgomp1 && \
+        apt-get autoremove -y && \
+        apt-get clean -y
+
+FROM base as build
+
+ENV         FFMPEG_VERSION=4.3.1 \
+            AOM_VERSION=v1.0.0 \
+            FDKAAC_VERSION=0.1.5 \
+            FONTCONFIG_VERSION=2.12.4 \
+            FREETYPE_VERSION=2.5.5 \
+            FRIBIDI_VERSION=0.19.7 \
+            KVAZAAR_VERSION=1.2.0 \
+            LAME_VERSION=3.100 \
+            LIBASS_VERSION=0.13.7 \
+            LIBPTHREAD_STUBS_VERSION=0.4 \
+            LIBVIDSTAB_VERSION=1.1.0 \
+            LIBXCB_VERSION=1.13.1 \
+            XCBPROTO_VERSION=1.13 \
+            OGG_VERSION=1.3.2 \
+            OPENCOREAMR_VERSION=0.1.5 \
+            OPUS_VERSION=1.2 \
+            OPENJPEG_VERSION=2.1.2 \
+            THEORA_VERSION=1.1.1 \
+            VORBIS_VERSION=1.3.5 \
+            VPX_VERSION=1.8.0 \
+            WEBP_VERSION=1.0.2 \
+            X264_VERSION=20170226-2245-stable \
+            X265_VERSION=3.1.1 \
+            XAU_VERSION=1.0.9 \
+            XORG_MACROS_VERSION=1.19.2 \
+            XPROTO_VERSION=7.0.31 \
+            XVID_VERSION=1.3.4 \
+            LIBXML2_VERSION=2.9.10 \
+            LIBBLURAY_VERSION=1.1.2 \
+            LIBZMQ_VERSION=4.3.3 \
+            SRC=/usr/local
+
+ARG         FREETYPE_SHA256SUM="5d03dd76c2171a7601e9ce10551d52d4471cf92cd205948e60289251daddffa8 freetype-2.5.5.tar.gz"
+ARG         FRIBIDI_SHA256SUM="3fc96fa9473bd31dcb5500bdf1aa78b337ba13eb8c301e7c28923fea982453a8 0.19.7.tar.gz"
+ARG         LIBASS_SHA256SUM="8fadf294bf701300d4605e6f1d92929304187fca4b8d8a47889315526adbafd7 0.13.7.tar.gz"
+ARG         LIBVIDSTAB_SHA256SUM="14d2a053e56edad4f397be0cb3ef8eb1ec3150404ce99a426c4eb641861dc0bb v1.1.0.tar.gz"
+ARG         OGG_SHA256SUM="e19ee34711d7af328cb26287f4137e70630e7261b17cbe3cd41011d73a654692 libogg-1.3.2.tar.gz"
+ARG         OPUS_SHA256SUM="77db45a87b51578fbc49555ef1b10926179861d854eb2613207dc79d9ec0a9a9 opus-1.2.tar.gz"
+ARG         THEORA_SHA256SUM="40952956c47811928d1e7922cda3bc1f427eb75680c3c37249c91e949054916b libtheora-1.1.1.tar.gz"
+ARG         VORBIS_SHA256SUM="6efbcecdd3e5dfbf090341b485da9d176eb250d893e3eb378c428a2db38301ce libvorbis-1.3.5.tar.gz"
+ARG         XVID_SHA256SUM="4e9fd62728885855bc5007fe1be58df42e5e274497591fec37249e1052ae316f xvidcore-1.3.4.tar.gz"
+ARG         LIBXML2_SHA256SUM="f07dab13bf42d2b8db80620cce7419b3b87827cc937c8bb20fe13b8571ee9501  libxml2-v2.9.10.tar.gz"
+ARG         LIBBLURAY_SHA256SUM="a3dd452239b100dc9da0d01b30e1692693e2a332a7d29917bf84bb10ea7c0b42 libbluray-1.1.2.tar.bz2"
+
+
+ARG         LD_LIBRARY_PATH=/opt/ffmpeg/lib
+ARG         MAKEFLAGS="-j2"
+ARG         PKG_CONFIG_PATH="/opt/ffmpeg/share/pkgconfig:/opt/ffmpeg/lib/pkgconfig:/opt/ffmpeg/lib64/pkgconfig:/opt/vc/lib/pkgconfig"
+ARG         PREFIX=/opt/ffmpeg
+ARG         LD_LIBRARY_PATH="/opt/ffmpeg/lib:/opt/ffmpeg/lib64:/usr/lib64:/usr/lib:/lib64:/lib:/opt/vc/lib"
+
+
+RUN      buildDeps="autoconf \
+                    automake \
+                    cmake \
+                    curl \
+                    bzip2 \
+                    libexpat1-dev \
+                    g++ \
+                    gcc \
+                    git \
+                    gperf \
+                    libtool \
+                    make \
+                    nasm \
+                    perl \
+                    pkg-config \
+                    python \
+                    sudo \
+                    libssl-dev \
+                    yasm \
+                    linux-headers-raspi2 \
+                    libomxil-bellagio-dev \
+                    libx265-dev \
+                    libaom-dev \
+                    zlib1g-dev" && \
+        apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends ${buildDeps}
+## opencore-amr https://sourceforge.net/projects/opencore-amr/
+RUN \
+        DIR=/tmp/opencore-amr && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/opencore-amr/opencore-amr/opencore-amr-${OPENCOREAMR_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## x264 http://www.videolan.org/developers/x264.html
+RUN \
+        DIR=/tmp/x264 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://download.videolan.org/pub/videolan/x264/snapshots/x264-snapshot-${X264_VERSION}.tar.bz2 | \
+        tar -jx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared --enable-pic --disable-cli && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+# ### x265 http://x265.org/
+# RUN \
+#         DIR=/tmp/x265 && \
+#         mkdir -p ${DIR} && \
+#         cd ${DIR} && \
+#         curl -sL https://download.videolan.org/pub/videolan/x265/x265_${X265_VERSION}.tar.gz  | \
+#         tar -zx && \
+#         cd x265_${X265_VERSION}/build/linux && \
+#         sed -i "/-DEXTRA_LIB/ s/$/ -DCMAKE_INSTALL_PREFIX=\${PREFIX}/" multilib.sh && \
+#         sed -i "/^cmake/ s/$/ -DENABLE_CLI=OFF/" multilib.sh && \
+#         # export CXXFLAGS="${CXXFLAGS} -fPIC" && \
+#         ./multilib.sh && \
+#         make -C 8bit install && \
+#         rm -rf ${DIR}
+### libogg https://www.xiph.org/ogg/
+RUN \
+        DIR=/tmp/ogg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/ogg/libogg-${OGG_VERSION}.tar.gz && \
+        echo ${OGG_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libogg-${OGG_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libopus https://www.opus-codec.org/
+RUN \
+        DIR=/tmp/opus && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://archive.mozilla.org/pub/opus/opus-${OPUS_VERSION}.tar.gz && \
+        echo ${OPUS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f opus-${OPUS_VERSION}.tar.gz && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libvorbis https://xiph.org/vorbis/
+RUN \
+        DIR=/tmp/vorbis && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/vorbis/libvorbis-${VORBIS_VERSION}.tar.gz && \
+        echo ${VORBIS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libvorbis-${VORBIS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libtheora http://www.theora.org/
+RUN \
+        DIR=/tmp/theora && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xiph.org/releases/theora/libtheora-${THEORA_VERSION}.tar.gz && \
+        echo ${THEORA_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f libtheora-${THEORA_VERSION}.tar.gz && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub && \
+        ./configure --prefix="${PREFIX}" --with-ogg="${PREFIX}" --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libvpx https://www.webmproject.org/code/
+RUN \
+        DIR=/tmp/vpx && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://codeload.github.com/webmproject/libvpx/tar.gz/v${VPX_VERSION} | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-vp8 --enable-vp9 --enable-vp9-highbitdepth --enable-pic --enable-shared \
+        --disable-debug --disable-examples --disable-docs --disable-install-bins  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libwebp https://developers.google.com/speed/webp/
+RUN \
+        DIR=/tmp/vebp && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --enable-shared  && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### libmp3lame http://lame.sourceforge.net/
+RUN \
+        DIR=/tmp/lame && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://versaweb.dl.sourceforge.net/project/lame/lame/$(echo ${LAME_VERSION} | sed -e 's/[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)/\1.\2/')/lame-${LAME_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" --enable-shared --enable-nasm --disable-frontend && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### xvid https://www.xvid.com/
+RUN \
+        DIR=/tmp/xvid && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO http://downloads.xvid.org/downloads/xvidcore-${XVID_VERSION}.tar.gz && \
+        echo ${XVID_SHA256SUM} | sha256sum --check && \
+        tar -zx -f xvidcore-${XVID_VERSION}.tar.gz && \
+        cd xvidcore/build/generic && \
+        ./configure --prefix="${PREFIX}" --bindir="${PREFIX}/bin" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+### fdk-aac https://github.com/mstorsjo/fdk-aac
+RUN \
+        DIR=/tmp/fdk-aac && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/mstorsjo/fdk-aac/archive/v${FDKAAC_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        autoreconf -fiv && \
+        ./configure --prefix="${PREFIX}" --enable-shared --datadir="${DIR}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## openjpeg https://github.com/uclouvain/openjpeg
+RUN \
+        DIR=/tmp/openjpeg && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | \
+        tar -zx --strip-components=1 && \
+        export CFLAGS="${CFLAGS} -DPNG_ARM_NEON_OPT=0" && \
+        cmake -DBUILD_THIRDPARTY:BOOL=ON -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## freetype https://www.freetype.org/
+RUN  \
+        DIR=/tmp/freetype && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.savannah.gnu.org/releases/freetype/freetype-${FREETYPE_VERSION}.tar.gz && \
+        echo ${FREETYPE_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f freetype-${FREETYPE_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## libvstab https://github.com/georgmartius/vid.stab
+RUN  \
+        DIR=/tmp/vid.stab && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/georgmartius/vid.stab/archive/v${LIBVIDSTAB_VERSION}.tar.gz && \
+        echo ${LIBVIDSTAB_SHA256SUM} | sha256sum --check &&  \
+        tar -zx --strip-components=1 -f v${LIBVIDSTAB_VERSION}.tar.gz && \
+        cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" . && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## fridibi https://www.fribidi.org/
+RUN  \
+        DIR=/tmp/fribidi && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/fribidi/fribidi/archive/${FRIBIDI_VERSION}.tar.gz && \
+        echo ${FRIBIDI_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${FRIBIDI_VERSION}.tar.gz && \
+        sed -i 's/^SUBDIRS =.*/SUBDIRS=gen.tab charset lib bin/' Makefile.am && \
+        ./bootstrap --no-config --auto && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j1 && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## fontconfig https://www.freedesktop.org/wiki/Software/fontconfig/
+RUN  \
+        DIR=/tmp/fontconfig && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.freedesktop.org/software/fontconfig/release/fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f fontconfig-${FONTCONFIG_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## libass https://github.com/libass/libass
+RUN  \
+        DIR=/tmp/libass && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/libass/libass/archive/${LIBASS_VERSION}.tar.gz && \
+        echo ${LIBASS_SHA256SUM} | sha256sum --check && \
+        tar -zx --strip-components=1 -f ${LIBASS_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+## kvazaar https://github.com/ultravideo/kvazaar
+RUN \
+        DIR=/tmp/kvazaar && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/ultravideo/kvazaar/archive/v${KVAZAAR_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f v${KVAZAAR_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+# RUN \
+#         DIR=/tmp/aom && \
+#         git clone --branch ${AOM_VERSION} --depth 1 https://aomedia.googlesource.com/aom ${DIR} ; \
+#         cd ${DIR} ; \
+#         rm -rf CMakeCache.txt CMakeFiles ; \
+#         mkdir -p ./aom_build ; \
+#         cd ./aom_build ; \
+#         cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=1 ..; \
+#         make ; \
+#         make install ; \
+#         rm -rf ${DIR}
+
+## libxcb (and supporting libraries) for screen capture https://xcb.freedesktop.org/
+RUN \
+        DIR=/tmp/xorg-macros && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive//individual/util/util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f util-macros-${XORG_MACROS_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/xproto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/proto/xproto-${XPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xproto-${XPROTO_VERSION}.tar.gz && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess && \
+        curl -sL 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libXau && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://www.x.org/archive/individual/lib/libXau-${XAU_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libXau-${XAU_VERSION}.tar.gz && \
+        ./configure --srcdir=${DIR} --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libpthread-stubs && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libpthread-stubs-${LIBPTHREAD_STUBS_VERSION}.tar.gz && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb-proto && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f xcb-proto-${XCBPROTO_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+RUN \
+        DIR=/tmp/libxcb && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://xcb.freedesktop.org/dist/libxcb-${LIBXCB_VERSION}.tar.gz && \
+        tar -zx --strip-components=1 -f libxcb-${LIBXCB_VERSION}.tar.gz && \
+        ACLOCAL_PATH="${PREFIX}/share/aclocal" ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libxml2 - for libbluray
+RUN \
+        DIR=/tmp/libxml2 && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://gitlab.gnome.org/GNOME/libxml2/-/archive/v${LIBXML2_VERSION}/libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        echo ${LIBXML2_SHA256SUM} | sha256sum --check && \
+        tar -xz --strip-components=1 -f libxml2-v${LIBXML2_VERSION}.tar.gz && \
+        ./autogen.sh --prefix="${PREFIX}" --with-ftp=no --with-http=no --with-python=no && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libbluray - Requires libxml, freetype, and fontconfig
+RUN \
+        DIR=/tmp/libbluray && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://download.videolan.org/pub/videolan/libbluray/${LIBBLURAY_VERSION}/libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        echo ${LIBBLURAY_SHA256SUM} | sha256sum --check && \
+        tar -jx --strip-components=1 -f libbluray-${LIBBLURAY_VERSION}.tar.bz2 && \
+        ./configure --prefix="${PREFIX}" --disable-examples --disable-bdjava-jar --disable-static --enable-shared && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## libzmq https://github.com/zeromq/libzmq/
+RUN \
+        DIR=/tmp/libzmq && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        curl -sLO https://github.com/zeromq/libzmq/archive/v${LIBZMQ_VERSION}.tar.gz && \
+        tar -xz --strip-components=1 -f v${LIBZMQ_VERSION}.tar.gz && \
+        ./autogen.sh && \
+        ./configure --prefix="${PREFIX}" && \
+        make -j $(nproc) && \
+        # make check && \
+        make -j $(nproc) install && \
+        rm -rf ${DIR}
+
+## userland https://github.com/raspberrypi/userland
+RUN \
+        DIR=/tmp/userland && \
+        mkdir -p ${DIR} && \
+        cd ${DIR} && \
+        git clone --depth 1 https://github.com/raspberrypi/userland.git . && \
+        ./buildme && \
+        rm -rf ${DIR}
+
+## ffmpeg https://ffmpeg.org/
+RUN  \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+        tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2
+
+RUN \
+        DIR=/tmp/ffmpeg && mkdir -p ${DIR} && cd ${DIR} && \
+        ./configure \
+        --disable-debug \
+        --disable-doc \
+        --disable-ffplay \
+        --enable-shared \
+        --enable-avresample \
+        --enable-libopencore-amrnb \
+        --enable-libopencore-amrwb \
+        --enable-gpl \
+        --enable-libass \
+        --enable-fontconfig \
+        --enable-libfreetype \
+        --enable-libvidstab \
+        --enable-libmp3lame \
+        --enable-libopus \
+        --enable-libtheora \
+        --enable-libvorbis \
+        --enable-libvpx \
+        --enable-libwebp \
+        --enable-libxcb \
+        --enable-libx265 \
+        --enable-libxvid \
+        --enable-libx264 \
+        --enable-nonfree \
+        --enable-openssl \
+        --enable-libfdk_aac \
+        --enable-postproc \
+        --enable-small \
+        --enable-version3 \
+        --enable-libbluray \
+        --enable-libzmq \
+        --extra-libs=-ldl \
+        --prefix="${PREFIX}" \
+        --enable-libopenjpeg \
+        --enable-libkvazaar \
+        --enable-libaom \
+        --extra-libs=-lpthread \
+        --enable-omx \
+        --enable-omx-rpi \
+        --enable-mmal \
+        --enable-v4l2_m2m \
+        --enable-neon \
+        --extra-cflags="-I${PREFIX}/include" \
+        --extra-ldflags="-L${PREFIX}/lib" && \
+        make -j $(nproc) && \
+        make -j $(nproc) install && \
+        make tools/zmqsend && cp tools/zmqsend ${PREFIX}/bin/ && \
+        make distclean && \
+        hash -r && \
+        cd tools && \
+        make qt-faststart && cp qt-faststart ${PREFIX}/bin/
+
+## cleanup
+RUN \
+        ldd ${PREFIX}/bin/ffmpeg | grep opt/ffmpeg | cut -d ' ' -f 3 | xargs -i cp {} /usr/local/lib/ && \
+        # copy userland lib too
+        ldd ${PREFIX}/bin/ffmpeg | grep opt/vc | cut -d ' ' -f 3 | xargs -i cp {} /usr/local/lib/ && \
+        for lib in /usr/local/lib/*.so.*; do ln -s "${lib##*/}" "${lib%%.so.*}".so; done && \
+        cp ${PREFIX}/bin/* /usr/local/bin/ && \
+        cp -r ${PREFIX}/share/ffmpeg /usr/local/share/ && \
+        LD_LIBRARY_PATH=/usr/local/lib ffmpeg -buildconf && \
+        cp -r ${PREFIX}/include/libav* ${PREFIX}/include/libpostproc ${PREFIX}/include/libsw* /usr/local/include && \
+        mkdir -p /usr/local/lib/pkgconfig && \
+        for pc in ${PREFIX}/lib/pkgconfig/libav*.pc ${PREFIX}/lib/pkgconfig/libpostproc.pc ${PREFIX}/lib/pkgconfig/libsw*.pc; do \
+          sed "s:${PREFIX}:/usr/local:g" <"$pc" >/usr/local/lib/pkgconfig/"${pc##*/}"; \
+        done
+
+FROM        base AS release
+
+ENV         LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/usr/lib:/usr/lib64:/lib:/lib64
+
+RUN \
+        apt-get -yqq update && \
+        apt-get install -yq --no-install-recommends libx265-dev libaom-dev && \
+        apt-get autoremove -y && \
+        apt-get clean -y
+
+CMD         ["--help"]
+ENTRYPOINT  ["ffmpeg"]
+
+COPY --from=build /usr/local /usr/local/

docker/Dockerfile.wheels

@@ -0,0 +1,39 @@
+FROM ubuntu:20.04 as build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -qq update \
+    && apt-get -qq install -y \
+    python3 \
+    python3-dev \
+    wget \
+    # opencv dependencies
+    build-essential cmake git pkg-config libgtk-3-dev \
+    libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
+    libxvidcore-dev libx264-dev libjpeg-dev libpng-dev libtiff-dev \
+    gfortran openexr libatlas-base-dev libssl-dev\
+    libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \
+    libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \
+    # scipy dependencies
+    gcc gfortran libopenblas-dev liblapack-dev cython
+
+RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+    && python3 get-pip.py
+
+RUN pip3 install scikit-build
+
+RUN pip3 wheel --wheel-dir=/wheels \
+    opencv-python-headless \
+    numpy \
+    imutils \
+    scipy \
+    psutil \
+    Flask \
+    paho-mqtt \
+    PyYAML \
+    matplotlib \
+    click
+
+FROM scratch
+
+COPY --from=build /wheels /wheels

docker/Dockerfile.wheels.arm64

@@ -0,0 +1,49 @@
+FROM ubuntu:20.04 as build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -qq update \
+    && apt-get -qq install -y \
+    python3 \
+    python3-dev \
+    wget \
+    # opencv dependencies
+    build-essential cmake git pkg-config libgtk-3-dev \
+    libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
+    libxvidcore-dev libx264-dev libjpeg-dev libpng-dev libtiff-dev \
+    gfortran openexr libatlas-base-dev libssl-dev\
+    libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \
+    libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \
+    # scipy dependencies
+    gcc gfortran libopenblas-dev liblapack-dev cython
+
+RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+    && python3 get-pip.py
+
+# need to build cmake from source because binary distribution is broken for arm64
+# https://github.com/scikit-build/cmake-python-distributions/issues/115
+# https://github.com/skvark/opencv-python/issues/366
+# https://github.com/scikit-build/cmake-python-distributions/issues/96#issuecomment-663062358
+RUN pip3 install scikit-build
+
+RUN git clone https://github.com/scikit-build/cmake-python-distributions.git \
+    && cd cmake-python-distributions/ \
+    && python3 setup.py bdist_wheel
+
+RUN pip3 install cmake-python-distributions/dist/*.whl
+
+RUN pip3 wheel --wheel-dir=/wheels \
+    opencv-python-headless \
+    numpy \
+    imutils \
+    scipy \
+    psutil \
+    Flask \
+    paho-mqtt \
+    PyYAML \
+    matplotlib \
+    click
+
+FROM scratch
+
+COPY --from=build /wheels /wheels

docs/CAMERAS.md

@@ -0,0 +1,23 @@
+# Camera Specific Configuration
+Frigate should work with most RTSP cameras and h264 feeds such as Dahua.
+
+## RTMP Cameras
+The input parameters need to be adjusted for RTMP cameras
+```yaml
+ffmpeg:
+input_args:
+    - -avoid_negative_ts
+    - make_zero
+    - -fflags
+    - nobuffer
+    - -flags
+    - low_delay
+    - -strict
+    - experimental
+    - -fflags
+    - +genpts+discardcorrupt
+    - -vsync
+    - drop
+    - -use_wallclock_as_timestamps
+    - '1'
+```

docs/DEVICES.md

@@ -1,74 +0,0 @@
-# Configuration Examples
-
-### Default (most RTSP cameras)
-This is the default ffmpeg command and should work with most RTSP cameras that send h264 video
-```yaml
-ffmpeg:
-  global_args:
-    - -hide_banner
-    - -loglevel
-    - panic
-  hwaccel_args: []
-  input_args:
-    - -avoid_negative_ts
-    - make_zero 
-    - -fflags
-    - nobuffer
-    - -flags
-    - low_delay
-    - -strict
-    - experimental
-    - -fflags
-    - +genpts+discardcorrupt
-    - -vsync
-    - drop
-    - -rtsp_transport
-    - tcp
-    - -stimeout
-    - '5000000' 
-    - -use_wallclock_as_timestamps
-    - '1'
-  output_args:
-    - -vf
-    - mpdecimate
-    - -f
-    - rawvideo
-    - -pix_fmt
-    - rgb24
-```
-
-### RTMP Cameras
-The input parameters need to be adjusted for RTMP cameras
-```yaml
-ffmpeg:
-input_args:
-    - -avoid_negative_ts
-    - make_zero
-    - -fflags
-    - nobuffer
-    - -flags
-    - low_delay
-    - -strict
-    - experimental
-    - -fflags
-    - +genpts+discardcorrupt
-    - -vsync
-    - drop
-    - -use_wallclock_as_timestamps
-    - '1'
-```
-
-
-### Hardware Acceleration
-
-Intel Quicksync
-```yaml
-ffmpeg:
-  hwaccel_args:
-    - -hwaccel
-    - vaapi
-    - -hwaccel_device
-    - /dev/dri/renderD128
-    - -hwaccel_output_format
-    - yuv420p
-```

docs/HWACCEL.md

@@ -0,0 +1,32 @@
+# Hardware Acceleration for Decoding Video
+
+FFmpeg is compiled to support hardware accelerated decoding of video streams.
+
+## Intel-based CPUs via Quicksync (https://trac.ffmpeg.org/wiki/Hardware/QuickSync)
+```yaml
+ffmpeg:
+  hwaccel_args:
+    - -hwaccel
+    - vaapi
+    - -hwaccel_device
+    - /dev/dri/renderD128
+    - -hwaccel_output_format
+    - yuv420p
+```
+
+## Raspberry Pi 3b and 4 (32bit OS)
+Ensure you increase the allocated RAM for your GPU to at least 128 (raspi-config > Advanced Options > Memory Split)
+```yaml
+ffmpeg:
+  hwaccel_args:
+    - -c:v
+    - h264_mmal
+```
+
+## Raspberry Pi 4 (64bit OS)
+```yaml
+ffmpeg:
+  hwaccel_args:
+    - -c:v
+    - h264_v4l2m2m
+```

frigate/edgetpu.py

@@ -2,12 +2,14 @@ import os
 import datetime
 import hashlib
 import multiprocessing as mp
+import queue
+from multiprocessing.connection import Connection
 from abc import ABC, abstractmethod
+from typing import Dict
 import numpy as np
-import pyarrow.plasma as plasma
 import tflite_runtime.interpreter as tflite
 from tflite_runtime.interpreter import load_delegate
-from frigate.util import EventsPerSecond, listen
+from frigate.util import EventsPerSecond, listen, SharedMemoryFrameManager
 
 def load_labels(path, encoding='utf-8'):
   """Loads labels from file (with or without index numbers).
@@ -46,15 +48,12 @@ class LocalObjectDetector(ObjectDetector):
             device_config = {"device": tf_device}
 
         edge_tpu_delegate = None
+
+        if tf_device != 'cpu':
             try:
                 print(f"Attempting to load TPU as {device_config['device']}")
                 edge_tpu_delegate = load_delegate('libedgetpu.so.1.0', device_config)
                 print("TPU found")
-        except ValueError:
-            try:
-                print(f"Attempting to load TPU as pci:0")
-                edge_tpu_delegate = load_delegate('libedgetpu.so.1.0', {"device": "pci:0"})
-                print("PCIe TPU found")
             except ValueError:
                 print("No EdgeTPU detected. Falling back to CPU.")
         
@@ -100,42 +99,49 @@ class LocalObjectDetector(ObjectDetector):
         
         return detections
 
-def run_detector(detection_queue, avg_speed, start, tf_device):
+def run_detector(detection_queue, out_events: Dict[str, mp.Event], avg_speed, start, tf_device):
     print(f"Starting detection process: {os.getpid()}")
     listen()
-    plasma_client = plasma.connect("/tmp/plasma")
+    frame_manager = SharedMemoryFrameManager()
     object_detector = LocalObjectDetector(tf_device=tf_device)
 
-    while True:
-        object_id_str = detection_queue.get()
-        object_id_hash = hashlib.sha1(str.encode(object_id_str))
-        object_id = plasma.ObjectID(object_id_hash.digest())
-        object_id_out = plasma.ObjectID(hashlib.sha1(str.encode(f"out-{object_id_str}")).digest())
-        input_frame = plasma_client.get(object_id, timeout_ms=0)
+    outputs = {}
+    for name in out_events.keys():
+        out_shm = mp.shared_memory.SharedMemory(name=f"out-{name}", create=False)
+        out_np = np.ndarray((20,6), dtype=np.float32, buffer=out_shm.buf)
+        outputs[name] = {
+            'shm': out_shm,
+            'np': out_np
+        }
     
-        if input_frame is plasma.ObjectNotAvailable:
+    while True:
+        connection_id = detection_queue.get()
+        input_frame = frame_manager.get(connection_id, (1,300,300,3))
+
+        if input_frame is None:
             continue
 
-        # detect and put the output in the plasma store
+        # detect and send the output
         start.value = datetime.datetime.now().timestamp()
-        plasma_client.put(object_detector.detect_raw(input_frame), object_id_out)
+        detections = object_detector.detect_raw(input_frame)
         duration = datetime.datetime.now().timestamp()-start.value
+        outputs[connection_id]['np'][:] = detections[:]
+        out_events[connection_id].set()
         start.value = 0.0
 
         avg_speed.value = (avg_speed.value*9 + duration)/10
         
 class EdgeTPUProcess():
-    def __init__(self, tf_device=None):
-        self.detection_queue = mp.Queue()
+    def __init__(self, detection_queue, out_events, tf_device=None):
+        self.out_events = out_events
+        self.detection_queue = detection_queue
         self.avg_inference_speed = mp.Value('d', 0.01)
         self.detection_start = mp.Value('d', 0.0)
         self.detect_process = None
         self.tf_device = tf_device
         self.start_or_restart()
     
-    def start_or_restart(self):
-        self.detection_start.value = 0.0
-        if (not self.detect_process is None) and self.detect_process.is_alive():
+    def stop(self):
         self.detect_process.terminate()
         print("Waiting for detection process to exit gracefully...")
         self.detect_process.join(timeout=30)
@@ -143,33 +149,41 @@ class EdgeTPUProcess():
             print("Detection process didnt exit. Force killing...")
             self.detect_process.kill()
             self.detect_process.join()
-        self.detect_process = mp.Process(target=run_detector, args=(self.detection_queue, self.avg_inference_speed, self.detection_start, self.tf_device))
+
+    def start_or_restart(self):
+        self.detection_start.value = 0.0
+        if (not self.detect_process is None) and self.detect_process.is_alive():
+            self.stop()
+        self.detect_process = mp.Process(target=run_detector, args=(self.detection_queue, self.out_events, self.avg_inference_speed, self.detection_start, self.tf_device))
         self.detect_process.daemon = True
         self.detect_process.start()
 
 class RemoteObjectDetector():
-    def __init__(self, name, labels, detection_queue):
+    def __init__(self, name, labels, detection_queue, event):
         self.labels = load_labels(labels)
         self.name = name
         self.fps = EventsPerSecond()
-        self.plasma_client = plasma.connect("/tmp/plasma")
         self.detection_queue = detection_queue
+        self.event = event
+        self.shm = mp.shared_memory.SharedMemory(name=self.name, create=False)
+        self.np_shm = np.ndarray((1,300,300,3), dtype=np.uint8, buffer=self.shm.buf)
+        self.out_shm = mp.shared_memory.SharedMemory(name=f"out-{self.name}", create=False)
+        self.out_np_shm = np.ndarray((20,6), dtype=np.float32, buffer=self.out_shm.buf)
     
     def detect(self, tensor_input, threshold=.4):
         detections = []
 
-        now = f"{self.name}-{str(datetime.datetime.now().timestamp())}"
-        object_id_frame = plasma.ObjectID(hashlib.sha1(str.encode(now)).digest())
-        object_id_detections = plasma.ObjectID(hashlib.sha1(str.encode(f"out-{now}")).digest())
-        self.plasma_client.put(tensor_input, object_id_frame)
-        self.detection_queue.put(now)
-        raw_detections = self.plasma_client.get(object_id_detections, timeout_ms=10000)
+        # copy input to shared memory
+        self.np_shm[:] = tensor_input[:]
+        self.event.clear()
+        self.detection_queue.put(self.name)
+        result = self.event.wait(timeout=10.0)
 
-        if raw_detections is plasma.ObjectNotAvailable:
-            self.plasma_client.delete([object_id_frame])
+        # if it timed out
+        if result is None:
             return detections
 
-        for d in raw_detections:
+        for d in self.out_np_shm:
             if d[1] < threshold:
                 break
             detections.append((
@@ -177,6 +191,9 @@ class RemoteObjectDetector():
                 float(d[1]),
                 (d[2], d[3], d[4], d[5])
             ))
-        self.plasma_client.delete([object_id_frame, object_id_detections])
         self.fps.update()
         return detections
+    
+    def cleanup(self):
+        self.shm.unlink()
+        self.out_shm.unlink()

frigate/motion.py

@@ -4,6 +4,7 @@ import numpy as np
 
 class MotionDetector():
     def __init__(self, frame_shape, mask, resize_factor=4):
+        self.frame_shape = frame_shape
         self.resize_factor = resize_factor
         self.motion_frame_size = (int(frame_shape[0]/resize_factor), int(frame_shape[1]/resize_factor))
         self.avg_frame = np.zeros(self.motion_frame_size, np.float)
@@ -16,14 +17,16 @@ class MotionDetector():
     def detect(self, frame):
         motion_boxes = []
 
+        gray = frame[0:self.frame_shape[0], 0:self.frame_shape[1]]
+
         # resize frame
-        resized_frame = cv2.resize(frame, dsize=(self.motion_frame_size[1], self.motion_frame_size[0]), interpolation=cv2.INTER_LINEAR)
+        resized_frame = cv2.resize(gray, dsize=(self.motion_frame_size[1], self.motion_frame_size[0]), interpolation=cv2.INTER_LINEAR)
 
         # convert to grayscale
-        gray = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY)
+        # resized_frame = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY)
 
         # mask frame
-        gray[self.mask] = [255]
+        resized_frame[self.mask] = [255]
 
         # it takes ~30 frames to establish a baseline
         # dont bother looking for motion
@@ -31,7 +34,7 @@ class MotionDetector():
             self.frame_counter += 1
         else:
             # compare to average
-            frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(self.avg_frame))
+            frameDelta = cv2.absdiff(resized_frame, cv2.convertScaleAbs(self.avg_frame))
 
             # compute the average delta over the past few frames
             # the alpha value can be modified to configure how sensitive the motion detection is.
@@ -70,10 +73,10 @@ class MotionDetector():
             # TODO: this really depends on FPS
             if self.motion_frame_count >= 10:
                 # only average in the current frame if the difference persists for at least 3 frames
-                cv2.accumulateWeighted(gray, self.avg_frame, 0.2)
+                cv2.accumulateWeighted(resized_frame, self.avg_frame, 0.2)
         else:
             # when no motion, just keep averaging the frames together
-            cv2.accumulateWeighted(gray, self.avg_frame, 0.2)
+            cv2.accumulateWeighted(resized_frame, self.avg_frame, 0.2)
             self.motion_frame_count = 0
 
         return motion_boxes

frigate/object_processing.py

@@ -10,9 +10,8 @@ import copy
 import numpy as np
 from collections import Counter, defaultdict
 import itertools
-import pyarrow.plasma as plasma
 import matplotlib.pyplot as plt
-from frigate.util import draw_box_with_label, PlasmaFrameManager
+from frigate.util import draw_box_with_label, SharedMemoryFrameManager
 from frigate.edgetpu import load_labels
 from typing import Callable, Dict
 from statistics import mean, median
@@ -59,11 +58,48 @@ class CameraState():
         self.object_status = defaultdict(lambda: 'OFF')
         self.tracked_objects = {}
         self.zone_objects = defaultdict(lambda: [])
-        self.current_frame = np.zeros((720,1280,3), np.uint8)
+        self._current_frame = np.zeros(self.config['frame_shape'], np.uint8)
+        self.current_frame_lock = threading.Lock()
         self.current_frame_time = 0.0
         self.previous_frame_id = None
         self.callbacks = defaultdict(lambda: [])
 
+    def get_current_frame(self, draw=False):
+        with self.current_frame_lock:
+            frame_copy = np.copy(self._current_frame)
+            frame_time = self.current_frame_time
+            tracked_objects = copy.deepcopy(self.tracked_objects)
+        
+        frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
+        # draw on the frame
+        if draw:
+            # draw the bounding boxes on the frame
+            for obj in tracked_objects.values():
+                thickness = 2
+                color = COLOR_MAP[obj['label']]
+                
+                if obj['frame_time'] != frame_time:
+                    thickness = 1
+                    color = (255,0,0)
+
+                # draw the bounding boxes on the frame
+                box = obj['box']
+                draw_box_with_label(frame_copy, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
+                # draw the regions on the frame
+                region = obj['region']
+                cv2.rectangle(frame_copy, (region[0], region[1]), (region[2], region[3]), (0,255,0), 1)
+            
+            if self.config['snapshots']['show_timestamp']:
+                time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
+                cv2.putText(frame_copy, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+
+            if self.config['snapshots']['draw_zones']:
+                for name, zone in self.config['zones'].items():
+                    thickness = 8 if any([name in obj['zones'] for obj in tracked_objects.values()]) else 2
+                    cv2.drawContours(frame_copy, [zone['contour']], -1, zone['color'], thickness)
+        
+        return frame_copy
+
     def false_positive(self, obj):
         # once a true positive, always a true positive
         if not obj.get('false_positive', True):
@@ -88,10 +124,7 @@ class CameraState():
         self.current_frame_time = frame_time
         # get the new frame and delete the old frame
         frame_id = f"{self.name}{frame_time}"
-        self.current_frame = self.frame_manager.get(frame_id)
-        if not self.previous_frame_id is None:
-            self.frame_manager.delete(self.previous_frame_id)
-        self.previous_frame_id = frame_id
+        current_frame = self.frame_manager.get(frame_id, (self.config['frame_shape'][0]*3//2, self.config['frame_shape'][1]))
 
         current_ids = tracked_objects.keys()
         previous_ids = self.tracked_objects.keys()
@@ -154,33 +187,6 @@ class CameraState():
                     current_zones.append(name)
             obj['zones'] = current_zones
 
-        # draw on the frame
-        if not self.current_frame is None:
-            # draw the bounding boxes on the frame
-            for obj in self.tracked_objects.values():
-                thickness = 2
-                color = COLOR_MAP[obj['label']]
-                
-                if obj['frame_time'] != frame_time:
-                    thickness = 1
-                    color = (255,0,0)
-
-                # draw the bounding boxes on the frame
-                box = obj['box']
-                draw_box_with_label(self.current_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
-                # draw the regions on the frame
-                region = obj['region']
-                cv2.rectangle(self.current_frame, (region[0], region[1]), (region[2], region[3]), (0,255,0), 1)
-            
-            if self.config['snapshots']['show_timestamp']:
-                time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
-                cv2.putText(self.current_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
-
-            if self.config['snapshots']['draw_zones']:
-                for name, zone in self.config['zones'].items():
-                    thickness = 8 if any([name in obj['zones'] for obj in self.tracked_objects.values()]) else 2
-                    cv2.drawContours(self.current_frame, [zone['contour']], -1, zone['color'], thickness)
-
         # maintain best objects
         for obj in self.tracked_objects.values():
             object_type = obj['label']
@@ -194,12 +200,12 @@ class CameraState():
                 # if the object is a higher score than the current best score 
                 # or the current object is older than desired, use the new object
                 if obj_copy['score'] > current_best['score'] or (now - current_best['frame_time']) > self.config.get('best_image_timeout', 60):
-                    obj_copy['frame'] = np.copy(self.current_frame)
+                    obj_copy['frame'] = np.copy(current_frame)
                     self.best_objects[object_type] = obj_copy
                     for c in self.callbacks['snapshot']:
                         c(self.name, self.best_objects[object_type])
             else:
-                obj_copy['frame'] = np.copy(self.current_frame)
+                obj_copy['frame'] = np.copy(current_frame)
                 self.best_objects[object_type] = obj_copy
                 for c in self.callbacks['snapshot']:
                     c(self.name, self.best_objects[object_type])
@@ -227,6 +233,11 @@ class CameraState():
             for c in self.callbacks['snapshot']:
                 c(self.name, self.best_objects[obj_name])
         
+        with self.current_frame_lock:
+            self._current_frame = current_frame
+            if not self.previous_frame_id is None:
+                self.frame_manager.delete(self.previous_frame_id)
+            self.previous_frame_id = frame_id
 
 class TrackedObjectProcessor(threading.Thread):
     def __init__(self, camera_config, client, topic_prefix, tracked_objects_queue, event_queue, stop_event):
@@ -238,7 +249,7 @@ class TrackedObjectProcessor(threading.Thread):
         self.event_queue = event_queue
         self.stop_event = stop_event
         self.camera_states: Dict[str, CameraState] = {}
-        self.plasma_client = PlasmaFrameManager(self.stop_event)
+        self.frame_manager = SharedMemoryFrameManager()
 
         def start(camera, obj):
             # publish events to mqtt
@@ -255,7 +266,7 @@ class TrackedObjectProcessor(threading.Thread):
         def snapshot(camera, obj):
             if not 'frame' in obj:
                 return
-            best_frame = cv2.cvtColor(obj['frame'], cv2.COLOR_RGB2BGR)
+            best_frame = cv2.cvtColor(obj['frame'], cv2.COLOR_YUV2BGR_I420)
             mqtt_config = self.camera_config[camera].get('mqtt', {'crop_to_region': False})
             if mqtt_config.get('crop_to_region'):
                 region = obj['region']
@@ -273,7 +284,7 @@ class TrackedObjectProcessor(threading.Thread):
             self.client.publish(f"{self.topic_prefix}/{camera}/{object_name}", status, retain=False)
 
         for camera in self.camera_config.keys():
-            camera_state = CameraState(camera, self.camera_config[camera], self.plasma_client)
+            camera_state = CameraState(camera, self.camera_config[camera], self.frame_manager)
             camera_state.on('start', start)
             camera_state.on('update', update)
             camera_state.on('end', end)
@@ -323,8 +334,8 @@ class TrackedObjectProcessor(threading.Thread):
         else:
             return {}
     
-    def get_current_frame(self, camera):
-        return self.camera_states[camera].current_frame
+    def get_current_frame(self, camera, draw=False):
+        return self.camera_states[camera].get_current_frame(draw)
 
     def run(self):
         while True:

frigate/util.py

@@ -9,7 +9,8 @@ import cv2
 import threading
 import matplotlib.pyplot as plt
 import hashlib
-import pyarrow.plasma as plasma
+from multiprocessing import shared_memory
+from typing import AnyStr
 
 def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'):
     if color is None:
@@ -69,6 +70,37 @@ def calculate_region(frame_shape, xmin, ymin, xmax, ymax, multiplier=2):
 
     return (x_offset, y_offset, x_offset+size, y_offset+size)
 
+def yuv_region_2_rgb(frame, region):
+    height = frame.shape[0]//3*2
+    width = frame.shape[1]
+    # make sure the size is a multiple of 4
+    size = (region[3] - region[1])//4*4
+
+    x1 = region[0] 
+    y1 = region[1]
+
+    uv_x1 = x1//2
+    uv_y1 = y1//4
+
+    uv_width = size//2
+    uv_height = size//4
+
+    u_y_start = height
+    v_y_start = height + height//4
+    two_x_offset = width//2
+
+    yuv_cropped_frame = np.zeros((size+size//2, size), np.uint8)
+    # y channel
+    yuv_cropped_frame[0:size, 0:size] = frame[y1:y1+size, x1:x1+size]
+    # u channel
+    yuv_cropped_frame[size:size+uv_height, 0:uv_width] = frame[uv_y1+u_y_start:uv_y1+u_y_start+uv_height, uv_x1:uv_x1+uv_width]
+    yuv_cropped_frame[size:size+uv_height, uv_width:size] = frame[uv_y1+u_y_start:uv_y1+u_y_start+uv_height, uv_x1+two_x_offset:uv_x1+two_x_offset+uv_width]
+    # v channel
+    yuv_cropped_frame[size+uv_height:size+uv_height*2, 0:uv_width] = frame[uv_y1+v_y_start:uv_y1+v_y_start+uv_height, uv_x1:uv_x1+uv_width]
+    yuv_cropped_frame[size+uv_height:size+uv_height*2, uv_width:size] = frame[uv_y1+v_y_start:uv_y1+v_y_start+uv_height, uv_x1+two_x_offset:uv_x1+two_x_offset+uv_width]
+
+    return cv2.cvtColor(yuv_cropped_frame, cv2.COLOR_YUV2RGB_I420)
+
 def intersection(box_a, box_b):
     return (
         max(box_a[0], box_b[0]),
@@ -148,12 +180,16 @@ def listen():
     signal.signal(signal.SIGUSR1, print_stack)
 
 class FrameManager(ABC):
+    @abstractmethod
+    def create(self, name, size) -> AnyStr:
+        pass
+
     @abstractmethod
     def get(self, name, timeout_ms=0):
         pass
 
     @abstractmethod
-    def put(self, name, frame):
+    def close(self, name):
         pass
 
     @abstractmethod
@@ -164,66 +200,45 @@ class DictFrameManager(FrameManager):
     def __init__(self):
         self.frames = {}
     
-    def get(self, name, timeout_ms=0):
-        return self.frames.get(name)
+    def create(self, name, size) -> AnyStr:
+        mem = bytearray(size)
+        self.frames[name] = mem
+        return mem
     
-    def put(self, name, frame):
-        self.frames[name] = frame
+    def get(self, name, shape):
+        mem = self.frames[name]
+        return np.ndarray(shape, dtype=np.uint8, buffer=mem)
+    
+    def close(self, name):
+        pass
     
     def delete(self, name):
         del self.frames[name]
 
-class PlasmaFrameManager(FrameManager):
-    def __init__(self, stop_event=None):
-        self.stop_event = stop_event
-        self.connect()
+class SharedMemoryFrameManager(FrameManager):
+    def __init__(self):
+        self.shm_store = {}
     
-    def connect(self):
-        while True:
-            if self.stop_event != None and self.stop_event.is_set():
-                return
-            try:
-                self.plasma_client = plasma.connect("/tmp/plasma")
-                return
-            except:
-                print(f"TrackedObjectProcessor: unable to connect plasma client")
-                time.sleep(10)
+    def create(self, name, size) -> AnyStr:
+        shm = shared_memory.SharedMemory(name=name, create=True, size=size)
+        self.shm_store[name] = shm
+        return shm.buf
 
-    def get(self, name, timeout_ms=0):
-        object_id = plasma.ObjectID(hashlib.sha1(str.encode(name)).digest())
-        while True:
-            if self.stop_event != None and self.stop_event.is_set():
-                return
-            try:
-                frame = self.plasma_client.get(object_id, timeout_ms=timeout_ms)
-                if frame is plasma.ObjectNotAvailable:
-                    return None
-                return frame
-            except:
-                self.connect()
-                time.sleep(1)
+    def get(self, name, shape):
+        if name in self.shm_store:
+            shm = self.shm_store[name]
+        else:
+            shm = shared_memory.SharedMemory(name=name)
+            self.shm_store[name] = shm
+        return np.ndarray(shape, dtype=np.uint8, buffer=shm.buf)
 
-    def put(self, name, frame):
-        object_id = plasma.ObjectID(hashlib.sha1(str.encode(name)).digest())
-        while True:
-            if self.stop_event != None and self.stop_event.is_set():
-                return
-            try:
-                self.plasma_client.put(frame, object_id)
-                return
-            except Exception as e:
-                print(f"Failed to put in plasma: {e}")
-                self.connect()
-                time.sleep(1)
+    def close(self, name):
+        if name in self.shm_store:
+            self.shm_store[name].close()
+            del self.shm_store[name]
 
     def delete(self, name):
-        object_id = plasma.ObjectID(hashlib.sha1(str.encode(name)).digest())
-        while True:
-            if self.stop_event != None and self.stop_event.is_set():
-                return
-            try:
-                self.plasma_client.delete([object_id])
-                return
-            except:
-                self.connect()
-                time.sleep(1)
+        if name in self.shm_store:
+            self.shm_store[name].close()
+            self.shm_store[name].unlink()
+            del self.shm_store[name]

frigate/video.py

@@ -5,7 +5,6 @@ import cv2
 import queue
 import threading
 import ctypes
-import pyarrow.plasma as plasma
 import multiprocessing as mp
 import subprocess as sp
 import numpy as np
@@ -15,7 +14,7 @@ import json
 import base64
 from typing import Dict, List
 from collections import defaultdict
-from frigate.util import draw_box_with_label, area, calculate_region, clipped, intersection_over_union, intersection, EventsPerSecond, listen, FrameManager, PlasmaFrameManager
+from frigate.util import draw_box_with_label, yuv_region_2_rgb, area, calculate_region, clipped, intersection_over_union, intersection, EventsPerSecond, listen, FrameManager, SharedMemoryFrameManager
 from frigate.objects import ObjectTracker
 from frigate.edgetpu import RemoteObjectDetector
 from frigate.motion import MotionDetector
@@ -89,7 +88,7 @@ def filtered(obj, objects_to_track, object_filters, mask=None):
     return False
 
 def create_tensor_input(frame, region):
-    cropped_frame = frame[region[1]:region[3], region[0]:region[2]]
+    cropped_frame = yuv_region_2_rgb(frame, region)
 
     # Resize to 300x300 if needed
     if cropped_frame.shape != (300, 300, 3):
@@ -118,11 +117,10 @@ def start_or_restart_ffmpeg(ffmpeg_cmd, frame_size, ffmpeg_process=None):
 
 def capture_frames(ffmpeg_process, camera_name, frame_shape, frame_manager: FrameManager, 
     frame_queue, take_frame: int, fps:EventsPerSecond, skipped_fps: EventsPerSecond, 
-    stop_event: mp.Event, detection_frame: mp.Value, current_frame: mp.Value):
+    stop_event: mp.Event, current_frame: mp.Value):
 
     frame_num = 0
-    last_frame = 0
-    frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
+    frame_size = frame_shape[0] * frame_shape[1] * 3 // 2
     skipped_fps.start()
     while True:
         if stop_event.is_set():
@@ -148,23 +146,21 @@ def capture_frames(ffmpeg_process, camera_name, frame_shape, frame_manager: Fram
             skipped_fps.update()
             continue
 
-        # if the detection process is more than 1 second behind, skip this frame
-        if detection_frame.value > 0.0 and (last_frame - detection_frame.value) > 1:
+        # if the queue is full, skip this frame
+        if frame_queue.full():
             skipped_fps.update()
             continue
 
         # put the frame in the frame manager
-        frame_manager.put(f"{camera_name}{current_frame.value}",
-                np
-                    .frombuffer(frame_bytes, np.uint8)
-                    .reshape(frame_shape)
-            )
+        frame_buffer = frame_manager.create(f"{camera_name}{current_frame.value}", frame_size)
+        frame_buffer[:] = frame_bytes[:]
+        frame_manager.close(f"{camera_name}{current_frame.value}")
+
         # add to the queue
         frame_queue.put(current_frame.value)
-        last_frame = current_frame.value
 
 class CameraCapture(threading.Thread):
-    def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps, detection_frame, stop_event):
+    def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps, stop_event):
         threading.Thread.__init__(self)
         self.name = name
         self.frame_shape = frame_shape
@@ -173,19 +169,18 @@ class CameraCapture(threading.Thread):
         self.take_frame = take_frame
         self.fps = fps
         self.skipped_fps = EventsPerSecond()
-        self.plasma_client = PlasmaFrameManager(stop_event)
+        self.frame_manager = SharedMemoryFrameManager()
         self.ffmpeg_process = ffmpeg_process
         self.current_frame = mp.Value('d', 0.0)
         self.last_frame = 0
-        self.detection_frame = detection_frame
         self.stop_event = stop_event
 
     def run(self):
         self.skipped_fps.start()
-        capture_frames(self.ffmpeg_process, self.name, self.frame_shape, self.plasma_client, self.frame_queue, self.take_frame,
-            self.fps, self.skipped_fps, self.stop_event, self.detection_frame, self.current_frame)
+        capture_frames(self.ffmpeg_process, self.name, self.frame_shape, self.frame_manager, self.frame_queue, self.take_frame,
+            self.fps, self.skipped_fps, self.stop_event, self.current_frame)
 
-def track_camera(name, config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, detection_fps, read_start, detection_frame, stop_event):
+def track_camera(name, config, frame_queue, frame_shape, detection_queue, result_connection, detected_objects_queue, fps, detection_fps, read_start, detection_frame, stop_event):
     print(f"Starting process for {name}: {os.getpid()}")
     listen()
 
@@ -218,13 +213,13 @@ def track_camera(name, config, frame_queue, frame_shape, detection_queue, detect
         mask[:] = 255
 
     motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
-    object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue)
+    object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue, result_connection)
 
     object_tracker = ObjectTracker(10)
 
-    plasma_client = PlasmaFrameManager()
+    frame_manager = SharedMemoryFrameManager()
 
-    process_frames(name, frame_queue, frame_shape, plasma_client, motion_detector, object_detector,
+    process_frames(name, frame_queue, frame_shape, frame_manager, motion_detector, object_detector,
         object_tracker, detected_objects_queue, fps, detection_fps, detection_frame, objects_to_track, object_filters, mask, stop_event)
 
     print(f"{name}: exiting subprocess")
@@ -281,7 +276,7 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape,
         
         current_frame_time.value = frame_time
 
-        frame = frame_manager.get(f"{camera_name}{frame_time}")
+        frame = frame_manager.get(f"{camera_name}{frame_time}", (frame_shape[0]*3//2, frame_shape[1]))
 
         if frame is None:
             print(f"{camera_name}: frame {frame_time} is not in memory store.")
@@ -364,3 +359,5 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape,
         detected_objects_queue.put((camera_name, frame_time, object_tracker.tracked_objects))
 
         detection_fps.value = object_detector.fps.eps()
+
+        frame_manager.close(f"{camera_name}{frame_time}")

process_clip.py

@@ -4,7 +4,7 @@ import os
 import datetime
 from unittest import TestCase, main
 from frigate.video import process_frames, start_or_restart_ffmpeg, capture_frames, get_frame_shape
-from frigate.util import DictFrameManager, EventsPerSecond, draw_box_with_label
+from frigate.util import DictFrameManager, SharedMemoryFrameManager, EventsPerSecond, draw_box_with_label
 from frigate.motion import MotionDetector
 from frigate.edgetpu import LocalObjectDetector
 from frigate.objects import ObjectTracker
@@ -19,6 +19,7 @@ class ProcessClip():
         self.frame_shape = frame_shape
         self.camera_name = 'camera'
         self.frame_manager = DictFrameManager()
+        # self.frame_manager = SharedMemoryFrameManager()
         self.frame_queue = mp.Queue()
         self.detected_objects_queue = mp.Queue()
         self.camera_state = CameraState(self.camera_name, config, self.frame_manager)
@@ -72,13 +73,15 @@ class ProcessClip():
             for obj in self.camera_state.tracked_objects.values():
                 print(f"{frame_time}: {obj['id']} - {obj['computed_score']} - {obj['score_history']}")
         
+        self.frame_manager.delete(self.camera_state.previous_frame_id)
+        
         return {
             'object_detected': obj_detected,
             'top_score': top_computed_score
         }
     
     def save_debug_frame(self, debug_path, frame_time, tracked_objects):
-        current_frame = self.frame_manager.get(f"{self.camera_name}{frame_time}")
+        current_frame = self.frame_manager.get(f"{self.camera_name}{frame_time}", self.frame_shape)
         # draw the bounding boxes on the frame
         for obj in tracked_objects:
             thickness = 2
@@ -132,6 +135,7 @@ def process(path, label, threshold, debug_path):
     results = []
     for c in clips:
         frame_shape = get_frame_shape(c)
+        config['frame_shape'] = frame_shape
         process_clip = ProcessClip(c, frame_shape, config)
         process_clip.load_frames()
         process_clip.process_frames(objects_to_track=config['objects']['track'])