init

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "cereal"]
+	path = cereal
+	url = https://github.com/commaai/cereal.git

bash_history.txt

@@ -0,0 +1,26 @@
+    1  sudo apt update
+    2  sudo apt upgrade
+   12  sudo apt install vim git 
+   14  sudo apt install python3-distutils
+   31  sudo apt update
+   33  sudo apt install tmux apt git
+   49  sudo apt install python-dev
+   50  sudo apt install python3-dev
+   91  sudo apt install iputils-tracepathj
+   92  sudo apt install iputils-tracepath
+  167  apt-get update && apt-get install -y --no-install-recommends     autoconf     build-essential     ca-certificates     capnproto     clang     cppcheck     curl     git     libbz2-dev     libcapnp-dev     libffi-dev     liblzma-dev     libncurses5-dev     libncursesw5-dev     libreadline-dev     libsqlite3-dev     libssl-dev     libtool     libzmq3-dev     llvm     make     ocl-icd-opencl-dev     opencl-headers      python-openssl     tk-dev     wget     xz-utils     zlib1g-dev
+  168  sudo apt-get update && sudo apt-get install -y --no-install-recommends     autoconf     build-essential     ca-certificates     capnproto     clang     cppcheck     curl     git     libbz2-dev     libcapnp-dev     libffi-dev     liblzma-dev     libncurses5-dev     libncursesw5-dev     libreadline-dev     libsqlite3-dev     libssl-dev     libtool     libzmq3-dev     llvm     make     ocl-icd-opencl-dev     opencl-headers      python-openssl     tk-dev     wget     xz-utils     zlib1g-dev
+  176  apt install libcapnp-dev
+  177  sudo apt install libcapnp-dev
+  178  apt install libcapnp-dev
+  180  sudo apt install capnp
+  181  sudo apt install capnproto
+  183  sudo apt install clang
+  185  sudo apt install libzmq3-dev 
+  187  sudo apt install opencl-headers
+  190  sudo apt install ocl-icd-opencl-dev
+
+# curl -sSL https://install.python-poetry.org | python3 -
+# poetry add ./cereal
+# poetry add pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==1.0.0 parameterized==0.7.4 numpy==1.21.1
+

client.js

@@ -0,0 +1,76 @@
+var pc = null;
+
+function negotiate() {
+    pc.addTransceiver('video', {direction: 'recvonly'});
+    pc.addTransceiver('audio', {direction: 'recvonly'});
+    return pc.createOffer().then(function(offer) {
+        return pc.setLocalDescription(offer);
+    }).then(function() {
+        // wait for ICE gathering to complete
+        return new Promise(function(resolve) {
+            if (pc.iceGatheringState === 'complete') {
+                resolve();
+            } else {
+                function checkState() {
+                    if (pc.iceGatheringState === 'complete') {
+                        pc.removeEventListener('icegatheringstatechange', checkState);
+                        resolve();
+                    }
+                }
+                pc.addEventListener('icegatheringstatechange', checkState);
+            }
+        });
+    }).then(function() {
+        var offer = pc.localDescription;
+        return fetch('/offer', {
+            body: JSON.stringify({
+                sdp: offer.sdp,
+                type: offer.type,
+            }),
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            method: 'POST'
+        });
+    }).then(function(response) {
+        return response.json();
+    }).then(function(answer) {
+        return pc.setRemoteDescription(answer);
+    }).catch(function(e) {
+        alert(e);
+    });
+}
+
+function start() {
+    var config = {
+        sdpSemantics: 'unified-plan'
+    };
+
+    if (document.getElementById('use-stun').checked) {
+        config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}];
+    }
+
+    pc = new RTCPeerConnection(config);
+
+    // connect audio / video
+    pc.addEventListener('track', function(evt) {
+        if (evt.track.kind == 'video') {
+            document.getElementById('video').srcObject = evt.streams[0];
+        } else {
+            document.getElementById('audio').srcObject = evt.streams[0];
+        }
+    });
+
+    document.getElementById('start').style.display = 'none';
+    negotiate();
+    document.getElementById('stop').style.display = 'inline-block';
+}
+
+function stop() {
+    document.getElementById('stop').style.display = 'none';
+
+    // close peer connection
+    setTimeout(function() {
+        pc.close();
+    }, 500);
+}

compressed_vipc_track.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import cereal.messaging as messaging
+from aiortc import VideoStreamTrack
+import asyncio
+import numpy as np
+import av
+import os
+import sys
+import time
+
+W, H = 1928, 1208
+V4L2_BUF_FLAG_KEYFRAME = 8
+
+class VisionIpcTrack(VideoStreamTrack):
+  def __init__(self, sock_name, addr):
+    super().__init__()
+    self.codec = av.CodecContext.create("hevc", "r")
+    os.environ["ZMQ"] = "1"
+    messaging.context = messaging.Context()
+    self.sock = messaging.sub_sock(sock_name, None, addr=addr, conflate=False)
+    self.cnt = 0
+    self.last_idx = -1
+    self.seen_iframe = False
+    self.time_q = []
+    self.sock_name = sock_name
+
+  async def recv(self):
+    pts, time_base = await self.next_timestamp()
+    frame = None
+    while frame is None:
+      msgs = messaging.drain_sock(self.sock, wait_for_one=True)
+      for evt in msgs:
+        print(type(evt).__name__)
+        evta = getattr(evt, evt.which())
+        if evta.idx.encodeId != 0 and evta.idx.encodeId != (self.last_idx+1):
+          print("DROP PACKET!")
+        self.last_idx = evta.idx.encodeId
+        if not self.seen_iframe and not (evta.idx.flags & V4L2_BUF_FLAG_KEYFRAME):
+          print("waiting for iframe")
+          continue
+        self.time_q.append(time.monotonic())
+
+        # put in header (first)
+        if not self.seen_iframe:
+          self.codec.decode(av.packet.Packet(evta.header))
+          self.seen_iframe = True
+
+        frames = self.codec.decode(av.packet.Packet(evta.data))
+        if len(frames) == 0:
+          print("DROP SURFACE")
+          continue
+        assert len(frames) == 1
+
+        frame = frames[0]
+        frame.pts = pts
+        frame.time_base = time_base
+    return frame
+
+if __name__ == "__main__":
+    from time import time_ns
+    import sys
+
+    async def test():
+        frame_count=0
+        start_time=time_ns()
+        track = VisionIpcTrack("roadEncodeData", "192.168.99.200")
+        while True:
+            await track.recv()
+            now = time_ns()
+            playtime = now - start_time
+            playtime_sec = playtime * 0.000000001
+            if playtime_sec >= 1:
+                print(f'fps: {frame_count}')
+                frame_count = 0
+                start_time = time_ns()
+            else:
+                frame_count+=1
+
+    # Run event loop
+    loop = asyncio.new_event_loop()
+    try:
+        loop.run_until_complete(test())
+    except KeyboardInterrupt:
+        sys.exit(0)

index.html

@@ -0,0 +1,40 @@
+<html>
+<head>
+    <meta charset="UTF-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>WebRTC webcam</title>
+    <style>
+    button {
+        padding: 8px 16px;
+    }
+
+    video {
+        width: 100%;
+    }
+
+    .option {
+        margin-bottom: 8px;
+    }
+
+    #media {
+        max-width: 1280px;
+    }
+    </style>
+</head>
+<body>
+
+<div class="option">
+    <input id="use-stun" type="checkbox" checked/>
+    <label for="use-stun">Use STUN server</label>
+</div>
+<button id="start" onclick="start()">Start</button>
+<button id="stop" style="display: none" onclick="stop()">Stop</button>
+
+<div id="media">
+    <audio id="audio" autoplay="true"></audio>
+    <video id="video" autoplay="true" playsinline="true"></video>
+</div>
+
+<script src="client.js"></script>
+</body>
+</html>

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "test"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+scons = "3.1.1"
+pycapnp = "1.0.0"
+pyyaml = "5.1.2"
+cython = "0.29.14"
+parameterized = "0.7.4"
+numpy = "1.21.1"
+aiohttp = "^3.8.3"
+aiortc = "^1.3.2"
+
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

server.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+import argparse
+import asyncio
+import json
+import logging
+import os
+import ssl
+from typing import OrderedDict
+from aiohttp import web
+from aiortc import RTCPeerConnection, RTCSessionDescription, RTCRtpCodecCapability
+from compressed_vipc_track import VisionIpcTrack
+
+ROOT = os.path.dirname(__file__)
+
+cams = ["roadEncodeData","wideRoadEncodeData","driverEncodeData"]
+
+async def index(request):
+    content = open(os.path.join(ROOT, "index.html"), "r").read()
+    return web.Response(content_type="text/html", text=content)
+
+async def javascript(request):
+    content = open(os.path.join(ROOT, "client.js"), "r").read()
+    return web.Response(content_type="application/javascript", text=content)
+
+
+async def offer(request):
+    params = await request.json()
+    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
+
+    pc = RTCPeerConnection()
+    pcs.add(pc)
+
+    @pc.on("connectionstatechange")
+    async def on_connectionstatechange():
+        print("Connection state is %s" % pc.connectionState)
+        if pc.connectionState == "failed":
+            await pc.close()
+            pcs.discard(pc)
+
+    # TODO: stream the microphone
+    audio = None
+
+    video = VisionIpcTrack(cams[int(args.cam)], args.addr)
+
+    video_sender = pc.addTrack(video)
+    transceiver = next(t for t in pc.getTransceivers() if t.sender == video_sender)
+    transceiver.setCodecPreferences(
+        # [codec for codec in codecs if codec.mimeType == forced_codec]
+        [RTCRtpCodecCapability(
+                        mimeType="video/H264",
+                        clockRate=90000,
+                        channels=None,
+                        parameters=OrderedDict([
+                        ("packetization-mode", "1"),
+                        ("level-asymmetry-allowed", "1"),
+                        ("profile-level-id", "42001f"),
+                        ])
+                    )]
+    )
+
+    await pc.setRemoteDescription(offer)
+
+    answer = await pc.createAnswer()
+    await pc.setLocalDescription(answer)
+
+    return web.Response(
+        content_type="application/json",
+        text=json.dumps(
+            {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
+        ),
+    )
+
+
+pcs = set()
+
+
+async def on_shutdown(app):
+    # close peer connections
+    coros = [pc.close() for pc in pcs]
+    await asyncio.gather(*coros)
+    pcs.clear()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Decode video streams and broadcast via WebRTC")
+    parser.add_argument("addr", help="Address of comma three")
+
+    # Not implemented (yet?). Geo already made the PoC for this, it should be possible.
+    # parser.add_argument("--nvidia", action="store_true", help="Use nvidia instead of ffmpeg")
+
+    parser.add_argument("--cam", default="0", help="Camera to stream")
+
+    parser.add_argument("--cert-file", help="SSL certificate file (for HTTPS)")
+    parser.add_argument("--key-file", help="SSL key file (for HTTPS)")
+    parser.add_argument(
+        "--host", default="0.0.0.0", help="Host for HTTP server (default: 0.0.0.0)"
+    )
+    parser.add_argument(
+        "--port", type=int, default=8080, help="Port for HTTP server (default: 8080)"
+    )
+    parser.add_argument("--verbose", "-v", action="count")
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    if args.cert_file:
+        ssl_context = ssl.SSLContext()
+        ssl_context.load_cert_chain(args.cert_file, args.key_file)
+    else:
+        ssl_context = None
+
+    app = web.Application()
+    app.on_shutdown.append(on_shutdown)
+    app.router.add_get("/", index)
+    app.router.add_get("/client.js", javascript)
+    app.router.add_post("/offer", offer)
+    web.run_app(app, host=args.host, port=args.port, ssl_context=ssl_context)