capnp generating, struggling to decode

TODO

@@ -2,6 +2,7 @@
 - [x] control interface
 - [x] camera switcher
 - [ ] replacetrack is broken https://github.com/aiortc/aiortc/issues/304 and the team seems uninterested, meanwhile Pion has been supporting it for 3 years https://github.com/pion/webrtc/pull/1527 so porting to go may really make sense. maybe other issues around performance will be resolved too by doing so.
+      currently struggling with decoding the hevc but perhaps we can remove goav now and start using pion for these functions...  https://github.com/pion/rtp/blob/master/codecs/h265_packet_test.go
 - battery view
 - lowcam view
 - multicam

compile.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+# Cap’n Proto 
+pushd cereal
+git checkout *.capnp
+rm -rf gen/go
+for f in *.capnp; do
+  name=$(echo $f | sed 's/.capnp//')
+  mv $f $f.bak
+  echo "using Go = import \"/go.capnp\";" > $name.capnp
+  echo "\$Go.package(\"cereal\");" >> $name.capnp
+  echo "\$Go.import(\"cereal\");" >> $name.capnp
+  cat $f.bak >> $name.capnp
+  rm $f.bak
+done
+mkdir gen/go
+go install capnproto.org/go/capnp/v3/capnpc-go@latest
+capnp compile -I$(go env GOPATH)/src/capnproto.org/go/capnp/std -o go:gen/go *.capnp
+git checkout *.capnp
+cat <<EOF > gen/go/go.mod
+module github.com/commaai/cereal
+
+go 1.19
+EOF
+popd
+
+
+# Go program
+go build -o webrtc-body main.go

go.mod

@@ -5,3 +5,12 @@ go 1.19
 require github.com/pebbe/zmq4 v1.2.9
 
 require github.com/giorgisio/goav v0.1.0
+
+require (
+	capnproto.org/go/capnp/v3 v3.0.0-alpha.24 // indirect
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	github.com/commaai/cereal v0.0.0
+)
+
+
+replace "github.com/commaai/cereal" v0.0.0 => "./cereal/gen/go"

go.sum

@@ -1,6 +1,10 @@
+capnproto.org/go/capnp/v3 v3.0.0-alpha.24 h1:Vxks4FsGgR6hE4A1wcrQC4BOkQANvgOXCcFmPYhHqls=
+capnproto.org/go/capnp/v3 v3.0.0-alpha.24/go.mod h1:Dynqh9/LE2Wy7jYd2wIoYgqFwh3hZHCmG7j6/uCWX6c=
 github.com/giorgisio/goav v0.1.0 h1:ZyfG3NfX7PMSimv4ulhmnQJf/XeHpMdGCn+afRmY5Oc=
 github.com/giorgisio/goav v0.1.0/go.mod h1:RtH8HyxLRLU1iY0pjfhWBKRhnbsnmfoI+FxMwb5bfEo=
 github.com/gosuri/uilive v0.0.0-20170323041506-ac356e6e42cd/go.mod h1:qkLSc0A5EXSP6B04TrN4oQoxqFI7A8XvoXSlJi8cwk8=
 github.com/gosuri/uiprogress v0.0.0-20170224063937-d0567a9d84a1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
 github.com/pebbe/zmq4 v1.2.9 h1:JlHcdgq6zpppNR1tH0wXJq0XK03pRUc4lBlHTD7aj/4=
 github.com/pebbe/zmq4 v1.2.9/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

main.go

@@ -3,10 +3,15 @@ package main
 import (
 	"fmt"
 	"os"
+	"unsafe"
 
+	"capnproto.org/go/capnp/v3"
 	"github.com/giorgisio/goav/avcodec"
+	"github.com/giorgisio/goav/avutil"
 
 	zmq "github.com/pebbe/zmq4"
+
+	"github.com/commaai/cereal"
 )
 
 type Service struct {
@@ -95,37 +100,97 @@ func main() {
 		os.Exit(1)
 	}
 
+	var lastIdx = -1
+	var seenIframe = false
+	var V4L2_BUF_FLAG_KEYFRAME = uint32(8)
 	for {
+		// Allocate video frame
+		pFrame := avutil.AvFrameAlloc()
 		var frame []byte
 		for frame == nil {
 			msgs := drainSock(subscriber, true)
 			if len(msgs) > 0 {
-				/* Port the following Python aiortc code to Golang for Pion
+				for _, rawMsg := range msgs {
-				   with evt_context as evt:
+					msg, err := capnp.Unmarshal(rawMsg)
-				       evta = getattr(evt, evt.which())
+					if err != nil {
-				       if evta.idx.encodeId != 0 and evta.idx.encodeId != (self.last_idx+1):
+						panic(err)
-				           print("DROP PACKET!")
+					}
-				       self.last_idx = evta.idx.encodeId
-				       if not self.seen_iframe and not (evta.idx.flags & V4L2_BUF_FLAG_KEYFRAME):
-				           print("waiting for iframe")
-				           continue
-				       self.time_q.append(time.monotonic())
 
-				       # put in header (first)
+					evt, err := cereal.ReadRootEncodeData(msg)
-				       if not self.seen_iframe:
+					if err != nil {
-				           self.codec.decode(av.packet.Packet(evta.header))
+						panic(err)
-				           self.seen_iframe = True
+					}
 
-				       frames = self.codec.decode(av.packet.Packet(evta.data))
+					ts := evt.UnixTimestampNanos()
-				       if len(frames) == 0:
-				           print("DROP SURFACE")
-				           continue
-				       assert len(frames) == 1
 
-				       frame = frames[0]
+					idx, err := evt.Idx()
-				       frame.pts = pts
+					if err != nil {
-				       frame.time_base = time_base
+						panic(err)
-				*/
+					}
+					encodeId := idx.EncodeId()
+
+					fmt.Printf("ts: %d,encodeId: %d\n", ts, encodeId)
+
+					if encodeId != 0 && encodeId != uint32(lastIdx+1) {
+						fmt.Println("DROP PACKET!")
+					} else {
+						lastIdx = int(encodeId)
+
+						idxFlags := idx.Flags()
+
+						if !seenIframe && (idxFlags&V4L2_BUF_FLAG_KEYFRAME) != 0 {
+							fmt.Println("waiting for iframe")
+							continue
+						}
+
+						if !seenIframe {
+							// Decode video frame
+							pkt := avcodec.AvPacketAlloc()
+							if pkt == nil {
+								panic("cannot allocate packet")
+							}
+
+							pkt.AvInitPacket()
+							pkt.SetFlags(pkt.Flags() | avcodec.AV_CODEC_FLAG_TRUNCATED)
+
+							response := codecContext.AvcodecSendPacket(packet)
+							if response < 0 {
+								fmt.Printf("Error while sending a packet to the decoder: %s\n", avutil.ErrorFromCode(response))
+								panic("decoding error")
+							}
+							if response >= 0 {
+								response = codecContext.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(pFrame)))
+								if response == avutil.AvErrorEAGAIN || response == avutil.AvErrorEOF {
+									break
+								} else if response < 0 {
+									fmt.Printf("Error while receiving a frame from the decoder: %s\n", avutil.ErrorFromCode(response))
+									fmt.Println("DROP SURFACE")
+									continue
+								}
+								seenIframe = true
+							}
+						}
+
+						// frames, err := codec.Decode(av.Packet{Data: evt.Data()})
+						// if err != nil {
+						// 	fmt.Println("DROP SURFACE")
+						// 	continue
+						// }
+						// if len(frames) == 0 {
+						// 	fmt.Println("DROP SURFACE")
+						// 	continue
+						// }
+						// assert(len(frames) == 1)
+
+						// frame = frames[0].Data
+						// framePts := frames[0].Pts
+						// frameTimebase := frames[0].Timebase
+
+						// frame = append(append(make([]byte, 0, len(frame)+FrameHeaderLen), encodeUint32(uint32(len(frame)))...), frame...)
+						// binary.LittleEndian.PutUint32(frame[4:], encodeUint64(framePts))
+						// binary.LittleEndian.PutUint32(frame[12:], encodeUint64(frameTimebase))
+					}
+				}
 			}
 		}
 		fmt.Println("Received frame with size:", len(frame), "bytes")