From 0ea21a1bb8230fc5c9d91f7e124d565a938d1707 Mon Sep 17 00:00:00 2001 From: Keyvan Fatehi Date: Mon, 13 Feb 2023 21:50:09 -0800 Subject: [PATCH] capnp generating, struggling to decode --- TODO | 1 + cereal | 2 +- compile.sh | 30 +++++++++++++++ go.mod | 9 +++++ go.sum | 4 ++ main.go | 111 ++++++++++++++++++++++++++++++++++++++++++----------- 6 files changed, 133 insertions(+), 24 deletions(-) create mode 100755 compile.sh diff --git a/TODO b/TODO index cf6cd0d..6c67b15 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,7 @@ - [x] control interface - [x] camera switcher - [ ] replacetrack is broken https://github.com/aiortc/aiortc/issues/304 and the team seems uninterested, meanwhile Pion has been supporting it for 3 years https://github.com/pion/webrtc/pull/1527 so porting to go may really make sense. maybe other issues around performance will be resolved too by doing so. + currently struggling with decoding the hevc but perhaps we can remove goav now and start using pion for these functions... https://github.com/pion/rtp/blob/master/codecs/h265_packet_test.go - battery view - lowcam view - multicam diff --git a/cereal b/cereal index f200875..92a98ff 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit f200875ca300d3a7b9293c4effcc9456e359e505 +Subproject commit 92a98ff3fd01dbc44029994faabb9e467846346c diff --git a/compile.sh b/compile.sh new file mode 100755 index 0000000..6d07353 --- /dev/null +++ b/compile.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +# Cap’n Proto +pushd cereal +git checkout *.capnp +rm -rf gen/go +for f in *.capnp; do + name=$(echo $f | sed 's/.capnp//') + mv $f $f.bak + echo "using Go = import \"/go.capnp\";" > $name.capnp + echo "\$Go.package(\"cereal\");" >> $name.capnp + echo "\$Go.import(\"cereal\");" >> $name.capnp + cat $f.bak >> $name.capnp + rm $f.bak +done +mkdir gen/go +go install capnproto.org/go/capnp/v3/capnpc-go@latest +capnp compile -I$(go env GOPATH)/src/capnproto.org/go/capnp/std -o go:gen/go *.capnp +git checkout *.capnp +cat < gen/go/go.mod +module github.com/commaai/cereal + +go 1.19 +EOF +popd + + +# Go program +go build -o webrtc-body main.go \ No newline at end of file diff --git a/go.mod b/go.mod index 846bf2b..e64a175 100644 --- a/go.mod +++ b/go.mod @@ -5,3 +5,12 @@ go 1.19 require github.com/pebbe/zmq4 v1.2.9 require github.com/giorgisio/goav v0.1.0 + +require ( + capnproto.org/go/capnp/v3 v3.0.0-alpha.24 // indirect + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect + github.com/commaai/cereal v0.0.0 +) + + +replace "github.com/commaai/cereal" v0.0.0 => "./cereal/gen/go" diff --git a/go.sum b/go.sum index 0535d50..fa0ba43 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,10 @@ +capnproto.org/go/capnp/v3 v3.0.0-alpha.24 h1:Vxks4FsGgR6hE4A1wcrQC4BOkQANvgOXCcFmPYhHqls= +capnproto.org/go/capnp/v3 v3.0.0-alpha.24/go.mod h1:Dynqh9/LE2Wy7jYd2wIoYgqFwh3hZHCmG7j6/uCWX6c= github.com/giorgisio/goav v0.1.0 h1:ZyfG3NfX7PMSimv4ulhmnQJf/XeHpMdGCn+afRmY5Oc= github.com/giorgisio/goav v0.1.0/go.mod h1:RtH8HyxLRLU1iY0pjfhWBKRhnbsnmfoI+FxMwb5bfEo= github.com/gosuri/uilive v0.0.0-20170323041506-ac356e6e42cd/go.mod h1:qkLSc0A5EXSP6B04TrN4oQoxqFI7A8XvoXSlJi8cwk8= github.com/gosuri/uiprogress v0.0.0-20170224063937-d0567a9d84a1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/pebbe/zmq4 v1.2.9 h1:JlHcdgq6zpppNR1tH0wXJq0XK03pRUc4lBlHTD7aj/4= github.com/pebbe/zmq4 v1.2.9/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/main.go b/main.go index 5a58ae9..cb1653b 100644 --- a/main.go +++ b/main.go @@ -3,10 +3,15 @@ package main import ( "fmt" "os" + "unsafe" + "capnproto.org/go/capnp/v3" "github.com/giorgisio/goav/avcodec" + "github.com/giorgisio/goav/avutil" zmq "github.com/pebbe/zmq4" + + "github.com/commaai/cereal" ) type Service struct { @@ -95,37 +100,97 @@ func main() { os.Exit(1) } + var lastIdx = -1 + var seenIframe = false + var V4L2_BUF_FLAG_KEYFRAME = uint32(8) for { + // Allocate video frame + pFrame := avutil.AvFrameAlloc() var frame []byte for frame == nil { msgs := drainSock(subscriber, true) if len(msgs) > 0 { - /* Port the following Python aiortc code to Golang for Pion - with evt_context as evt: - evta = getattr(evt, evt.which()) - if evta.idx.encodeId != 0 and evta.idx.encodeId != (self.last_idx+1): - print("DROP PACKET!") - self.last_idx = evta.idx.encodeId - if not self.seen_iframe and not (evta.idx.flags & V4L2_BUF_FLAG_KEYFRAME): - print("waiting for iframe") - continue - self.time_q.append(time.monotonic()) + for _, rawMsg := range msgs { + msg, err := capnp.Unmarshal(rawMsg) + if err != nil { + panic(err) + } - # put in header (first) - if not self.seen_iframe: - self.codec.decode(av.packet.Packet(evta.header)) - self.seen_iframe = True + evt, err := cereal.ReadRootEncodeData(msg) + if err != nil { + panic(err) + } - frames = self.codec.decode(av.packet.Packet(evta.data)) - if len(frames) == 0: - print("DROP SURFACE") - continue - assert len(frames) == 1 + ts := evt.UnixTimestampNanos() - frame = frames[0] - frame.pts = pts - frame.time_base = time_base - */ + idx, err := evt.Idx() + if err != nil { + panic(err) + } + encodeId := idx.EncodeId() + + fmt.Printf("ts: %d,encodeId: %d\n", ts, encodeId) + + if encodeId != 0 && encodeId != uint32(lastIdx+1) { + fmt.Println("DROP PACKET!") + } else { + lastIdx = int(encodeId) + + idxFlags := idx.Flags() + + if !seenIframe && (idxFlags&V4L2_BUF_FLAG_KEYFRAME) != 0 { + fmt.Println("waiting for iframe") + continue + } + + if !seenIframe { + // Decode video frame + pkt := avcodec.AvPacketAlloc() + if pkt == nil { + panic("cannot allocate packet") + } + + pkt.AvInitPacket() + pkt.SetFlags(pkt.Flags() | avcodec.AV_CODEC_FLAG_TRUNCATED) + + response := codecContext.AvcodecSendPacket(packet) + if response < 0 { + fmt.Printf("Error while sending a packet to the decoder: %s\n", avutil.ErrorFromCode(response)) + panic("decoding error") + } + if response >= 0 { + response = codecContext.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(pFrame))) + if response == avutil.AvErrorEAGAIN || response == avutil.AvErrorEOF { + break + } else if response < 0 { + fmt.Printf("Error while receiving a frame from the decoder: %s\n", avutil.ErrorFromCode(response)) + fmt.Println("DROP SURFACE") + continue + } + seenIframe = true + } + } + + // frames, err := codec.Decode(av.Packet{Data: evt.Data()}) + // if err != nil { + // fmt.Println("DROP SURFACE") + // continue + // } + // if len(frames) == 0 { + // fmt.Println("DROP SURFACE") + // continue + // } + // assert(len(frames) == 1) + + // frame = frames[0].Data + // framePts := frames[0].Pts + // frameTimebase := frames[0].Timebase + + // frame = append(append(make([]byte, 0, len(frame)+FrameHeaderLen), encodeUint32(uint32(len(frame)))...), frame...) + // binary.LittleEndian.PutUint32(frame[4:], encodeUint64(framePts)) + // binary.LittleEndian.PutUint32(frame[12:], encodeUint64(frameTimebase)) + } + } } } fmt.Println("Received frame with size:", len(frame), "bytes")