as easy as 1, 2, 3...?

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "pion/webrtc"]
 	path = pion/webrtc
 	url = https://github.com/pion/webrtc.git
+[submodule "pion/rtp"]
+	path = pion/rtp
+	url = https://github.com/pion/rtp.git

TODO

@@ -3,7 +3,7 @@
 - [x] camera switcher: discovered that it is broken https://github.com/aiortc/aiortc/issues/304 meanwhile Pion has been supporting it for 3 years https://github.com/pion/webrtc/pull/1527
 beginning of port from python/aiortc to go/pion ... maybe other issues around performance will be resolved too by doing so.
 - [x] write a go version of the compressed_vipc decoder demo
-- [ ] write a Pion-compatible vision IPC track
+- [ ] write a Pion-compatible vision IPC track by looking at https://github.com/pion/webrtc/blob/master/examples/rtp-to-webrtc/main.go and https://github.com/asticode/go-astiav/blob/master/examples/transcoding/main.go
 - battery view
 - lowcam view
 - multicam

go.mod

@@ -2,7 +2,10 @@ module github.com/kfatehi/webrtc-body
 
 go 1.19
 
-require github.com/pebbe/zmq4 v1.2.9
+require (
+	github.com/pebbe/zmq4 v1.2.9
+	github.com/pion/webrtc/v3 v3.1.55
+)
 
 require (
 	github.com/google/uuid v1.3.0 // indirect
@@ -22,8 +25,6 @@ require (
 	github.com/pion/transport/v2 v2.0.1 // indirect
 	github.com/pion/turn/v2 v2.1.0 // indirect
 	github.com/pion/udp v0.1.4 // indirect
-	github.com/pion/webrtc/v3 v3.1.55 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/net v0.6.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect

go.work

@@ -7,4 +7,5 @@ use (
 	./go-capnproto2
 	./cereal/gen/go
 	./pion/webrtc
+	./pion/rtp
 )

go.work.sum

@@ -5,6 +5,7 @@ github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/otiai10/gosseract v2.2.1+incompatible/go.mod h1:XrzWItCzCpFRZ35n3YtVTgq5bLAhFIkascoRo8G32QE=
+github.com/pion/webrtc/v3 v3.1.55/go.mod h1:M1gU5mnvvo4e1nnLvF23esYz0nZAFOtbU/wq44MSfbc=
 github.com/robotn/gohook v0.31.3 h1:kGX8iukJ9ensVRwRKnTtdojAMQOpa6KFnXDi4OA4RaI=
 github.com/robotn/gohook v0.31.3/go.mod h1:wyGik0yb4iwCfJjDprtNkTyxkgQWuKoVPQ3hkz6+6js=
 github.com/robotn/xgb v0.0.0-20190912153532-2cb92d044934 h1:2lhSR8N3T6I30q096DT7/5AKEIcf1vvnnWAmS0wfnNY=
@@ -29,4 +30,3 @@ golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=

main.go

@@ -1,35 +1,99 @@
 package main
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"log"
 	"secureput"
+	"strings"
 
+	"github.com/asticode/go-astiav"
 	"github.com/pion/webrtc/v3"
 )
 
-var vipctrack *VisionIpcTrack
+var visionTrack *VisionIpcTrack
 
-func StopTrack() {
-	if vipctrack != nil {
-		vipctrack.Stop()
-		vipctrack = nil
-	}
-}
-
-func ReplaceTrack(prefix string, pc *webrtc.PeerConnection) {
-	StopTrack()
+func ReplaceTrack(prefix string, peerConnection *webrtc.PeerConnection) {
 	var err error
-	vipctrack, err = NewVisionIpcTrack(prefix + "EncodeData")
+	if visionTrack != nil {
+		visionTrack.Stop()
+	}
+	visionTrack, err = NewVisionIpcTrack(prefix + "EncodeData")
 	if err != nil {
 		log.Fatal(fmt.Errorf("main: creating track failed: %w", err))
 	}
-	go vipctrack.Start()
-
-	webrtc.newtrack
-	for frame := range vipctrack.Frame {
-		// Do something with decoded frame
+	// Create a video track
+	videoTrack, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
+	if err != nil {
+		panic(err)
 	}
+	rtpSender, err := peerConnection.AddTrack(videoTrack)
+	if err != nil {
+		panic(err)
+	}
+	// Later on, we will use rtpSender.ReplaceTrack() for graceful track replacement
+
+	// Read incoming RTCP packets
+	// Before these packets are returned they are processed by interceptors. For things
+	// like NACK this needs to be called.
+	go func() {
+		rtcpBuf := make([]byte, 1500)
+		for {
+			if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
+				return
+			}
+		}
+	}()
+
+	// Set the handler for ICE connection state
+	// This will notify you when the peer has connected/disconnected
+	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
+		fmt.Printf("Connection State has changed %s \n", connectionState.String())
+		if connectionState.String() == "disconnected" {
+			visionTrack.Stop()
+		} else if connectionState == webrtc.ICEConnectionStateFailed {
+			visionTrack.Stop()
+			if closeErr := peerConnection.Close(); closeErr != nil {
+				log.Println(fmt.Errorf("main: peer connection closed due to error: %w", err))
+			}
+		} else if connectionState.String() == "connected" {
+			astiav.SetLogLevel(astiav.LogLevelDebug)
+			astiav.SetLogCallback(func(l astiav.LogLevel, fmt, msg, parent string) {
+				log.Printf("ffmpeg log: %s (level: %d)\n", strings.TrimSpace(msg), l)
+			})
+			go func() {
+				go visionTrack.Start()
+				defer visionTrack.Stop()
+				for visionTrack != nil {
+					for frame := range visionTrack.Frame {
+						// Do something with decoded frame
+						fmt.Println(frame.Roll)
+
+						// so right now frame is a raw decoded AVFrame
+						// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavutil/frame.h#L317
+						// avframe := frame.Frame
+
+						// we need to:
+						// 1. transcode to h264 with adaptive bitrate using astiav
+
+						// 2. create an RTP packet with h264 data inside using pion's rtp packetizer
+						var rtpPacket []byte
+						// 3. write the RTP packet to the videoTrack
+
+						if _, err = videoTrack.Write(rtpPacket); err != nil {
+							if errors.Is(err, io.ErrClosedPipe) {
+								// The peerConnection has been closed.
+								return
+							}
+
+							log.Println(fmt.Errorf("rtp write error: %w", err))
+						}
+					}
+				}
+			}()
+		}
+	})
 }
 
 func main() {
@@ -45,11 +109,7 @@ func main() {
 	signal.OnPeerConnectionCreated = func(pc *webrtc.PeerConnection) {
 		ReplaceTrack("road", pc)
 	}
-	signal.OnICEConnectionStateChange = func(connectionState webrtc.ICEConnectionState) {
-		if connectionState.String() == "disconnected" {
-			StopTrack()
-		}
-	}
 	for {
+		select {}
 	}
 }