client: parse incoming RTP/H264 packets; fix RTCP receiver jitter

This commit is contained in:
aler9
2022-04-07 20:14:45 +02:00
committed by Alessandro Ros
parent 3c104d3727
commit f4efe9ceb5
13 changed files with 174 additions and 128 deletions

109
client.go
View File

@@ -25,12 +25,14 @@ import (
"github.com/aler9/gortsplib/pkg/auth"
"github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/h264"
"github.com/aler9/gortsplib/pkg/headers"
"github.com/aler9/gortsplib/pkg/liberrors"
"github.com/aler9/gortsplib/pkg/multibuffer"
"github.com/aler9/gortsplib/pkg/ringbuffer"
"github.com/aler9/gortsplib/pkg/rtcpreceiver"
"github.com/aler9/gortsplib/pkg/rtcpsender"
"github.com/aler9/gortsplib/pkg/rtph264"
)
const (
@@ -59,6 +61,7 @@ type clientTrack struct {
tcpChannel int
rtcpReceiver *rtcpreceiver.RTCPReceiver
rtcpSender *rtcpsender.RTCPSender
h264Decoder *rtph264.Decoder
}
func (s clientState) String() string {
@@ -122,6 +125,21 @@ type clientRes struct {
err error
}
// ClientOnPacketRTPCtx is the context of a RTP packet.
type ClientOnPacketRTPCtx struct {
TrackID int
Packet *rtp.Packet
PTSEqualsDTS bool
H264NALUs [][]byte
H264PTS time.Duration
}
// ClientOnPacketRTCPCtx is the context of a RTCP packet.
type ClientOnPacketRTCPCtx struct {
TrackID int
Packet rtcp.Packet
}
// Client is a RTSP client.
type Client struct {
//
@@ -132,9 +150,9 @@ type Client struct {
// called after every response.
OnResponse func(*base.Response)
// called when a RTP packet arrives.
OnPacketRTP func(int, *rtp.Packet)
OnPacketRTP func(*ClientOnPacketRTPCtx)
// called when a RTCP packet arrives.
OnPacketRTCP func(int, rtcp.Packet)
OnPacketRTCP func(*ClientOnPacketRTCPCtx)
//
// RTSP parameters
@@ -252,11 +270,11 @@ type Client struct {
func (c *Client) Start(scheme string, host string) error {
// callbacks
if c.OnPacketRTP == nil {
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
c.OnPacketRTP = func(ctx *ClientOnPacketRTPCtx) {
}
}
if c.OnPacketRTCP == nil {
c.OnPacketRTCP = func(trackID int, pkt rtcp.Packet) {
c.OnPacketRTCP = func(ctx *ClientOnPacketRTCPCtx) {
}
}
@@ -698,7 +716,15 @@ func (c *Client) playRecordStart() {
v := time.Now().Unix()
c.tcpLastFrameTime = &v
}
} else if *c.effectiveTransport == TransportUDP {
} else {
for _, ct := range c.tracks {
if _, ok := ct.track.(*TrackH264); ok {
ct.h264Decoder = &rtph264.Decoder{}
ct.h264Decoder.Init()
}
}
if *c.effectiveTransport == TransportUDP {
for trackID, cct := range c.tracks {
ctrackID := trackID
@@ -713,6 +739,7 @@ func (c *Client) playRecordStart() {
cct.udpRTCPListener.start(false)
}
}
}
// for some reason, SetReadDeadline() must always be called in the same
// goroutine, otherwise Read() freezes.
@@ -753,11 +780,7 @@ func (c *Client) runReader() {
return
}
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
c.OnPacketRTP(trackID, pkt)
c.onPacketRTP(trackID, pkt)
} else {
packets, err := rtcp.Unmarshal(payload)
if err != nil {
@@ -765,7 +788,7 @@ func (c *Client) runReader() {
}
for _, pkt := range packets {
c.OnPacketRTCP(trackID, pkt)
c.onPacketRTCP(trackID, pkt)
}
}
}
@@ -783,7 +806,7 @@ func (c *Client) runReader() {
}
for _, pkt := range packets {
c.OnPacketRTCP(trackID, pkt)
c.onPacketRTCP(trackID, pkt)
}
}
}
@@ -851,6 +874,10 @@ func (c *Client) playRecordStop(isClosing bool) {
}
}
for _, ct := range c.tracks {
ct.h264Decoder = nil
}
// stop timers
c.checkStreamTimer = emptyTimer()
c.keepaliveTimer = emptyTimer()
@@ -1822,6 +1849,64 @@ func (c *Client) runWriter() {
}
}
func (c *Client) onPacketRTP(trackID int, pkt *rtp.Packet) {
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
ct := c.tracks[trackID]
if ct.h264Decoder != nil {
nalus, pts, err := ct.h264Decoder.DecodeUntilMarker(pkt)
if err == nil {
ptsEqualsDTS := h264.IDRPresent(nalus)
rr := ct.rtcpReceiver
if rr != nil {
rr.ProcessPacketRTP(time.Now(), pkt, ptsEqualsDTS)
}
c.OnPacketRTP(&ClientOnPacketRTPCtx{
TrackID: trackID,
Packet: pkt,
PTSEqualsDTS: ptsEqualsDTS,
H264NALUs: append([][]byte(nil), nalus...),
H264PTS: pts,
})
} else {
rr := ct.rtcpReceiver
if rr != nil {
rr.ProcessPacketRTP(time.Now(), pkt, false)
}
c.OnPacketRTP(&ClientOnPacketRTPCtx{
TrackID: trackID,
Packet: pkt,
PTSEqualsDTS: false,
})
}
return
}
rr := ct.rtcpReceiver
if rr != nil {
rr.ProcessPacketRTP(time.Now(), pkt, true)
}
c.OnPacketRTP(&ClientOnPacketRTPCtx{
TrackID: trackID,
Packet: pkt,
PTSEqualsDTS: true,
})
}
func (c *Client) onPacketRTCP(trackID int, pkt rtcp.Packet) {
c.OnPacketRTCP(&ClientOnPacketRTCPCtx{
TrackID: trackID,
Packet: pkt,
})
}
// WritePacketRTP writes a RTP packet.
func (c *Client) WritePacketRTP(trackID int, pkt *rtp.Packet, ptsEqualsDTS bool) error {
c.writeMutex.RLock()

View File

@@ -233,9 +233,9 @@ func TestClientPublishSerial(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
require.Equal(t, 0, trackID)
require.Equal(t, &testRTCPPacket, pkt)
OnPacketRTCP: func(ctx *ClientOnPacketRTCPCtx) {
require.Equal(t, 0, ctx.TrackID)
require.Equal(t, &testRTCPPacket, ctx.Packet)
close(recvDone)
},
}
@@ -1119,10 +1119,10 @@ func TestClientPublishIgnoreTCPRTPPackets(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
t.Errorf("should not happen")
},
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
OnPacketRTCP: func(ctx *ClientOnPacketRTCPCtx) {
close(rtcpReceived)
},
}

View File

@@ -404,7 +404,7 @@ func TestClientRead(t *testing.T) {
}(),
}
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
c.OnPacketRTP = func(ctx *ClientOnPacketRTPCtx) {
// ignore multicast loopback
if transport == "multicast" {
counter++
@@ -413,8 +413,8 @@ func TestClientRead(t *testing.T) {
}
}
require.Equal(t, 0, trackID)
require.Equal(t, &testRTPPacket, pkt)
require.Equal(t, 0, ctx.TrackID)
require.Equal(t, &testRTPPacket, ctx.Packet)
err := c.WritePacketRTCP(0, &testRTCPPacket)
require.NoError(t, err)
@@ -546,9 +546,9 @@ func TestClientReadNonStandardFrameSize(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
require.Equal(t, 0, trackID)
require.Equal(t, &refRTPPacket, pkt)
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
require.Equal(t, 0, ctx.TrackID)
require.Equal(t, &refRTPPacket, ctx.Packet)
close(packetRecv)
},
}
@@ -684,9 +684,9 @@ func TestClientReadPartial(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
require.Equal(t, 0, trackID)
require.Equal(t, &testRTPPacket, pkt)
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
require.Equal(t, 0, ctx.TrackID)
require.Equal(t, &testRTPPacket, ctx.Packet)
close(packetRecv)
},
}
@@ -970,8 +970,8 @@ func TestClientReadAnyPort(t *testing.T) {
c := &Client{
AnyPortEnable: true,
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
require.Equal(t, &testRTPPacket, pkt)
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
require.Equal(t, &testRTPPacket, ctx.Packet)
close(packetRecv)
},
}
@@ -1101,7 +1101,7 @@ func TestClientReadAutomaticProtocol(t *testing.T) {
packetRecv := make(chan struct{})
c := Client{
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
close(packetRecv)
},
}
@@ -1349,7 +1349,7 @@ func TestClientReadAutomaticProtocol(t *testing.T) {
c := &Client{
ReadTimeout: 1 * time.Second,
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
close(packetRecv)
},
}
@@ -1481,8 +1481,8 @@ func TestClientReadDifferentInterleavedIDs(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
require.Equal(t, 0, trackID)
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
require.Equal(t, 0, ctx.TrackID)
close(packetRecv)
},
}
@@ -1634,7 +1634,7 @@ func TestClientReadRedirect(t *testing.T) {
packetRecv := make(chan struct{})
c := Client{
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
close(packetRecv)
},
}
@@ -1845,7 +1845,7 @@ func TestClientReadPause(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
if atomic.SwapInt32(&firstFrame, 1) == 0 {
close(packetRecv)
}
@@ -1911,7 +1911,8 @@ func TestClientReadRTCPReport(t *testing.T) {
require.NoError(t, err)
require.Equal(t, base.Describe, req.Method)
track, err := NewTrackH264(96, []byte{0x01, 0x02, 0x03, 0x04}, []byte{0x01, 0x02, 0x03, 0x04}, nil)
track, err := NewTrackH264(96, []byte{0x01, 0x02, 0x03, 0x04},
[]byte{0x01, 0x02, 0x03, 0x04}, nil)
require.NoError(t, err)
tracks := Tracks{track}
@@ -1985,7 +1986,7 @@ func TestClientReadRTCPReport(t *testing.T) {
Timestamp: 54352,
SSRC: 753621,
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
Payload: []byte{0x05, 0x02, 0x03, 0x04},
}
byts, _ := pkt.Marshal()
_, err = l1.WriteTo(byts, &net.UDPAddr{
@@ -1998,8 +1999,8 @@ func TestClientReadRTCPReport(t *testing.T) {
SSRC: 753621,
NTPTime: 0,
RTPTime: 0,
PacketCount: 0,
OctetCount: 0,
PacketCount: 1,
OctetCount: 4,
}
byts, _ = sr.Marshal()
_, err = l2.WriteTo(byts, &net.UDPAddr{
@@ -2335,7 +2336,7 @@ func TestClientReadIgnoreTCPInvalidTrack(t *testing.T) {
v := TransportTCP
return &v
}(),
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
OnPacketRTP: func(ctx *ClientOnPacketRTPCtx) {
close(recv)
},
}
@@ -2791,9 +2792,9 @@ func TestClientReadDifferentSource(t *testing.T) {
}(),
}
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
require.Equal(t, 0, trackID)
require.Equal(t, &testRTPPacket, pkt)
c.OnPacketRTP = func(ctx *ClientOnPacketRTPCtx) {
require.Equal(t, 0, ctx.TrackID)
require.Equal(t, &testRTPPacket, ctx.Packet)
close(packetRecv)
}

View File

@@ -182,13 +182,7 @@ func (u *clientUDPListener) processPlayRTP(now time.Time, payload []byte) {
return
}
u.c.tracks[u.trackID].rtcpReceiver.ProcessPacketRTP(now, pkt, true)
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
u.c.OnPacketRTP(u.trackID, pkt)
u.c.onPacketRTP(u.trackID, pkt)
}
func (u *clientUDPListener) processPlayRTCP(now time.Time, payload []byte) {
@@ -199,7 +193,7 @@ func (u *clientUDPListener) processPlayRTCP(now time.Time, payload []byte) {
for _, pkt := range packets {
u.c.tracks[u.trackID].rtcpReceiver.ProcessPacketRTCP(now, pkt)
u.c.OnPacketRTCP(u.trackID, pkt)
u.c.onPacketRTCP(u.trackID, pkt)
}
}
@@ -210,7 +204,7 @@ func (u *clientUDPListener) processRecordRTCP(now time.Time, payload []byte) {
}
for _, pkt := range packets {
u.c.OnPacketRTCP(u.trackID, pkt)
u.c.onPacketRTCP(u.trackID, pkt)
}
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/rtpaac"
"github.com/pion/rtp"
)
// This example shows how to
@@ -58,13 +57,13 @@ func main() {
dec.Init()
// called when a RTP packet arrives
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
if trackID != aacTrack {
c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
if ctx.TrackID != aacTrack {
return
}
// decode AAC AUs from the RTP packet
aus, _, err := dec.Decode(pkt)
aus, _, err := dec.Decode(ctx.Packet)
if err != nil {
return
}

View File

@@ -10,8 +10,6 @@ import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/rtph264"
"github.com/pion/rtp"
)
// This example shows how to
@@ -75,10 +73,6 @@ func main() {
panic("H264 track not found")
}
// setup RTP->H264 decoder
rtpDec := &rtph264.Decoder{}
rtpDec.Init()
// setup H264->raw frames decoder
h264dec, err := newH264Decoder()
if err != nil {
@@ -96,18 +90,16 @@ func main() {
// called when a RTP packet arrives
saveCount := 0
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
if trackID != h264TrackID {
c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
if ctx.TrackID != h264TrackID {
return
}
// convert RTP packet to H264 NALUs
nalus, _, err := rtpDec.Decode(pkt)
if err != nil {
if ctx.H264NALUs == nil {
return
}
for _, nalu := range nalus {
for _, nalu := range ctx.H264NALUs {
// convert H264 NALUs to RGBA frames
img, err := h264dec.decode(nalu)
if err != nil {

View File

@@ -3,8 +3,6 @@ package main
import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/rtph264"
"github.com/pion/rtp"
)
// This example shows how to
@@ -47,10 +45,6 @@ func main() {
panic("H264 track not found")
}
// setup RTP->H264 decoder
rtpDec := &rtph264.Decoder{}
rtpDec.Init()
// setup H264->MPEGTS encoder
enc, err := newMPEGTSEncoder(h264track.SPS(), h264track.PPS())
if err != nil {
@@ -58,19 +52,17 @@ func main() {
}
// called when a RTP packet arrives
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
if trackID != h264TrackID {
c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
if ctx.TrackID != h264TrackID {
return
}
// convert RTP packet to H264 NALUs
nalus, pts, err := rtpDec.DecodeUntilMarker(pkt)
if err != nil {
if ctx.H264NALUs == nil {
return
}
// encode H264 NALUs into MPEG-TS
err = enc.encode(nalus, pts)
err = enc.encode(ctx.H264NALUs, ctx.H264PTS)
if err != nil {
return
}

View File

@@ -5,8 +5,6 @@ import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/rtph264"
"github.com/pion/rtp"
)
// This example shows how to
@@ -52,10 +50,6 @@ func main() {
panic("H264 track not found")
}
// setup RTP->H264 decoder
rtpDec := &rtph264.Decoder{}
rtpDec.Init()
// setup H264->raw frames decoder
h264dec, err := newH264Decoder()
if err != nil {
@@ -72,18 +66,16 @@ func main() {
}
// called when a RTP packet arrives
c.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
if trackID != h264TrackID {
c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
if ctx.TrackID != h264TrackID {
return
}
// convert RTP packet to H264 NALUs
nalus, _, err := rtpDec.Decode(pkt)
if err != nil {
if ctx.H264NALUs == nil {
return
}
for _, nalu := range nalus {
for _, nalu := range ctx.H264NALUs {
// convert H264 NALUs to RGBA frames
img, err := h264dec.decode(nalu)
if err != nil {

View File

@@ -5,8 +5,6 @@ import (
"time"
"github.com/aler9/gortsplib"
"github.com/pion/rtcp"
"github.com/pion/rtp"
)
// This example shows how to
@@ -23,12 +21,12 @@ func main() {
// timeout of write operations
WriteTimeout: 10 * time.Second,
// called when a RTP packet arrives
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
log.Printf("RTP packet from track %d, payload type %d\n", trackID, pkt.Header.PayloadType)
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
log.Printf("RTP packet from track %d, payload type %d\n", ctx.TrackID, ctx.Packet.Header.PayloadType)
},
// called when a RTCP packet arrives
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
log.Printf("RTCP packet from track %d, type %T\n", trackID, pkt)
OnPacketRTCP: func(ctx *gortsplib.ClientOnPacketRTCPCtx) {
log.Printf("RTCP packet from track %d, type %T\n", ctx.TrackID, ctx.Packet)
},
}

View File

@@ -5,8 +5,6 @@ import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/pion/rtcp"
"github.com/pion/rtp"
)
// This example shows how to
@@ -17,12 +15,12 @@ import (
func main() {
c := gortsplib.Client{
// called when a RTP packet arrives
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
log.Printf("RTP packet from track %d, payload type %d\n", trackID, pkt.Header.PayloadType)
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
log.Printf("RTP packet from track %d, payload type %d\n", ctx.TrackID, ctx.Packet.Header.PayloadType)
},
// called when a RTCP packet arrives
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
log.Printf("RTCP packet from track %d, type %T\n", trackID, pkt)
OnPacketRTCP: func(ctx *gortsplib.ClientOnPacketRTCPCtx) {
log.Printf("RTCP packet from track %d, type %T\n", ctx.TrackID, ctx.Packet)
},
}

View File

@@ -5,8 +5,6 @@ import (
"time"
"github.com/aler9/gortsplib"
"github.com/pion/rtcp"
"github.com/pion/rtp"
)
// This example shows how to
@@ -18,12 +16,12 @@ import (
func main() {
c := gortsplib.Client{
// called when a RTP packet arrives
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
log.Printf("RTP packet from track %d, payload type %d\n", trackID, pkt.Header.PayloadType)
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
log.Printf("RTP packet from track %d, payload type %d\n", ctx.TrackID, ctx.Packet.Header.PayloadType)
},
// called when a RTCP packet arrives
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
log.Printf("RTCP packet from track %d, type %T\n", trackID, pkt)
OnPacketRTCP: func(ctx *gortsplib.ClientOnPacketRTCPCtx) {
log.Printf("RTCP packet from track %d, type %T\n", ctx.TrackID, ctx.Packet)
},
}

View File

@@ -5,7 +5,6 @@ import (
"github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base"
"github.com/pion/rtp"
)
// This example shows how to
@@ -46,8 +45,8 @@ func main() {
defer publisher.Close()
// called when a RTP packet arrives
reader.OnPacketRTP = func(trackID int, pkt *rtp.Packet) {
publisher.WritePacketRTP(trackID, pkt, true)
reader.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
publisher.WritePacketRTP(ctx.TrackID, ctx.Packet, ctx.PTSEqualsDTS)
}
// start reading tracks

View File

@@ -4,8 +4,6 @@ import (
"log"
"github.com/aler9/gortsplib"
"github.com/pion/rtcp"
"github.com/pion/rtp"
)
// This example shows how to connect to a RTSP server
@@ -14,12 +12,12 @@ import (
func main() {
c := gortsplib.Client{
// called when a RTP packet arrives
OnPacketRTP: func(trackID int, pkt *rtp.Packet) {
log.Printf("RTP packet from track %d, payload type %d\n", trackID, pkt.Header.PayloadType)
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
log.Printf("RTP packet from track %d, payload type %d\n", ctx.TrackID, ctx.Packet.Header.PayloadType)
},
// called when a RTCP packet arrives
OnPacketRTCP: func(trackID int, pkt rtcp.Packet) {
log.Printf("RTCP packet from track %d, type %T\n", trackID, pkt)
OnPacketRTCP: func(ctx *gortsplib.ClientOnPacketRTCPCtx) {
log.Printf("RTCP packet from track %d, type %T\n", ctx.TrackID, ctx.Packet)
},
}