formatprocessor: prevent generating empty H264/H265 RTP packets (#1505)

internal/formatprocessor/h264.go

@@ -259,11 +259,15 @@ func (t *formatProcessorH264) Process(dat Data, hasNonRTSPReaders bool) error {
 		tdata.AU = t.remuxAccessUnit(tdata.AU)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
+	if len(tdata.AU) != 0 {
-	if err != nil {
+		pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
-		return err
+		if err != nil {
+			return err
+		}
+		tdata.RTPPackets = pkts
+	} else {
+		tdata.RTPPackets = nil
 	}
 
-	tdata.RTPPackets = pkts
 	return nil
 }

internal/formatprocessor/h264_test.go

@@ -151,3 +151,25 @@ func TestH264OversizedPackets(t *testing.T) {
 		},
 	}, out)
 }
+
+func TestH264EmptyPacket(t *testing.T) {
+	forma := &format.H264{
+		PayloadTyp:        96,
+		PacketizationMode: 1,
+	}
+
+	p, err := New(forma, true)
+	require.NoError(t, err)
+
+	unit := &DataH264{
+		AU: [][]byte{
+			{0x07, 0x01, 0x02, 0x03}, // SPS
+			{0x08, 0x01, 0x02},       // PPS
+		},
+	}
+
+	p.Process(unit, false)
+
+	// if all NALUs have been removed, no RTP packets must be generated.
+	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
+}

internal/formatprocessor/h265.go

@@ -280,11 +280,15 @@ func (t *formatProcessorH265) Process(dat Data, hasNonRTSPReaders bool) error {
 		tdata.AU = t.remuxAccessUnit(tdata.AU)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
+	if len(tdata.AU) != 0 {
-	if err != nil {
+		pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
-		return err
+		if err != nil {
+			return err
+		}
+		tdata.RTPPackets = pkts
+	} else {
+		tdata.RTPPackets = nil
 	}
 
-	tdata.RTPPackets = pkts
 	return nil
 }

internal/formatprocessor/h265_test.go

@@ -144,3 +144,25 @@ func TestH265OversizedPackets(t *testing.T) {
 		},
 	}, out)
 }
+
+func TestH265EmptyPacket(t *testing.T) {
+	forma := &format.H265{
+		PayloadTyp: 96,
+	}
+
+	p, err := New(forma, true)
+	require.NoError(t, err)
+
+	unit := &DataH265{
+		AU: [][]byte{
+			{byte(h265.NALUType_VPS_NUT) << 1, 10, 11, 12}, // VPS
+			{byte(h265.NALUType_SPS_NUT) << 1, 13, 14, 15}, // SPS
+			{byte(h265.NALUType_PPS_NUT) << 1, 16, 17, 18}, // PPS
+		},
+	}
+
+	p.Process(unit, false)
+
+	// if all NALUs have been removed, no RTP packets must be generated.
+	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
+}

internal/formatprocessor/processor.go

@@ -17,7 +17,7 @@ type Data interface {
 
 // Processor allows to cleanup and normalize streams.
 type Processor interface {
-	// cleanups and normalizes a data unit.
+	// clears and normalizes a data unit.
 	Process(Data, bool) error
 }