fix invalid TID=0 in writeAggregationUnit of rtph265 encoder (#782)

* fix invalid TID=0 in writeAggregationUnit of rtph265 encoder

* return error in case of invalid NALUs

---------

Co-authored-by: aler9 <46489434+aler9@users.noreply.github.com>

pkg/format/rtph265/encoder.go

@@ -212,13 +212,25 @@ func (e *Encoder) lenAggregationUnit(nalus [][]byte, addNALU []byte) int {
 func (e *Encoder) writeAggregationUnit(nalus [][]byte, marker bool) ([]*rtp.Packet, error) {
 	payload := make([]byte, e.lenAggregationUnit(nalus, nil))
 
-	// header
-	h := uint16(48) << 9
-	payload[0] = byte(h >> 8)
-	payload[1] = byte(h)
+	layerID := byte(0xFF)
+	temporalID := byte(0xFF)
 	pos := 2
 
 	for _, nalu := range nalus {
+		if len(nalu) < 2 {
+			return nil, fmt.Errorf("invalid NALU")
+		}
+
+		// select lowest layerID & temporalID
+		nalLayerID := ((nalu[0] & 0x01) << 5) | ((nalu[1] >> 3) & 0x1F)
+		nalTemporalID := nalu[1] & 0x07
+		if nalLayerID < layerID {
+			layerID = nalLayerID
+		}
+		if nalTemporalID < temporalID {
+			temporalID = nalTemporalID
+		}
+
 		// size
 		naluLen := len(nalu)
 		payload[pos] = uint8(naluLen >> 8)
@@ -230,6 +242,10 @@ func (e *Encoder) writeAggregationUnit(nalus [][]byte, marker bool) ([]*rtp.Pack
 		pos += naluLen
 	}
 
+	// header
+	payload[0] = (48 << 1) | (layerID & 0x20)
+	payload[1] = ((layerID & 0x1F) << 3) | (temporalID & 0x07)
+
 	pkt := &rtp.Packet{
 		Header: rtp.Header{
 			Version:        rtpVersion,

pkg/format/rtph265/encoder_test.go

@@ -56,9 +56,9 @@ var cases = []struct {
 	{
 		"aggregated",
 		[][]byte{
-			{0x07, 0x07},
-			{0x08, 0x08},
-			{0x09, 0x09},
+			{0x07, 0x57},
+			{0x08, 0x66},
+			{0x09, 0x75},
 		},
 		[]*rtp.Packet{
 			{
@@ -70,8 +70,8 @@ var cases = []struct {
 					SSRC:           0x9dbb7812,
 				},
 				Payload: []byte{
-					0x60, 0x00, 0x00, 0x02, 0x07, 0x07, 0x00, 0x02,
-					0x08, 0x08, 0x00, 0x02, 0x09, 0x09,
+					0x60, 0x65, 0x00, 0x02, 0x07, 0x57, 0x00, 0x02,
+					0x08, 0x66, 0x00, 0x02, 0x09, 0x75,
 				},
 			},
 		},