Fix empty SPS for mp4 format

README.md

@@ -547,6 +547,30 @@ If you need Web interface protection without Home Assistant Add-on - you need to
 
 PS. Additionally WebRTC opens a lot of random UDP ports for transmit encrypted media. They work without problems on the local network. And sometimes work for external access, even if you haven't opened ports on your router. But for stable external WebRTC access, you need to configure the TCP port.
 
+## Codecs madness
+
+`AVC/H.264` codec can be played almost anywhere. But `HEVC/H.265` has a lot of limitations in supporting with different devices and browsers. It's all about patents and money, you can't do anything about it.
+
+Device | WebRTC | MSE | MP4
+-------|--------|-----|----
+*latency*       | best        | medium      | bad
+Desktop Chrome  | H264        | H264, H265* | H264, H265*
+Desktop Safari  | H264, H265* | H264        | no
+Desktop Edge    | H264        | H264, H265* | H264, H265*
+Desktop Firefox | H264        | H264        | H264
+Desktop Opera   | no          | H264        | H264
+iPhone Safari   | H264, H265* | no          | no
+iPad Safari     | H264, H265* | H264        | no
+Android Chrome  | H264        | H264        | H264
+masOS Hass App  | no          | no          | no
+
+- WebRTC audio codecs: `PCMU/8000`, `PCMA/8000`, `OPUS/48000/2`
+- MSE/MP4 audio codecs: not supported yet (should be: `AAC`)
+- Chrome H265: [read this](https://github.com/StaZhu/enable-chromium-hevc-hardware-decoding)
+- Edge H265: [read this](https://www.reddit.com/r/MicrosoftEdge/comments/v9iw8k/enable_hevc_support_in_edge/)
+- Desktop Safari H265: Menu > Develop > Experimental > WebRTC H265
+- iOS Safari H265: Settings > Safari > Advanced > Experimental > WebRTC H265
+
 ## FAQ
 
 **Q. What's the difference between go2rtc, WebRTC Camera and RTSPtoWebRTC?**

cmd/ffmpeg/ffmpeg.go

@@ -121,7 +121,7 @@ func Init() {
 
 			for _, audio := range query["audio"] {
 				if audio == "copy" {
-					s += " -codec:v copy"
+					s += " -codec:a copy"
 				} else {
 					s += " " + tpl[audio]
 				}

pkg/h264/README.md

@@ -1,3 +1,11 @@
+# H264
+
+Access Unit (AU) can contain one or multiple NAL Unit:
+
+1. [SEI,] SPS, PPS, IFrame, [IFrame...]
+2. BFrame, [BFrame...]
+3. IFrame, [IFrame...]
+
 ## RTP H264
 
 Camera | NALu

pkg/h264/avc.go

@@ -12,49 +12,38 @@ func IsAVC(codec *streamer.Codec) bool {
 	return codec.PayloadType == PayloadTypeAVC
 }
 
-func EncodeAVC(raw []byte) (avc []byte) {
-	avc = make([]byte, len(raw)+4)
-	binary.BigEndian.PutUint32(avc, uint32(len(raw)))
-	copy(avc[4:], raw)
+func EncodeAVC(nals ...[]byte) (avc []byte) {
+	var i, n int
+
+	for _, nal := range nals {
+		if i = len(nal); i > 0 {
+			n += 4 + i
+		}
+	}
+
+	avc = make([]byte, n)
+
+	n = 0
+	for _, nal := range nals {
+		if i = len(nal); i > 0 {
+			binary.BigEndian.PutUint32(avc[n:], uint32(i))
+			n += 4 + copy(avc[n+4:], nal)
+		}
+	}
+
 	return
 }
 
 func RepairAVC(track *streamer.Track) streamer.WrapperFunc {
 	sps, pps := GetParameterSet(track.Codec.FmtpLine)
-	sps = EncodeAVC(sps)
-	pps = EncodeAVC(pps)
+	ps := EncodeAVC(sps, pps)
 
 	return func(push streamer.WriterFunc) streamer.WriterFunc {
 		return func(packet *rtp.Packet) (err error) {
-			naluType := NALUType(packet.Payload)
-			switch naluType {
-			case NALUTypeSPS:
-				sps = packet.Payload
-				return
-			case NALUTypePPS:
-				pps = packet.Payload
-				return
+			if NALUType(packet.Payload) == NALUTypeIFrame {
+				packet.Payload = Join(ps, packet.Payload)
 			}
-
-			var clone rtp.Packet
-
-			if naluType == NALUTypeIFrame {
-				clone = *packet
-				clone.Payload = sps
-				if err = push(&clone); err != nil {
-					return
-				}
-
-				clone = *packet
-				clone.Payload = pps
-				if err = push(&clone); err != nil {
-					return
-				}
-			}
-
-			clone = *packet
-			clone.Payload = packet.Payload
-			return push(&clone)
+			return push(packet)
 		}
 	}
 }
@@ -63,12 +52,12 @@ func SplitAVC(data []byte) [][]byte {
 	var nals [][]byte
 	for {
 		// get AVC length
-		size := int(binary.BigEndian.Uint32(data))
+		size := int(binary.BigEndian.Uint32(data)) + 4
 
 		// check if multiple items in one packet
-		if size+4 < len(data) {
-			nals = append(nals, data[:size+4])
-			data = data[size+4:]
+		if size < len(data) {
+			nals = append(nals, data[:size])
+			data = data[size:]
 		} else {
 			nals = append(nals, data)
 			break
@@ -76,3 +65,18 @@ func SplitAVC(data []byte) [][]byte {
 	}
 	return nals
 }
+
+func Types(data []byte) []byte {
+	var types []byte
+	for {
+		types = append(types, NALUType(data))
+
+		size := 4 + int(binary.BigEndian.Uint32(data))
+		if size < len(data) {
+			data = data[size:]
+		} else {
+			break
+		}
+	}
+	return types
+}

pkg/h264/helper.go

@@ -2,6 +2,7 @@ package h264
 
 import (
 	"encoding/base64"
+	"encoding/binary"
 	"github.com/AlexxIT/go2rtc/pkg/streamer"
 	"strings"
 )
@@ -18,8 +19,31 @@ func NALUType(b []byte) byte {
 	return b[4] & 0x1F
 }
 
+// IsKeyframe - check if any NALU in one AU is Keyframe
 func IsKeyframe(b []byte) bool {
-	return NALUType(b) == NALUTypeIFrame
+	for {
+		switch NALUType(b) {
+		case NALUTypePFrame:
+			return false
+		case NALUTypeIFrame:
+			return true
+		}
+
+		size := int(binary.BigEndian.Uint32(b)) + 4
+		if size < len(b) {
+			b = b[size:]
+			continue
+		} else {
+			return false
+		}
+	}
+}
+
+func Join(ps, iframe []byte) []byte {
+	b := make([]byte, len(ps)+len(iframe))
+	i := copy(b, ps)
+	copy(b[i:], iframe)
+	return b
 }
 
 func GetProfileLevelID(fmtp string) string {

pkg/h264/rtp.go

@@ -13,8 +13,7 @@ func RTPDepay(track *streamer.Track) streamer.WrapperFunc {
 	depack := &codecs.H264Packet{IsAVC: true}
 
 	sps, pps := GetParameterSet(track.Codec.FmtpLine)
-	sps = EncodeAVC(sps)
-	pps = EncodeAVC(pps)
+	ps := EncodeAVC(sps, pps)
 
 	var buffer []byte
 
@@ -27,83 +26,41 @@ func RTPDepay(track *streamer.Track) streamer.WrapperFunc {
 			//	packet.PayloadType, packet.SSRC, packet.SequenceNumber,
 			//)
 
-			data, err := depack.Unmarshal(packet.Payload)
-			if len(data) == 0 || err != nil {
+			payload, err := depack.Unmarshal(packet.Payload)
+			if len(payload) == 0 || err != nil {
 				return nil
 			}
 
-			for {
-				unitType := NALUType(data)
-				//fmt.Printf("[H264] nalu: %2d, size: %6d\n", unitType, len(data))
-
-				// multiple 5 and 1 in one payload is OK
-				if unitType != NALUTypeIFrame && unitType != NALUTypePFrame {
-					i := int(binary.BigEndian.Uint32(data)) + 4
-					if i < len(data) {
-						data0 := data[:i] // NAL Unit with AVC header
-						data = data[i:]
-						switch unitType {
-						case NALUTypeSPS:
-							sps = data0
-							continue
-						case NALUTypePPS:
-							pps = data0
-							continue
-						case NALUTypeSEI:
-							// some unnecessary text information
-							continue
-						}
-					}
-				}
-
-				switch unitType {
-				case NALUTypeSPS:
-					sps = data
-					return nil
-				case NALUTypePPS:
-					pps = data
-					return nil
-				case NALUTypeSEI:
-					// some unnecessary text information
-					return nil
-				}
-
-				// ffmpeg with `-tune zerolatency` enable option `-x264opts sliced-threads=1`
-				// and every NALU will be sliced to multiple NALUs
-				if !packet.Marker {
-					buffer = append(buffer, data...)
-					return nil
-				}
-
-				if buffer != nil {
-					buffer = append(buffer, data...)
-					data = buffer
-					buffer = nil
-				}
-
-				var clone rtp.Packet
-
-				if unitType == NALUTypeIFrame {
-					clone = *packet
-					clone.Version = RTPPacketVersionAVC
-					clone.Payload = sps
-					if err = push(&clone); err != nil {
-						return err
-					}
-
-					clone = *packet
-					clone.Version = RTPPacketVersionAVC
-					clone.Payload = pps
-					if err = push(&clone); err != nil {
-						return err
-					}
-				}
-
-				clone = *packet
-				clone.Version = RTPPacketVersionAVC
-				clone.Payload = data
-				return push(&clone)
+			// ffmpeg with `-tune zerolatency` enable option `-x264opts sliced-threads=1`
+			// and every NALU will be sliced to multiple NALUs
+			if !packet.Marker {
+				buffer = append(buffer, payload...)
+				return nil
 			}
+
+			if buffer != nil {
+				payload = append(buffer, payload...)
+				buffer = nil
+			}
+
+			//fmt.Printf("[AVC] %v, len: %d\n", Types(payload), len(payload))
+
+			switch NALUType(payload) {
+			case NALUTypeIFrame:
+				payload = Join(ps, payload)
+			case NALUTypeSEI:
+				// ffmpeg with transcoding
+				i := 4 + binary.BigEndian.Uint32(payload)
+				payload = payload[i:]
+				if NALUType(payload) == NALUTypeIFrame {
+					payload = Join(ps, payload)
+				}
+			}
+
+			clone := *packet
+			clone.Version = RTPPacketVersionAVC
+			clone.Payload = payload
+			return push(&clone)
 		}
 	}
 }
@@ -117,11 +74,12 @@ func RTPPay(mtu uint16) streamer.WrapperFunc {
 		return func(packet *rtp.Packet) error {
 			if packet.Version == RTPPacketVersionAVC {
 				payloads := payloader.Payload(mtu, packet.Payload)
+				last := len(payloads) - 1
 				for i, payload := range payloads {
 					clone := rtp.Packet{
 						Header: rtp.Header{
 							Version: 2,
-							Marker:  i == len(payloads)-1,
+							Marker:  i == last,
 							//PayloadType:    packet.PayloadType,
 							SequenceNumber: sequencer.NextSequenceNumber(),
 							Timestamp:      packet.Timestamp,

pkg/ivideon/client.go

@@ -245,14 +245,12 @@ func (c *Client) worker() {
 			time.Sleep(d)
 
 			// can be SPS, PPS and IFrame in one packet
-			for _, payload := range h264.SplitAVC(data[:entry.Size]) {
-				packet := &rtp.Packet{
-					// ivideon clockrate=1000, RTP clockrate=90000
-					Header:  rtp.Header{Timestamp: ts * 90},
-					Payload: payload,
-				}
-				_ = track.WriteRTP(packet)
+			packet := &rtp.Packet{
+				// ivideon clockrate=1000, RTP clockrate=90000
+				Header:  rtp.Header{Timestamp: ts * 90},
+				Payload: data[:entry.Size],
 			}
+			_ = track.WriteRTP(packet)
 
 			data = data[entry.Size:]
 			ts += entry.Duration

pkg/mp4/consumer.go

@@ -53,15 +53,16 @@ func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *strea
 				return nil
 			}
 
-			switch h264.NALUType(packet.Payload) {
-			case h264.NALUTypeIFrame:
-				c.start = true
-			case h264.NALUTypePFrame:
-				if !c.start {
+			if c.muxer == nil {
+				return nil
+			}
+
+			if !c.start {
+				if h264.IsKeyframe(packet.Payload) {
+					c.start = true
+				} else {
 					return nil
 				}
-			default:
-				return nil
 			}
 
 			buf := c.muxer.Marshal(packet)
@@ -71,10 +72,13 @@ func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *strea
 			return nil
 		}
 
-		if !h264.IsAVC(codec) {
-			wrapper := h264.RTPDepay(track)
-			push = wrapper(push)
+		var wrapper streamer.WrapperFunc
+		if h264.IsAVC(codec) {
+			wrapper = h264.RepairAVC(track)
+		} else {
+			wrapper = h264.RTPDepay(track)
 		}
+		push = wrapper(push)
 
 		return track.Bind(push)
 

pkg/mp4/muxer.go

@@ -46,10 +46,11 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
 		case streamer.CodecH264:
 			sps, pps := h264.GetParameterSet(codec.FmtpLine)
 			if sps == nil {
-				return nil, fmt.Errorf("empty SPS: %#v", codec)
+				// some dummy SPS and PPS not a problem
+				sps = []byte{0x67, 0x42, 0x00, 0x0a, 0xf8, 0x41, 0xa2}
+				pps = []byte{0x68, 0xce, 0x38, 0x80}
 			}
 
-			// TODO: remove
 			codecData, err := h264parser.NewCodecDataFromSPSAndPPS(sps, pps)
 			if err != nil {
 				return nil, err

pkg/rtmp/client.go

@@ -47,10 +47,14 @@ func (c *Client) Dial() (err error) {
 	for _, stream := range streams {
 		switch stream.Type() {
 		case av.H264:
-			cd := stream.(h264parser.CodecData)
-			fmtp := "sprop-parameter-sets=" +
-				base64.StdEncoding.EncodeToString(cd.RecordInfo.SPS[0]) + "," +
-				base64.StdEncoding.EncodeToString(cd.RecordInfo.PPS[0])
+			info := stream.(h264parser.CodecData).RecordInfo
+
+			fmtp := fmt.Sprintf(
+				"profile-level-id=%02X%02X%02X;sprop-parameter-sets=%s,%s",
+				info.AVCProfileIndication, info.ProfileCompatibility, info.AVCLevelIndication,
+				base64.StdEncoding.EncodeToString(info.SPS[0]),
+				base64.StdEncoding.EncodeToString(info.PPS[0]),
+			)
 
 			codec := &streamer.Codec{
 				Name:        streamer.CodecH264,
@@ -129,22 +133,14 @@ func (c *Client) Handle() (err error) {
 
 		track := c.tracks[int(pkt.Idx)]
 
-		timestamp := uint32(pkt.Time / time.Duration(track.Codec.ClockRate))
+		// convert seconds to RTP timestamp
+		timestamp := uint32(pkt.Time * time.Duration(track.Codec.ClockRate) / time.Second)
 
-		var payloads [][]byte
-		if track.Codec.Name == streamer.CodecH264 {
-			payloads = h264.SplitAVC(pkt.Data)
-		} else {
-			payloads = [][]byte{pkt.Data}
-		}
-
-		for _, payload := range payloads {
-			packet := &rtp.Packet{
-				Header:  rtp.Header{Timestamp: timestamp},
-				Payload: payload,
-			}
-			_ = track.WriteRTP(packet)
+		packet := &rtp.Packet{
+			Header:  rtp.Header{Timestamp: timestamp},
+			Payload: pkt.Data,
 		}
+		_ = track.WriteRTP(packet)
 	}
 }
 

pkg/rtsp/conn.go

@@ -392,12 +392,16 @@ func (c *Conn) SetupMedia(
 	// we send our `interleaved`, but camera can answer with another
 
 	// Transport: RTP/AVP/TCP;unicast;interleaved=10-11;ssrc=10117CB7
-	// Transport: RTP/AVP/TCP;unicast;destination=192.168.1.123;source=192.168.10.12;interleaved=0
+	// Transport: RTP/AVP/TCP;unicast;destination=192.168.1.111;source=192.168.1.222;interleaved=0
 	// Transport: RTP/AVP/TCP;ssrc=22345682;interleaved=0-1
 	s := res.Header.Get("Transport")
 	// TODO: rewrite
 	if !strings.HasPrefix(s, "RTP/AVP/TCP;") {
-		return nil, fmt.Errorf("wrong transport: %s", s)
+		// Escam Q6 has a bug:
+		// Transport: RTP/AVP;unicast;destination=192.168.1.111;source=192.168.1.222;interleaved=0-1
+		if !strings.Contains(s, ";interleaved=") {
+			return nil, fmt.Errorf("wrong transport: %s", s)
+		}
 	}
 
 	i := strings.Index(s, "interleaved=")