feat: add screenshare

feat: add camera selector

desc: 增加屏幕分享和摄像头选择功能

README.en.md

@@ -69,6 +69,13 @@ Response Body: `SDP`
 `/webrtc/test/publish`
 - `?streamPath=xxx` The streamPath to publish, default is `live/webrtc`
 - you can add other query parameters to the URL
+
+### ScreenShare Test Page
+
+`/webrtc/test/screenshare`
+- `?streamPath=xxx` The streamPath to publish, default is `live/webrtc`
+- you can add other query parameters to the URL
+
 ### Play Test Page
 
 `/webrtc/test/subscribe`

README.md

@@ -66,6 +66,12 @@ Response Body: `SDP`
 `/webrtc/test/publish`
 - 可增加参数`?streamPath=xxx`指定推流地址，默认为`live/webrtc`
 - 可以增加其他推流参数
+
+### 屏幕分享测试
+  
+`/webrtc/test/screenshare`
+- 可增加参数`?streamPath=xxx`指定推流地址，默认为`live/webrtc`
+- 可以增加其他推流参数
 ### 播放测试页面
 
 `/webrtc/test/subscribe`

batcher.go

@@ -73,13 +73,13 @@ func (suber *WebRTCBatcher) Signal(msg DataChannelMessage) {
 				sub.WebRTCIO = suber.WebRTCIO
 				if err = WebRTCPlugin.SubscribeExist(streamPath, sub); err == nil {
 					suber.subscribers = append(suber.subscribers, sub)
-					go func() {
+					go func(streamPath string) {
 						sub.PlayRTP()
-						if sub.audioSender != nil {
-							suber.RemoveTrack(sub.audioSender)
+						if sub.audio.RTPSender != nil {
+							suber.RemoveTrack(sub.audio.RTPSender )
 						}
-						if sub.videoSender != nil {
-							suber.RemoveTrack(sub.videoSender)
+						if sub.video.RTPSender != nil {
+							suber.RemoveTrack(sub.video.RTPSender)
 						}
 						if sub.DC != nil {
 							sub.DC.Close()
@@ -87,7 +87,7 @@ func (suber *WebRTCBatcher) Signal(msg DataChannelMessage) {
 						removeMap["streamPath"] = streamPath
 						b, _ := json.Marshal(removeMap)
 						suber.signalChannel.SendText(string(b))
-					}()
+					}(streamPath)
 				} else {
 					removeMap["streamPath"] = streamPath
 					b, _ := json.Marshal(removeMap)

io.go

@@ -7,6 +7,7 @@ import (
 type WebRTCIO struct {
 	*PeerConnection
 	SDP string
+	// LocalSDP *sdp.SessionDescription
 }
 
 func (IO *WebRTCIO) GetAnswer() (string, error) {
@@ -15,10 +16,14 @@ func (IO *WebRTCIO) GetAnswer() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	// IO.LocalSDP, err = answer.Unmarshal()
+	// if err != nil {
+	// 	return "", err
+	// }
 	gatherComplete := GatheringCompletePromise(IO.PeerConnection)
 	if err := IO.SetLocalDescription(answer); err != nil {
 		return "", err
 	}
 	<-gatherComplete
 	return IO.LocalDescription().SDP, nil
-}
+}

main.go

@@ -230,6 +230,9 @@ func (conf *WebRTCConfig) Push_(w http.ResponseWriter, r *http.Request) {
 func (conf *WebRTCConfig) Test_Publish(w http.ResponseWriter, r *http.Request) {
 	w.Write(publishHTML)
 }
+func (conf *WebRTCConfig) Test_ScreenShare(w http.ResponseWriter, r *http.Request) {
+	w.Write(publishHTML)
+}
 func (conf *WebRTCConfig) Test_Subscribe(w http.ResponseWriter, r *http.Request) {
 	w.Write(subscribeHTML)
 }

publish.html

@@ -12,6 +12,7 @@
   <video id="video" width="640" height="480" autoplay muted>
   </video>
   <!-- <button id="sw" onclick="action()" type="button" style="width:100px;height:30px;display: block;">unpublish</button> -->
+  <div id="camera"></div>
   <pre>
   <code id="remoteSdp">
 
@@ -20,16 +21,29 @@
 </body>
 <script>
   let action = () => { console.log('action not set'); };
+  const screenshare = location.pathname.endsWith("screenshare");
   (async () => {
+    const searchParams = new URLSearchParams(location.search);
+    const $camera = document.getElementById('camera');
     navigator.mediaDevices.enumerateDevices().then((devices) => {
       devices.forEach((device) => {
         console.log(device.kind + ": " + device.label + " id = " + device.deviceId);
+        if (device.kind == 'videoinput' && !screenshare) {
+          const a = document.createElement('a');
+          a.href = location.pathname + '?videoinput=' + device.deviceId;
+          a.innerHTML = device.label;
+          $camera.appendChild(a);
+          $camera.appendChild(document.createElement('br'));
+        }
       });
     });
-    const mediaStream = await navigator.mediaDevices.getUserMedia({
-      video: true,
+    const mediaStream = await (screenshare ? navigator.mediaDevices.getDisplayMedia() : navigator.mediaDevices.getUserMedia({
+      video: searchParams.get('videoinput') ? {
+        deviceId: searchParams.get('videoinput'),
+      } : true,
       audio: true,
-    });
+    }));
+    searchParams.delete('videoinput');
     document.getElementById('video').srcObject = mediaStream;
     const pc = new RTCPeerConnection();
 
@@ -39,9 +53,8 @@
     pc.onicecandidate = (e) => {
       console.log('onicecandidate', e.candidate);
     };
-    const searchParams = new URLSearchParams(location.search);
     const streamPath = searchParams.get('streamPath') || 'live/webrtc';
-    searchParams.delete('streamPath')
+    searchParams.delete('streamPath');
     mediaStream.id = streamPath;
     mediaStream.getTracks().forEach((t) => {
       pc.addTrack(t, mediaStream);
@@ -52,7 +65,7 @@
     const offer = await pc.createOffer();
     await pc.setLocalDescription(offer);
     const result = await fetch(
-      `/webrtc/push/${streamPath}${location.search?`?${searchParams.toString()}`:''}`,
+      `/webrtc/push/${streamPath}${location.search ? `?${searchParams.toString()}` : ''}`,
       {
         method: 'POST',
         mode: 'cors',

publisher.go

@@ -68,6 +68,11 @@ func (puber *WebRTCPublisher) onTrack(track *TrackRemote, receiver *RTPReceiver)
 			rtpItem := puber.VideoTrack.GetRTPFromPool()
 			if i, _, err := track.Read(rtpItem.Value.Raw); err == nil {
 				rtpItem.Value.Unmarshal(rtpItem.Value.Raw[:i])
+				if rtpItem.Value.Extension {
+					for _, id := range rtpItem.Value.GetExtensionIDs() {
+						puber.Debug("extension", zap.Uint8("id", id), zap.Binary("value", rtpItem.Value.GetExtension(id)))
+					}
+				}
 				puber.VideoTrack.WriteRTP(rtpItem)
 			} else {
 				puber.Info("track stop", zap.String("kind", track.Kind().String()), zap.Error(err))

subscriber.go

@@ -2,27 +2,35 @@ package webrtc
 
 import (
 	"fmt"
-	"net"
 	"strings"
 
 	"github.com/pion/rtcp"
 	. "github.com/pion/webrtc/v3"
-	"go.uber.org/zap"
 	. "m7s.live/engine/v4"
 	"m7s.live/engine/v4/codec"
 	"m7s.live/engine/v4/track"
 	"m7s.live/engine/v4/util"
 )
 
+type trackSender struct {
+	*TrackLocalStaticRTP
+	*RTPSender
+	// seq uint32
+}
+
 type WebRTCSubscriber struct {
 	Subscriber
 	WebRTCIO
-	videoTrack   *TrackLocalStaticRTP
-	audioTrack   *TrackLocalStaticRTP
-	videoSender  *RTPSender
-	audioSender  *RTPSender
-	DC           *DataChannel
-	flvHeadCache []byte
+	audio trackSender
+	video trackSender
+	DC    *DataChannel
+	// flvHeadCache []byte
+}
+
+func (suber *WebRTCSubscriber) queueDCData(data ...[]byte) {
+	for _, d := range data {
+		suber.DC.Send(d)
+	}
 }
 
 func (suber *WebRTCSubscriber) createDataChannel() {
@@ -30,27 +38,27 @@ func (suber *WebRTCSubscriber) createDataChannel() {
 		return
 	}
 	suber.DC, _ = suber.PeerConnection.CreateDataChannel(suber.Subscriber.Stream.Path, nil)
-	suber.flvHeadCache = make([]byte, 15)
-	suber.DC.Send(codec.FLVHeader)
-}
-func (suber *WebRTCSubscriber) sendAvByDatachannel(t byte, reader *track.AVRingReader) {
-	suber.flvHeadCache[0] = t
-	frame := reader.Frame
-	dataSize := uint32(frame.AVCC.ByteLength)
-	result := net.Buffers{suber.flvHeadCache[:11]}
-	result = append(result, frame.AVCC.ToBuffers()...)
-	ts := reader.AbsTime
-	util.PutBE(suber.flvHeadCache[1:4], dataSize)
-	util.PutBE(suber.flvHeadCache[4:7], ts)
-	suber.flvHeadCache[7] = byte(ts >> 24)
-	result = append(result, util.PutBE(suber.flvHeadCache[11:15], dataSize+11))
-	for _, data := range util.SplitBuffers(result, 65535) {
-		for _, d := range data {
-			suber.DC.Send(d)
-		}
-	}
+	// suber.flvHeadCache = make([]byte, 15)
 }
 
+// func (suber *WebRTCSubscriber) sendAvByDatachannel(t byte, reader *track.AVRingReader) {
+// 	suber.flvHeadCache[0] = t
+// 	frame := reader.Frame
+// 	dataSize := uint32(frame.AVCC.ByteLength)
+// 	result := net.Buffers{suber.flvHeadCache[:11]}
+// 	result = append(result, frame.AVCC.ToBuffers()...)
+// 	ts := reader.AbsTime
+// 	util.PutBE(suber.flvHeadCache[1:4], dataSize)
+// 	util.PutBE(suber.flvHeadCache[4:7], ts)
+// 	suber.flvHeadCache[7] = byte(ts >> 24)
+// 	result = append(result, util.PutBE(suber.flvHeadCache[11:15], dataSize+11))
+// 	for _, data := range util.SplitBuffers(result, 65535) {
+// 		for _, d := range data {
+// 			suber.queueDCData(d)
+// 		}
+// 	}
+// }
+
 func (suber *WebRTCSubscriber) OnEvent(event any) {
 	switch v := event.(type) {
 	case *track.Video:
@@ -64,35 +72,13 @@ func (suber *WebRTCSubscriber) OnEvent(event any) {
 					pli = list[0][1]
 				}
 			}
-			suber.videoTrack, _ = NewTrackLocalStaticRTP(RTPCodecCapability{MimeType: MimeTypeH264, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=" + pli}, v.Name, suber.Subscriber.Stream.Path)
+			suber.video.TrackLocalStaticRTP, _ = NewTrackLocalStaticRTP(RTPCodecCapability{MimeType: MimeTypeH264, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=" + pli}, v.Name, suber.Subscriber.Stream.Path)
 		case codec.CodecID_H265:
+			suber.createDataChannel()
 			// suber.videoTrack, _ = NewTrackLocalStaticRTP(RTPCodecCapability{MimeType: MimeTypeH265, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=" + pli}, "video", suber.Subscriber.Stream.Path)
 		default:
 			return
 		}
-		if suber.videoTrack == nil {
-			suber.createDataChannel()
-		} else {
-			suber.videoSender, _ = suber.PeerConnection.AddTrack(suber.videoTrack)
-			go func() {
-				rtcpBuf := make([]byte, 1500)
-				for {
-					if n, _, rtcpErr := suber.videoSender.Read(rtcpBuf); rtcpErr != nil {
-
-						return
-					} else {
-						if p, err := rtcp.Unmarshal(rtcpBuf[:n]); err == nil {
-							for _, pp := range p {
-								switch pp.(type) {
-								case *rtcp.PictureLossIndication:
-									// fmt.Println("PictureLossIndication")
-								}
-							}
-						}
-					}
-				}
-			}()
-		}
 		suber.Subscriber.AddTrack(v) //接受这个track
 	case *track.Audio:
 		audioMimeType := MimeTypePCMA
@@ -103,37 +89,77 @@ func (suber *WebRTCSubscriber) OnEvent(event any) {
 		case codec.CodecID_AAC:
 			suber.createDataChannel()
 		case codec.CodecID_PCMA, codec.CodecID_PCMU:
-			suber.audioTrack, _ = NewTrackLocalStaticRTP(RTPCodecCapability{MimeType: audioMimeType}, v.Name, suber.Subscriber.Stream.Path)
-			suber.audioSender, _ = suber.PeerConnection.AddTrack(suber.audioTrack)
-			suber.Subscriber.AddTrack(v) //接受这个track
-		}
-	case VideoDeConf:
-		if suber.DC != nil {
-			suber.DC.Send(util.ConcatBuffers(codec.VideoAVCC2FLV(0, v)))
-		}
-	case AudioDeConf:
-		if suber.DC != nil {
-			suber.DC.Send(util.ConcatBuffers(codec.AudioAVCC2FLV(0, v)))
+			suber.audio.TrackLocalStaticRTP, _ = NewTrackLocalStaticRTP(RTPCodecCapability{MimeType: audioMimeType}, v.Name, suber.Subscriber.Stream.Path)
+			//suber.audio.RTPSender, _ = suber.PeerConnection.AddTrack(suber.audio.TrackLocalStaticRTP)
 		}
+		suber.Subscriber.AddTrack(v) //接受这个track
+	// case VideoDeConf:
+	// 	if suber.DC != nil {
+	// 		suber.queueDCData(codec.VideoAVCC2FLV(0, v)...)
+	// 	}
+	// case AudioDeConf:
+	// 	if suber.DC != nil {
+	// 		suber.queueDCData(codec.AudioAVCC2FLV(0, v)...)
+	// 	}
 	case VideoRTP:
-		if suber.videoTrack != nil {
-			suber.Trace("video rtp", zap.Any("packet", v.Packet.Header))
-			suber.videoTrack.WriteRTP(v.Packet)
-		} else if suber.DC != nil {
-			suber.sendAvByDatachannel(9, suber.VideoReader)
-		}
+		// if suber.video.TrackLocalStaticRTP != nil {
+		suber.video.WriteRTP(v.Packet)
+		// } else if suber.DC != nil && suber.VideoReader.Frame.Sequence != suber.video.seq {
+		// 	suber.video.seq = suber.VideoReader.Frame.Sequence
+		// 	suber.sendAvByDatachannel(9, suber.VideoReader)
+		// }
 	case AudioRTP:
-		if suber.audioTrack != nil {
-			suber.audioTrack.WriteRTP(v.Packet)
-		} else if suber.DC != nil {
-			suber.sendAvByDatachannel(8, suber.AudioReader)
+		// if suber.audio.TrackLocalStaticRTP != nil {
+		suber.audio.WriteRTP(v.Packet)
+		// } else if suber.DC != nil && suber.AudioReader.Frame.Sequence != suber.audio.seq {
+		// 	suber.audio.seq = suber.AudioReader.Frame.Sequence
+		// 	suber.sendAvByDatachannel(8, suber.AudioReader)
+		// }
+	case FLVFrame:
+		for _, data := range util.SplitBuffers(v, 65535) {
+			suber.queueDCData(data...)
 		}
 	case ISubscriber:
+		if suber.DC == nil {
+			if suber.audio.TrackLocalStaticRTP != nil {
+				suber.audio.RTPSender, _ = suber.PeerConnection.AddTrack(suber.audio.TrackLocalStaticRTP)
+			}
+			if suber.video.TrackLocalStaticRTP != nil {
+				suber.video.RTPSender, _ = suber.PeerConnection.AddTrack(suber.video.TrackLocalStaticRTP)
+				go func() {
+					rtcpBuf := make([]byte, 1500)
+					for {
+						if n, _, rtcpErr := suber.video.Read(rtcpBuf); rtcpErr != nil {
+
+							return
+						} else {
+							if p, err := rtcp.Unmarshal(rtcpBuf[:n]); err == nil {
+								for _, pp := range p {
+									switch pp.(type) {
+									case *rtcp.PictureLossIndication:
+										// fmt.Println("PictureLossIndication")
+									}
+								}
+							}
+						}
+					}
+				}()
+			}
+		} else {
+			suber.DC.OnOpen(func() {
+				suber.DC.Send(codec.FLVHeader)
+				go suber.PlayFLV()
+			})
+		}
 		suber.OnConnectionStateChange(func(pcs PeerConnectionState) {
 			suber.Info("Connection State has changed:" + pcs.String())
 			switch pcs {
 			case PeerConnectionStateConnected:
-				go suber.PlayRTP()
+				if suber.DC != nil {
+					// go suber.PlayFLV()
+				} else {
+					go suber.PlayRTP()
+				}
 			case PeerConnectionStateDisconnected, PeerConnectionStateFailed:
 				suber.Stop()
 				suber.PeerConnection.Close()