apps/monibuca
1
0
Fork 0
mirror of https://github.com/langhuihui/monibuca.git synced 2025-10-06 11:26:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add webrtc h265 support

Browse Source
This commit is contained in:
langhuihui
2025-05-15 19:16:40 +08:00
parent ecc3947016
commit bcc7defa97
7 changed files with 116 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
plugin/rtp/pkg/video.go
View File

@@ -167,8 +167,6 @@ func (r *Video) Parse(t *AVTrack) (err error) {
if ctx.CodecData, err = h265parser.NewCodecDataFromVPSAndSPSAndPPS(vps, sps, pps); err != nil {
return
}
} else {
return
}
if sprop_donl, ok := ctx.Fmtp["sprop-max-don-diff"]; ok {
if sprop_donl != "0" {

4
plugin/rtsp/pkg/transceiver.go
View File

@@ -475,7 +475,7 @@ func (r *Receiver) Receive() (err error) {
if r.lastAudioPacketTS == 0 {
r.lastAudioPacketTS = packet.Timestamp
r.audioTSCheckStart = now
r.Stream.Debug("check audio timestamp start", "firsttime", "timestamp", packet.Timestamp)
r.Stream.Debug("check audio timestamp start firsttime", "timestamp", packet.Timestamp)
} else if !r.useVideoTS {
r.Stream.Debug("debug audio timestamp", "current", packet.Timestamp, "last", r.lastAudioPacketTS, "duration", now.Sub(r.audioTSCheckStart))
// 如果3秒内时间戳没有变化，切换到使用视频时间戳
@@ -493,7 +493,7 @@ func (r *Receiver) Receive() (err error) {
// 时间戳有变化，重置检查
r.lastAudioPacketTS = packet.Timestamp
r.audioTSCheckStart = now
r.Stream.Debug("check audio timestamp start", "reset audioTSCheckStart", "lastAudioPacketTS", r.lastAudioPacketTS)
r.Stream.Debug("reset audioTSCheckStart", "lastAudioPacketTS", r.lastAudioPacketTS)
}
}

6
plugin/webrtc/api.go
View File

@@ -85,6 +85,10 @@ func (conf *WebRTCPlugin) servePlay(w http.ResponseWriter, r *http.Request) {
return
}
conn.SDP = string(bytes)
// Check if client supports H265
if strings.Contains(strings.ToLower(conn.SDP), "h265") {
conn.SupportsH265 = true
}
if conn.PeerConnection, err = conf.api.NewPeerConnection(Configuration{
ICEServers: conf.ICEServers,
}); err != nil {
@@ -93,7 +97,7 @@ func (conf *WebRTCPlugin) servePlay(w http.ResponseWriter, r *http.Request) {
if rawQuery != "" {
streamPath += "?" + rawQuery
}
if conn.Subscriber, err = conf.Subscribe(conn.Context, streamPath); err != nil {
if conn.Subscriber, err = conf.Subscribe(conf.Context, streamPath); err != nil {
return
}
conn.Subscriber.RemoteAddr = r.RemoteAddr

17
plugin/webrtc/batchv2.go
View File

@@ -86,7 +86,9 @@ func (wsh *WebSocketHandler) Go() (err error) {
if !wsh.validateSDP(initialSignal.SDP) {
return wsh.sendError("Invalid SDP: missing ICE credentials")
}
if strings.Contains(strings.ToLower(wsh.SDP), "h265") {
wsh.SupportsH265 = true
}
// 设置远程描述
if err = wsh.SetRemoteDescription(SessionDescription{
Type: SDPTypeOffer,
@@ -171,7 +173,7 @@ func (wsh *WebSocketHandler) sendError(message string) error {
// handlePublish 处理发布信号
func (wsh *WebSocketHandler) handlePublish(signal Signal) {
if publisher, err := wsh.config.Publish(wsh.config.Context, signal.StreamPath); err == nil {
if publisher, err := wsh.config.Publish(wsh, signal.StreamPath); err == nil {
wsh.Publisher = publisher
wsh.Receive()
@@ -363,6 +365,17 @@ func (wsh *WebSocketHandler) handleGetStreamList() {
Height: uint32(ctx.Height()),
Fps: uint32(publisher.VideoTrack.FPS),
})
case *codec.H265Ctx:
if wsh.SupportsH265 {
// 获取视频信息
streams = append(streams, StreamInfo{
Path: publisher.StreamPath,
Codec: "H265",
Width: uint32(ctx.Width()),
Height: uint32(ctx.Height()),
Fps: uint32(publisher.VideoTrack.FPS),
})
}
}
}
}

4
plugin/webrtc/pkg/config.go
View File

@@ -91,6 +91,10 @@ func RegisterCodecs(m *MediaEngine) error {
// RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=123", nil},
// PayloadType: 118,
// },
{
RTPCodecCapability: RTPCodecCapability{MimeTypeH265, 90000, 0, "level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST", videoRTCPFeedback},
PayloadType: 49,
},
} {
if err := m.RegisterCodec(codec, RTPCodecTypeVideo); err != nil {
return err

273
plugin/webrtc/pkg/connection.go
View File

@@ -3,9 +3,8 @@ package webrtc
import (
"errors"
"fmt"
"net"
"regexp"
"strings"
"net" // Add this import
"strings" // Add this import
"time"
"github.com/pion/rtcp"
@@ -22,8 +21,9 @@ import (
type Connection struct {
*PeerConnection
Publisher *m7s.Publisher
SDP string
SupportsH265 bool // Add this field
Publisher *m7s.Publisher
SDP string
}
func (IO *Connection) GetOffer() (*SessionDescription, error) {
@@ -64,10 +64,12 @@ type MultipleConnection struct {
func (IO *MultipleConnection) Start() (err error) {
if IO.Publisher != nil {
IO.Depend(IO.Publisher)
IO.Publisher.Depend(IO)
IO.Receive()
}
if IO.Subscriber != nil {
IO.Depend(IO.Subscriber)
IO.Subscriber.Depend(IO)
IO.Send()
}
IO.OnICECandidate(func(ice *ICECandidate) {
@@ -204,213 +206,23 @@ func (IO *MultipleConnection) Receive() {
})
}
// H264CodecParams represents the parameters for an H.264 codec
type H264CodecParams struct {
ProfileLevelID string
PacketizationMode string
LevelAsymmetryAllowed string
SpropParameterSets string
OtherParams map[string]string
}
// parseH264Params parses H.264 codec parameters from an fmtp line
func parseH264Params(fmtpLine string) H264CodecParams {
params := H264CodecParams{
OtherParams: make(map[string]string),
}
// Split the fmtp line into key-value pairs
kvPairs := strings.Split(fmtpLine, ";")
for _, kv := range kvPairs {
kv = strings.TrimSpace(kv)
if kv == "" {
continue
}
parts := strings.SplitN(kv, "=", 2)
key := strings.TrimSpace(parts[0])
var value string
if len(parts) > 1 {
value = strings.TrimSpace(parts[1])
}
switch key {
case "profile-level-id":
params.ProfileLevelID = value
case "packetization-mode":
params.PacketizationMode = value
case "level-asymmetry-allowed":
params.LevelAsymmetryAllowed = value
case "sprop-parameter-sets":
params.SpropParameterSets = value
default:
params.OtherParams[key] = value
}
}
return params
}
// extractH264CodecParams extracts all H.264 codec parameters from an SDP
func extractH264CodecParams(sdp string) []H264CodecParams {
var result []H264CodecParams
// Find all fmtp lines for H.264 codecs
// First, find all a=rtpmap lines for H.264
rtpmapRegex := regexp.MustCompile(`a=rtpmap:(\d+) H264/\d+`)
rtpmapMatches := rtpmapRegex.FindAllStringSubmatch(sdp, -1)
for _, rtpmapMatch := range rtpmapMatches {
if len(rtpmapMatch) < 2 {
continue
}
// Get the payload type
payloadType := rtpmapMatch[1]
// Find the corresponding fmtp line
fmtpRegex := regexp.MustCompile(`a=fmtp:` + payloadType + ` ([^\r\n]+)`)
fmtpMatch := fmtpRegex.FindStringSubmatch(sdp)
if len(fmtpMatch) >= 2 {
// Parse the fmtp line
params := parseH264Params(fmtpMatch[1])
result = append(result, params)
}
}
return result
}
// findClosestProfileLevelID finds the closest matching profile-level-id
func findClosestProfileLevelID(availableIDs []string, currentID string) string {
// If current ID is empty, return the first available one
if currentID == "" && len(availableIDs) > 0 {
return availableIDs[0]
}
// If current ID is in the available ones, use it
for _, id := range availableIDs {
if strings.EqualFold(id, currentID) {
return currentID
}
}
// Try to match the profile part (first two characters)
if len(currentID) >= 2 {
currentProfile := currentID[:2]
for _, id := range availableIDs {
if len(id) >= 2 && strings.EqualFold(id[:2], currentProfile) {
return id
}
}
}
// If no match found, return the first available one
if len(availableIDs) > 0 {
return availableIDs[0]
}
// Fallback to the current one
return currentID
}
// findBestMatchingH264Codec finds the best matching H.264 codec configuration
func findBestMatchingH264Codec(sdp string, currentFmtpLine string) string {
// If no SDP or no current fmtp line, return the current one
if sdp == "" || currentFmtpLine == "" {
return currentFmtpLine
}
// Parse current parameters
currentParams := parseH264Params(currentFmtpLine)
// Extract all H.264 codec parameters from the SDP
availableParams := extractH264CodecParams(sdp)
// If no available parameters found, return the current one
if len(availableParams) == 0 {
return currentFmtpLine
}
// Extract all available profile-level-ids
var availableProfileLevelIDs []string
var packetizationModeMap = make(map[string]string)
for _, params := range availableParams {
if params.ProfileLevelID != "" {
availableProfileLevelIDs = append(availableProfileLevelIDs, params.ProfileLevelID)
// Store packetization mode for each profile-level-id
if params.PacketizationMode != "" {
packetizationModeMap[params.ProfileLevelID] = params.PacketizationMode
}
}
}
// Find the closest matching profile-level-id
closestProfileLevelID := findClosestProfileLevelID(availableProfileLevelIDs, currentParams.ProfileLevelID)
// Create result parameters
resultParams := H264CodecParams{
ProfileLevelID: closestProfileLevelID,
SpropParameterSets: currentParams.SpropParameterSets, // Always use original sprop-parameter-sets
LevelAsymmetryAllowed: "1", // Default to 1
}
// Use matching packetization mode if available
if mode, ok := packetizationModeMap[closestProfileLevelID]; ok {
resultParams.PacketizationMode = mode
} else if currentParams.PacketizationMode != "" {
resultParams.PacketizationMode = currentParams.PacketizationMode
} else {
resultParams.PacketizationMode = "1" // Default to 1
}
// Build and return the fmtp line
return buildFmtpLine(resultParams)
}
// buildFmtpLine builds an fmtp line from H.264 codec parameters
func buildFmtpLine(params H264CodecParams) string {
var parts []string
// Add profile-level-id if present
if params.ProfileLevelID != "" {
parts = append(parts, "profile-level-id="+params.ProfileLevelID)
}
// Add packetization-mode if present
if params.PacketizationMode != "" {
parts = append(parts, "packetization-mode="+params.PacketizationMode)
}
// Add level-asymmetry-allowed if present
if params.LevelAsymmetryAllowed != "" {
parts = append(parts, "level-asymmetry-allowed="+params.LevelAsymmetryAllowed)
}
// Add sprop-parameter-sets if present
if params.SpropParameterSets != "" {
parts = append(parts, "sprop-parameter-sets="+params.SpropParameterSets)
}
// Add other parameters
for k, v := range params.OtherParams {
parts = append(parts, k+"="+v)
}
return strings.Join(parts, ";")
}
func (IO *MultipleConnection) SendSubscriber(subscriber *m7s.Subscriber) (audioSender, videoSender *RTPSender, err error) {
var useDC bool
var audioTLSRTP, videoTLSRTP *TrackLocalStaticRTP
vctx, actx := subscriber.Publisher.GetVideoCodecCtx(), subscriber.Publisher.GetAudioCodecCtx()
if IO.EnableDC {
if IO.EnableDC && vctx != nil && vctx.FourCC() == codec.FourCC_H265 {
useDC = true
}
if IO.EnableDC && actx != nil && actx.FourCC() == codec.FourCC_MP4A {
// If H265 is supported by the client, we do NOT use DataChannel for H265 video.
// DataChannel will be used for H265 video only if the client does NOT support H265 (potentially for transcoding or specific handling).
// Or if video is not H265 but DC is enabled for other codecs like MP4A audio.
if vctx != nil && vctx.FourCC() == codec.FourCC_H265 {
if !IO.SupportsH265 { // Client does not support H265, so use DC
useDC = true
IO.Info("Client does not support H265, using DataChannel for H265 video.")
} else {
// Client supports H265, so we will use RTP. useDC remains false.
IO.Info("Client supports H265, using RTP for H265 video.")
}
} else if actx != nil && actx.FourCC() == codec.FourCC_MP4A { // For MP4A audio, use DC if enabled
useDC = true
}
}
@@ -430,19 +242,6 @@ func (IO *MultipleConnection) SendSubscriber(subscriber *m7s.Subscriber) (audioS
}
}
// // For H.264, adjust codec parameters based on SDP
// if rcc.MimeType == MimeTypeH264 && IO.SDP != "" {
// // Find best matching codec configuration
// originalFmtpLine := rcc.SDPFmtpLine
// bestMatchingFmtpLine := findBestMatchingH264Codec(IO.SDP, rcc.SDPFmtpLine)
// // Update the codec parameters if a better match was found
// if bestMatchingFmtpLine != originalFmtpLine {
// rcc.SDPFmtpLine = bestMatchingFmtpLine
// IO.Info("Adjusted H.264 codec parameters", "from", originalFmtpLine, "to", bestMatchingFmtpLine)
// }
// }
videoTLSRTP, err = NewTrackLocalStaticRTP(rcc.RTPCodecCapability, videoCodec.String(), subscriber.StreamPath)
if err != nil {
return
@@ -470,7 +269,7 @@ func (IO *MultipleConnection) SendSubscriber(subscriber *m7s.Subscriber) (audioS
}
}()
}
if actx != nil && !useDC {
if actx != nil && !useDC && actx.FourCC() != codec.FourCC_MP4A {
audioCodec := actx.FourCC()
var rcc RTPCodecParameters
if ctx, ok := actx.(mrtp.IRTPCtx); ok {
@@ -485,6 +284,26 @@ func (IO *MultipleConnection) SendSubscriber(subscriber *m7s.Subscriber) (audioS
return
}
}
// Transform SDPFmtpLine for WebRTC compatibility (primarily for video codecs, but general logic)
mimeTypeLower := strings.ToLower(rcc.RTPCodecCapability.MimeType)
if strings.Contains(mimeTypeLower, "h264") || strings.Contains(mimeTypeLower, "h265") { // This condition will likely not match for typical audio codecs
originalFmtpLine := rcc.RTPCodecCapability.SDPFmtpLine
parts := strings.Split(originalFmtpLine, ";")
var newParts []string
for _, part := range parts {
trimmedPart := strings.TrimSpace(part)
if !strings.HasPrefix(trimmedPart, "sprop-parameter-sets=") {
newParts = append(newParts, trimmedPart)
}
}
transformedFmtpLine := strings.Join(newParts, ";")
if transformedFmtpLine != originalFmtpLine {
rcc.RTPCodecCapability.SDPFmtpLine = transformedFmtpLine
IO.Info("Adjusted SDPFmtpLine for WebRTC (audio track context)", "codec", rcc.RTPCodecCapability.MimeType, "from", originalFmtpLine, "to", transformedFmtpLine)
}
}
audioTLSRTP, err = NewTrackLocalStaticRTP(rcc.RTPCodecCapability, audioCodec.String(), subscriber.StreamPath)
if err != nil {
return
@@ -589,18 +408,6 @@ func (r *RemoteStream) Start() (err error) {
return
}
}
// // For H.264, adjust codec parameters based on SDP
// if rcc.MimeType == MimeTypeH264 && r.pc.SDP != "" {
// // Find best matching codec configuration
// originalFmtpLine := rcc.SDPFmtpLine
// bestMatchingFmtpLine := findBestMatchingH264Codec(r.pc.SDP, rcc.SDPFmtpLine)
// // Update the codec parameters if a better match was found
// if bestMatchingFmtpLine != originalFmtpLine {
// rcc.SDPFmtpLine = bestMatchingFmtpLine
// r.Info("Adjusted H.264 codec parameters", "from", originalFmtpLine, "to", bestMatchingFmtpLine)
// }
// }
r.videoTLSRTP, err = NewTrackLocalStaticRTP(rcc.RTPCodecCapability, videoCodec.String(), r.suber.StreamPath)
if err != nil {

50
plugin/webrtc/web/publish.html
View File

@@ -59,6 +59,56 @@
mediaStream.getTracks().forEach((t) => {
pc.addTrack(t, mediaStream);
});
const preferH265 = searchParams.has('h265');
if (preferH265) {
const videoTransceiver = pc.getTransceivers().find(
t => t.sender.track && t.sender.track.kind === 'video'
);
if (videoTransceiver && typeof videoTransceiver.setCodecPreferences === 'function') {
const capabilities = RTCRtpSender.getCapabilities('video');
if (capabilities && capabilities.codecs) {
const h265Codec = capabilities.codecs.find(c => c.mimeType.toLowerCase() === 'video/h265');
if (h265Codec) {
const preferredCodecs = [h265Codec];
videoTransceiver.setCodecPreferences(preferredCodecs);
console.log('Attempted to set H.265 as preferred codec.');
} else {
console.warn('H.265 codec not found in sender capabilities.');
}
} else {
console.warn('Could not get video sender capabilities for H.265 preference.');
}
} else if (videoTransceiver && typeof videoTransceiver.setCodecPreferences !== 'function') {
console.warn('videoTransceiver.setCodecPreferences is not a function. Cannot set H.265 preference.');
} else {
console.warn('Video transceiver not found. Cannot set H.265 preference.');
}
}
const audioTransceiver = pc.getTransceivers().find(
t => t.sender.track && t.sender.track.kind === 'audio'
);
if (audioTransceiver && typeof audioTransceiver.setCodecPreferences === 'function') {
const capabilities = RTCRtpSender.getCapabilities('audio');
if (capabilities && capabilities.codecs) {
const pcmaCodec = capabilities.codecs.find(c => c.mimeType.toLowerCase() === 'audio/pcma');
if (pcmaCodec) {
const preferredCodecs = [pcmaCodec];
audioTransceiver.setCodecPreferences(preferredCodecs);
console.log('Attempted to set PCMA as preferred audio codec.');
} else {
console.warn('PCMA codec not found in sender capabilities.');
}
} else {
console.warn('Could not get audio sender capabilities for PCMA preference.');
}
} else if (audioTransceiver && typeof audioTransceiver.setCodecPreferences !== 'function') {
console.warn('audioTransceiver.setCodecPreferences is not a function. Cannot set PCMA preference.');
} else {
console.warn('Audio transceiver not found. Cannot set PCMA preference.');
}
// const videoTransceiver = pc.addTransceiver(mediaStream.getVideoTracks()[0], { direction: 'sendonly' });
// const audioTransceiver = pc.addTransceiver(mediaStream.getAudioTracks()[0], { direction: 'sendonly' });
// const dc = pc.createDataChannel('sdp');