diff --git a/mediaengine.go b/mediaengine.go index ab735e9f..d1264c15 100644 --- a/mediaengine.go +++ b/mediaengine.go @@ -630,6 +630,23 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e return err } + addIfNew := func(existingCodecs []RTPCodecParameters, codec RTPCodecParameters) []RTPCodecParameters { + found := false + for _, existingCodec := range existingCodecs { + if existingCodec.PayloadType == codec.PayloadType { + found = true + + break + } + } + + if !found { + existingCodecs = append(existingCodecs, codec) + } + + return existingCodecs + } + exactMatches := make([]RTPCodecParameters, 0, len(codecs)) partialMatches := make([]RTPCodecParameters, 0, len(codecs)) @@ -642,9 +659,24 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e remoteCodec.RTCPFeedback = rtcpFeedbackIntersection(localCodec.RTCPFeedback, remoteCodec.RTCPFeedback) if matchType == codecMatchExact { - exactMatches = append(exactMatches, remoteCodec) + exactMatches = addIfNew(exactMatches, remoteCodec) } else if matchType == codecMatchPartial { - partialMatches = append(partialMatches, remoteCodec) + partialMatches = addIfNew(partialMatches, remoteCodec) + } + } + // second pass in case there were missed RTX codecs + for _, remoteCodec := range codecs { + localCodec, matchType, mErr := m.matchRemoteCodec(remoteCodec, typ, exactMatches, partialMatches) + if mErr != nil { + return mErr + } + + remoteCodec.RTCPFeedback = rtcpFeedbackIntersection(localCodec.RTCPFeedback, remoteCodec.RTCPFeedback) + + if matchType == codecMatchExact { + exactMatches = addIfNew(exactMatches, remoteCodec) + } else if matchType == codecMatchPartial { + partialMatches = addIfNew(partialMatches, remoteCodec) } } diff --git a/peerconnection.go b/peerconnection.go index 069c2144..1434e7dc 100644 --- a/peerconnection.go +++ b/peerconnection.go @@ -1157,35 +1157,11 @@ func (pc *PeerConnection) SetRemoteDescription(desc SessionDescription) error { transceiver = newRTPTransceiver(receiver, nil, localDirection, kind, pc.api) transceiver.setCurrentRemoteDirection(direction) + transceiver.setCodecPreferencesFromRemoteDescription(media) pc.mu.Lock() pc.addRTPTransceiver(transceiver) pc.mu.Unlock() - // if transceiver is create by remote sdp, set prefer codec same as remote peer - if codecs, err := codecsFromMediaDescription(media); err == nil { - filteredCodecs := []RTPCodecParameters{} - filteredCodecsPartial := []RTPCodecParameters{} - for _, codec := range codecs { - c, matchType := codecParametersFuzzySearch( - codec, - pc.api.mediaEngine.getCodecsByKind(kind), - ) - switch matchType { - case codecMatchExact: - // if codec match exact, use payloadtype register to mediaengine - codec.PayloadType = c.PayloadType - filteredCodecs = append(filteredCodecs, codec) - case codecMatchPartial: - codec.PayloadType = c.PayloadType - filteredCodecsPartial = append(filteredCodecsPartial, codec) - - default: - } - } - filteredCodecs = append(filteredCodecs, filteredCodecsPartial...) - _ = transceiver.SetCodecPreferences(filteredCodecs) - } - case direction == RTPTransceiverDirectionRecvonly: if transceiver.Direction() == RTPTransceiverDirectionSendrecv { transceiver.setDirection(RTPTransceiverDirectionSendonly) diff --git a/peerconnection_go_test.go b/peerconnection_go_test.go index 15857747..bf57f45c 100644 --- a/peerconnection_go_test.go +++ b/peerconnection_go_test.go @@ -1502,13 +1502,13 @@ func TestPeerConnectionNilCallback(t *testing.T) { } func TestTransceiverCreatedByRemoteSdpHasSameCodecOrderAsRemote(t *testing.T) { - t.Run("Codec MatchExact", func(t *testing.T) { //nolint:dupl + t.Run("Codec MatchExact and MatchPartial", func(t *testing.T) { //nolint:dupl const remoteSdp = `v=0 o=- 4596489990601351948 2 IN IP4 127.0.0.1 s=- t=0 0 a=group:BUNDLE 0 1 -m=video 60323 UDP/TLS/RTP/SAVPF 98 94 106 +m=video 60323 UDP/TLS/RTP/SAVPF 98 94 106 49 a=ice-ufrag:1/MvHwjAyVf27aLu a=ice-pwd:3dBU7cFOBl120v33cynDvN1E a=ice-options:google-ice @@ -1519,8 +1519,10 @@ a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f a=rtpmap:94 VP8/90000 a=rtpmap:106 H264/90000 a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:49 H265/90000 +a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST a=sendonly -m=video 60323 UDP/TLS/RTP/SAVPF 108 98 125 +m=video 60323 UDP/TLS/RTP/SAVPF 49 108 98 125 a=ice-ufrag:1/MvHwjAyVf27aLu a=ice-pwd:3dBU7cFOBl120v33cynDvN1E a=ice-options:google-ice @@ -1532,6 +1534,8 @@ a=rtpmap:108 VP8/90000 a=sendonly a=rtpmap:125 H264/90000 a=fmtp:125 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:49 H265/90000 +a=fmtp:49 level-id=93;profile-id=1;tier-flag=0;tx-mode=SRST ` mediaEngine := MediaEngine{} assert.NoError(t, mediaEngine.RegisterCodec(RTPCodecParameters{ @@ -1544,6 +1548,12 @@ a=fmtp:125 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01 }, PayloadType: 98, }, RTPCodecTypeVideo)) + assert.NoError(t, mediaEngine.RegisterCodec(RTPCodecParameters{ + RTPCodecCapability: RTPCodecCapability{ + MimeTypeH265, 90000, 0, "level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST", nil, + }, + PayloadType: 49, + }, RTPCodecTypeVideo)) api := NewAPI(WithMediaEngine(&mediaEngine)) pc, err := api.NewPeerConnection(Configuration{}) @@ -1552,20 +1562,35 @@ a=fmtp:125 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01 Type: SDPTypeOffer, SDP: remoteSdp, })) + ans, _ := pc.CreateAnswer(nil) assert.NoError(t, pc.SetLocalDescription(ans)) - codecOfTr1 := pc.GetTransceivers()[0].getCodecs()[0] + codecs := pc.api.mediaEngine.getCodecsByKind(RTPCodecTypeVideo) - _, matchType := codecParametersFuzzySearch(codecOfTr1, codecs) + + codecsOfTr1 := pc.GetTransceivers()[0].getCodecs() + _, matchType := codecParametersFuzzySearch(codecsOfTr1[0], codecs) assert.Equal(t, codecMatchExact, matchType) - codecOfTr2 := pc.GetTransceivers()[1].getCodecs()[0] - _, matchType = codecParametersFuzzySearch(codecOfTr2, codecs) + assert.EqualValues(t, 98, codecsOfTr1[0].PayloadType) + _, matchType = codecParametersFuzzySearch(codecsOfTr1[1], codecs) assert.Equal(t, codecMatchExact, matchType) - assert.EqualValues(t, 94, codecOfTr2.PayloadType) - codecPartialMatchOfTr2 := pc.GetTransceivers()[1].getCodecs()[2] - _, matchType = codecParametersFuzzySearch(codecPartialMatchOfTr2, codecs) + assert.EqualValues(t, 94, codecsOfTr1[1].PayloadType) + _, matchType = codecParametersFuzzySearch(codecsOfTr1[2], codecs) + assert.Equal(t, codecMatchExact, matchType) + assert.EqualValues(t, 49, codecsOfTr1[2].PayloadType) + + codecsOfTr2 := pc.GetTransceivers()[1].getCodecs() + _, matchType = codecParametersFuzzySearch(codecsOfTr2[0], codecs) + assert.Equal(t, codecMatchExact, matchType) + assert.EqualValues(t, 94, codecsOfTr2[0].PayloadType) + _, matchType = codecParametersFuzzySearch(codecsOfTr2[1], codecs) + assert.Equal(t, codecMatchExact, matchType) + assert.EqualValues(t, 98, codecsOfTr2[1].PayloadType) + // as H.265 (49) is a partial match, it gets pushed to the end + _, matchType = codecParametersFuzzySearch(codecsOfTr2[2], codecs) assert.Equal(t, codecMatchPartial, matchType) - assert.EqualValues(t, 98, codecPartialMatchOfTr2.PayloadType) + assert.EqualValues(t, 49, codecsOfTr2[2].PayloadType) + assert.NoError(t, pc.Close()) }) @@ -1613,21 +1638,34 @@ a=sendonly api := NewAPI(WithMediaEngine(&mediaEngine)) pc, err := api.NewPeerConnection(Configuration{}) assert.NoError(t, err) + assert.NoError(t, pc.SetRemoteDescription(SessionDescription{ Type: SDPTypeOffer, SDP: remoteSdp, })) + ans, _ := pc.CreateAnswer(nil) assert.NoError(t, pc.SetLocalDescription(ans)) - codecOfTr1 := pc.GetTransceivers()[0].getCodecs()[0] + codecs := pc.api.mediaEngine.getCodecsByKind(RTPCodecTypeVideo) - _, matchType := codecParametersFuzzySearch(codecOfTr1, codecs) + + codecsOfTr1 := pc.GetTransceivers()[0].getCodecs() + _, matchType := codecParametersFuzzySearch(codecsOfTr1[0], codecs) assert.Equal(t, codecMatchExact, matchType) - codecOfTr2 := pc.GetTransceivers()[1].getCodecs()[0] - _, matchType = codecParametersFuzzySearch(codecOfTr2, codecs) + assert.EqualValues(t, 98, codecsOfTr1[0].PayloadType) + _, matchType = codecParametersFuzzySearch(codecsOfTr1[1], codecs) + assert.Equal(t, codecMatchExact, matchType) + assert.EqualValues(t, 106, codecsOfTr1[1].PayloadType) + + codecsOfTr2 := pc.GetTransceivers()[1].getCodecs() + _, matchType = codecParametersFuzzySearch(codecsOfTr2[0], codecs) assert.Equal(t, codecMatchExact, matchType) // h.264/profile-id=640032 should be remap to 106 as same as transceiver 1 - assert.EqualValues(t, 106, codecOfTr2.PayloadType) + assert.EqualValues(t, 106, codecsOfTr2[0].PayloadType) + _, matchType = codecParametersFuzzySearch(codecsOfTr2[1], codecs) + assert.Equal(t, codecMatchExact, matchType) + assert.EqualValues(t, 98, codecsOfTr2[1].PayloadType) + assert.NoError(t, pc.Close()) }) } diff --git a/peerconnection_media_test.go b/peerconnection_media_test.go index c25d60b3..1005bcbe 100644 --- a/peerconnection_media_test.go +++ b/peerconnection_media_test.go @@ -27,6 +27,7 @@ import ( "github.com/pion/sdp/v3" "github.com/pion/transport/v3/test" "github.com/pion/transport/v3/vnet" + "github.com/pion/webrtc/v4/internal/fmtp" "github.com/pion/webrtc/v4/internal/util" "github.com/pion/webrtc/v4/pkg/media" "github.com/stretchr/testify/assert" @@ -766,7 +767,6 @@ func TestAddTransceiverAddTrack_Reuse(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.remoteDirection.String(), func(t *testing.T) { pcOffer, pcAnswer, err := newPair() - require.NoError(t, err) assert.NoError(t, err) remoteTrack, err := NewTrackLocalStaticSample( @@ -883,6 +883,184 @@ func TestAddTransceiverAddTrack_Reuse(t *testing.T) { }) } +func TestAddTransceiverFromRemoteDescription(t *testing.T) { + lim := test.TimeOut(time.Second * 30) + defer lim.Stop() + + report := test.CheckRoutines(t) + defer report() + + // offer side + se := SettingEngine{} + se.DisableMediaEngineCopy(true) + mediaEngineOffer := &MediaEngine{} + // offer side has fewer codecs than answer side + for _, codec := range []RTPCodecParameters{ + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=50", nil}, + PayloadType: 51, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/vp9", 90000, 0, "", nil}, + PayloadType: 50, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/vp8", 90000, 0, "", nil}, + PayloadType: 52, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=52", nil}, + PayloadType: 53, + }, + } { + assert.NoError(t, mediaEngineOffer.RegisterCodec(codec, RTPCodecTypeVideo)) + } + pcOffer, err := NewAPI(WithSettingEngine(se), WithMediaEngine(mediaEngineOffer)).NewPeerConnection(Configuration{}) + assert.NoError(t, err) + + // answer side + mediaEngineAnswer := &MediaEngine{} + // answer has more codecs than offer side and in different order + for _, codec := range []RTPCodecParameters{ + { + RTPCodecCapability: RTPCodecCapability{"video/vp8", 90000, 0, "", nil}, + PayloadType: 82, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=82", nil}, + PayloadType: 83, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/vp9", 90000, 0, "", nil}, + PayloadType: 80, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=80", nil}, + PayloadType: 81, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/av1", 90000, 0, "", nil}, + PayloadType: 84, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=84", nil}, + PayloadType: 85, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/h265", 90000, 0, "", nil}, + PayloadType: 86, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=86", nil}, + PayloadType: 87, + }, + } { + assert.NoError(t, mediaEngineAnswer.RegisterCodec(codec, RTPCodecTypeVideo)) + } + pcAnswer, err := NewAPI(WithMediaEngine(mediaEngineAnswer)).NewPeerConnection(Configuration{}) + assert.NoError(t, err) + + track1, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, "video", "pion1") + require.NoError(t, err) + + _, err = pcOffer.AddTransceiverFromTrack(track1) + assert.NoError(t, err) + + offer, err := pcOffer.CreateOffer(nil) + assert.NoError(t, err) + assert.NoError(t, pcOffer.SetLocalDescription(offer)) + + // this should create a transceiver on answer side from remote description and + // set codec prefereces with order of codecs in offer using the corresponding + // payload types from the media engine codecs + assert.NoError(t, pcAnswer.SetRemoteDescription(offer)) + + answerSideTransceivers := pcAnswer.GetTransceivers() + assert.Equal(t, 1, len(answerSideTransceivers)) + + // media engine updates negotiated codecs from remote description, + // so payload type will be what is in the offer + // all rtx are placed later and could be in any order + checkPreferredCodecs := func( + actualPreferredCodecs []RTPCodecParameters, + expectedPreferredCodecsPrimary []RTPCodecParameters, + expectedPreferredCodecsRTX []RTPCodecParameters, + ) { + assert.Equal( + t, + len(expectedPreferredCodecsPrimary)+len(expectedPreferredCodecsRTX), + len(actualPreferredCodecs), + ) + + for i, expectedPreferredCodec := range expectedPreferredCodecsPrimary { + expectedFmtp := fmtp.Parse( + expectedPreferredCodec.RTPCodecCapability.MimeType, + expectedPreferredCodec.RTPCodecCapability.ClockRate, + expectedPreferredCodec.RTPCodecCapability.Channels, + expectedPreferredCodec.RTPCodecCapability.SDPFmtpLine, + ) + actualFmtp := fmtp.Parse( + actualPreferredCodecs[i].RTPCodecCapability.MimeType, + actualPreferredCodecs[i].RTPCodecCapability.ClockRate, + actualPreferredCodecs[i].RTPCodecCapability.Channels, + actualPreferredCodecs[i].RTPCodecCapability.SDPFmtpLine, + ) + assert.True(t, expectedFmtp.Match(actualFmtp)) + } + + for _, expectedPreferredCodec := range expectedPreferredCodecsRTX { + expectedFmtp := fmtp.Parse( + expectedPreferredCodec.RTPCodecCapability.MimeType, + expectedPreferredCodec.RTPCodecCapability.ClockRate, + expectedPreferredCodec.RTPCodecCapability.Channels, + expectedPreferredCodec.RTPCodecCapability.SDPFmtpLine, + ) + + found := false + for j := len(expectedPreferredCodecsPrimary); j < len(actualPreferredCodecs); j++ { + actualFmtp := fmtp.Parse( + actualPreferredCodecs[j].RTPCodecCapability.MimeType, + actualPreferredCodecs[j].RTPCodecCapability.ClockRate, + actualPreferredCodecs[j].RTPCodecCapability.Channels, + actualPreferredCodecs[j].RTPCodecCapability.SDPFmtpLine, + ) + if expectedFmtp.Match(actualFmtp) { + found = true + + break + } + } + assert.True(t, found) + } + } + + preferredCodecsPrimary := []RTPCodecParameters{ + { + RTPCodecCapability: RTPCodecCapability{"video/vp9", 90000, 0, "", nil}, + PayloadType: 50, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/vp8", 90000, 0, "", nil}, + PayloadType: 52, + }, + } + preferredCodecsRTX := []RTPCodecParameters{ + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=50", nil}, + PayloadType: 51, + }, + { + RTPCodecCapability: RTPCodecCapability{"video/rtx", 90000, 0, "apt=52", nil}, + PayloadType: 53, + }, + } + + checkPreferredCodecs(answerSideTransceivers[0].getCodecs(), preferredCodecsPrimary, preferredCodecsRTX) + + assert.NoError(t, pcOffer.Close()) + assert.NoError(t, pcAnswer.Close()) +} + func TestAddTransceiverAddTrack_NewRTPSender_Error(t *testing.T) { pc, err := NewPeerConnection(Configuration{}) assert.NoError(t, err) diff --git a/rtptransceiver.go b/rtptransceiver.go index 2afc20b0..70ad4c78 100644 --- a/rtptransceiver.go +++ b/rtptransceiver.go @@ -8,10 +8,13 @@ package webrtc import ( "fmt" + "strings" "sync" "sync/atomic" "github.com/pion/rtp" + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v4/internal/fmtp" ) // RTPTransceiver represents a combination of an RTPSender and an RTPReceiver that share a common mid. @@ -73,7 +76,7 @@ func (t *RTPTransceiver) getCodecs() []RTPCodecParameters { mediaEngineCodecs := t.api.mediaEngine.getCodecsByKind(t.kind) if len(t.codecs) == 0 { - return mediaEngineCodecs + return filterUnattachedRTX(mediaEngineCodecs) } filteredCodecs := []RTPCodecParameters{} @@ -90,6 +93,96 @@ func (t *RTPTransceiver) getCodecs() []RTPCodecParameters { return filterUnattachedRTX(filteredCodecs) } +// match codecs from remote description, used when remote is offerer and creating a transceiver +// from remote description with the aim of keeping order of codecs in remote description. +func (t *RTPTransceiver) setCodecPreferencesFromRemoteDescription(media *sdp.MediaDescription) { //nolint:cyclop + remoteCodecs, err := codecsFromMediaDescription(media) + if err != nil { + return + } + + // make a copy as this slice is modified + leftCodecs := append([]RTPCodecParameters{}, t.api.mediaEngine.getCodecsByKind(t.kind)...) + + // find codec matches between what is in remote description and + // the transceivers codecs and use payload type registered to + // media engine. + payloadMapping := make(map[PayloadType]PayloadType) // for RTX re-mapping later + filterByMatchType := func(matchFilter codecMatchType) []RTPCodecParameters { + filteredCodecs := []RTPCodecParameters{} + for remoteCodecIdx := len(remoteCodecs) - 1; remoteCodecIdx >= 0; remoteCodecIdx-- { + remoteCodec := remoteCodecs[remoteCodecIdx] + if strings.EqualFold(remoteCodec.RTPCodecCapability.MimeType, MimeTypeRTX) { + continue + } + + matchCodec, matchType := codecParametersFuzzySearch( + remoteCodec, + leftCodecs, + ) + if matchType == matchFilter { + payloadMapping[remoteCodec.PayloadType] = matchCodec.PayloadType + + remoteCodec.PayloadType = matchCodec.PayloadType + filteredCodecs = append([]RTPCodecParameters{remoteCodec}, filteredCodecs...) + + // removed matched codec for next round + remoteCodecs = append(remoteCodecs[:remoteCodecIdx], remoteCodecs[remoteCodecIdx+1:]...) + + needleFmtp := fmtp.Parse( + matchCodec.RTPCodecCapability.MimeType, + matchCodec.RTPCodecCapability.ClockRate, + matchCodec.RTPCodecCapability.Channels, + matchCodec.RTPCodecCapability.SDPFmtpLine, + ) + + for leftCodecIdx := len(leftCodecs) - 1; leftCodecIdx >= 0; leftCodecIdx-- { + leftCodec := leftCodecs[leftCodecIdx] + leftCodecFmtp := fmtp.Parse( + leftCodec.RTPCodecCapability.MimeType, + leftCodec.RTPCodecCapability.ClockRate, + leftCodec.RTPCodecCapability.Channels, + leftCodec.RTPCodecCapability.SDPFmtpLine, + ) + + if needleFmtp.Match(leftCodecFmtp) { + leftCodecs = append(leftCodecs[:leftCodecIdx], leftCodecs[leftCodecIdx+1:]...) + + break + } + } + } + } + + return filteredCodecs + } + + filteredCodecs := filterByMatchType(codecMatchExact) + filteredCodecs = append(filteredCodecs, filterByMatchType(codecMatchPartial)...) + + // find RTX associations and add those + for remotePayloadType, mediaEnginePayloadType := range payloadMapping { + remoteRTX := findRTXPayloadType(remotePayloadType, remoteCodecs) + if remoteRTX == PayloadType(0) { + continue + } + + mediaEngineRTX := findRTXPayloadType(mediaEnginePayloadType, leftCodecs) + if mediaEngineRTX == PayloadType(0) { + continue + } + + for _, rtxCodec := range leftCodecs { + if rtxCodec.PayloadType == mediaEngineRTX { + filteredCodecs = append(filteredCodecs, rtxCodec) + + break + } + } + } + _ = t.SetCodecPreferences(filteredCodecs) +} + // Sender returns the RTPTransceiver's RTPSender if it has one. func (t *RTPTransceiver) Sender() *RTPSender { if v, ok := t.sender.Load().(*RTPSender); ok {