apps/go2rtc
1
0
Fork 0
mirror of https://github.com/AlexxIT/go2rtc.git synced 2025-09-26 20:31:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support OPUS audio for MSE/MP4

Browse Source
This commit is contained in:
Alexey Khit
2023-01-27 12:36:36 +03:00
parent 073acdfec9
commit a1fec1c6f6
9 changed files with 98 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/mp4/ws.go
View File

@@ -106,15 +106,18 @@ func parseMedias(codecs string, parseAudio bool) (medias []*streamer.Media) {
for _, name := range strings.Split(codecs, ",") {
switch name {
case "avc1.640029":
case mp4.MimeH264:
codec := &streamer.Codec{Name: streamer.CodecH264}
videos = append(videos, codec)
case "hvc1.1.6.L153.B0":
case mp4.MimeH265:
codec := &streamer.Codec{Name: streamer.CodecH265}
videos = append(videos, codec)
case "mp4a.40.2":
case mp4.MimeAAC:
codec := &streamer.Codec{Name: streamer.CodecAAC}
audios = append(audios, codec)
case mp4.MimeOpus:
codec := &streamer.Codec{Name: streamer.CodecOpus}
audios = append(audios, codec)
}
}

10
pkg/mov/atoms.go → pkg/iso/atoms.go
View File

@@ -1,4 +1,4 @@
package mov
package iso
const (
Ftyp = "ftyp"
@@ -210,7 +210,7 @@ func (m *Movie) WriteTrackExtend(id uint32) {
m.EndAtom()
}
func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf []byte, h264 bool) {
func (m *Movie) WriteVideoTrack(id uint32, codec string, timescale uint32, width, height uint16, conf []byte) {
m.StartAtom(MoovTrak)
m.WriteTrackHeader(id, width, height)
@@ -222,7 +222,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
m.WriteVideoMediaInfo()
m.WriteDataInfo()
m.WriteSampleTable(func() {
m.WriteH26X(width, height, conf, h264)
m.WriteVideo(codec, width, height, conf)
})
m.EndAtom() // MINF
@@ -230,7 +230,7 @@ func (m *Movie) WriteVideoTrack(id, timescale uint32, width, height uint16, conf
m.EndAtom() // TRAK
}
func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSize uint16, conf []byte) {
func (m *Movie) WriteAudioTrack(id uint32, codec string, timescale uint32, channels uint16, conf []byte) {
m.StartAtom(MoovTrak)
m.WriteTrackHeader(id, 0, 0)
@@ -242,7 +242,7 @@ func (m *Movie) WriteAudioTrack(id uint32, timescale uint32, channels, sampleSiz
m.WriteAudioMediaInfo()
m.WriteDataInfo()
m.WriteSampleTable(func() {
m.WriteMP4A(channels, sampleSize, timescale, conf)
m.WriteAudio(codec, channels, timescale, conf)
})
m.EndAtom() // MINF

50
pkg/mov/codecs.go → pkg/iso/codecs.go
View File

@@ -1,4 +1,6 @@
package mov
package iso
import "github.com/AlexxIT/go2rtc/pkg/streamer"
const (
MoovTrakMdiaMinfStblStsdAvc1 = "avc1"
@@ -6,14 +8,18 @@ const (
MoovTrakMdiaMinfStblStsdHev1 = "hev1"
MoovTrakMdiaMinfStblStsdHev1HvcC = "hvcC"
MoovTrakMdiaMinfStblStsdMp4a = "mp4a"
MoovTrakMdiaMinfStblStsdOpus = "Opus"
)
func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
func (m *Movie) WriteVideo(codec string, width, height uint16, conf []byte) {
// https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap3/qtff3.html
if h264 {
switch codec {
case streamer.CodecH264:
m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1)
} else {
case streamer.CodecH265:
m.StartAtom(MoovTrakMdiaMinfStblStsdHev1)
default:
panic("unsupported iso video: " + codec)
}
m.Skip(6)
m.WriteUint16(1) // data_reference_index
@@ -32,9 +38,10 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
m.WriteUint16(24) // depth
m.WriteUint16(0xFFFF) // color table id (-1)
if h264 {
switch codec {
case streamer.CodecH264:
m.StartAtom(MoovTrakMdiaMinfStblStsdAvc1AvcC)
} else {
case streamer.CodecH265:
m.StartAtom(MoovTrakMdiaMinfStblStsdHev1HvcC)
}
m.Write(conf)
@@ -43,25 +50,37 @@ func (m *Movie) WriteH26X(width, height uint16, conf []byte, h264 bool) {
m.EndAtom() // AVC1
}
func (m *Movie) WriteMP4A(channels, sampleSize uint16, sampleRate uint32, conf []byte) {
m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
func (m *Movie) WriteAudio(codec string, channels uint16, sampleRate uint32, conf []byte) {
switch codec {
case streamer.CodecAAC:
m.StartAtom(MoovTrakMdiaMinfStblStsdMp4a)
case streamer.CodecOpus:
m.StartAtom(MoovTrakMdiaMinfStblStsdOpus)
default:
panic("unsupported iso audio: " + codec)
}
m.Skip(6)
m.WriteUint16(1) // data_reference_index
m.Skip(2) // version
m.Skip(2) // revision
m.Skip(4) // vendor
m.WriteUint16(channels) // channel_count
m.WriteUint16(sampleSize) // sample_size
m.WriteUint16(16) // sample_size
m.Skip(2) // compression id
m.Skip(2) // reserved
m.WriteFloat32(float64(sampleRate)) // sample_rate
m.WriteESDS(conf)
switch codec {
case streamer.CodecAAC:
m.WriteEsdsAAC(conf)
case streamer.CodecOpus:
m.WriteDops()
}
m.EndAtom() // MP4A
m.EndAtom() // MP4A/OPUS
}
func (m *Movie) WriteESDS(conf []byte) {
func (m *Movie) WriteEsdsAAC(conf []byte) {
m.StartAtom("esds")
m.Skip(1) // version
m.Skip(3) // flags
@@ -95,3 +114,10 @@ func (m *Movie) WriteESDS(conf []byte) {
m.EndAtom() // ESDS
}
func (m *Movie) WriteDops() {
// don't know what means this magic
m.StartAtom("dOps")
m.WriteBytes(0, 0x02, 0x01, 0x38, 0, 0, 0xBB, 0x80, 0, 0, 0)
m.EndAtom()
}

2
pkg/mov/mov.go → pkg/iso/iso.go
View File

@@ -1,4 +1,4 @@
package mov
package iso
import (
"encoding/binary"

16
pkg/mp4/consumer.go
View File

@@ -50,6 +50,7 @@ func (c *Consumer) GetMedias() []*streamer.Media {
Direction: streamer.DirectionRecvonly,
Codecs: []*streamer.Codec{
{Name: streamer.CodecAAC},
{Name: streamer.CodecOpus},
},
},
}
@@ -140,6 +141,21 @@ func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *strea
push = wrapper(push)
}
return track.Bind(push)
case streamer.CodecOpus:
push := func(packet *rtp.Packet) error {
if c.wait != waitNone {
return nil
}
buf := c.muxer.Marshal(trackID, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
return nil
}
return track.Bind(push)
}

40
pkg/mp4/muxer.go
View File

@@ -4,7 +4,7 @@ import (
"encoding/hex"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/mov"
"github.com/AlexxIT/go2rtc/pkg/iso"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/deepch/vdk/codec/h264parser"
"github.com/deepch/vdk/codec/h265parser"
@@ -18,6 +18,13 @@ type Muxer struct {
pts []uint32
}
const (
MimeH264 = "avc1.640029"
MimeH265 = "hvc1.1.6.L153.B0"
MimeAAC = "mp4a.40.2"
MimeOpus = "opus"
)
func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
s := `video/mp4; codecs="`
@@ -32,9 +39,11 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
case streamer.CodecH265:
// H.265 profile=main level=5.1
// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
s += "hvc1.1.6.L153.B0"
s += MimeH265
case streamer.CodecAAC:
s += "mp4a.40.2"
s += MimeAAC
case streamer.CodecOpus:
s += MimeOpus
}
}
@@ -42,10 +51,10 @@ func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
}
func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
mv := mov.NewMovie(1024)
mv := iso.NewMovie(1024)
mv.WriteFileType()
mv.StartAtom(mov.Moov)
mv.StartAtom(iso.Moov)
mv.WriteMovieHeader()
for i, codec := range codecs {
@@ -64,9 +73,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteVideoTrack(
uint32(i+1), codec.ClockRate,
uint32(i+1), codec.Name, codec.ClockRate,
uint16(codecData.Width()), uint16(codecData.Height()),
codecData.AVCDecoderConfRecordBytes(), true,
codecData.AVCDecoderConfRecordBytes(),
)
m.flags = append(m.flags, 0x1010000)
@@ -86,9 +95,9 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteVideoTrack(
uint32(i+1), codec.ClockRate,
uint32(i+1), codec.Name, codec.ClockRate,
uint16(codecData.Width()), uint16(codecData.Height()),
codecData.AVCDecoderConfRecordBytes(), false,
codecData.AVCDecoderConfRecordBytes(),
)
m.flags = append(m.flags, 0x1010000)
@@ -101,7 +110,14 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
}
mv.WriteAudioTrack(
uint32(i+1), codec.ClockRate, codec.Channels, 16, b,
uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, b,
)
m.flags = append(m.flags, 0x2000000)
case streamer.CodecOpus:
mv.WriteAudioTrack(
uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, nil,
)
m.flags = append(m.flags, 0x2000000)
@@ -111,7 +127,7 @@ func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
m.dts = append(m.dts, 0)
}
mv.StartAtom(mov.MoovMvex)
mv.StartAtom(iso.MoovMvex)
for i := range codecs {
mv.WriteTrackExtend(uint32(i + 1))
}
@@ -147,7 +163,7 @@ func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
}
m.pts[trackID] = newTime
mv := mov.NewMovie(1024 + len(packet.Payload))
mv := iso.NewMovie(1024 + len(packet.Payload))
mv.WriteMovieFragment(
m.fragIndex, uint32(trackID+1), duration,
uint32(len(packet.Payload)),

2
www/README.md
View File

@@ -56,3 +56,5 @@ pc.ontrack = ev => {
- https://web.dev/i18n/en/fast-playback-with-preload/#manual_buffering
- https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API
- https://chromium.googlesource.com/external/w3c/web-platform-tests/+/refs/heads/master/media-source/mediasource-is-type-supported.html
- https://googlechrome.github.io/samples/media/sourcebuffer-changetype.html
- https://chromestatus.com/feature/5100845653819392

2
www/codecs.html
View File

@@ -53,7 +53,7 @@
const video = document.createElement("video");
out.innerText += "video.canPlayType\n";
types.forEach(type => {
out.innerText += type + "=" + (video.canPlayType(type) ? "true" : "false") + "\n";
out.innerText += `${type} = ${MediaSource.isTypeSupported(type)} / ${video.canPlayType(type)}\n`;
})
</script>

3
www/video-rtc.js
View File

@@ -26,8 +26,7 @@ export class VideoRTC extends HTMLElement {
"hvc1.1.6.L153.B0", // H.265 main 5.1 (Chromecast Ultra)
"mp4a.40.2", // AAC LC
"mp4a.40.5", // AAC HE
"mp4a.69", // MP3
"mp4a.6B", // MP3
"opus", // OPUS Chrome
];
/**