hls muxer: support reading Opus tracks (#1338)

This commit is contained in:
aler9
2023-01-03 18:36:13 +01:00
parent cce3fb440f
commit 034e42f463
23 changed files with 447 additions and 193 deletions

View File

@@ -22,7 +22,7 @@ And can be read from the server with:
|--------|--------|------| |--------|--------|------|
|RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec| |RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec|
|RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)| |RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)|
|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC)| |HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC), Opus|
|WebRTC||H264, VP8, VP9, Opus, G711, G722| |WebRTC||H264, VP8, VP9, Opus, G711, G722|
Features: Features:
@@ -908,7 +908,7 @@ where `mystream` is the name of a stream that is being published.
Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginning of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page: Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginning of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page:
https://jsfiddle.net/7nwxmLto https://jsfiddle.net/4msrhudv
If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_: If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_:
@@ -1067,9 +1067,9 @@ Related projects
Standards Standards
* RTSP 1.0 https://datatracker.ietf.org/doc/html/rfc2326 * RTSP/RTP standards https://github.com/aler9/gortsplib#links
* RTSP 2.0 https://datatracker.ietf.org/doc/html/rfc7826
* HTTP 1.1 https://datatracker.ietf.org/doc/html/rfc2616 * HTTP 1.1 https://datatracker.ietf.org/doc/html/rfc2616
* HLS https://datatracker.ietf.org/doc/html/rfc8216 * HLS https://datatracker.ietf.org/doc/html/rfc8216
* HLS v2 https://datatracker.ietf.org/doc/html/draft-pantos-hls-rfc8216bis * HLS v2 https://datatracker.ietf.org/doc/html/draft-pantos-hls-rfc8216bis
* Opus in MP4/ISOBMFF https://opus-codec.org/docs/opus_in_isobmff.html
* Golang project layout https://github.com/golang-standards/project-layout * Golang project layout https://github.com/golang-standards/project-layout

2
go.mod
View File

@@ -5,7 +5,7 @@ go 1.18
require ( require (
code.cloudfoundry.org/bytefmt v0.0.0 code.cloudfoundry.org/bytefmt v0.0.0
github.com/abema/go-mp4 v0.0.0 github.com/abema/go-mp4 v0.0.0
github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414
github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757 github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
github.com/fsnotify/fsnotify v1.4.9 github.com/fsnotify/fsnotify v1.4.9
github.com/gin-gonic/gin v1.8.1 github.com/gin-gonic/gin v1.8.1

4
go.sum
View File

@@ -4,8 +4,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968 h1:wU8pLx4dc8bLB+JuVPWuGp+BoMkOabj98a0RmO3gqvw= github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968 h1:wU8pLx4dc8bLB+JuVPWuGp+BoMkOabj98a0RmO3gqvw=
github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 h1:EFq9LqgA15drNgXj3hNlmAouxjMYb9jyyBq6hmjDO8U= github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414 h1:pVyJ7Uuk5kdU/RhCepxJQJEC9hsrFgxIIw1mIHn02Zs=
github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA= github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8= github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=

View File

@@ -25,6 +25,9 @@ func newFormatProcessor(forma format.Format, generateRTPPackets bool) (formatPro
case *format.MPEG4Audio: case *format.MPEG4Audio:
return newFormatProcessorMPEG4Audio(forma, generateRTPPackets) return newFormatProcessorMPEG4Audio(forma, generateRTPPackets)
case *format.Opus:
return newFormatProcessorOpus(forma, generateRTPPackets)
default: default:
return newFormatProcessorGeneric(forma, generateRTPPackets) return newFormatProcessorGeneric(forma, generateRTPPackets)
} }

View File

@@ -0,0 +1,89 @@
package core
import (
"fmt"
"time"
"github.com/aler9/gortsplib/v2/pkg/format"
"github.com/aler9/gortsplib/v2/pkg/formatdecenc/rtpsimpleaudio"
"github.com/pion/rtp"
)
type dataOpus struct {
rtpPackets []*rtp.Packet
ntp time.Time
pts time.Duration
frame []byte
}
func (d *dataOpus) getRTPPackets() []*rtp.Packet {
return d.rtpPackets
}
func (d *dataOpus) getNTP() time.Time {
return d.ntp
}
type formatProcessorOpus struct {
format *format.Opus
encoder *rtpsimpleaudio.Encoder
decoder *rtpsimpleaudio.Decoder
}
func newFormatProcessorOpus(
forma *format.Opus,
allocateEncoder bool,
) (*formatProcessorOpus, error) {
t := &formatProcessorOpus{
format: forma,
}
if allocateEncoder {
t.encoder = forma.CreateEncoder()
}
return t, nil
}
func (t *formatProcessorOpus) process(dat data, hasNonRTSPReaders bool) error { //nolint:dupl
tdata := dat.(*dataOpus)
if tdata.rtpPackets != nil {
pkt := tdata.rtpPackets[0]
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
if pkt.MarshalSize() > maxPacketSize {
return fmt.Errorf("payload size (%d) is greater than maximum allowed (%d)",
pkt.MarshalSize(), maxPacketSize)
}
// decode from RTP
if hasNonRTSPReaders {
if t.decoder == nil {
t.decoder = t.format.CreateDecoder()
}
frame, pts, err := t.decoder.Decode(pkt)
if err != nil {
return err
}
tdata.frame = frame
tdata.pts = pts
}
// route packet as is
return nil
}
pkt, err := t.encoder.Encode(tdata.frame, tdata.pts)
if err != nil {
return err
}
tdata.rtpPackets = []*rtp.Packet{pkt}
return nil
}

View File

@@ -1,4 +1,4 @@
package core package core //nolint:dupl
import ( import (
"fmt" "fmt"

View File

@@ -1,4 +1,4 @@
package core package core //nolint:dupl
import ( import (
"fmt" "fmt"

View File

@@ -20,7 +20,7 @@ html, body {
<video id="video" muted controls autoplay playsinline></video> <video id="video" muted controls autoplay playsinline></video>
<script src="https://cdn.jsdelivr.net/npm/hls.js@1.1.5"></script> <script src="https://cdn.jsdelivr.net/npm/hls.js@1.2.9"></script>
<script> <script>

View File

@@ -247,120 +247,21 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
var medias media.Medias var medias media.Medias
var videoFormat format.Format videoMedia, videoFormat := m.setupVideoMedia(res.stream)
var videoMedia *media.Media if videoMedia != nil {
var videoFormatH265 *format.H265
videoMedia = res.stream.medias().FindFormat(&videoFormatH265)
if videoFormatH265 != nil {
videoFormat = videoFormatH265
medias = append(medias, videoMedia) medias = append(medias, videoMedia)
videoStartPTSFilled := false
var videoStartPTS time.Duration
res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataH265)
if tdata.au == nil {
return nil
} }
if !videoStartPTSFilled { audioMedia, audioFormat := m.setupAudioMedia(res.stream)
videoStartPTSFilled = true if audioMedia != nil {
videoStartPTS = tdata.pts
}
pts := tdata.pts - videoStartPTS
err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
return nil
})
})
} else {
var videoFormatH264 *format.H264
videoMedia = res.stream.medias().FindFormat(&videoFormatH264)
if videoFormatH264 != nil {
videoFormat = videoFormatH264
medias = append(medias, videoMedia)
videoStartPTSFilled := false
var videoStartPTS time.Duration
res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataH264)
if tdata.au == nil {
return nil
}
if !videoStartPTSFilled {
videoStartPTSFilled = true
videoStartPTS = tdata.pts
}
pts := tdata.pts - videoStartPTS
err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
return nil
})
})
}
}
var audioFormat *format.MPEG4Audio
audioMedia := res.stream.medias().FindFormat(&audioFormat)
if audioFormat != nil {
medias = append(medias, audioMedia) medias = append(medias, audioMedia)
audioStartPTSFilled := false
var audioStartPTS time.Duration
res.stream.readerAdd(m, audioMedia, audioFormat, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataMPEG4Audio)
if tdata.aus == nil {
return nil
}
if !audioStartPTSFilled {
audioStartPTSFilled = true
audioStartPTS = tdata.pts
}
pts := tdata.pts - audioStartPTS
for i, au := range tdata.aus {
err := m.muxer.WriteAAC(
tdata.ntp,
pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/time.Duration(audioFormat.ClockRate()),
au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
}
return nil
})
})
} }
defer res.stream.readerRemove(m) defer res.stream.readerRemove(m)
if medias == nil { if medias == nil {
return fmt.Errorf("the stream doesn't contain a supported video or audio track") return fmt.Errorf(
"the stream doesn't contain any supported codec (which are currently H264, H265, MPEG4-Audio, Opus)")
} }
var err error var err error
@@ -412,6 +313,151 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
} }
} }
func (m *hlsMuxer) setupVideoMedia(stream *stream) (*media.Media, format.Format) {
var videoFormatH265 *format.H265
videoMedia := stream.medias().FindFormat(&videoFormatH265)
if videoFormatH265 != nil {
videoStartPTSFilled := false
var videoStartPTS time.Duration
stream.readerAdd(m, videoMedia, videoFormatH265, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataH265)
if tdata.au == nil {
return nil
}
if !videoStartPTSFilled {
videoStartPTSFilled = true
videoStartPTS = tdata.pts
}
pts := tdata.pts - videoStartPTS
err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
return nil
})
})
return videoMedia, videoFormatH265
}
var videoFormatH264 *format.H264
videoMedia = stream.medias().FindFormat(&videoFormatH264)
if videoFormatH264 != nil {
videoStartPTSFilled := false
var videoStartPTS time.Duration
stream.readerAdd(m, videoMedia, videoFormatH264, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataH264)
if tdata.au == nil {
return nil
}
if !videoStartPTSFilled {
videoStartPTSFilled = true
videoStartPTS = tdata.pts
}
pts := tdata.pts - videoStartPTS
err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
return nil
})
})
return videoMedia, videoFormatH264
}
return nil, nil
}
func (m *hlsMuxer) setupAudioMedia(stream *stream) (*media.Media, format.Format) {
var audioFormatMPEG4Audio *format.MPEG4Audio
audioMedia := stream.medias().FindFormat(&audioFormatMPEG4Audio)
if audioFormatMPEG4Audio != nil {
audioStartPTSFilled := false
var audioStartPTS time.Duration
stream.readerAdd(m, audioMedia, audioFormatMPEG4Audio, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataMPEG4Audio)
if tdata.aus == nil {
return nil
}
if !audioStartPTSFilled {
audioStartPTSFilled = true
audioStartPTS = tdata.pts
}
pts := tdata.pts - audioStartPTS
for i, au := range tdata.aus {
err := m.muxer.WriteAudio(
tdata.ntp,
pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/time.Duration(audioFormatMPEG4Audio.ClockRate()),
au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
}
return nil
})
})
return audioMedia, audioFormatMPEG4Audio
}
var audioFormatOpus *format.Opus
audioMedia = stream.medias().FindFormat(&audioFormatOpus)
if audioFormatOpus != nil {
audioStartPTSFilled := false
var audioStartPTS time.Duration
stream.readerAdd(m, audioMedia, audioFormatOpus, func(dat data) {
m.ringBuffer.Push(func() error {
tdata := dat.(*dataOpus)
if !audioStartPTSFilled {
audioStartPTSFilled = true
audioStartPTS = tdata.pts
}
pts := tdata.pts - audioStartPTS
err := m.muxer.WriteAudio(
tdata.ntp,
pts,
tdata.frame)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
return nil
})
})
return audioMedia, audioFormatOpus
}
return nil, nil
}
func (m *hlsMuxer) runWriter() error { func (m *hlsMuxer) runWriter() error {
for { for {
item, ok := m.ringBuffer.Pull() item, ok := m.ringBuffer.Pull()

View File

@@ -363,6 +363,17 @@ func (s *rtspSession) onRecord(ctx *gortsplib.ServerHandlerOnRecordCtx) (*base.R
} }
}) })
case *format.Opus:
ctx.Session.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
err := s.stream.writeData(cmedia, cformat, &dataOpus{
rtpPackets: []*rtp.Packet{pkt},
ntp: time.Now(),
})
if err != nil {
s.log(logger.Warn, "%v", err)
}
})
default: default:
ctx.Session.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) { ctx.Session.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
err := s.stream.writeData(cmedia, cformat, &dataGeneric{ err := s.stream.writeData(cmedia, cformat, &dataGeneric{

View File

@@ -203,6 +203,17 @@ func (s *rtspSource) run(ctx context.Context) error {
} }
}) })
case *format.Opus:
c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
err := res.stream.writeData(cmedia, cformat, &dataOpus{
rtpPackets: []*rtp.Packet{pkt},
ntp: time.Now(),
})
if err != nil {
s.Log(logger.Warn, "%v", err)
}
})
default: default:
c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) { c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
err := res.stream.writeData(cmedia, cformat, &dataGeneric{ err := res.stream.writeData(cmedia, cformat, &dataGeneric{

View File

@@ -22,20 +22,24 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
- mdhd - mdhd
- hdlr - hdlr
- minf - minf
- vmhd (video only) - vmhd (video)
- smhd (audio only) - smhd (audio)
- dinf - dinf
- dref - dref
- url - url
- stbl - stbl
- stsd - stsd
- avc1 (h264 only) - avc1 (h264)
- avcC - avcC
- pasp
- btrt - btrt
- mp4a (mpeg4audio only) - hev1 (h265)
- hvcC
- mp4a (mpeg4audio)
- esds - esds
- btrt - btrt
- Opus (opus)
- dOps
- btrt
- stts - stts
- stsc - stsc
- stsz - stsz
@@ -72,19 +76,6 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
width = h264SPSP.Width() width = h264SPSP.Width()
height = h264SPSP.Height() height = h264SPSP.Height()
_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
FullBox: gomp4.FullBox{
Flags: [3]byte{0, 0, 3},
},
TrackID: uint32(track.ID),
Width: uint32(width * 65536),
Height: uint32(height * 65536),
Matrix: [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
})
if err != nil {
return err
}
case *format.H265: case *format.H265:
h265VPS = ttrack.SafeVPS() h265VPS = ttrack.SafeVPS()
h265SPS = ttrack.SafeSPS() h265SPS = ttrack.SafeSPS()
@@ -97,7 +88,10 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
width = h265SPSP.Width() width = h265SPSP.Width()
height = h265SPSP.Height() height = h265SPSP.Height()
}
switch track.Format.(type) {
case *format.H264, *format.H265:
_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/> _, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
FullBox: gomp4.FullBox{ FullBox: gomp4.FullBox{
Flags: [3]byte{0, 0, 3}, Flags: [3]byte{0, 0, 3},
@@ -111,7 +105,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
return err return err
} }
case *format.MPEG4Audio: case *format.MPEG4Audio, *format.Opus:
_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/> _, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
FullBox: gomp4.FullBox{ FullBox: gomp4.FullBox{
Flags: [3]byte{0, 0, 3}, Flags: [3]byte{0, 0, 3},
@@ -149,7 +143,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
return err return err
} }
case *format.MPEG4Audio: case *format.MPEG4Audio, *format.Opus:
_, err = w.WriteBox(&gomp4.Hdlr{ // <hdlr/> _, err = w.WriteBox(&gomp4.Hdlr{ // <hdlr/>
HandlerType: [4]byte{'s', 'o', 'u', 'n'}, HandlerType: [4]byte{'s', 'o', 'u', 'n'},
Name: "SoundHandler", Name: "SoundHandler",
@@ -175,7 +169,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
return err return err
} }
case *format.MPEG4Audio: case *format.MPEG4Audio, *format.Opus:
_, err = w.WriteBox(&gomp4.Smhd{ // <smhd/> _, err = w.WriteBox(&gomp4.Smhd{ // <smhd/>
}) })
if err != nil { if err != nil {
@@ -391,10 +385,6 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
enc, _ := ttrack.Config.Marshal() enc, _ := ttrack.Config.Marshal()
_, err = w.WriteBox(&gomp4.Esds{ // <esds/> _, err = w.WriteBox(&gomp4.Esds{ // <esds/>
FullBox: gomp4.FullBox{
Version: 0,
Flags: [3]byte{0x00, 0x00, 0x00},
},
Descriptors: []gomp4.Descriptor{ Descriptors: []gomp4.Descriptor{
{ {
Tag: gomp4.ESDescrTag, Tag: gomp4.ESDescrTag,
@@ -443,6 +433,44 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
if err != nil { if err != nil {
return err return err
} }
case *format.Opus:
_, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ // <Opus>
SampleEntry: gomp4.SampleEntry{
AnyTypeBox: gomp4.AnyTypeBox{
Type: BoxTypeOpus(),
},
DataReferenceIndex: 1,
},
ChannelCount: uint16(ttrack.ChannelCount),
SampleSize: 16,
SampleRate: uint32(ttrack.ClockRate() * 65536),
})
if err != nil {
return err
}
_, err = w.WriteBox(&DOps{ // <dOps/>
OutputChannelCount: uint8(ttrack.ChannelCount),
PreSkip: 312,
InputSampleRate: uint32(ttrack.ClockRate()),
})
if err != nil {
return err
}
_, err = w.WriteBox(&gomp4.Btrt{ // <btrt/>
MaxBitrate: 128825,
AvgBitrate: 128825,
})
if err != nil {
return err
}
err = w.writeBoxEnd() // </Opus>
if err != nil {
return err
}
} }
err = w.writeBoxEnd() // </stsd> err = w.writeBoxEnd() // </stsd>

53
internal/hls/fmp4/opus.go Normal file
View File

@@ -0,0 +1,53 @@
//nolint:gochecknoinits,revive,gocritic
package fmp4
import (
gomp4 "github.com/abema/go-mp4"
)
func BoxTypeOpus() gomp4.BoxType { return gomp4.StrToBoxType("Opus") }
func init() {
gomp4.AddAnyTypeBoxDef(&gomp4.AudioSampleEntry{}, BoxTypeOpus())
}
func BoxTypeDOps() gomp4.BoxType { return gomp4.StrToBoxType("dOps") }
func init() {
gomp4.AddBoxDef(&DOps{})
}
type DOpsChannelMappingTable struct{}
type DOps struct {
gomp4.Box
Version uint8 `mp4:"0,size=8"`
OutputChannelCount uint8 `mp4:"1,size=8"`
PreSkip uint16 `mp4:"2,size=16"`
InputSampleRate uint32 `mp4:"3,size=32"`
OutputGain int16 `mp4:"4,size=16"`
ChannelMappingFamily uint8 `mp4:"5,size=8"`
StreamCount uint8 `mp4:"6,opt=dynamic,size=8"`
CoupledCount uint8 `mp4:"7,opt=dynamic,size=8"`
ChannelMapping []uint8 `mp4:"8,opt=dynamic,size=8,len=dynamic"`
}
func (DOps) GetType() gomp4.BoxType {
return BoxTypeDOps()
}
func (dops DOps) IsOptFieldEnabled(name string, ctx gomp4.Context) bool {
switch name {
case "StreamCount", "CoupledCount", "ChannelMapping":
return dops.ChannelMappingFamily != 0
}
return false
}
func (ops DOps) GetFieldLength(name string, ctx gomp4.Context) uint {
switch name {
case "ChannelMapping":
return uint(ops.OutputChannelCount)
}
return 0
}

View File

@@ -29,7 +29,7 @@ func NewMuxer(
partDuration time.Duration, partDuration time.Duration,
segmentMaxSize uint64, segmentMaxSize uint64,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
) (*Muxer, error) { ) (*Muxer, error) {
m := &Muxer{} m := &Muxer{}
@@ -85,9 +85,9 @@ func (m *Muxer) WriteH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
return m.variant.writeH26x(ntp, pts, au) return m.variant.writeH26x(ntp, pts, au)
} }
// WriteAAC writes an AAC access unit. // WriteAudio writes an audio access unit.
func (m *Muxer) WriteAAC(ntp time.Time, pts time.Duration, au []byte) error { func (m *Muxer) WriteAudio(ntp time.Time, pts time.Duration, au []byte) error {
return m.variant.writeAAC(ntp, pts, au) return m.variant.writeAudio(ntp, pts, au)
} }
// File returns a file reader. // File returns a file reader.

View File

@@ -31,6 +31,9 @@ func codecParameters(track format.Format) string {
case *format.MPEG4Audio: case *format.MPEG4Audio:
// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
return "mp4a.40." + strconv.FormatInt(int64(ttrack.Config.Type), 10) return "mp4a.40." + strconv.FormatInt(int64(ttrack.Config.Type), 10)
case *format.Opus:
return "opus"
} }
return "" return ""
@@ -39,13 +42,13 @@ func codecParameters(track format.Format) string {
type muxerPrimaryPlaylist struct { type muxerPrimaryPlaylist struct {
fmp4 bool fmp4 bool
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
} }
func newMuxerPrimaryPlaylist( func newMuxerPrimaryPlaylist(
fmp4 bool, fmp4 bool,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
) *muxerPrimaryPlaylist { ) *muxerPrimaryPlaylist {
return &muxerPrimaryPlaylist{ return &muxerPrimaryPlaylist{
fmp4: fmp4, fmp4: fmp4,

View File

@@ -75,13 +75,13 @@ func TestMuxerVideoAudio(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
d = 3 * time.Second d = 3 * time.Second
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)
d = 3500 * time.Millisecond d = 3500 * time.Millisecond
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -94,7 +94,7 @@ func TestMuxerVideoAudio(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
d = 4500 * time.Millisecond d = 4500 * time.Millisecond
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -323,20 +323,20 @@ func TestMuxerAudioOnly(t *testing.T) {
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
d := 1 * time.Second d := 1 * time.Second
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)
} }
d := 2 * time.Second d := 2 * time.Second
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)
d = 3 * time.Second d = 3 * time.Second
err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{ err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04,
}) })
require.NoError(t, err) require.NoError(t, err)

View File

@@ -17,6 +17,6 @@ const (
type muxerVariant interface { type muxerVariant interface {
close() close()
writeH26x(time.Time, time.Duration, [][]byte) error writeH26x(time.Time, time.Duration, [][]byte) error
writeAAC(time.Time, time.Duration, []byte) error writeAudio(time.Time, time.Duration, []byte) error
file(name string, msn string, part string, skip string) *MuxerFileResponse file(name string, msn string, part string, skip string) *MuxerFileResponse
} }

View File

@@ -48,7 +48,7 @@ type muxerVariantFMP4 struct {
playlist *muxerVariantFMP4Playlist playlist *muxerVariantFMP4Playlist
segmenter *muxerVariantFMP4Segmenter segmenter *muxerVariantFMP4Segmenter
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
mutex sync.Mutex mutex sync.Mutex
lastVideoParams [][]byte lastVideoParams [][]byte
@@ -62,7 +62,7 @@ func newMuxerVariantFMP4(
partDuration time.Duration, partDuration time.Duration,
segmentMaxSize uint64, segmentMaxSize uint64,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
) *muxerVariantFMP4 { ) *muxerVariantFMP4 {
v := &muxerVariantFMP4{ v := &muxerVariantFMP4{
videoTrack: videoTrack, videoTrack: videoTrack,
@@ -99,8 +99,8 @@ func (v *muxerVariantFMP4) writeH26x(ntp time.Time, pts time.Duration, au [][]by
return v.segmenter.writeH26x(ntp, pts, au) return v.segmenter.writeH26x(ntp, pts, au)
} }
func (v *muxerVariantFMP4) writeAAC(ntp time.Time, pts time.Duration, au []byte) error { func (v *muxerVariantFMP4) writeAudio(ntp time.Time, pts time.Duration, au []byte) error {
return v.segmenter.writeAAC(ntp, pts, au) return v.segmenter.writeAudio(ntp, pts, au)
} }
func (v *muxerVariantFMP4) mustRegenerateInit() bool { func (v *muxerVariantFMP4) mustRegenerateInit() bool {

View File

@@ -18,7 +18,7 @@ func fmp4PartName(id uint64) string {
type muxerVariantFMP4Part struct { type muxerVariantFMP4Part struct {
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
id uint64 id uint64
isIndependent bool isIndependent bool
@@ -34,7 +34,7 @@ type muxerVariantFMP4Part struct {
func newMuxerVariantFMP4Part( func newMuxerVariantFMP4Part(
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
id uint64, id uint64,
) *muxerVariantFMP4Part { ) *muxerVariantFMP4Part {
p := &muxerVariantFMP4Part{ p := &muxerVariantFMP4Part{
@@ -130,7 +130,7 @@ func (p *muxerVariantFMP4Part) writeH264(sample *augmentedVideoSample) {
p.videoSamples = append(p.videoSamples, &sample.PartSample) p.videoSamples = append(p.videoSamples, &sample.PartSample)
} }
func (p *muxerVariantFMP4Part) writeAAC(sample *augmentedAudioSample) { func (p *muxerVariantFMP4Part) writeAudio(sample *augmentedAudioSample) {
if !p.audioStartDTSFilled { if !p.audioStartDTSFilled {
p.audioStartDTSFilled = true p.audioStartDTSFilled = true
p.audioStartDTS = sample.dts p.audioStartDTS = sample.dts

View File

@@ -71,7 +71,7 @@ type muxerVariantFMP4Playlist struct {
lowLatency bool lowLatency bool
segmentCount int segmentCount int
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
mutex sync.Mutex mutex sync.Mutex
cond *sync.Cond cond *sync.Cond
@@ -90,7 +90,7 @@ func newMuxerVariantFMP4Playlist(
lowLatency bool, lowLatency bool,
segmentCount int, segmentCount int,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
) *muxerVariantFMP4Playlist { ) *muxerVariantFMP4Playlist {
p := &muxerVariantFMP4Playlist{ p := &muxerVariantFMP4Playlist{
lowLatency: lowLatency, lowLatency: lowLatency,

View File

@@ -46,7 +46,7 @@ type muxerVariantFMP4Segment struct {
startDTS time.Duration startDTS time.Duration
segmentMaxSize uint64 segmentMaxSize uint64
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
genPartID func() uint64 genPartID func() uint64
onPartFinalized func(*muxerVariantFMP4Part) onPartFinalized func(*muxerVariantFMP4Part)
@@ -64,7 +64,7 @@ func newMuxerVariantFMP4Segment(
startDTS time.Duration, startDTS time.Duration,
segmentMaxSize uint64, segmentMaxSize uint64,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
genPartID func() uint64, genPartID func() uint64,
onPartFinalized func(*muxerVariantFMP4Part), onPartFinalized func(*muxerVariantFMP4Part),
) *muxerVariantFMP4Segment { ) *muxerVariantFMP4Segment {
@@ -155,14 +155,14 @@ func (s *muxerVariantFMP4Segment) writeH264(sample *augmentedVideoSample, adjust
return nil return nil
} }
func (s *muxerVariantFMP4Segment) writeAAC(sample *augmentedAudioSample, adjustedPartDuration time.Duration) error { func (s *muxerVariantFMP4Segment) writeAudio(sample *augmentedAudioSample, adjustedPartDuration time.Duration) error {
size := uint64(len(sample.Payload)) size := uint64(len(sample.Payload))
if (s.size + size) > s.segmentMaxSize { if (s.size + size) > s.segmentMaxSize {
return fmt.Errorf("reached maximum segment size") return fmt.Errorf("reached maximum segment size")
} }
s.size += size s.size += size
s.currentPart.writeAAC(sample) s.currentPart.writeAudio(sample)
// switch part // switch part
if s.lowLatency && s.videoTrack == nil && if s.lowLatency && s.videoTrack == nil &&

View File

@@ -79,7 +79,7 @@ type muxerVariantFMP4Segmenter struct {
partDuration time.Duration partDuration time.Duration
segmentMaxSize uint64 segmentMaxSize uint64
videoTrack format.Format videoTrack format.Format
audioTrack *format.MPEG4Audio audioTrack format.Format
onSegmentFinalized func(*muxerVariantFMP4Segment) onSegmentFinalized func(*muxerVariantFMP4Segment)
onPartFinalized func(*muxerVariantFMP4Part) onPartFinalized func(*muxerVariantFMP4Part)
@@ -104,7 +104,7 @@ func newMuxerVariantFMP4Segmenter(
partDuration time.Duration, partDuration time.Duration,
segmentMaxSize uint64, segmentMaxSize uint64,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
onSegmentFinalized func(*muxerVariantFMP4Segment), onSegmentFinalized func(*muxerVariantFMP4Segment),
onPartFinalized func(*muxerVariantFMP4Part), onPartFinalized func(*muxerVariantFMP4Part),
) *muxerVariantFMP4Segmenter { ) *muxerVariantFMP4Segmenter {
@@ -317,7 +317,7 @@ func (m *muxerVariantFMP4Segmenter) writeH26xEntry(
return nil return nil
} }
func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, au []byte) error { func (m *muxerVariantFMP4Segmenter) writeAudio(ntp time.Time, dts time.Duration, au []byte) error {
if m.videoTrack != nil { if m.videoTrack != nil {
// wait for the video track // wait for the video track
if !m.videoFirstRandomAccessReceived { if !m.videoFirstRandomAccessReceived {
@@ -367,7 +367,7 @@ func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, a
} }
} }
err := m.currentSegment.writeAAC(sample, m.partDuration) err := m.currentSegment.writeAudio(sample, m.partDuration)
if err != nil { if err != nil {
return err return err
} }

View File

@@ -17,7 +17,7 @@ func newMuxerVariantMPEGTS(
segmentDuration time.Duration, segmentDuration time.Duration,
segmentMaxSize uint64, segmentMaxSize uint64,
videoTrack format.Format, videoTrack format.Format,
audioTrack *format.MPEG4Audio, audioTrack format.Format,
) (*muxerVariantMPEGTS, error) { ) (*muxerVariantMPEGTS, error) {
var videoTrackH264 *format.H264 var videoTrackH264 *format.H264
if videoTrack != nil { if videoTrack != nil {
@@ -25,7 +25,17 @@ func newMuxerVariantMPEGTS(
videoTrackH264, ok = videoTrack.(*format.H264) videoTrackH264, ok = videoTrack.(*format.H264)
if !ok { if !ok {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"the MPEG-TS variant of HLS doesn't support H265. Use the fMP4 or Low-Latency variants instead") "the MPEG-TS variant of HLS only supports H264 video. Use the fMP4 or Low-Latency variants instead")
}
}
var audioTrackMPEG4Audio *format.MPEG4Audio
if audioTrack != nil {
var ok bool
audioTrackMPEG4Audio, ok = audioTrack.(*format.MPEG4Audio)
if !ok {
return nil, fmt.Errorf(
"the MPEG-TS variant of HLS only supports MPEG4-audio. Use the fMP4 or Low-Latency variants instead")
} }
} }
@@ -37,7 +47,7 @@ func newMuxerVariantMPEGTS(
segmentDuration, segmentDuration,
segmentMaxSize, segmentMaxSize,
videoTrackH264, videoTrackH264,
audioTrack, audioTrackMPEG4Audio,
func(seg *muxerVariantMPEGTSSegment) { func(seg *muxerVariantMPEGTSSegment) {
v.playlist.pushSegment(seg) v.playlist.pushSegment(seg)
}, },
@@ -54,7 +64,7 @@ func (v *muxerVariantMPEGTS) writeH26x(ntp time.Time, pts time.Duration, nalus [
return v.segmenter.writeH264(ntp, pts, nalus) return v.segmenter.writeH264(ntp, pts, nalus)
} }
func (v *muxerVariantMPEGTS) writeAAC(ntp time.Time, pts time.Duration, au []byte) error { func (v *muxerVariantMPEGTS) writeAudio(ntp time.Time, pts time.Duration, au []byte) error {
return v.segmenter.writeAAC(ntp, pts, au) return v.segmenter.writeAAC(ntp, pts, au)
} }