package rtp import ( "encoding/hex" "fmt" "strings" "time" "unsafe" "github.com/bluenviron/mediacommon/pkg/bits" "github.com/pion/rtp" "github.com/pion/webrtc/v4" . "m7s.live/v5/pkg" "m7s.live/v5/pkg/codec" "m7s.live/v5/pkg/util" ) type RTPData struct { Sample Packets util.ReuseArray[rtp.Packet] } func (r *RTPData) Recycle() { r.RecyclableMemory.Recycle() r.Packets.Reset() } func (r *RTPData) String() (s string) { for p := range r.Packets.RangePoint { s += fmt.Sprintf("t: %d, s: %d, p: %02X %d\n", p.Timestamp, p.SequenceNumber, p.Payload[0:2], len(p.Payload)) } return } func (r *RTPData) GetSize() (s int) { for p := range r.Packets.RangePoint { s += p.MarshalSize() } return } type ( RTPCtx struct { webrtc.RTPCodecParameters Fmtp map[string]string SequenceNumber uint16 SSRC uint32 } PCMACtx struct { RTPCtx *codec.PCMACtx } PCMUCtx struct { RTPCtx *codec.PCMUCtx } OPUSCtx struct { RTPCtx *codec.OPUSCtx } AACCtx struct { RTPCtx *codec.AACCtx SizeLength int // 通常为13 IndexLength int IndexDeltaLength int } IRTPCtx interface { GetRTPCodecParameter() webrtc.RTPCodecParameters } ) func (r *RTPCtx) ParseFmtpLine(cp *webrtc.RTPCodecParameters) { r.RTPCodecParameters = *cp r.Fmtp = make(map[string]string) kvs := strings.Split(r.SDPFmtpLine, ";") for _, kv := range kvs { if kv = strings.TrimSpace(kv); kv == "" { continue } if key, value, found := strings.Cut(kv, "="); found { r.Fmtp[strings.TrimSpace(key)] = strings.TrimSpace(value) } } } func (r *RTPCtx) GetInfo() string { return r.GetRTPCodecParameter().SDPFmtpLine } func (r *AACCtx) GetInfo() string { return r.AACCtx.GetInfo() } func (r *OPUSCtx) GetInfo() string { return r.OPUSCtx.GetInfo() } func (r *RTPCtx) GetRTPCodecParameter() webrtc.RTPCodecParameters { return r.RTPCodecParameters } func (r *RTPData) Append(ctx *RTPCtx, ts uint32, payload []byte) (packet *rtp.Packet) { ctx.SequenceNumber++ packet = r.Packets.GetNextPointer() packet.Header = rtp.Header{ Version: 2, SequenceNumber: ctx.SequenceNumber, Timestamp: ts, SSRC: ctx.SSRC, PayloadType: uint8(ctx.PayloadType), } packet.Payload = payload return } var _ IAVFrame = (*AudioFrame)(nil) type AudioFrame struct { RTPData } func payloadLengthInfoDecode(buf []byte) (int, int, error) { lb := len(buf) l := 0 n := 0 for { if (lb - n) == 0 { return 0, 0, fmt.Errorf("not enough bytes") } b := buf[n] n++ l += int(b) if b != 255 { break } } return l, n, nil } func (r *AudioFrame) Demux() (err error) { if len(r.Packets) == 0 { return ErrSkip } data := r.GetAudioData() // 从编解码器上下文获取 MimeType var mimeType string if rtpCtx, ok := r.ICodecCtx.(IRTPCtx); ok { mimeType = rtpCtx.GetRTPCodecParameter().MimeType } switch mimeType { case "audio/MP4A-LATM": var fragments util.Memory var fragmentsExpected int var fragmentsSize int for packet := range r.Packets.RangePoint { if len(packet.Payload) == 0 { continue } if packet.Padding { packet.Padding = false } buf := packet.Payload if fragments.Size == 0 { pl, n, err := payloadLengthInfoDecode(buf) if err != nil { return err } buf = buf[n:] bl := len(buf) if pl <= bl { data.PushOne(buf[:pl]) // there could be other data, due to otherDataPresent. Ignore it. } else { if pl > 5*1024 { fragments = util.Memory{} // discard pending fragments return fmt.Errorf("access unit size (%d) is too big, maximum is %d", pl, 5*1024) } fragments.PushOne(buf) fragmentsSize = pl fragmentsExpected = pl - bl continue } } else { bl := len(buf) if fragmentsExpected > bl { fragments.PushOne(buf) fragmentsExpected -= bl continue } fragments.PushOne(buf[:fragmentsExpected]) // there could be other data, due to otherDataPresent. Ignore it. data.Push(fragments.Buffers...) if fragments.Size != fragmentsSize { return fmt.Errorf("fragmented AU size is not correct %d != %d", data.Size, fragmentsSize) } fragments = util.Memory{} } } case "audio/MPEG4-GENERIC": var fragments util.Memory for packet := range r.Packets.RangePoint { if len(packet.Payload) < 2 { continue } auHeaderLen := util.ReadBE[int](packet.Payload[:2]) if auHeaderLen == 0 { data.PushOne(packet.Payload) } else { dataLens, err := r.readAUHeaders(r.ICodecCtx.(*AACCtx), packet.Payload[2:], auHeaderLen) if err != nil { return err } payload := packet.Payload[2:] pos := auHeaderLen >> 3 if (auHeaderLen % 8) != 0 { pos++ } payload = payload[pos:] if fragments.Size == 0 { if packet.Marker { for _, dataLen := range dataLens { if len(payload) < int(dataLen) { return fmt.Errorf("invalid data len %d", dataLen) } data.PushOne(payload[:dataLen]) payload = payload[dataLen:] } } else { if len(dataLens) != 1 { return fmt.Errorf("a fragmented packet can only contain one AU") } fragments.PushOne(payload) } } else { if len(dataLens) != 1 { return fmt.Errorf("a fragmented packet can only contain one AU") } fragments.PushOne(payload) if !packet.Header.Marker { continue } if uint64(fragments.Size) != dataLens[0] { return fmt.Errorf("fragmented AU size is not correct %d != %d", dataLens[0], fragments.Size) } data.Push(fragments.Buffers...) fragments = util.Memory{} } } break } default: for packet := range r.Packets.RangePoint { data.PushOne(packet.Payload) } } return nil } func (r *AudioFrame) Mux(from *Sample) (err error) { data := from.Raw.(*AudioData) var ctx *RTPCtx var lastPacket *rtp.Packet switch base := from.GetBase().(type) { case *codec.AACCtx: var c *AACCtx if r.ICodecCtx == nil { c = &AACCtx{} c.SSRC = uint32(uintptr(unsafe.Pointer(&ctx))) c.AACCtx = base c.MimeType = "audio/MPEG4-GENERIC" c.SDPFmtpLine = fmt.Sprintf("profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3;config=%s", hex.EncodeToString(c.ConfigBytes)) c.IndexLength = 3 c.IndexDeltaLength = 3 c.SizeLength = 13 c.RTPCtx.Channels = uint16(base.GetChannels()) c.PayloadType = 97 c.ClockRate = uint32(base.CodecData.SampleRate()) r.ICodecCtx = c } else { c = r.ICodecCtx.(*AACCtx) } ctx = &c.RTPCtx pts := uint32(from.Timestamp * time.Duration(ctx.ClockRate) / time.Second) //AU_HEADER_LENGTH,因为单位是bit, 除以8就是auHeader的字节长度;又因为单个auheader字节长度2字节,所以再除以2就是auheader的个数。 auHeaderLen := []byte{0x00, 0x10, (byte)((data.Size & 0x1fe0) >> 5), (byte)((data.Size & 0x1f) << 3)} // 3 = 16-13, 5 = 8-3 for reader := data.NewReader(); reader.Length > 0; { payloadLen := MTUSize if reader.Length+4 < MTUSize { payloadLen = reader.Length + 4 } mem := r.NextN(payloadLen) copy(mem, auHeaderLen) reader.Read(mem[4:]) lastPacket = r.Append(ctx, pts, mem) } lastPacket.Header.Marker = true return case *codec.PCMACtx: if r.ICodecCtx == nil { var ctx PCMACtx ctx.SSRC = uint32(uintptr(unsafe.Pointer(&ctx))) ctx.PCMACtx = base ctx.MimeType = webrtc.MimeTypePCMA ctx.PayloadType = 8 ctx.ClockRate = uint32(ctx.SampleRate) r.ICodecCtx = &ctx } ctx = &r.ICodecCtx.(*PCMACtx).RTPCtx case *codec.PCMUCtx: if r.ICodecCtx == nil { var ctx PCMUCtx ctx.SSRC = uint32(uintptr(unsafe.Pointer(&ctx))) ctx.PCMUCtx = base ctx.MimeType = webrtc.MimeTypePCMU ctx.PayloadType = 0 ctx.ClockRate = uint32(ctx.SampleRate) r.ICodecCtx = &ctx } ctx = &r.ICodecCtx.(*PCMUCtx).RTPCtx } pts := uint32(from.Timestamp * time.Duration(ctx.ClockRate) / time.Second) if reader := data.NewReader(); reader.Length > MTUSize { for reader.Length > 0 { payloadLen := MTUSize if reader.Length < MTUSize { payloadLen = reader.Length } mem := r.NextN(payloadLen) reader.Read(mem) lastPacket = r.Append(ctx, pts, mem) } } else { mem := r.NextN(reader.Length) reader.Read(mem) lastPacket = r.Append(ctx, pts, mem) } lastPacket.Header.Marker = true return } func (r *AudioFrame) readAUHeaders(ctx *AACCtx, buf []byte, headersLen int) ([]uint64, error) { firstRead := false count := 0 for i := 0; i < headersLen; { if i == 0 { i += ctx.SizeLength i += ctx.IndexLength } else { i += ctx.SizeLength i += ctx.IndexDeltaLength } count++ } dataLens := make([]uint64, count) pos := 0 i := 0 for headersLen > 0 { dataLen, err := bits.ReadBits(buf, &pos, ctx.SizeLength) if err != nil { return nil, err } headersLen -= ctx.SizeLength if !firstRead { firstRead = true if ctx.IndexLength > 0 { auIndex, err := bits.ReadBits(buf, &pos, ctx.IndexLength) if err != nil { return nil, err } headersLen -= ctx.IndexLength if auIndex != 0 { return nil, fmt.Errorf("AU-index different than zero is not supported") } } } else if ctx.IndexDeltaLength > 0 { auIndexDelta, err := bits.ReadBits(buf, &pos, ctx.IndexDeltaLength) if err != nil { return nil, err } headersLen -= ctx.IndexDeltaLength if auIndexDelta != 0 { return nil, fmt.Errorf("AU-index-delta different than zero is not supported") } } dataLens[i] = dataLen i++ } return dataLens, nil }