diff --git a/plugin/mp4/api_extract.go b/plugin/mp4/api_extract.go new file mode 100644 index 0000000..35f50d3 --- /dev/null +++ b/plugin/mp4/api_extract.go @@ -0,0 +1,1209 @@ +/** + * @file 文件名.h + * @brief MP4 文件查询提取功能,GOP提取新的MP4，片段提取图片等，已验证测试H264,H265 + * @author erroot + * @date 250614 + * @version 1.0.0 + */ + +package plugin_mp4 + +import ( + "bytes" + "fmt" + "image" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + m7s "m7s.live/v5" + "m7s.live/v5/pkg" + "m7s.live/v5/pkg/util" + mp4 "m7s.live/v5/plugin/mp4/pkg" + "m7s.live/v5/plugin/mp4/pkg/box" +) + +/* +根据时间范围提取视频片段 +njtv/glgc.mp4? +start=1748620153000& +end=1748620453000& +outputPath=/opt/njtv/1748620153000.mp4 +*/ +func (p *MP4Plugin) extractClipToFile(streamPath string, startTime, endTime time.Time, outputPath string) error { + if p.DB == nil { + return pkg.ErrNoDB + } + + var flag mp4.Flag + if strings.HasSuffix(streamPath, ".fmp4") { + flag = mp4.FLAG_FRAGMENT + streamPath = strings.TrimSuffix(streamPath, ".fmp4") + } else { + streamPath = strings.TrimSuffix(streamPath, ".mp4") + } + + // 查询数据库获取符合条件的片段 + queryRecord := m7s.RecordStream{ + Type: "mp4", + } + var streams []m7s.RecordStream + p.DB.Where(&queryRecord).Find(&streams, "end_time>? AND start_time 0 { + t.Samplelist = audioHistory[len(audioHistory)-1].Track.Samplelist + } + audioTrack = t + audioHistory = append(audioHistory, TrackHistory{Track: t, ExtraData: track.ExtraData}) + } + + addVideoTrack := func(track *mp4.Track) { + t := muxer.AddTrack(track.Cid) + t.ExtraData = track.ExtraData + t.Width = track.Width + t.Height = track.Height + if len(videoHistory) > 0 { + t.Samplelist = videoHistory[len(videoHistory)-1].Track.Samplelist + } + videoTrack = t + videoHistory = append(videoHistory, TrackHistory{Track: t, ExtraData: track.ExtraData}) + } + + addTrack := func(track *mp4.Track) { + var lastAudioTrack, lastVideoTrack *TrackHistory + if len(audioHistory) > 0 { + lastAudioTrack = &audioHistory[len(audioHistory)-1] + } + if len(videoHistory) > 0 { + lastVideoTrack = &videoHistory[len(videoHistory)-1] + } + if track.Cid.IsAudio() { + if lastAudioTrack == nil { + addAudioTrack(track) + } else if !bytes.Equal(lastAudioTrack.ExtraData, track.ExtraData) { + for _, history := range audioHistory { + if bytes.Equal(history.ExtraData, track.ExtraData) { + audioTrack = history.Track + audioTrack.Samplelist = audioHistory[len(audioHistory)-1].Track.Samplelist + return + } + } + addAudioTrack(track) + } + } else if track.Cid.IsVideo() { + if lastVideoTrack == nil { + addVideoTrack(track) + } else if !bytes.Equal(lastVideoTrack.ExtraData, track.ExtraData) { + for _, history := range videoHistory { + if bytes.Equal(history.ExtraData, track.ExtraData) { + videoTrack = history.Track + videoTrack.Samplelist = videoHistory[len(videoHistory)-1].Track.Samplelist + return + } + } + addVideoTrack(track) + } + } + } + + // 处理每个片段 + for i, stream := range streams { + tsOffset = lastTs + file, err = os.Open(stream.FilePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %v", stream.FilePath, err) + } + defer file.Close() + + p.Info("processing segment", "file", file.Name()) + demuxer := mp4.NewDemuxer(file) + err = demuxer.Demux() + if err != nil { + return fmt.Errorf("demux error: %v", err) + } + + trackCount := len(demuxer.Tracks) + if i == 0 || flag == mp4.FLAG_FRAGMENT { + for _, track := range demuxer.Tracks { + addTrack(track) + } + } + + if trackCount != len(muxer.Tracks) { + if flag == mp4.FLAG_FRAGMENT { + moov = muxer.MakeMoov() + } + } + + if i == 0 { + startTimestamp := startTime.Sub(stream.StartTime).Milliseconds() + if startTimestamp < 0 { + startTimestamp = 0 + } + var startSample *box.Sample + if startSample, err = demuxer.SeekTimePreIDR(uint64(startTimestamp)); err != nil { + tsOffset = 0 + continue + } + tsOffset = -int64(startSample.Timestamp) + } + + var part *ContentPart + for track, sample := range demuxer.RangeSample { + if i == streamCount-1 && int64(sample.Timestamp) > endTime.Sub(stream.StartTime).Milliseconds() { + break + } + + if part == nil { + part = &ContentPart{ + File: file, + Start: sample.Offset, + } + } + + lastTs = int64(sample.Timestamp + uint32(tsOffset)) + fixSample := *sample + fixSample.Timestamp += uint32(tsOffset) + + if flag == 0 { + fixSample.Offset = sampleOffset + (fixSample.Offset - part.Start) + part.Size += sample.Size + if track.Cid.IsAudio() { + audioTrack.AddSampleEntry(fixSample) + } else if track.Cid.IsVideo() { + videoTrack.AddSampleEntry(fixSample) + } + } else { + part.Seek(sample.Offset, io.SeekStart) + fixSample.Data = make([]byte, sample.Size) + part.Read(fixSample.Data) + var moof, mdat box.IBox + if track.Cid.IsAudio() { + moof, mdat = muxer.CreateFlagment(audioTrack, fixSample) + } else if track.Cid.IsVideo() { + moof, mdat = muxer.CreateFlagment(videoTrack, fixSample) + } + if moof != nil { + part.boxies = append(part.boxies, moof, mdat) + part.Size += int(moof.Size() + mdat.Size()) + } + } + } + + if part != nil { + sampleOffset += int64(part.Size) + parts = append(parts, part) + } + } + + // 写入输出文件 + if flag == 0 { + moovSize := muxer.MakeMoov().Size() + dataSize := uint64(sampleOffset - mdatOffset) + + // 调整sample偏移量 + for _, track := range muxer.Tracks { + for i := range track.Samplelist { + track.Samplelist[i].Offset += int64(moovSize) + } + } + + mdatBox := box.CreateBaseBox(box.TypeMDAT, dataSize+box.BasicBoxLen) + + var freeBox *box.FreeBox + if mdatBox.HeaderSize() == box.BasicBoxLen { + freeBox = box.CreateFreeBox(nil) + } + + // 写入文件头 + _, err = box.WriteTo(outputFile, ftyp, muxer.MakeMoov(), freeBox, mdatBox) + if err != nil { + return fmt.Errorf("failed to write header: %v", err) + } + + // 写入媒体数据 + for _, part := range parts { + part.Seek(part.Start, io.SeekStart) + _, err = io.CopyN(outputFile, part.File, int64(part.Size)) + if err != nil { + return fmt.Errorf("failed to write media data: %v", err) + } + part.Close() + } + } else { + var children []box.IBox + children = append(children, ftyp, moov) + for _, part := range parts { + children = append(children, part.boxies...) + part.Close() + } + _, err = box.WriteTo(outputFile, children...) + if err != nil { + return fmt.Errorf("failed to write fragmented MP4: %v", err) + } + } + + p.Info("clip saved successfully", "path", outputPath) + return nil +} + +// bytes2hexStr 将字节数组前n个字节转为16进制字符串 +// data: 原始字节数组 +// length: 需要转换的字节数（超过实际长度时自动截断） +func Bytes2HexStr(data []byte, length int) string { + if length > len(data) { + length = len(data) + } + + var builder strings.Builder + for i := 0; i < length; i++ { + if i > 0 { + builder.WriteString(" ") + } + builder.WriteString(fmt.Sprintf("%02X", data[i])) + } + return builder.String() +} + +/* +提取压缩视频（快放视频） + +njtv/glgc.mp4? +start=1748620153000& +end=1748620453000& +outputPath=/opt/njtv/1748620153000.mp4 +gopSeconds=1& +gopInterval=1& + +FLAG_FRAGMENT 暂时不支持没有调试 + +假设原生帧率25fps GOP = 50 frame +时间范围: endTime-startTime = 300s = 7500 frame = 150 GOP +gopSeconds=0.2 6 frame +gopInterval=10 +提取结果15 gop, 90 frame , 90/25 = 3.6 s + +反过推算 要求 5范围分钟 压缩到15s 播放完 +当gopSeconds=0.1， 推算 gopInterval=1 +当gopSeconds=0.2， 推算 gopInterval=2 +*/ +func (p *MP4Plugin) extractCompressedVideo(streamPath string, startTime, endTime time.Time, outputPath string, gopSeconds float64, gopInterval int) error { + if p.DB == nil { + return pkg.ErrNoDB + } + + var flag mp4.Flag + if strings.HasSuffix(streamPath, ".fmp4") { + flag = mp4.FLAG_FRAGMENT + streamPath = strings.TrimSuffix(streamPath, ".fmp4") + } else { + streamPath = strings.TrimSuffix(streamPath, ".mp4") + } + + // 查询数据库获取符合条件的片段 + queryRecord := m7s.RecordStream{ + Type: "mp4", + } + var streams []m7s.RecordStream + p.DB.Where(&queryRecord).Find(&streams, "end_time>? AND start_time 0 && gopElapsed > gopSeconds { + continue + } + + // 处理结束时间边界 + if i == len(streams)-1 && int64(adjustedTimestamp) > endTime.Sub(streams[0].StartTime).Milliseconds() { + continue + } + + // 确保样本数据有效 + if sample.Size <= 0 || sample.Size > 10*1024*1024 { // 10MB限制 + p.Warn("invalid sample size", "size", sample.Size, "timestamp", sample.Timestamp) + continue + } + + // 读取样本数据 + if _, err := file.Seek(sample.Offset, io.SeekStart); err != nil { + p.Warn("seek error", "error", err, "offset", sample.Offset) + continue + } + data := make([]byte, sample.Size) + if _, err := io.ReadFull(file, data); err != nil { + p.Warn("read sample error", "error", err, "size", sample.Size) + continue + } + + // 创建新的样本 + newSample := box.Sample{ + KeyFrame: sample.KeyFrame, + Data: data, + Timestamp: adjustedTimestamp, + Offset: sampleOffset, + Size: sample.Size, + Duration: sample.Duration, + } + + // p.Info("Compressed", "KeyFrame", newSample.KeyFrame, + // "CTS", newSample.CTS, + // "Timestamp", newSample.Timestamp, + // "Offset", newSample.Offset, + // "Size", newSample.Size, + // "Duration", newSample.Duration, + // "Data", Bytes2HexStr(newSample.Data, 16)) + + sampleOffset += int64(newSample.Size) + filteredSamples = append(filteredSamples, newSample) + + } + } + + if len(filteredSamples) == 0 { + return fmt.Errorf("no valid video samples found") + } + + // 按25fps重新计算时间戳 + for i := range filteredSamples { + filteredSamples[i].Timestamp = uint32(i * targetFrameInterval) + } + + // 添加样本到轨道 + for _, sample := range filteredSamples { + videoTrack.AddSampleEntry(sample) + } + + // 计算视频时长 + videoDuration := uint32(len(filteredSamples) * targetFrameInterval) + + // 写入输出文件 + if flag == 0 { + // 非分片MP4处理 + moovSize := muxer.MakeMoov().Size() + dataSize := uint64(sampleOffset - mdatOffset) + + // 调整sample偏移量 + for _, track := range muxer.Tracks { + for i := range track.Samplelist { + track.Samplelist[i].Offset += int64(moovSize) + } + } + + // 创建MDAT盒子 (添加8字节头) + mdatHeaderSize := uint64(8) + mdatBox := box.CreateBaseBox(box.TypeMDAT, dataSize+mdatHeaderSize) + + var freeBox *box.FreeBox + if mdatBox.HeaderSize() == box.BasicBoxLen { + freeBox = box.CreateFreeBox(nil) + } + + // 写入文件头 + _, err = box.WriteTo(outputFile, ftyp, muxer.MakeMoov(), freeBox, mdatBox) + if err != nil { + return fmt.Errorf("failed to write header: %v", err) + } + + for _, track := range muxer.Tracks { + for i := range track.Samplelist { + track.Samplelist[i].Offset += int64(moovSize) + if _, err := outputFile.Write(track.Samplelist[i].Data); err != nil { + return err + } + } + } + } else { + // 分片MP4处理 + var children []box.IBox + moov := muxer.MakeMoov() + children = append(children, ftyp, moov) + + // 创建分片 + for _, sample := range filteredSamples { + moof, mdat := muxer.CreateFlagment(videoTrack, sample) + children = append(children, moof, mdat) + } + + _, err = box.WriteTo(outputFile, children...) + if err != nil { + return fmt.Errorf("failed to write fragmented MP4: %v", err) + } + } + + p.Info("compressed video saved", "path", outputPath, + "originalDuration", (endTime.Sub(startTime)).Milliseconds(), + "compressedDuration", videoDuration, + "frameCount", len(filteredSamples), + "fps", 25) + return nil +} + +/* +根据时间范围提取视频片段 +njtv/glgc.mp4? +timest=1748620153000& +outputPath=/opt/njtv/gop_tmp_1748620153000.mp4 + +原理：根据时间戳找到最近的mp4文件，再从mp4 文件中找到最近gop 生成mp4 文件 +*/ +func (p *MP4Plugin) extractGopVideo(streamPath string, targetTime time.Time, outputPath string) (float64, error) { + if p.DB == nil { + return 0, pkg.ErrNoDB + } + + var flag mp4.Flag + if strings.HasSuffix(streamPath, ".fmp4") { + flag = mp4.FLAG_FRAGMENT + streamPath = strings.TrimSuffix(streamPath, ".fmp4") + } else { + streamPath = strings.TrimSuffix(streamPath, ".mp4") + } + + // 查询数据库获取符合条件的片段 + queryRecord := m7s.RecordStream{ + Type: "mp4", + } + var streams []m7s.RecordStream + p.DB.Where(&queryRecord).Find(&streams, "end_time>=? AND start_time<=? AND stream_path=?", targetTime, targetTime, streamPath) + if len(streams) == 0 { + return 0, fmt.Errorf("no matching MP4 segments found") + } + + // 创建输出文件 + outputFile, err := os.Create(outputPath) + if err != nil { + return 0, fmt.Errorf("failed to create output file: %v", err) + } + defer outputFile.Close() + + p.Info("extracting compressed video", "streamPath", streamPath, "targetTime", targetTime, + "output", outputPath) + + muxer := mp4.NewMuxer(flag) + ftyp := muxer.CreateFTYPBox() + n := ftyp.Size() + muxer.CurrentOffset = int64(n) + var videoTrack *mp4.Track + sampleOffset := muxer.CurrentOffset + mp4.BeforeMdatData + mdatOffset := sampleOffset + + //var audioTrack *mp4.Track + var extraData []byte + + // 压缩相关变量 + findGOP := false + targetFrameInterval := 40 // 25fps对应的毫秒间隔 (1000/25=40ms) + var filteredSamples []box.Sample + //var lastVideoTimestamp uint32 + var timescale uint32 = 1000 // 默认时间刻度为1000 (毫秒) + var currentGopStartTime int64 = -1 + var gopElapsed float64 = 0 + // 仅处理视频轨道 + for _, stream := range streams { + file, err := os.Open(stream.FilePath) + if err != nil { + return 0, fmt.Errorf("failed to open file %s: %v", stream.FilePath, err) + } + defer file.Close() + + p.Info("processing segment", "file", file.Name()) + demuxer := mp4.NewDemuxer(file) + err = demuxer.Demux() + if err != nil { + p.Warn("demux error, skipping segment", "error", err, "file", stream.FilePath) + continue + } + + // 确保有视频轨道 + var hasVideo bool + for _, track := range demuxer.Tracks { + if track.Cid.IsVideo() { + hasVideo = true + // 只在第一个片段或关键帧变化时更新extraData + if extraData == nil || !bytes.Equal(extraData, track.ExtraData) { + extraData = track.ExtraData + if videoTrack == nil { + videoTrack = muxer.AddTrack(track.Cid) + videoTrack.ExtraData = extraData + videoTrack.Width = track.Width + videoTrack.Height = track.Height + } + } + break + } + } + + if !hasVideo { + p.Warn("no video track found in segment", "file", stream.FilePath) + continue + } + + // 处理起始时间边界 + var tsOffset int64 + + startTimestamp := targetTime.Sub(stream.StartTime).Milliseconds() + + // p.Info("extractGop", "targetTime", targetTime, + // "stream.StartTime", stream.StartTime, + // "startTimestamp", startTimestamp) + + if startTimestamp < 0 { + startTimestamp = 0 + } + //通过时间戳定位到最近的‌关键帧‌（如视频IDR帧），返回的startSample是该关键帧对应的样本 + startSample, err := demuxer.SeekTimePreIDR(uint64(startTimestamp)) + if err == nil { + tsOffset = -int64(startSample.Timestamp) + } + + //p.Info("extractGop", "startSample", startSample) + + // 处理样本 + //RangeSample迭代的是‌当前时间范围内的所有样本‌（可能包含非关键帧），顺序取决于MP4文件中样本的物理存储顺序 + for track, sample := range demuxer.RangeSample { + if !track.Cid.IsVideo() { + continue + } + + if sample.Timestamp < startSample.Timestamp { + continue + } + + //for _, sample := range samples { + adjustedTimestamp := sample.Timestamp + uint32(tsOffset) + + // 处理GOP逻辑,已经处理完上一个gop + if sample.KeyFrame && findGOP { + break + } + + // 处理GOP逻辑 + if sample.KeyFrame && !findGOP { + findGOP = true + currentGopStartTime = int64(sample.Timestamp) + } + + // 跳过不在当前GOP的帧 + if !findGOP { + currentGopStartTime = -1 + continue + } + // 检查是否超过gopSeconds限制 + currentTime := int64(sample.Timestamp) + gopElapsed = float64(currentTime-currentGopStartTime) / float64(timescale) + + // 确保样本数据有效 + if sample.Size <= 0 || sample.Size > 10*1024*1024 { // 10MB限制 + p.Warn("invalid sample size", "size", sample.Size, "timestamp", sample.Timestamp) + continue + } + + // 读取样本数据 + if _, err := file.Seek(sample.Offset, io.SeekStart); err != nil { + p.Warn("seek error", "error", err, "offset", sample.Offset) + continue + } + data := make([]byte, sample.Size) + if _, err := io.ReadFull(file, data); err != nil { + p.Warn("read sample error", "error", err, "size", sample.Size) + continue + } + + // 创建新的样本 + newSample := box.Sample{ + KeyFrame: sample.KeyFrame, + Data: data, + Timestamp: adjustedTimestamp, + Offset: sampleOffset, + Size: sample.Size, + Duration: sample.Duration, + } + + // p.Info("extractGop", "KeyFrame", newSample.KeyFrame, + // "CTS", newSample.CTS, + // "Timestamp", newSample.Timestamp, + // "Offset", newSample.Offset, + // "Size", newSample.Size, + // "Duration", newSample.Duration, + // "Data", Bytes2HexStr(newSample.Data, 16)) + + sampleOffset += int64(newSample.Size) + filteredSamples = append(filteredSamples, newSample) + + } + } + + if len(filteredSamples) == 0 { + return 0, fmt.Errorf("no valid video samples found") + } + + // 按25fps重新计算时间戳 + for i := range filteredSamples { + filteredSamples[i].Timestamp = uint32(i * targetFrameInterval) + } + + // 添加样本到轨道 + for _, sample := range filteredSamples { + videoTrack.AddSampleEntry(sample) + } + + // 计算视频时长 + videoDuration := uint32(len(filteredSamples) * targetFrameInterval) + + // 写入输出文件 + if flag == 0 { + // 非分片MP4处理 + moovSize := muxer.MakeMoov().Size() + dataSize := uint64(sampleOffset - mdatOffset) + + // 调整sample偏移量 + for _, track := range muxer.Tracks { + for i := range track.Samplelist { + track.Samplelist[i].Offset += int64(moovSize) + } + } + + // 创建MDAT盒子 (添加8字节头) + mdatHeaderSize := uint64(8) + mdatBox := box.CreateBaseBox(box.TypeMDAT, dataSize+mdatHeaderSize) + + var freeBox *box.FreeBox + if mdatBox.HeaderSize() == box.BasicBoxLen { + freeBox = box.CreateFreeBox(nil) + } + + // 写入文件头 + _, err = box.WriteTo(outputFile, ftyp, muxer.MakeMoov(), freeBox, mdatBox) + if err != nil { + return 0, fmt.Errorf("failed to write header: %v", err) + } + + for _, track := range muxer.Tracks { + for i := range track.Samplelist { + track.Samplelist[i].Offset += int64(moovSize) + if _, err := outputFile.Write(track.Samplelist[i].Data); err != nil { + return 0, err + } + } + } + } else { + // 分片MP4处理 + var children []box.IBox + moov := muxer.MakeMoov() + children = append(children, ftyp, moov) + + // 创建分片 + for _, sample := range filteredSamples { + moof, mdat := muxer.CreateFlagment(videoTrack, sample) + children = append(children, moof, mdat) + } + + _, err = box.WriteTo(outputFile, children...) + if err != nil { + return 0, fmt.Errorf("failed to write fragmented MP4: %v", err) + } + } + p.Info("extract gop video saved", "path", outputPath, + "targetTime", targetTime, + "compressedDuration", videoDuration, + "gopElapsed", gopElapsed, + "frameCount", len(filteredSamples), + "fps", 25) + return gopElapsed, nil +} + +/* +根据时间范围提取视频片段 +njtv/glgc.mp4? +timest=1748620153000& +outputPath=/opt/njtv/gop_tmp_1748620153000.mp4 + +原理：根据时间戳找到最近的mp4文件，再从mp4 文件中找到最近gop 生成mp4 文件 +*/ +func (p *MP4Plugin) snapImage(streamPath string, targetTime time.Time) (image.Image, error) { + if p.DB == nil { + return nil, pkg.ErrNoDB + } + + var flag mp4.Flag + if strings.HasSuffix(streamPath, ".fmp4") { + flag = mp4.FLAG_FRAGMENT + streamPath = strings.TrimSuffix(streamPath, ".fmp4") + } else { + streamPath = strings.TrimSuffix(streamPath, ".mp4") + } + + // 查询数据库获取符合条件的片段 + queryRecord := m7s.RecordStream{ + Type: "mp4", + } + var streams []m7s.RecordStream + p.DB.Where(&queryRecord).Find(&streams, "end_time>=? AND start_time<=? AND stream_path=?", targetTime, targetTime, streamPath) + if len(streams) == 0 { + return nil, fmt.Errorf("no matching MP4 segments found") + } + + muxer := mp4.NewMuxer(flag) + ftyp := muxer.CreateFTYPBox() + n := ftyp.Size() + muxer.CurrentOffset = int64(n) + var videoTrack *mp4.Track + sampleOffset := muxer.CurrentOffset + mp4.BeforeMdatData + + //var audioTrack *mp4.Track + var extraData []byte + + // 压缩相关变量 + findGOP := false + targetFrameInterval := 40 // 25fps对应的毫秒间隔 (1000/25=40ms) + var filteredSamples []box.Sample + var sampleIdx = 0 + // 仅处理视频轨道 + for _, stream := range streams { + file, err := os.Open(stream.FilePath) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %v", stream.FilePath, err) + } + defer file.Close() + + p.Info("processing segment", "file", file.Name()) + demuxer := mp4.NewDemuxer(file) + err = demuxer.Demux() + if err != nil { + p.Warn("demux error, skipping segment", "error", err, "file", stream.FilePath) + continue + } + + // 确保有视频轨道 + var hasVideo bool + for _, track := range demuxer.Tracks { + if track.Cid.IsVideo() { + hasVideo = true + // 只在第一个片段或关键帧变化时更新extraData + if extraData == nil || !bytes.Equal(extraData, track.ExtraData) { + extraData = track.ExtraData + if videoTrack == nil { + videoTrack = muxer.AddTrack(track.Cid) + videoTrack.ExtraData = extraData + videoTrack.Width = track.Width + videoTrack.Height = track.Height + } + } + break + } + } + + if !hasVideo { + p.Warn("no video track found in segment", "file", stream.FilePath) + continue + } + + //p.Info("extractGop", "SPS PPS", Bytes2HexStr(videoTrack.ExtraData, len(videoTrack.ExtraData))) + + // 处理起始时间边界 + var tsOffset int64 + + startTimestamp := targetTime.Sub(stream.StartTime).Milliseconds() + + // p.Info("extractGop", + // "Timescale", videoTrack.Timescale, + // "targetTime", targetTime, + // "stream.StartTime", stream.StartTime, + // "startTimestamp", startTimestamp) + + if startTimestamp < 0 { + startTimestamp = 0 + } + //通过时间戳定位到最近的‌关键帧‌（如视频IDR帧），返回的startSample是该关键帧对应的样本 + startSample, err := demuxer.SeekTimePreIDR(uint64(startTimestamp)) + if err == nil { + tsOffset = -int64(startSample.Timestamp) + } + + // p.Info("extractGop", "startSample Timestamp", + // startSample.Timestamp) + + // 处理样本 + //RangeSample迭代的是‌当前时间范围内的所有样本‌（可能包含非关键帧），顺序取决于MP4文件中样本的物理存储顺序 + for track, sample := range demuxer.RangeSample { + if !track.Cid.IsVideo() { + continue + } + + if sample.Timestamp < startSample.Timestamp { + p.Info("extractGop", "KeyFrame", sample.KeyFrame, + "CTS", sample.CTS, + "Timestamp", sample.Timestamp, + "Offset", sample.Offset, + "Size", sample.Size, + "Duration", sample.Duration) + + continue + } + //记录GOP内帧的序号，没有考虑B帧的情况 + if sample.Timestamp < uint32(startTimestamp) { + sampleIdx++ + } + + adjustedTimestamp := sample.Timestamp + uint32(tsOffset) + + // 处理GOP逻辑,已经处理完上一个gop + if sample.KeyFrame && findGOP { + break + } + + // 处理GOP逻辑 + if sample.KeyFrame && !findGOP { + findGOP = true + } + + // 跳过不在当前GOP的帧 + if !findGOP { + continue + } + // 检查是否超过gopSeconds限制 + + // 确保样本数据有效 + if sample.Size <= 0 || sample.Size > 10*1024*1024 { // 10MB限制 + p.Warn("invalid sample size", "size", sample.Size, "timestamp", sample.Timestamp) + continue + } + + // 读取样本数据 + if _, err := file.Seek(sample.Offset, io.SeekStart); err != nil { + p.Warn("seek error", "error", err, "offset", sample.Offset) + continue + } + data := make([]byte, sample.Size) + if _, err := io.ReadFull(file, data); err != nil { + p.Warn("read sample error", "error", err, "size", sample.Size) + continue + } + + // p.Info("extractGop", "KeyFrame", sample.KeyFrame, + // "CTS", sample.CTS, + // "Timestamp", sample.Timestamp, + // "Offset", sample.Offset, + // "Size", sample.Size, + // "Duration", sample.Duration, + // "Data", Bytes2HexStr(data, 32)) + + // 创建新的样本 + newSample := box.Sample{ + KeyFrame: sample.KeyFrame, + Data: data, + Timestamp: adjustedTimestamp, + Offset: sampleOffset, + Size: sample.Size, + Duration: sample.Duration, + } + + sampleOffset += int64(newSample.Size) + filteredSamples = append(filteredSamples, newSample) + } + } + + if len(filteredSamples) == 0 { + return nil, fmt.Errorf("no valid video samples found") + } + + // 按25fps重新计算时间戳 + for i := range filteredSamples { + filteredSamples[i].Timestamp = uint32(i * targetFrameInterval) + } + + p.Info("extract gop and snap", + "targetTime", targetTime, + "frist", filteredSamples[0].Timestamp, + "sampleIdx", sampleIdx, + "frameCount", len(filteredSamples)) + + img, err := ProcessWithFFmpeg(filteredSamples, sampleIdx, videoTrack) + if err != nil { + return nil, err + } + // 添加样本到轨道 + p.Info("extract gop and snap saved", + "targetTime", targetTime, + "frameCount", len(filteredSamples)) + + return img, nil +} + +/* +提取普通MP4视频 +GET http://192.168.0.238:8080/mp4/extractClip/njtv/glgc.mp4? + + start=1748620153000& + end=1748620453000& + outputPath=/opt/njtv/1748620153000.mp4 +*/ +func (p *MP4Plugin) extractClipToFileHandel(w http.ResponseWriter, r *http.Request) { + streamPath := r.PathValue("streamPath") + query := r.URL.Query() + // 合并多个 mp4 + startTime, endTime, err := util.TimeRangeQueryParse(query) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + p.Info("extractClipToFileHandel", "streamPath", streamPath, "start", startTime, "end", endTime) + + outputPath := query.Get("outputPath") + + p.extractClipToFile(streamPath, startTime, endTime, outputPath) + + // 返回成功响应 + w.WriteHeader(http.StatusOK) +} + +/* +提取压缩视频 + +GET http://192.168.0.238:8080/mp4/extractCompressed/ +njtv/glgc.mp4? +start=1748620153000& +end=1748620453000& +outputPath=/opt/njtv/1748620153000.mp4 +gopSeconds=1& +gopInterval=1& +*/ +func (p *MP4Plugin) extractCompressedVideoHandel(w http.ResponseWriter, r *http.Request) { + streamPath := r.PathValue("streamPath") + query := r.URL.Query() + // 合并多个 mp4 + startTime, endTime, err := util.TimeRangeQueryParse(query) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + p.Info("extractClipToFileHandel", "streamPath", streamPath, "start", startTime, "end", endTime) + + outputPath := query.Get("outputPath") + gopSeconds, _ := strconv.ParseFloat(query.Get("gopSeconds"), 64) + gopInterval, _ := strconv.Atoi(query.Get("gopInterval")) + + if gopSeconds == 0 { + gopSeconds = 1 + } + if gopInterval == 0 { + gopInterval = 1 + } + + p.extractCompressedVideo(streamPath, startTime, endTime, outputPath, gopSeconds, gopInterval) + + // 返回成功响应 + w.WriteHeader(http.StatusOK) +} + +func (p *MP4Plugin) extractGopVideoHandel(w http.ResponseWriter, r *http.Request) { + streamPath := r.PathValue("streamPath") + query := r.URL.Query() + + targetTimeString := query.Get("targetTime") + // 合并多个 mp4 + targetTime, err := util.UnixTimeQueryParse(targetTimeString) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + p.Info("extractGopVideoHandel", "streamPath", streamPath, "targetTime", targetTime) + + outputPath := query.Get("outputPath") + p.extractGopVideo(streamPath, targetTime, outputPath) + + // 返回成功响应 + w.WriteHeader(http.StatusOK) +} + +func (p *MP4Plugin) snapHandel(w http.ResponseWriter, r *http.Request) { + streamPath := r.PathValue("streamPath") + query := r.URL.Query() + + targetTimeString := query.Get("targetTime") + // 合并多个 mp4 + targetTime, err := util.UnixTimeQueryParse(targetTimeString) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + p.Info("snapHandel", "streamPath", streamPath, "targetTime", targetTime) + + outputPath := query.Get("outputPath") + img, err := p.snapImage(streamPath, targetTime) + if err == nil { + //水印测试 + // wImg, err := watermark.WatermarkTest(img) + // if err != nil { + // p.Info("watermarkTest", "err", err) + // http.Error(w, err.Error(), http.StatusBadRequest) + // return + // } + //saveAsJPG(wImg, outputPath) + saveAsJPG(img, outputPath) + } else { + p.Info("snapHandel", "err", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // 返回成功响应 + w.WriteHeader(http.StatusOK) +} diff --git a/plugin/mp4/index.go b/plugin/mp4/index.go index 3e97bbf..eb4e77b 100644 --- a/plugin/mp4/index.go +++ b/plugin/mp4/index.go @@ -76,7 +76,11 @@ var _ = m7s.InstallPlugin[MP4Plugin](m7s.PluginMeta{ func (p *MP4Plugin) RegisterHandler() map[string]http.HandlerFunc { return map[string]http.HandlerFunc{ - "/download/{streamPath...}": p.download, + "/download/{streamPath...}": p.download, + "/extractClip/{streamPath...}": p.extractClipToFileHandel, + "/extractCompressed/{streamPath...}": p.extractCompressedVideoHandel, + "/extractGop/{streamPath...}": p.extractGopVideoHandel, + "/snap/{streamPath...}": p.snapHandel, } } diff --git a/plugin/mp4/pkg/demuxer.go b/plugin/mp4/pkg/demuxer.go index 687bdc6..442acc4 100644 --- a/plugin/mp4/pkg/demuxer.go +++ b/plugin/mp4/pkg/demuxer.go @@ -218,6 +218,54 @@ func (d *Demuxer) SeekTime(dts uint64) (sample *Sample, err error) { return } +/** +* @brief 函数跳帧到dts 前面的第一个关键帧位置 +* +* @param 参数名dts 跳帧位置 +* +* @todo 待实现的功能或改进点 audioTrack 没有同步改进 +* @author erroot +* @date 250614 +* +**/ +func (d *Demuxer) SeekTimePreIDR(dts uint64) (sample *Sample, err error) { + var audioTrack, videoTrack *Track + for _, track := range d.Tracks { + if track.Cid.IsAudio() { + audioTrack = track + } else if track.Cid.IsVideo() { + videoTrack = track + } + } + if videoTrack != nil { + idx := videoTrack.SeekPreIDR(dts) + if idx == -1 { + return nil, errors.New("seek failed") + } + d.ReadSampleIdx[videoTrack.TrackId-1] = uint32(idx) + sample = &videoTrack.Samplelist[idx] + if audioTrack != nil { + for i, sample := range audioTrack.Samplelist { + if sample.Offset < int64(videoTrack.Samplelist[idx].Offset) { + continue + } + d.ReadSampleIdx[audioTrack.TrackId-1] = uint32(i) + break + } + } + } else if audioTrack != nil { + idx := audioTrack.Seek(dts) + if idx == -1 { + return nil, errors.New("seek failed") + } + d.ReadSampleIdx[audioTrack.TrackId-1] = uint32(idx) + sample = &audioTrack.Samplelist[idx] + } else { + return nil, pkg.ErrNoTrack + } + return +} + // func (d *Demuxer) decodeTRUN(trun *TrackRunBox) { // dataOffset := trun.Dataoffset // nextDts := d.currentTrack.StartDts diff --git a/plugin/mp4/pkg/track.go b/plugin/mp4/pkg/track.go index 9a9f8b7..d2b9532 100644 --- a/plugin/mp4/pkg/track.go +++ b/plugin/mp4/pkg/track.go @@ -102,6 +102,28 @@ func (track *Track) Seek(dts uint64) int { return -1 } +/** +* @brief 函数跳帧到dts 前面的第一个关键帧位置 +* +* @param 参数名dts 跳帧位置 +* +* @author erroot +* @date 250614 +* +**/ +func (track *Track) SeekPreIDR(dts uint64) int { + idx := 0 + for i, sample := range track.Samplelist { + if track.Cid.IsVideo() && sample.KeyFrame { + idx = i + } + if sample.Timestamp*1000/uint32(track.Timescale) > uint32(dts) { + break + } + } + return idx +} + func (track *Track) makeEdtsBox() *ContainerBox { return CreateContainerBox(TypeEDTS, track.makeElstBox()) } diff --git a/plugin/mp4/util.go b/plugin/mp4/util.go new file mode 100644 index 0000000..c232580 --- /dev/null +++ b/plugin/mp4/util.go @@ -0,0 +1,338 @@ +package plugin_mp4 + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "image" + "image/color" + "image/jpeg" + "io" + "log" + "os" + "os/exec" + + mp4 "m7s.live/v5/plugin/mp4/pkg" + "m7s.live/v5/plugin/mp4/pkg/box" +) + +func saveAsJPG(img image.Image, path string) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + opt := jpeg.Options{Quality: 90} + return jpeg.Encode(file, img, &opt) +} + +func ExtractH264SPSPPS(extraData []byte) (sps, pps []byte, err error) { + if len(extraData) < 7 { + return nil, nil, fmt.Errorf("extradata too short") + } + + // 解析 SPS 数量 (第6字节低5位) + spsCount := int(extraData[5] & 0x1F) + offset := 6 // 当前解析位置 + + // 提取 SPS + for i := 0; i < spsCount; i++ { + if offset+2 > len(extraData) { + return nil, nil, fmt.Errorf("invalid sps length") + } + spsLen := int(binary.BigEndian.Uint16(extraData[offset : offset+2])) + offset += 2 + if offset+spsLen > len(extraData) { + return nil, nil, fmt.Errorf("sps data overflow") + } + sps = extraData[offset : offset+spsLen] + offset += spsLen + } + + // 提取 PPS 数量 + if offset >= len(extraData) { + return nil, nil, fmt.Errorf("missing pps count") + } + ppsCount := int(extraData[offset]) + offset++ + + // 提取 PPS + for i := 0; i < ppsCount; i++ { + if offset+2 > len(extraData) { + return nil, nil, fmt.Errorf("invalid pps length") + } + ppsLen := int(binary.BigEndian.Uint16(extraData[offset : offset+2])) + offset += 2 + if offset+ppsLen > len(extraData) { + return nil, nil, fmt.Errorf("pps data overflow") + } + pps = extraData[offset : offset+ppsLen] + offset += ppsLen + } + return sps, pps, nil +} + +// 转换函数（支持动态插入参数集） +func ConvertAVCCH264ToAnnexB(data []byte, extraData []byte, isFirst *bool) ([]byte, error) { + var buf bytes.Buffer + pos := 0 + + for pos < len(data) { + if pos+4 > len(data) { + break + } + nalSize := binary.BigEndian.Uint32(data[pos : pos+4]) + pos += 4 + nalStart := pos + pos += int(nalSize) + if pos > len(data) { + break + } + nalu := data[nalStart:pos] + nalType := nalu[0] & 0x1F + + // 关键帧前插入SPS/PPS（仅需执行一次） + if *isFirst && nalType == 5 { + sps, pps, err := ExtractH264SPSPPS(extraData) + if err != nil { + //panic(err) + return nil, err + } + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + buf.Write(sps) + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + buf.Write(pps) + //buf.Write(videoTrack.ExtraData) + *isFirst = false // 仅首帧插入 + } + + // 保留SEI单元（类型6）和所有其他单元 + if nalType == 5 || nalType == 6 { // IDR/SEI用4字节起始码 + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + } else { + buf.Write([]byte{0x00, 0x00, 0x01}) // 其他用3字节 + } + buf.Write(nalu) + } + return buf.Bytes(), nil +} + +/* +H.264与H.265的AVCC格式差异​​ +VPS引入​​：H.265新增视频参数集（VPS），用于描述多层编码、时序等信息 +*/ +// 提取H.265的VPS/SPS/PPS（HEVCDecoderConfigurationRecord格式） +func ExtractHEVCParams(extraData []byte) (vps, sps, pps []byte, err error) { + if len(extraData) < 22 { + return nil, nil, nil, errors.New("extra data too short") + } + + // HEVC的extradata格式参考ISO/IEC 14496-15 + offset := 22 // 跳过头部22字节 + if offset+2 > len(extraData) { + return nil, nil, nil, errors.New("invalid extra data") + } + + numOfArrays := int(extraData[offset]) + offset++ + + for i := 0; i < numOfArrays; i++ { + if offset+3 > len(extraData) { + break + } + + naluType := extraData[offset] & 0x3F + offset++ + count := int(binary.BigEndian.Uint16(extraData[offset:])) + offset += 2 + + for j := 0; j < count; j++ { + if offset+2 > len(extraData) { + break + } + + naluSize := int(binary.BigEndian.Uint16(extraData[offset:])) + offset += 2 + + if offset+naluSize > len(extraData) { + break + } + + naluData := extraData[offset : offset+naluSize] + offset += naluSize + + // 根据类型存储参数集 + switch naluType { + case 32: // VPS + if vps == nil { + vps = make([]byte, len(naluData)) + copy(vps, naluData) + } + case 33: // SPS + if sps == nil { + sps = make([]byte, len(naluData)) + copy(sps, naluData) + } + case 34: // PPS + if pps == nil { + pps = make([]byte, len(naluData)) + copy(pps, naluData) + } + } + } + } + + if vps == nil || sps == nil || pps == nil { + return nil, nil, nil, errors.New("missing required parameter sets") + } + + return vps, sps, pps, nil +} + +// H.265的AVCC转Annex B +func ConvertAVCCHEVCToAnnexB(data []byte, extraData []byte, isFirst *bool) ([]byte, error) { + var buf bytes.Buffer + pos := 0 + + // 首帧插入VPS/SPS/PPS + if *isFirst { + vps, sps, pps, err := ExtractHEVCParams(extraData) + if err == nil { + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + buf.Write(vps) + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + buf.Write(sps) + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + buf.Write(pps) + } else { + return nil, err + } + } + + // 处理NALU + for pos < len(data) { + if pos+4 > len(data) { + break + } + nalSize := binary.BigEndian.Uint32(data[pos : pos+4]) + pos += 4 + nalStart := pos + pos += int(nalSize) + if pos > len(data) { + break + } + nalu := data[nalStart:pos] + nalType := (nalu[0] >> 1) & 0x3F // H.265的NALU类型在头部的第2-7位 + + // 关键帧或参数集使用4字节起始码 + if nalType == 19 || nalType == 20 || nalType >= 32 && nalType <= 34 { + buf.Write([]byte{0x00, 0x00, 0x00, 0x01}) + } else { + buf.Write([]byte{0x00, 0x00, 0x01}) + } + buf.Write(nalu) + } + return buf.Bytes(), nil +} + +// ffmpeg -hide_banner -i gop.mp4 -vf "select=eq(n\,15)" -vframes 1 -f image2 -pix_fmt bgr24 output.bmp +func ProcessWithFFmpeg(samples []box.Sample, index int, videoTrack *mp4.Track) (image.Image, error) { + // code := "h264" + // if videoTrack.Cid == box.MP4_CODEC_H265 { + // code = "hevc" + // } + cmd := exec.Command("ffmpeg", + "-hide_banner", + //"-f", code, //"h264" 强制指定输入格式为H.264裸流 + "-i", "pipe:0", + "-vf", fmt.Sprintf("select=eq(n\\,%d)", index), + "-vframes", "1", + "-pix_fmt", "bgr24", + "-f", "rawvideo", + "pipe:1") + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + go func() { + errOutput, _ := io.ReadAll(stderr) + log.Printf("FFmpeg stderr: %s", errOutput) + }() + + if err = cmd.Start(); err != nil { + log.Printf("cmd.Start失败: %v", err) + return nil, err + } + + go func() { + defer stdin.Close() + isFirst := true + for _, sample := range samples { + + if videoTrack.Cid == box.MP4_CODEC_H264 { + annexb, _ := ConvertAVCCH264ToAnnexB(sample.Data, videoTrack.ExtraData, &isFirst) + if _, err := stdin.Write(annexb); err != nil { + log.Printf("写入失败: %v", err) + break + } + } else { + annexb, _ := ConvertAVCCHEVCToAnnexB(sample.Data, videoTrack.ExtraData, &isFirst) + if _, err := stdin.Write(annexb); err != nil { + log.Printf("写入失败: %v", err) + break + } + } + } + }() + + // 读取原始RGB数据 + var buf bytes.Buffer + if _, err = io.Copy(&buf, stdout); err != nil { + log.Printf("读取失败: %v", err) + return nil, err + } + if err = cmd.Wait(); err != nil { + log.Printf("cmd.Wait失败: %v", err) + return nil, err + } + + //log.Printf("ffmpeg 提取成功: data size:%v", buf.Len()) + + // 转换为image.Image对象 + data := buf.Bytes() + //width, height := parseBMPDimensions(data) + + width := int(videoTrack.Width) + height := int(videoTrack.Height) + + log.Printf("ffmpeg size: %v,%v", width, height) + + //FFmpeg的 rawvideo 输出默认采用​​从上到下​​的扫描方式 + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + //pos := (height-y-1)*width*3 + x*3 + pos := (y*width + x) * 3 // 关键修复：按行顺序读取 + img.Set(x, y, color.RGBA{ + R: data[pos+2], + G: data[pos+1], + B: data[pos], + A: 255, + }) + } + } + return img, nil +}