refactor: frame converter and mp4 track improvements

- Refactor frame converter implementation - Update mp4 track to use ICodex - General refactoring and code improvements 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-12-24 13:48:04 +08:00 · 2025-08-04 09:17:12 +08:00
parent b6ee2843b0
commit 8a9fffb987
262 changed files with 20831 additions and 12141 deletions
--- a/plugin/README.md
+++ b/plugin/README.md
@@ -6,6 +6,12 @@
 - Visual Studio Code
 - Goland
 - Cursor
+- CodeBuddy
+- Trae
+- Qoder
+- Claude Code
+- Kiro
+- Windsurf

 ### Install gRPC
 ```shell
@@ -53,14 +59,16 @@ Example:
 const defaultConfig = m7s.DefaultYaml(`tcp:
  listenaddr: :5554`)

-var _ = m7s.InstallPlugin[MyPlugin](defaultConfig)
+var _ = m7s.InstallPlugin[MyPlugin](m7s.PluginMeta{
+    DefaultYaml: defaultConfig,
+})
 ```

 ## 3. Implement Event Callbacks (Optional)

 ### Initialization Callback
 ```go
-func (config *MyPlugin) OnInit() (err error) {
+func (config *MyPlugin) Start() (err error) {
    // Initialize things
    return
 }
@@ -121,22 +129,25 @@ func (config *MyPlugin) test1(rw http.ResponseWriter, r *http.Request) {
 Push client needs to implement IPusher interface and pass the creation method to InstallPlugin.
 ```go
 type Pusher struct {
-    pullCtx m7s.PullJob
+    task.Task
+    pushJob m7s.PushJob
 }

-func (c *Pusher) GetPullJob() *m7s.PullJob {
-    return &c.pullCtx
+func (c *Pusher) GetPushJob() *m7s.PushJob {
+    return &c.pushJob
 }

 func NewPusher(_ config.Push) m7s.IPusher {
    return &Pusher{}
 }
-var _ = m7s.InstallPlugin[MyPlugin](NewPusher)
+var _ = m7s.InstallPlugin[MyPlugin](m7s.PluginMeta{
+    NewPusher: NewPusher,
+})
 ```

 ### Implement Pull Client
 Pull client needs to implement IPuller interface and pass the creation method to InstallPlugin.
-The following Puller inherits from m7s.HTTPFilePuller for basic file and HTTP pulling:
+The following Puller inherits from m7s.HTTPFilePuller for basic file and HTTP pulling. You need to override the Start method for specific pulling logic:
 ```go
 type Puller struct {
    m7s.HTTPFilePuller
@@ -145,7 +156,9 @@ type Puller struct {
 func NewPuller(_ config.Pull) m7s.IPuller {
    return &Puller{}
 }
-var _ = m7s.InstallPlugin[MyPlugin](NewPuller)
+var _ = m7s.InstallPlugin[MyPlugin](m7s.PluginMeta{
+    NewPuller: NewPuller,
+})
 ```

 ## 6. Implement gRPC Service
@@ -226,7 +239,10 @@ import (
    "m7s.live/v5/plugin/myplugin/pb"
 )

-var _ = m7s.InstallPlugin[MyPlugin](&pb.Api_ServiceDesc, pb.RegisterApiHandler)
+var _ = m7s.InstallPlugin[MyPlugin](m7s.PluginMeta{
+	ServiceDesc:         &pb.Api_ServiceDesc,
+	RegisterGRPCHandler: pb.RegisterApiHandler,
+})

 type MyPlugin struct {
    pb.UnimplementedApiServer
@@ -247,43 +263,72 @@ Accessible via GET request to `/myplugin/api/test1`
 ## 7. Publishing Streams

 ```go
-publisher, err = p.Publish(streamPath, connectInfo)
+publisher, err := p.Publish(ctx, streamPath)
 ```
-The last two parameters are optional.
+The `ctx` parameter is required, `streamPath` parameter is required.

-After obtaining the `publisher`, you can publish audio/video data using `publisher.WriteAudio` and `publisher.WriteVideo`.
+### Writing Audio/Video Data
+
+The old `WriteAudio` and `WriteVideo` methods have been replaced with a more structured writer pattern using generics:
+
+#### **Create Writers**
+```go
+// Audio writer
+audioWriter := m7s.NewPublishAudioWriter[*AudioFrame](publisher, allocator)
+
+// Video writer  
+videoWriter := m7s.NewPublishVideoWriter[*VideoFrame](publisher, allocator)
+
+// Combined audio/video writer
+writer := m7s.NewPublisherWriter[*AudioFrame, *VideoFrame](publisher, allocator)
+```
+
+#### **Write Frames**
+```go
+// Set timestamp and write audio frame
+writer.AudioFrame.SetTS32(timestamp)
+err := writer.NextAudio()
+
+// Set timestamp and write video frame
+writer.VideoFrame.SetTS32(timestamp)
+err := writer.NextVideo()
+```
+
+#### **Write Custom Data**
+```go
+// For custom data frames
+err := publisher.WriteData(data IDataFrame)
+```

 ### Define Audio/Video Data
 If existing audio/video data formats don't meet your needs, you can define custom formats by implementing this interface:
 ```go
 IAVFrame interface {
-    GetAllocator() *util.ScalableMemoryAllocator
-    SetAllocator(*util.ScalableMemoryAllocator)
-    Parse(*AVTrack) error
-    ConvertCtx(codec.ICodecCtx) (codec.ICodecCtx, IAVFrame, error)
-    Demux(codec.ICodecCtx) (any, error)
-    Mux(codec.ICodecCtx, *AVFrame)
-    GetTimestamp() time.Duration
-    GetCTS() time.Duration
+    GetSample() *Sample
    GetSize() int
+    CheckCodecChange() error
+    Demux() error      // demux to raw format
+    Mux(*Sample) error // mux from origin format
    Recycle()
    String() string
-    Dump(byte, io.Writer)
 }
 ```
 > Define separate types for audio and video

- GetAllocator/SetAllocator: Automatically implemented when embedding RecyclableMemory
- Parse: Identifies key frames, sequence frames, and other important information
- ConvertCtx: Called when protocol conversion is needed
- Demux: Called when audio/video data needs to be demuxed
- Mux: Called when audio/video data needs to be muxed
- Recycle: Automatically implemented when embedding RecyclableMemory
- String: Prints audio/video data information
+The methods serve the following purposes:
+- GetSample: Gets the Sample object containing codec context and raw data
 - GetSize: Gets the size of audio/video data
- GetTimestamp: Gets the timestamp in nanoseconds
- GetCTS: Gets the Composition Time Stamp in nanoseconds (PTS = DTS+CTS)
- Dump: Prints binary audio/video data
+- CheckCodecChange: Checks if the codec has changed
+- Demux: Demuxes audio/video data to raw format for use by other formats
+- Mux: Muxes from original format to custom audio/video data format
+- Recycle: Recycles resources, automatically implemented when embedding RecyclableMemory
+- String: Prints audio/video data information
+
+### Memory Management
+The new pattern includes built-in memory management:
+- `util.ScalableMemoryAllocator` - For efficient memory allocation
+- Frame recycling through `Recycle()` method
+- Automatic memory pool management

 ## 8. Subscribing to Streams
 ```go
@@ -293,7 +338,245 @@ go m7s.PlayBlock(suber, handleAudio, handleVideo)
 ```
 Note that handleAudio and handleVideo are callback functions you need to implement. They take an audio/video format type as input and return an error. If the error is not nil, the subscription is terminated.

-## 9. Prometheus Integration
+## 9. Working with H26xFrame for Raw Stream Data
+
+### 9.1 Understanding H26xFrame Structure
+
+The `H26xFrame` struct is used for handling H.264/H.265 raw stream data:
+
+```go
+type H26xFrame struct {
+    pkg.Sample
+}
+```
+
+Key characteristics:
+- Inherits from `pkg.Sample` - contains codec context, memory management, and timing
+- Uses `Raw.(*pkg.Nalus)` to store NALU (Network Abstraction Layer Unit) data
+- Supports both H.264 (AVC) and H.265 (HEVC) formats
+- Uses efficient memory allocators for zero-copy operations
+
+### 9.2 Creating H26xFrame for Publishing
+
+```go
+import (
+    "m7s.live/v5"
+    "m7s.live/v5/pkg/format"
+    "m7s.live/v5/pkg/util"
+    "time"
+)
+
+// Create publisher with H26xFrame support
+func publishRawH264Stream(streamPath string, h264Frames [][]byte) error {
+    // Get publisher
+    publisher, err := p.Publish(streamPath)
+    if err != nil {
+        return err
+    }
+    
+    // Create memory allocator
+    allocator := util.NewScalableMemoryAllocator(1 << util.MinPowerOf2)
+    defer allocator.Recycle()
+    
+    // Create writer for H26xFrame
+    writer := m7s.NewPublisherWriter[*format.RawAudio, *format.H26xFrame](publisher, allocator)
+    
+    // Set up H264 codec context
+    writer.VideoFrame.ICodecCtx = &format.H264{}
+    
+    // Publish multiple frames
+    // Note: This is a demonstration of multi-frame writing. In actual scenarios, 
+    // frames should be written gradually as they are received from the video source.
+    startTime := time.Now()
+    for i, frameData := range h264Frames {
+        // Create H26xFrame for each frame
+        frame := writer.VideoFrame
+        
+        // Set timestamp with proper interval
+        frame.Timestamp = startTime.Add(time.Duration(i) * time.Second / 30) // 30 FPS
+        
+        // Write NALU data
+        nalus := frame.GetNalus()
+        // if frameData is a single NALU, otherwise need to loop
+        p := nalus.GetNextPointer()
+        mem := frame.NextN(len(frameData))
+        copy(mem, frameData)
+        p.PushOne(mem)	
+        // Publish frame
+        if err := writer.NextVideo(); err != nil {
+            return err
+        }
+    }
+    
+    return nil
+}
+
+// Example usage with continuous streaming
+func continuousH264Publishing(streamPath string, frameSource <-chan []byte, stopChan <-chan struct{}) error {
+    // Get publisher
+    publisher, err := p.Publish(streamPath)
+    if err != nil {
+        return err
+    }
+    defer publisher.Dispose()
+    
+    // Create memory allocator
+    allocator := util.NewScalableMemoryAllocator(1 << util.MinPowerOf2)
+    defer allocator.Recycle()
+    
+    // Create writer for H26xFrame
+    writer := m7s.NewPublisherWriter[*format.RawAudio, *format.H26xFrame](publisher, allocator)
+    
+    // Set up H264 codec context
+    writer.VideoFrame.ICodecCtx = &format.H264{}
+    
+    startTime := time.Now()
+    frameCount := 0
+    
+    for {
+        select {
+        case frameData := <-frameSource:
+            // Create H26xFrame for each frame
+            frame := writer.VideoFrame
+            
+            // Set timestamp with proper interval
+            frame.Timestamp = startTime.Add(time.Duration(frameCount) * time.Second / 30) // 30 FPS
+            
+            // Write NALU data
+            nalus := frame.GetNalus()
+            mem := frame.NextN(len(frameData))
+            copy(mem, frameData)
+            
+            // Publish frame
+            if err := writer.NextVideo(); err != nil {
+                return err
+            }
+            
+            frameCount++
+            
+        case <-stopChan:
+            // Stop publishing
+            return nil
+        }
+    }
+}
+```
+
+### 9.3 Processing H26xFrame (Transform Pattern)
+
+```go
+type MyTransform struct {
+    m7s.DefaultTransformer
+    Writer *m7s.PublishWriter[*format.RawAudio, *format.H26xFrame]
+}
+
+func (t *MyTransform) Go() {
+    defer t.Dispose()
+    
+    for video := range t.Video {
+        if err := t.processH26xFrame(video); err != nil {
+            t.Error("process frame failed", "error", err)
+            break
+        }
+    }
+}
+
+func (t *MyTransform) processH26xFrame(video *format.H26xFrame) error {
+    // Copy frame metadata
+    copyVideo := t.Writer.VideoFrame
+    copyVideo.ICodecCtx = video.ICodecCtx
+    *copyVideo.BaseSample = *video.BaseSample
+    nalus := copyVideo.GetNalus()
+    
+    // Process each NALU unit
+    for nalu := range video.Raw.(*pkg.Nalus).RangePoint {
+        p := nalus.GetNextPointer()
+        mem := copyVideo.NextN(nalu.Size)
+        nalu.CopyTo(mem)
+        
+        // Example: Filter or modify specific NALU types
+        if video.FourCC() == codec.FourCC_H264 {
+            switch codec.ParseH264NALUType(mem[0]) {
+            case codec.NALU_IDR_Picture, codec.NALU_Non_IDR_Picture:
+                // Process video frame NALUs
+                // Example: Apply transformations, filters, etc.
+            case codec.NALU_SPS, codec.NALU_PPS:
+                // Process parameter set NALUs
+            }
+        } else if video.FourCC() == codec.FourCC_H265 {
+            switch codec.ParseH265NALUType(mem[0]) {
+            case h265parser.NAL_UNIT_CODED_SLICE_IDR_W_RADL:
+                // Process H.265 IDR frames
+            }
+        }
+        
+        // Push processed NALU
+        p.PushOne(mem)
+    }
+    
+    return t.Writer.NextVideo()
+}
+```
+
+### 9.4 Common NALU Types for H.264/H.265
+
+#### H.264 NALU Types
+```go
+const (
+    NALU_Non_IDR_Picture = 1  // Non-IDR picture (P-frames)
+    NALU_IDR_Picture     = 5  // IDR picture (I-frames)
+    NALU_SEI             = 6  // Supplemental enhancement information
+    NALU_SPS             = 7  // Sequence parameter set
+    NALU_PPS             = 8  // Picture parameter set
+)
+
+// Parse NALU type from first byte
+naluType := codec.ParseH264NALUType(mem[0])
+```
+
+#### H.265 NALU Types
+```go
+// Parse H.265 NALU type from first byte
+naluType := codec.ParseH265NALUType(mem[0])
+```
+
+### 9.5 Memory Management Best Practices
+
+```go
+// Use memory allocators for efficient operations
+allocator := util.NewScalableMemoryAllocator(1 << 20) // 1MB initial size
+defer allocator.Recycle()
+
+// When processing multiple frames, reuse the same allocator
+writer := m7s.NewPublisherWriter[*format.RawAudio, *format.H26xFrame](publisher, allocator)
+```
+
+### 9.6 Error Handling and Validation
+
+```go
+func processFrame(video *format.H26xFrame) error {
+    // Check codec changes
+    if err := video.CheckCodecChange(); err != nil {
+        return err
+    }
+    
+    // Validate frame data
+    if video.Raw == nil {
+        return fmt.Errorf("empty frame data")
+    }
+    
+    // Process NALUs safely
+    nalus, ok := video.Raw.(*pkg.Nalus)
+    if !ok {
+        return fmt.Errorf("invalid NALUs format")
+    }
+    
+    // Process frame...
+    return nil
+}
+```
+
+## 10. Prometheus Integration
 Just implement the Collector interface, and the system will automatically collect metrics from all plugins:
 ```go
 func (p *MyPlugin) Describe(ch chan<- *prometheus.Desc) {