Now handling audio samples in frame data as well

README.md

@@ -27,7 +27,7 @@ Examples are located in the [examples](examples) directory and mirror as much as
 |Custom IO Demuxing|[see](examples/custom_io_demuxing/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/avio_reading.c)
 |Demuxing/Decoding|[see](examples/demuxing_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/demuxing_decoding.c)
 |Filtering|[see](examples/filtering/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/filtering_video.c)
-|Frame data manipulating|[see](examples/frame_data_manipulating/main.go)|X
+|Frame data manipulation|[see](examples/frame_data_manipulation/main.go)|X
 |Hardware Decoding|[see](examples/hardware_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/hw_decode.c)
 |Remuxing|[see](examples/remuxing/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/remuxing.c)
 |Scaling|[see](examples/scaling/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n7.0/doc/examples/scaling_video.c)

astiav_test.go

@@ -50,7 +50,7 @@ type helperInput struct {
 	lastFrame     *Frame
 }
 
-func (h *helper) inputFormatContext(name string) (fc *FormatContext, err error) {
+func (h *helper) inputFormatContext(name string, ifmt *InputFormat) (fc *FormatContext, err error) {
 	h.m.Lock()
 	i, ok := h.inputs[name]
 	if ok && i.formatContext != nil {
@@ -65,7 +65,7 @@ func (h *helper) inputFormatContext(name string) (fc *FormatContext, err error)
 	}
 	h.closer.Add(fc.Free)
 
-	if err = fc.OpenInput("testdata/"+name, nil, nil); err != nil {
+	if err = fc.OpenInput("testdata/"+name, ifmt, nil); err != nil {
 		err = fmt.Errorf("astiav_test: opening input failed: %w", err)
 		return
 	}
@@ -95,7 +95,7 @@ func (h *helper) inputFirstPacket(name string) (pkt *Packet, err error) {
 	h.m.Unlock()
 
 	var fc *FormatContext
-	if fc, err = h.inputFormatContext(name); err != nil {
+	if fc, err = h.inputFormatContext(name, nil); err != nil {
 		err = fmt.Errorf("astiav_test: getting input format context failed")
 		return
 	}
@@ -118,7 +118,7 @@ func (h *helper) inputFirstPacket(name string) (pkt *Packet, err error) {
 	return
 }
 
-func (h *helper) inputLastFrame(name string, mediaType MediaType) (f *Frame, err error) {
+func (h *helper) inputLastFrame(name string, mediaType MediaType, ifmt *InputFormat) (f *Frame, err error) {
 	h.m.Lock()
 	i, ok := h.inputs[name]
 	if ok && i.lastFrame != nil {
@@ -128,7 +128,7 @@ func (h *helper) inputLastFrame(name string, mediaType MediaType) (f *Frame, err
 	h.m.Unlock()
 
 	var fc *FormatContext
-	if fc, err = h.inputFormatContext(name); err != nil {
+	if fc, err = h.inputFormatContext(name, ifmt); err != nil {
 		err = fmt.Errorf("astiav_test: getting input format context failed: %w", err)
 		return
 	}

codec_context_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestCodecContext(t *testing.T) {
-	fc, err := globalHelper.inputFormatContext("video.mp4")
+	fc, err := globalHelper.inputFormatContext("video.mp4", nil)
 	require.NoError(t, err)
 	ss := fc.Streams()
 	require.Len(t, ss, 2)

codec_parameters_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestCodecParameters(t *testing.T) {
-	fc, err := globalHelper.inputFormatContext("video.mp4")
+	fc, err := globalHelper.inputFormatContext("video.mp4", nil)
 	require.NoError(t, err)
 	ss := fc.Streams()
 	require.Len(t, ss, 2)

examples/frame_data_manipulating/main.go

@@ -1,108 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"image/png"
-	"log"
-	"os"
-	"strings"
-
-	"github.com/asticode/go-astiav"
-)
-
-const (
-	align         = 1
-	pngPath       = "testdata/image-rgba.png"
-	rawBufferPath = "testdata/image-rgba.rgba"
-)
-
-func main() {
-	// Handle ffmpeg logs
-	astiav.SetLogLevel(astiav.LogLevelDebug)
-	astiav.SetLogCallback(func(c astiav.Classer, l astiav.LogLevel, fmt, msg string) {
-		var cs string
-		if c != nil {
-			if cl := c.Class(); cl != nil {
-				cs = " - class: " + cl.String()
-			}
-		}
-		log.Printf("ffmpeg log: %s%s - level: %d\n", strings.TrimSpace(msg), cs, l)
-	})
-
-	// Alloc frames
-	f1 := astiav.AllocFrame()
-	defer f1.Free()
-	f2 := astiav.AllocFrame()
-	defer f2.Free()
-
-	// To write data manually into a frame, proper attributes need to be set and allocated
-	for _, f := range []*astiav.Frame{f1, f2} {
-		// Set attributes
-		f.SetHeight(256)
-		f.SetPixelFormat(astiav.PixelFormatRgba)
-		f.SetWidth(256)
-
-		// Alloc buffer
-		if err := f.AllocBuffer(align); err != nil {
-			log.Fatal(fmt.Errorf("main: allocating buffer failed: %w", err))
-		}
-
-		// Alloc image
-		if err := f.AllocImage(align); err != nil {
-			log.Fatal(fmt.Errorf("main: allocating image failed: %w", err))
-		}
-	}
-
-	// When writing data manually into a frame, you usually need to make sure the frame is writable
-	// Don't forget this step above all if the frame's buffer is referenced elsewhere
-	for _, f := range []*astiav.Frame{f1, f2} {
-		// Make writable
-		if err := f.MakeWritable(); err != nil {
-			log.Fatal(fmt.Errorf("main: making frame writable failed: %w", err))
-		}
-	}
-
-	// As an example, we're going to write data manually into the first frame based on a buffer (i.e. raw data)
-	b, err := os.ReadFile(rawBufferPath)
-	if err != nil {
-		log.Fatal(fmt.Errorf("main: reading %s failed: %w", rawBufferPath, err))
-	}
-	if err := f1.Data().SetBytes(b, align); err != nil {
-		log.Fatal(fmt.Errorf("main: setting frame's data based on bytes failed: %w", err))
-	}
-
-	// As an example, we're going to write data manually into the second frame based on a Go image
-	fl1, err := os.Open(pngPath)
-	if err != nil {
-		log.Fatal(fmt.Errorf("main: opening %s failed: %w", pngPath, err))
-	}
-	defer fl1.Close()
-	i1, err := png.Decode(fl1)
-	if err != nil {
-		log.Fatal(fmt.Errorf("main: decoding %s failed: %w", pngPath, err))
-	}
-	if err := f2.Data().FromImage(i1); err != nil {
-		log.Fatal(fmt.Errorf("main: setting frame's data based on Go image failed: %w", err))
-	}
-
-	// This is the place where you do stuff with the frames
-
-	// As an example, we're going to read the first frame's data as a buffer (i.e. raw data)
-	if _, err = f1.Data().Bytes(align); err != nil {
-		log.Fatal(fmt.Errorf("main: getting frame's data as bytes failed: %w", err))
-	}
-
-	// As an example, we're going to read the second frame's data as a Go image
-	// For that we first need to guess the Go image format based on the frame's attributes before providing
-	// it to .ToImage(). You may not need this and can provide your own image.Image to .ToImage()
-	i2, err := f2.Data().GuessImageFormat()
-	if err != nil {
-		log.Fatal(fmt.Errorf("main: guessing image format failed: %w", err))
-	}
-	if err := f2.Data().ToImage(i2); err != nil {
-		log.Fatal(fmt.Errorf("main: getting frame's data as Go image failed: %w", err))
-	}
-
-	// Success
-	log.Println("success")
-}

examples/frame_data_manipulation/main.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"log"
+	"strings"
+
+	"github.com/asticode/go-astiav"
+)
+
+func main() {
+	// Handle ffmpeg logs
+	astiav.SetLogLevel(astiav.LogLevelDebug)
+	astiav.SetLogCallback(func(c astiav.Classer, l astiav.LogLevel, fmt, msg string) {
+		var cs string
+		if c != nil {
+			if cl := c.Class(); cl != nil {
+				cs = " - class: " + cl.String()
+			}
+		}
+		log.Printf("ffmpeg log: %s%s - level: %d\n", strings.TrimSpace(msg), cs, l)
+	})
+
+	/*
+
+		In this first part we're going to manipulate an audio frame
+
+	*/
+
+	// Alloc frame
+	audioFrame := astiav.AllocFrame()
+	defer audioFrame.Free()
+
+	// To write data manually into a frame, proper attributes need to be set and allocated
+	audioFrame.SetChannelLayout(astiav.ChannelLayoutStereo)
+	audioFrame.SetNbSamples(960)
+	audioFrame.SetSampleFormat(astiav.SampleFormatFlt)
+	audioFrame.SetSampleRate(48000)
+
+	// Alloc buffer
+	align := 0
+	if err := audioFrame.AllocBuffer(align); err != nil {
+		log.Fatal(fmt.Errorf("main: allocating buffer failed: %w", err))
+	}
+
+	// Alloc samples
+	if err := audioFrame.AllocSamples(align); err != nil {
+		log.Fatal(fmt.Errorf("main: allocating image failed: %w", err))
+	}
+
+	// When writing data manually into a frame, you need to make sure the frame is writable
+	if err := audioFrame.MakeWritable(); err != nil {
+		log.Fatal(fmt.Errorf("main: making frame writable failed: %w", err))
+	}
+
+	// Let's say b1 contains an actual audio buffer, we can update the audio frame's data based on the buffer
+	var b1 []byte
+	if err := audioFrame.Data().SetBytes(b1, align); err != nil {
+		log.Fatal(fmt.Errorf("main: setting frame's data based from bytes failed: %w", err))
+	}
+
+	// We can also retrieve the audio frame's data as buffer
+	if _, err := audioFrame.Data().Bytes(align); err != nil {
+		log.Fatal(fmt.Errorf("main: getting frame's data as bytes failed: %w", err))
+	}
+
+	/*
+
+		In this second part we're going to manipulate a video frame
+
+	*/
+
+	// Alloc frame
+	videoFrame := astiav.AllocFrame()
+	defer videoFrame.Free()
+
+	// To write data manually into a frame, proper attributes need to be set and allocated
+	videoFrame.SetHeight(256)
+	videoFrame.SetPixelFormat(astiav.PixelFormatRgba)
+	videoFrame.SetWidth(256)
+
+	// Alloc buffer
+	align = 1
+	if err := videoFrame.AllocBuffer(align); err != nil {
+		log.Fatal(fmt.Errorf("main: allocating buffer failed: %w", err))
+	}
+
+	// Alloc image
+	if err := videoFrame.AllocImage(align); err != nil {
+		log.Fatal(fmt.Errorf("main: allocating image failed: %w", err))
+	}
+
+	// When writing data manually into a frame, you need to make sure the frame is writable
+	if err := videoFrame.MakeWritable(); err != nil {
+		log.Fatal(fmt.Errorf("main: making frame writable failed: %w", err))
+	}
+
+	// Let's say b2 contains an actual video buffer, we can update the video frame's data based on the buffer
+	var b2 []byte
+	if err := videoFrame.Data().SetBytes(b2, align); err != nil {
+		log.Fatal(fmt.Errorf("main: setting frame's data based from bytes failed: %w", err))
+	}
+
+	// We can also retrieve the video frame's data as buffer
+	if _, err := videoFrame.Data().Bytes(align); err != nil {
+		log.Fatal(fmt.Errorf("main: getting frame's data as bytes failed: %w", err))
+	}
+
+	// Let's say i1 is an actual Go image.Image, we can update the video frame's data based on the image
+	var i1 image.Image
+	if err := videoFrame.Data().FromImage(i1); err != nil {
+		log.Fatal(fmt.Errorf("main: setting frame's data based on Go image failed: %w", err))
+	}
+
+	// We can also retrieve the video frame's data as a Go image
+	// For that we first need to guess the Go image format based on the frame's attributes before providing
+	// it to .ToImage(). You may not need this and can provide your own image.Image to .ToImage()
+	i2, err := videoFrame.Data().GuessImageFormat()
+	if err != nil {
+		log.Fatal(fmt.Errorf("main: guessing image format failed: %w", err))
+	}
+	if err := videoFrame.Data().ToImage(i2); err != nil {
+		log.Fatal(fmt.Errorf("main: getting frame's data as Go image failed: %w", err))
+	}
+
+	// Success
+	log.Println("success")
+}

format_context_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestFormatContext(t *testing.T) {
-	fc1, err := globalHelper.inputFormatContext("video.mp4")
+	fc1, err := globalHelper.inputFormatContext("video.mp4", nil)
 	require.NoError(t, err)
 	ss := fc1.Streams()
 	require.Len(t, ss, 2)

frame.c

@@ -0,0 +1,22 @@
+#include <errno.h>
+#include <libavutil/avutil.h>
+#include <libavutil/samplefmt.h>
+#include <stdint.h>
+#include <string.h>
+
+int astiavSamplesCopyToBuffer(uint8_t* dst, int dst_size, const uint8_t * const src_data[8], int nb_channels, int nb_samples, enum AVSampleFormat sample_fmt, int align) {
+    int linesize, buffer_size, nb_planes, i;
+    
+    buffer_size = av_samples_get_buffer_size(&linesize, nb_channels, nb_samples, sample_fmt, align);
+    if (buffer_size > dst_size || buffer_size < 0) return AVERROR(EINVAL);
+
+    nb_planes = buffer_size / linesize;
+
+    for (i = 0; i < nb_planes; i++) {
+        const uint8_t *src = src_data[i];
+        memcpy(dst, src, linesize);
+        dst += linesize;
+        src += linesize;
+    }
+    return buffer_size;
+}

frame.go

@@ -5,6 +5,7 @@ package astiav
 //#include <libavutil/imgutils.h>
 //#include <libavutil/samplefmt.h>
 //#include <libavutil/hwcontext.h>
+//#include "frame.h"
 import "C"
 import (
 	"unsafe"
@@ -113,6 +114,26 @@ func (f *Frame) ImageFillBlack() error {
 	return newError(C.av_image_fill_black(&f.c.data[0], &linesize[0], (C.enum_AVPixelFormat)(f.c.format), (C.enum_AVColorRange)(f.c.color_range), f.c.width, f.c.height))
 }
 
+func (f *Frame) SamplesBufferSize(align int) (int, error) {
+	ret := C.av_samples_get_buffer_size(nil, f.c.ch_layout.nb_channels, f.c.nb_samples, (C.enum_AVSampleFormat)(f.c.format), C.int(align))
+	if err := newError(ret); err != nil {
+		return 0, err
+	}
+	return int(ret), nil
+}
+
+func (f *Frame) SamplesCopyToBuffer(b []byte, align int) (int, error) {
+	ret := C.astiavSamplesCopyToBuffer((*C.uint8_t)(unsafe.Pointer(&b[0])), C.int(len(b)), &f.c.data[0], f.c.ch_layout.nb_channels, f.c.nb_samples, (C.enum_AVSampleFormat)(f.c.format), C.int(align))
+	if err := newError(ret); err != nil {
+		return 0, err
+	}
+	return int(ret), nil
+}
+
+func (f *Frame) SamplesFillSilence() error {
+	return newError(C.av_samples_set_silence(&f.c.data[0], 0, f.c.nb_samples, f.c.ch_layout.nb_channels, (C.enum_AVSampleFormat)(f.c.format)))
+}
+
 func (f *Frame) Linesize() [NumDataPointers]int {
 	o := [NumDataPointers]int{}
 	for i := 0; i < int(NumDataPointers); i++ {

frame.h

@@ -0,0 +1,4 @@
+#include <libavutil/samplefmt.h>
+#include <stdint.h>
+
+int astiavSamplesCopyToBuffer(uint8_t* dst, int dst_size, const uint8_t * const src_data[8], int nb_channels, int nb_samples, enum AVSampleFormat sample_fmt, int align);

frame_data.go

@@ -1,6 +1,7 @@
 package astiav
 
 //#include <libavutil/imgutils.h>
+//#include <libavutil/samplefmt.h>
 //#include <stdlib.h>
 //#include "macros.h"
 import "C"
@@ -254,31 +255,40 @@ func newFrameDataFrame(f *Frame) *frameDataFrame {
 }
 
 func (f *frameDataFrame) bytes(align int) ([]byte, error) {
-	switch {
-	// Video
-	case f.height() > 0 && f.width() > 0:
+	// Get funcs
+	var bufferSizeFunc func(int) (int, error)
+	var copyToBufferFunc func([]byte, int) (int, error)
+	switch f.mediaType() {
+	case MediaTypeAudio:
+		bufferSizeFunc = f.f.SamplesBufferSize
+		copyToBufferFunc = f.f.SamplesCopyToBuffer
+	case MediaTypeVideo:
+		bufferSizeFunc = f.f.ImageBufferSize
+		copyToBufferFunc = f.f.ImageCopyToBuffer
+	default:
+		return nil, errors.New("astiav: media type not implemented")
+	}
+
 	// Get buffer size
-		s, err := f.f.ImageBufferSize(align)
+	s, err := bufferSizeFunc(align)
 	if err != nil {
-			return nil, fmt.Errorf("astiav: getting image buffer size failed: %w", err)
+		return nil, fmt.Errorf("astiav: getting buffer size failed: %w", err)
 	}
 
 	// Invalid buffer size
 	if s == 0 {
-			return nil, errors.New("astiav: invalid image buffer size")
+		return nil, errors.New("astiav: invalid buffer size")
 	}
 
 	// Create buffer
 	b := make([]byte, s)
 
-		// Copy image to buffer
-		if _, err = f.f.ImageCopyToBuffer(b, align); err != nil {
-			return nil, fmt.Errorf("astiav: copying image to buffer failed: %w", err)
+	// Copy to buffer
+	if _, err = copyToBufferFunc(b, align); err != nil {
+		return nil, fmt.Errorf("astiav: copying to buffer failed: %w", err)
 	}
 	return b, nil
 }
-	return nil, errors.New("astiav: frame type not implemented")
-}
 
 func (f *frameDataFrame) copyPlanes(ps []frameDataPlane) error {
 	// Check writability
@@ -286,10 +296,7 @@ func (f *frameDataFrame) copyPlanes(ps []frameDataPlane) error {
 		return errors.New("astiav: frame is not writable")
 	}
 
-	switch {
-	// Video
-	case f.height() > 0 && f.width() > 0:
-		// Loop through planes
+	// Prepare data
 	var cdata [8]*C.uint8_t
 	var clinesizes [8]C.int
 	for i, p := range ps {
@@ -303,9 +310,14 @@ func (f *frameDataFrame) copyPlanes(ps []frameDataPlane) error {
 		clinesizes[i] = C.int(p.linesize)
 	}
 
-		// Copy image
+	// Copy data
+	switch f.mediaType() {
+	case MediaTypeAudio:
+		C.av_samples_copy(&f.f.c.data[0], &cdata[0], 0, 0, f.f.c.nb_samples, f.f.c.ch_layout.nb_channels, (C.enum_AVSampleFormat)(f.f.c.format))
+	case MediaTypeVideo:
 		C.av_image_copy(&f.f.c.data[0], &f.f.c.linesize[0], &cdata[0], &clinesizes[0], (C.enum_AVPixelFormat)(f.f.c.format), f.f.c.width, f.f.c.height)
-		return nil
+	default:
+		return errors.New("astiav: media type not implemented")
 	}
 	return nil
 }
@@ -314,40 +326,77 @@ func (f *frameDataFrame) height() int {
 	return f.f.Height()
 }
 
+func (f *frameDataFrame) mediaType() MediaType {
+	switch {
+	// Audio
+	case f.f.NbSamples() > 0:
+		return MediaTypeAudio
+	// Video
+	case f.f.Height() > 0 && f.f.Width() > 0:
+		return MediaTypeVideo
+	default:
+		return MediaTypeUnknown
+	}
+}
+
 func (f *frameDataFrame) pixelFormat() PixelFormat {
 	return f.f.PixelFormat()
 }
 
 func (f *frameDataFrame) planes(b []byte, align int) ([]frameDataPlane, error) {
-	switch {
-	// Video
-	case f.height() > 0 && f.width() > 0:
+	// Get line and plane sizes
+	var linesizes [8]int
+	var planeSizes [8]int
+	switch f.mediaType() {
+	case MediaTypeAudio:
+		// Get buffer size
+		var cLinesize C.int
+		cBufferSize := C.av_samples_get_buffer_size(&cLinesize, f.f.c.ch_layout.nb_channels, f.f.c.nb_samples, (C.enum_AVSampleFormat)(f.f.c.format), C.int(align))
+		if err := newError(cBufferSize); err != nil {
+			return nil, fmt.Errorf("astiav: getting buffer size failed: %w", err)
+		}
+
+		// Update line and plane sizes
+		for i := 0; i < int(cBufferSize/cLinesize); i++ {
+			linesizes[i] = int(cLinesize)
+			planeSizes[i] = int(cLinesize)
+		}
+	case MediaTypeVideo:
 		// Below is mostly inspired by https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/libavutil/imgutils.c#L466
 
 		// Get linesize
-		var linesizes [4]C.int
-		if err := newError(C.av_image_fill_linesizes(&linesizes[0], (C.enum_AVPixelFormat)(f.f.c.format), f.f.c.width)); err != nil {
+		var cLinesizes [8]C.int
+		if err := newError(C.av_image_fill_linesizes(&cLinesizes[0], (C.enum_AVPixelFormat)(f.f.c.format), f.f.c.width)); err != nil {
 			return nil, fmt.Errorf("astiav: getting linesize failed: %w", err)
 		}
 
 		// Align linesize
-		var alignedLinesizes [4]C.ptrdiff_t
+		var cAlignedLinesizes [8]C.ptrdiff_t
 		for i := 0; i < 4; i++ {
-			alignedLinesizes[i] = C.astiavFFAlign(linesizes[i], C.int(align))
+			cAlignedLinesizes[i] = C.astiavFFAlign(cLinesizes[i], C.int(align))
 		}
 
 		// Get plane sizes
-		var planeSizes [4]C.size_t
-		if err := newError(C.av_image_fill_plane_sizes(&planeSizes[0], (C.enum_AVPixelFormat)(f.f.c.format), f.f.c.height, &alignedLinesizes[0])); err != nil {
+		var cPlaneSizes [8]C.size_t
+		if err := newError(C.av_image_fill_plane_sizes(&cPlaneSizes[0], (C.enum_AVPixelFormat)(f.f.c.format), f.f.c.height, &cAlignedLinesizes[0])); err != nil {
 			return nil, fmt.Errorf("astiav: getting plane sizes failed: %w", err)
 		}
 
-		// Loop through planes
+		// Update line and plane sizes
+		for i := range cPlaneSizes {
+			linesizes[i] = int(cAlignedLinesizes[i])
+			planeSizes[i] = int(cPlaneSizes[i])
+		}
+	default:
+		return nil, errors.New("astiav: media type not implemented")
+	}
+
+	// Loop through plane sizes
 	var ps []frameDataPlane
 	start := 0
 	for i := range planeSizes {
 		// Get end
-			end := start + int(planeSizes[i])
+		end := start + planeSizes[i]
 		if len(b) < end {
 			return nil, fmt.Errorf("astiav: buffer length %d is invalid for [%d:%d]", len(b), start, end)
 		}
@@ -355,16 +404,13 @@ func (f *frameDataFrame) planes(b []byte, align int) ([]frameDataPlane, error) {
 		// Append plane
 		ps = append(ps, frameDataPlane{
 			bytes:    b[start:end],
-				linesize: int(linesizes[i]),
+			linesize: linesizes[i],
 		})
 
 		// Update start
 		start = end
 	}
 	return ps, nil
-	default:
-		return nil, errors.New("astiav: frame type not implemented")
-	}
 }
 
 func (f *frameDataFrame) width() int {

frame_data_test.go

@@ -119,14 +119,14 @@ func TestFrameDataInternal(t *testing.T) {
 		}
 	}
 
-	fdf.h = 1
 	b1 := []byte{0, 1, 2, 3}
 	fdf.onBytes = func(align int) ([]byte, error) { return b1, nil }
-	fdf.w = 2
 	b2, err := fd.Bytes(0)
 	require.NoError(t, err)
 	require.Equal(t, b1, b2)
 
+	fdf.h = 1
+	fdf.w = 2
 	for _, v := range []struct {
 		e           image.Image
 		i           image.Image
@@ -465,18 +465,28 @@ func TestFrameDataInternal(t *testing.T) {
 func TestFrameData(t *testing.T) {
 	for _, v := range []struct {
 		ext  string
+		ifmt *InputFormat
+		md   MediaType
 		name string
 	}{
+		{
+			ext:  "pcm",
+			ifmt: FindInputFormat("s16le"),
+			md:   MediaTypeAudio,
+			name: "audio-s16le",
+		},
 		{
 			ext:  "png",
+			md:   MediaTypeVideo,
 			name: "image-rgba",
 		},
 		{
 			ext:  "h264",
+			md:   MediaTypeVideo,
 			name: "video-yuv420p",
 		},
 	} {
-		f1, err := globalHelper.inputLastFrame(v.name+"."+v.ext, MediaTypeVideo)
+		f1, err := globalHelper.inputLastFrame(v.name+"."+v.ext, v.md, v.ifmt)
 		require.NoError(t, err)
 		fd1 := f1.Data()
 
@@ -487,31 +497,55 @@ func TestFrameData(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, b3, b2)
 
-		i1, err := fd1.GuessImageFormat()
+		var i1 image.Image
+		switch v.md {
+		case MediaTypeVideo:
+			i1, err = fd1.GuessImageFormat()
 			require.NoError(t, err)
 			require.NoError(t, fd1.ToImage(i1))
 			b4 := []byte(fmt.Sprintf("%+v", i1))
 			b5, err := os.ReadFile("testdata/" + v.name + "-struct")
 			require.NoError(t, err)
 			require.Equal(t, b5, b4)
+		}
 
 		f2 := AllocFrame()
 		defer f2.Free()
+		fd2 := f2.Data()
+
+		align := 0
+		switch v.md {
+		case MediaTypeAudio:
+			f2.SetChannelLayout(f1.ChannelLayout())
+			f2.SetNbSamples(f1.NbSamples())
+			f2.SetSampleFormat(f1.SampleFormat())
+			f2.SetSampleRate(f1.SampleRate())
+			require.NoError(t, f2.AllocBuffer(align))
+			require.NoError(t, f2.AllocSamples(align))
+		case MediaTypeVideo:
+			align = 1
 			f2.SetHeight(f1.Height())
 			f2.SetPixelFormat(f1.PixelFormat())
 			f2.SetWidth(f1.Width())
-		const align = 1
 			require.NoError(t, f2.AllocBuffer(align))
 			require.NoError(t, f2.AllocImage(align))
-		fd2 := f2.Data()
+		}
 
+		switch v.md {
+		case MediaTypeVideo:
 			require.NoError(t, fd2.FromImage(i1))
 			b6, err := fd2.Bytes(align)
 			require.NoError(t, err)
 			b7 := []byte(fmt.Sprintf("%+v", b6))
 			require.Equal(t, b3, b7)
+		}
 
+		switch v.md {
+		case MediaTypeAudio:
+			require.NoError(t, f2.SamplesFillSilence())
+		case MediaTypeVideo:
 			require.NoError(t, f2.ImageFillBlack())
+		}
 		require.NoError(t, fd2.SetBytes(b1, align))
 		b1[0] -= 1
 		b8, err := fd2.Bytes(align)
@@ -522,10 +556,16 @@ func TestFrameData(t *testing.T) {
 		f3 := AllocFrame()
 		defer f3.Free()
 		require.NoError(t, f3.Ref(f2))
+		switch v.md {
+		case MediaTypeVideo:
 			require.Error(t, fd2.FromImage(i1))
+		}
 		require.Error(t, fd2.SetBytes(b1, align))
 		f2.MakeWritable()
+		switch v.md {
+		case MediaTypeVideo:
 			require.NoError(t, fd2.FromImage(i1))
+		}
 		require.NoError(t, fd2.SetBytes(b1, align))
 	}
 }

software_scale_context_test.go

@@ -93,7 +93,7 @@ func TestSoftwareScaleContext(t *testing.T) {
 	require.Equal(t, w, srcW)
 	require.Equal(t, h, srcH)
 
-	f4, err := globalHelper.inputLastFrame("image-rgba.png", MediaTypeVideo)
+	f4, err := globalHelper.inputLastFrame("image-rgba.png", MediaTypeVideo, nil)
 	require.NoError(t, err)
 
 	f5 := AllocFrame()

stream_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestStream(t *testing.T) {
-	fc, err := globalHelper.inputFormatContext("video.mp4")
+	fc, err := globalHelper.inputFormatContext("video.mp4", nil)
 	require.NoError(t, err)
 	ss := fc.Streams()
 	require.Len(t, ss, 2)