Implementation for HW Context (#32)

* Adding HW Context

* Check for hw pixel format and transfer data on ReciveFrame when its hw pixel format

* Remove hw_decoding example and update demuxing_dec example

* unref hw_device_ctx, and improve example

Unref correctly hw_device_ctx on free
decoding example has now a flag for hw decoding
Use error on hw context because its good to know the error that happen
Add a method to list supported hw codecs so user could check if system has a specif hwcodec before the create a hw context

* Fix test to use err, and update hw with device to also return error

* revert changes in demux example

* Add/Modify according to review

Seperate Hardware Device Context and Test
Renaming HW to Hardware
Create dedicated hardware_decoding Example
Update Readme
Add TransferHardwareData to frame.go
Add SetHardwareDeviceContext to codec_context.go

* Review changes

add HardwareConfigs to codec
create codec_hardware_config
create codec_hardware_config_method_flag
update example
update flags
better error handling in TransferHardwareData
update CreateHardwareDeviceContext, better null handling
remove hw ctx test because it fails when no nvidia is present
some reordering of conss in hardware_device_type

* Minor tweaks (review changes)

HardwareConfigs only returns a slice an no error anymore
remove if HardwareDeviceType checking in codec.go this should be done by user
order codec hw_config_method_flags constants alphabetically
some small changes in example
##  hardware_device_context.go
change link
update optionsC

README.md

@@ -25,6 +25,7 @@ Examples are located in the [examples](examples) directory and mirror as much as
 |---|---|---|
 |Demuxing/Decoding|[see](examples/demuxing_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/demuxing_decoding.c)
 |Filtering|[see](examples/filtering/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/filtering_video.c)
+|Hardware Decoding|[see](examples/hardware_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/hw_decode.c)
 |Remuxing|[see](examples/remuxing/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/remuxing.c)
 |Transcoding|[see](examples/transcoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/transcoding.c)
 

codec.go

@@ -100,3 +100,20 @@ func FindEncoderByName(n string) *Codec {
 	defer C.free(unsafe.Pointer(cn))
 	return newCodecFromC(C.avcodec_find_encoder_by_name(cn))
 }
+
+func (c *Codec) HardwareConfigs(dt HardwareDeviceType) []CodecHardwareConfig {
+	var configs []CodecHardwareConfig
+	var i int
+
+	for {
+		config := C.avcodec_get_hw_config(c.c, C.int(i))
+		if config == nil {
+			break
+		}
+
+		configs = append(configs, CodecHardwareConfig{c: config})
+
+		i++
+	}
+	return configs
+}

codec_context.go

@@ -261,3 +261,7 @@ func (cc *CodecContext) SendFrame(f *Frame) error {
 	}
 	return newError(C.avcodec_send_frame(cc.c, fc))
 }
+
+func (cc *CodecContext) SetHardwareDeviceContext(hdc *HardwareDeviceContext) {
+	cc.c.hw_device_ctx = hdc.c
+}

codec_hardware_config.go

@@ -0,0 +1,22 @@
+package astiav
+
+//#cgo pkg-config: libavcodec
+//#include <libavcodec/avcodec.h>
+import "C"
+
+// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavcodec/codec.h#L460
+type CodecHardwareConfig struct {
+	c *C.AVCodecHWConfig
+}
+
+func (chc CodecHardwareConfig) HardwareDeviceType() HardwareDeviceType {
+	return HardwareDeviceType(chc.c.device_type)
+}
+
+func (chc CodecHardwareConfig) MethodFlags() CodecHardwareConfigMethodFlags {
+	return CodecHardwareConfigMethodFlags(chc.c.methods)
+}
+
+func (chc CodecHardwareConfig) PixelFormat() PixelFormat {
+	return PixelFormat(chc.c.pix_fmt)
+}

codec_hardware_config_method_flag.go

@@ -0,0 +1,15 @@
+package astiav
+
+//#cgo pkg-config: libavcodec
+//#include <libavcodec/avcodec.h>
+import "C"
+
+// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavcodec/codec.h#L420
+type CodecHardwareConfigMethodFlag int
+
+const (
+	CodecHardwareConfigMethodAdHoc       = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_AD_HOC)
+	CodecHardwareConfigMethodHwDeviceCtx = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)
+	CodecHardwareConfigMethodHwFramesCtx = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX)
+	CodecHardwareConfigMethodInternal    = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_INTERNAL)
+)

examples/hardware_decoding/main.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/asticode/go-astiav"
+)
+
+var (
+	hardwareDeviceTypeName = flag.String("d", "", "the hardware device type like: cuda")
+	input                  = flag.String("i", "", "the input path")
+)
+
+type stream struct {
+	decCodec              *astiav.Codec
+	decCodecContext       *astiav.CodecContext
+	hardwareDeviceContext *astiav.HardwareDeviceContext
+	hardwarePixelFormat   astiav.PixelFormat
+	inputStream           *astiav.Stream
+}
+
+func main() {
+	// Handle ffmpeg logs
+	astiav.SetLogLevel(astiav.LogLevelDebug)
+	astiav.SetLogCallback(func(l astiav.LogLevel, fmt, msg, parent string) {
+		log.Printf("ffmpeg log: %s (level: %d)\n", strings.TrimSpace(msg), l)
+	})
+
+	// Parse flags
+	flag.Parse()
+
+	// Usage
+	if *input == "" || *hardwareDeviceTypeName == "" {
+		log.Println("Usage: <binary path> -d <device type> -i <input path>")
+		return
+	}
+
+	// Get hardware device type
+	hardwareDeviceType := astiav.FindHardwareDeviceTypeByName(*hardwareDeviceTypeName)
+	if hardwareDeviceType == astiav.HardwareDeviceTypeNone {
+		log.Fatal(errors.New("main: hardware device not found"))
+	}
+
+	// Alloc packet
+	pkt := astiav.AllocPacket()
+	defer pkt.Free()
+
+	// Alloc hardware frame
+	hardwareFrame := astiav.AllocFrame()
+	defer hardwareFrame.Free()
+
+	// Alloc software frame
+	softwareFrame := astiav.AllocFrame()
+	defer softwareFrame.Free()
+
+	// Alloc input format context
+	inputFormatContext := astiav.AllocFormatContext()
+	if inputFormatContext == nil {
+		log.Fatal(errors.New("main: input format context is nil"))
+	}
+	defer inputFormatContext.Free()
+
+	// Open input
+	if err := inputFormatContext.OpenInput(*input, nil, nil); err != nil {
+		log.Fatal(fmt.Errorf("main: opening input failed: %w", err))
+	}
+	defer inputFormatContext.CloseInput()
+
+	// Find stream info
+	if err := inputFormatContext.FindStreamInfo(nil); err != nil {
+		log.Fatal(fmt.Errorf("main: finding stream info failed: %w", err))
+	}
+
+	// Loop through streams
+	streams := make(map[int]*stream) // Indexed by input stream index
+	for _, is := range inputFormatContext.Streams() {
+		var err error
+
+		// Only process video
+		if is.CodecParameters().MediaType() != astiav.MediaTypeVideo {
+			continue
+		}
+
+		// Create stream
+		s := &stream{inputStream: is}
+
+		// Find decoder
+		if s.decCodec = astiav.FindDecoder(is.CodecParameters().CodecID()); s.decCodec == nil {
+			log.Fatal(errors.New("main: codec is nil"))
+		}
+
+		// Alloc codec context
+		if s.decCodecContext = astiav.AllocCodecContext(s.decCodec); s.decCodecContext == nil {
+			log.Fatal(errors.New("main: codec context is nil"))
+		}
+		defer s.decCodecContext.Free()
+
+		// Get codec hardware configs
+		hardwareConfigs := s.decCodec.HardwareConfigs(hardwareDeviceType)
+
+		// Loop through codec hardware configs
+		for _, p := range hardwareConfigs {
+			// Valid hardware config
+			if p.MethodFlags().Has(astiav.CodecHardwareConfigMethodHwDeviceCtx) && p.HardwareDeviceType() == hardwareDeviceType {
+				s.hardwarePixelFormat = p.PixelFormat()
+				break
+			}
+		}
+
+		// No valid hardware pixel format
+		if s.hardwarePixelFormat == astiav.PixelFormatNone {
+			log.Fatal(errors.New("main: hardware device type not supported by decoder"))
+		}
+
+		// Update codec context
+		if err := is.CodecParameters().ToCodecContext(s.decCodecContext); err != nil {
+			log.Fatal(fmt.Errorf("main: updating codec context failed: %w", err))
+		}
+
+		// Create hardware device context
+		s.hardwareDeviceContext, err = astiav.CreateHardwareDeviceContext(hardwareDeviceType, "", nil)
+		if err != nil {
+			log.Fatal(fmt.Errorf("main: creating hardware device context failed: %w", err))
+		}
+		s.decCodecContext.SetHardwareDeviceContext(s.hardwareDeviceContext)
+
+		// Open codec context
+		if err := s.decCodecContext.Open(s.decCodec, nil); err != nil {
+			log.Fatal(fmt.Errorf("main: opening codec context failed: %w", err))
+		}
+
+		// Add stream
+		streams[is.Index()] = s
+	}
+
+	// Loop through packets
+	for {
+		// Read frame
+		if err := inputFormatContext.ReadFrame(pkt); err != nil {
+			if errors.Is(err, astiav.ErrEof) {
+				break
+			}
+			log.Fatal(fmt.Errorf("main: reading frame failed: %w", err))
+		}
+
+		// Get stream
+		s, ok := streams[pkt.StreamIndex()]
+		if !ok {
+			continue
+		}
+
+		// Send packet
+		if err := s.decCodecContext.SendPacket(pkt); err != nil {
+			log.Fatal(fmt.Errorf("main: sending packet failed: %w", err))
+		}
+
+		// Loop
+		for {
+			// Receive frame
+			if err := s.decCodecContext.ReceiveFrame(hardwareFrame); err != nil {
+				if errors.Is(err, astiav.ErrEof) || errors.Is(err, astiav.ErrEagain) {
+					break
+				}
+				log.Fatal(fmt.Errorf("main: receiving frame failed: %w", err))
+			}
+
+			// Get final frame
+			var finalFrame *astiav.Frame
+			if hardwareFrame.PixelFormat() == s.hardwarePixelFormat {
+				// Transfer hardware data
+				if err := hardwareFrame.TransferHardwareData(softwareFrame); err != nil {
+					log.Fatal(fmt.Errorf("main: transferring hardware data failed: %w", err))
+				}
+
+				// Update pts
+				softwareFrame.SetPts(hardwareFrame.Pts())
+
+				// Update final frame
+				finalFrame = softwareFrame
+			} else {
+				// Update final frame
+				finalFrame = hardwareFrame
+			}
+
+			// Do something with decoded frame
+			log.Printf("new frame: stream %d - pts: %d", pkt.StreamIndex(), finalFrame.Pts())
+		}
+	}
+}

flags.go

@@ -85,6 +85,26 @@ func (fs CodecContextFlags2) Del(f CodecContextFlag2) CodecContextFlags2 {
 
 func (fs CodecContextFlags2) Has(f CodecContextFlag2) bool { return astikit.BitFlags(fs).Has(uint64(f)) }
 
+type CodecHardwareConfigMethodFlags astikit.BitFlags
+
+func NewCodecHardwareConfigMethodFlags(fs ...CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags {
+	o := CodecHardwareConfigMethodFlags(0)
+	for _, f := range fs {
+		o = o.Add(f)
+	}
+	return o
+}
+
+func (fs CodecHardwareConfigMethodFlags) Add(f CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags {
+	return CodecHardwareConfigMethodFlags(astikit.BitFlags(fs).Add(uint64(f)))
+}
+
+func (fs CodecHardwareConfigMethodFlags) Del(f CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags {
+	return CodecHardwareConfigMethodFlags(astikit.BitFlags(fs).Del(uint64(f)))
+}
+
+func (fs CodecHardwareConfigMethodFlags) Has(f CodecHardwareConfigMethodFlag) bool { return astikit.BitFlags(fs).Has(uint64(f)) }
+
 type DictionaryFlags astikit.BitFlags
 
 func NewDictionaryFlags(fs ...DictionaryFlag) DictionaryFlags {

flags_test.go

@@ -43,6 +43,15 @@ func TestCodecContextFlags2(t *testing.T) {
 	require.False(t, fs.Has(astiav.CodecContextFlag2(2)))
 }
 
+func TestCodecHardwareConfigMethodFlags(t *testing.T) {
+	fs := astiav.NewCodecHardwareConfigMethodFlags(astiav.CodecHardwareConfigMethodFlag(1))
+	require.True(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(1)))
+	fs = fs.Add(astiav.CodecHardwareConfigMethodFlag(2))
+	require.True(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(2)))
+	fs = fs.Del(astiav.CodecHardwareConfigMethodFlag(2))
+	require.False(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(2)))
+}
+
 func TestDictionaryFlags(t *testing.T) {
 	fs := astiav.NewDictionaryFlags(astiav.DictionaryFlag(1))
 	require.True(t, fs.Has(astiav.DictionaryFlag(1)))

frame.go

@@ -5,8 +5,11 @@ package astiav
 //#include <libavutil/frame.h>
 //#include <libavutil/imgutils.h>
 //#include <libavutil/samplefmt.h>
+//#include <libavutil/hwcontext.h>
 import "C"
-import "unsafe"
+import (
+	"unsafe"
+)
 
 const NumDataPointers = uint(C.AV_NUM_DATA_POINTERS)
 
@@ -187,6 +190,10 @@ func (f *Frame) SetWidth(w int) {
 	f.c.width = C.int(w)
 }
 
+func (f *Frame) TransferHardwareData(dst *Frame) error {
+	return newError(C.av_hwframe_transfer_data(dst.c, f.c, 0))
+}
+
 func (f *Frame) Free() {
 	C.av_frame_free(&f.c)
 }

hardware_device_context.go

@@ -0,0 +1,37 @@
+package astiav
+
+//#cgo pkg-config: libavutil libavcodec
+//#include <libavcodec/avcodec.h>
+//#include <libavutil/hwcontext.h>
+import "C"
+import (
+	"unsafe"
+)
+
+// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavutil/hwcontext.h#L61
+type HardwareDeviceContext struct {
+	c *C.AVBufferRef
+}
+
+func CreateHardwareDeviceContext(t HardwareDeviceType, device string, options *Dictionary) (*HardwareDeviceContext, error) {
+	hdc := HardwareDeviceContext{}
+	deviceC := (*C.char)(nil)
+	if device != "" {
+		deviceC = C.CString(device)
+		defer C.free(unsafe.Pointer(deviceC))
+	}
+	optionsC := (*C.struct_AVDictionary)(nil)
+	if options != nil {
+		optionsC = options.c
+	}
+	if err := newError(C.av_hwdevice_ctx_create(&hdc.c, (C.enum_AVHWDeviceType)(t), deviceC, optionsC, 0)); err != nil {
+		return nil, err
+	}
+	return &hdc, nil
+}
+
+func (hdc *HardwareDeviceContext) Free() {
+	if hdc.c != nil {
+		C.av_buffer_unref(&hdc.c)
+	}
+}

hardware_device_type.go

@@ -0,0 +1,36 @@
+package astiav
+
+//#cgo pkg-config: libavutil
+//#include <libavutil/hwcontext.h>
+import "C"
+import (
+	"unsafe"
+)
+
+// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavutil/hwcontext.h#L27
+type HardwareDeviceType C.enum_AVHWDeviceType
+
+const (
+	HardwareDeviceTypeCUDA         = HardwareDeviceType(C.AV_HWDEVICE_TYPE_CUDA)
+	HardwareDeviceTypeD3D11VA      = HardwareDeviceType(C.AV_HWDEVICE_TYPE_D3D11VA)
+	HardwareDeviceTypeDRM          = HardwareDeviceType(C.AV_HWDEVICE_TYPE_DRM)
+	HardwareDeviceTypeDXVA2        = HardwareDeviceType(C.AV_HWDEVICE_TYPE_DXVA2)
+	HardwareDeviceTypeMediaCodec   = HardwareDeviceType(C.AV_HWDEVICE_TYPE_MEDIACODEC)
+	HardwareDeviceTypeNone         = HardwareDeviceType(C.AV_HWDEVICE_TYPE_NONE)
+	HardwareDeviceTypeOpenCL       = HardwareDeviceType(C.AV_HWDEVICE_TYPE_OPENCL)
+	HardwareDeviceTypeQSV          = HardwareDeviceType(C.AV_HWDEVICE_TYPE_QSV)
+	HardwareDeviceTypeVAAPI        = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VAAPI)
+	HardwareDeviceTypeVDPAU        = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VDPAU)
+	HardwareDeviceTypeVideoToolbox = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VIDEOTOOLBOX)
+	HardwareDeviceTypeVulkan       = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VULKAN)
+)
+
+func (t HardwareDeviceType) String() string {
+	return C.GoString(C.av_hwdevice_get_type_name((C.enum_AVHWDeviceType)(t)))
+}
+
+func FindHardwareDeviceTypeByName(n string) HardwareDeviceType {
+	cn := C.CString(n)
+	defer C.free(unsafe.Pointer(cn))
+	return HardwareDeviceType(C.av_hwdevice_find_type_by_name(cn))
+}

hardware_device_type_test.go

@@ -0,0 +1,13 @@
+package astiav_test
+
+import (
+	"testing"
+
+	"github.com/asticode/go-astiav"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHardwareDeviceType(t *testing.T) {
+	require.Equal(t, "cuda", astiav.HardwareDeviceTypeCUDA.String())
+	require.Equal(t, astiav.FindHardwareDeviceTypeByName("cuda"), astiav.HardwareDeviceTypeCUDA)
+}

internal/cmd/flags/main.go

@@ -18,6 +18,7 @@ var list = []listItem{
 	{Name: "Buffersrc"},
 	{Name: "CodecContext"},
 	{Name: "CodecContext", Suffix: "2"},
+	{Name: "CodecHardwareConfigMethod"},
 	{Name: "Dictionary"},
 	{Name: "FilterCommand"},
 	{Name: "FormatContextCtx"},