Add hardware acceleration support to FFmpeg

cmd/ffmpeg/ffmpeg.go

@@ -1,6 +1,8 @@
 package ffmpeg
 
 import (
+	"bytes"
+	"errors"
 	"github.com/AlexxIT/go2rtc/cmd/app"
 	"github.com/AlexxIT/go2rtc/cmd/exec"
 	"github.com/AlexxIT/go2rtc/cmd/ffmpeg/device"
@@ -17,10 +19,29 @@ func Init() {
 		Mod map[string]string `yaml:"ffmpeg"`
 	}
 
-	// defaults
+	cfg.Mod = defaults // will be overriden from yaml
 
-	cfg.Mod = map[string]string{
+	app.LoadConfig(&cfg)
+
+	if app.GetLogger("exec").GetLevel() >= 0 {
+		defaults["global"] += " -v error"
+	}
+
+	streams.HandleFunc("ffmpeg", func(url string) (streamer.Producer, error) {
+		args := parseArgs(url[7:]) // remove `ffmpeg:`
+		if args == nil {
+			return nil, errors.New("can't generate ffmpeg command")
+		}
+		return exec.Handle("exec:" + args.String())
+	})
+
+	device.Bin = defaults["bin"]
+	device.Init()
+}
+
+var defaults = map[string]string{
 	"bin":    "ffmpeg",
+	"global": "-hide_banner",
 
 	// inputs
 	"file": "-re -stream_loop -1 -i {input}",
@@ -30,16 +51,13 @@ func Init() {
 	// output
 	"output": "-user_agent ffmpeg/go2rtc -rtsp_transport tcp -f rtsp {output}",
 
-		// `-g 30` - group of picture, GOP, keyframe interval
 	// `-preset superfast` - we can't use ultrafast because it doesn't support `-profile main -level 4.1`
 	// `-tune zerolatency` - for minimal latency
-		// `-profile main -level 4.1` - most used streaming profile
-		// `-pix_fmt yuv420p` - if input pix format 4:2:2
-		"h264":       "-c:v libx264 -g:v 30 -preset:v superfast -tune:v zerolatency -profile:v main -level:v 4.1 -pix_fmt:v yuv420p",
-		"h264/ultra": "-c:v libx264 -g:v 30 -preset:v ultrafast -tune:v zerolatency",
-		"h264/high":  "-c:v libx264 -g:v 30 -preset:v superfast -tune:v zerolatency",
-		"h265":       "-c:v libx265 -g:v 30 -preset:v superfast -tune:v zerolatency -profile:v main -level:v 5.1 -pix_fmt:v yuv420p",
+	// `-profile high -level 4.1` - most used streaming profile
+	"h264":  "-c:v libx264 -g 50 -profile:v high -level:v 4.1 -preset:v superfast -tune:v zerolatency",
+	"h265":  "-c:v libx265 -g 50 -profile:v high -level:v 5.1 -preset:v superfast -tune:v zerolatency",
 	"mjpeg": "-c:v mjpeg -force_duplicated_matrix:v 1 -huffman:v 0 -pix_fmt:v yuvj420p",
+
 	"opus":       "-c:a libopus -ar:a 48000 -ac:a 2",
 	"pcmu":       "-c:a pcm_mulaw -ar:a 8000 -ac:a 1",
 	"pcmu/16000": "-c:a pcm_mulaw -ar:a 16000 -ac:a 1",
@@ -49,90 +67,107 @@ func Init() {
 	"pcma/48000": "-c:a pcm_alaw -ar:a 48000 -ac:a 1",
 	"aac":        "-c:a aac", // keep sample rate and channels
 	"aac/16000":  "-c:a aac -ar:a 16000 -ac:a 1",
+
+	// hardware Intel and AMD on Linux
+	"h264/vaapi":  "-c:v h264_vaapi -profile:v high -level:v 4.1 -async_depth:v 1 -sei:v 0",
+	"h265/vaapi":  "-c:v hevc_vaapi -profile:v high -level:v 5.1 -async_depth:v 1 -sei:v 0",
+	"mjpeg/vaapi": "-c:v mjpeg_vaapi",
+
+	// hardware Raspberry
+	"h264/v4l2m2m": "-c:v h264_v4l2m2m",
+	"h265/v4l2m2m": "-c:v hevc_v4l2m2m",
+
+	// hardware NVidia on Linux and Windows
+	// preset=p2 - faster, tune=ll - low latency
+	"h264/cuda": "-c:v h264_nvenc -g 50 -profile:v high -level:v auto -preset:v p2 -tune:v ll",
+	"h265/cuda": "-c:v hevc_nvenc -g 50 -profile:v high -level:v auto",
+
+	// hardware Intel on Windows
+	"h264/dxva2":  "-c:v h264_qsv -profile:v high -level:v 4.1 -async_depth:v 1",
+	"h265/dxva2":  "-c:v hevc_qsv -profile:v high -level:v 5.1 -async_depth:v 1",
+	"mjpeg/dxva2": "-c:v mjpeg_qsv -profile:v high -level:v 5.1",
+
+	// hardware macOS
+	"h264/videotoolbox": "-c:v h264_videotoolbox -profile:v high -level:v 4.1",
+	"h265/videotoolbox": "-c:v hevc_videotoolbox -profile:v high -level:v 5.1",
+}
+
+func parseArgs(s string) *Args {
+	// init FFmpeg arguments
+	args := &Args{
+		bin:    defaults["bin"],
+		global: defaults["global"],
+		output: defaults["output"],
 	}
 
-	app.LoadConfig(&cfg)
-
-	tpl := cfg.Mod
-
-	cmd := "exec:" + tpl["bin"] + " -hide_banner "
-
-	if app.GetLogger("exec").GetLevel() >= 0 {
-		cmd += "-v error "
-	}
-
-	streams.HandleFunc("ffmpeg", func(s string) (streamer.Producer, error) {
-		s = s[7:] // remove `ffmpeg:`
-
 	var query url.Values
-		var queryVideo, queryAudio bool
-
 	if i := strings.IndexByte(s, '#'); i > 0 {
 		query = parseQuery(s[i+1:])
-			queryVideo = query["video"] != nil
-			queryAudio = query["audio"] != nil
+		args.video = len(query["video"])
+		args.audio = len(query["audio"])
 		s = s[:i]
-		} else {
-			// by default query both video and audio
-			queryVideo = true
-			queryAudio = true
 	}
 
-		var input string
+	// Parse input:
+	//   1. Input as xxxx:// link (http or rtsp or any other)
+	//   2. Input as stream name
+	//   3. Input as FFmpeg device (local USB camera)
 	if i := strings.Index(s, "://"); i > 0 {
 		switch s[:i] {
 		case "http", "https", "rtmp":
-				input = strings.Replace(tpl["http"], "{input}", s, 1)
+			args.input = strings.Replace(defaults["http"], "{input}", s, 1)
 		case "rtsp", "rtsps":
 			// https://ffmpeg.org/ffmpeg-protocols.html#rtsp
 			// skip unnecessary input tracks
 			switch {
-				case queryVideo && queryAudio:
-					input = "-allowed_media_types video+audio "
-				case queryVideo:
-					input = "-allowed_media_types video "
-				case queryAudio:
-					input = "-allowed_media_types audio "
+			case (args.video > 0 && args.audio > 0) || (args.video == 0 && args.audio == 0):
+				args.input = "-allowed_media_types video+audio "
+			case args.video > 0:
+				args.input = "-allowed_media_types video "
+			case args.audio > 0:
+				args.input = "-allowed_media_types audio "
 			}
 
-				input += strings.Replace(tpl["rtsp"], "{input}", s, 1)
+			args.input += strings.Replace(defaults["rtsp"], "{input}", s, 1)
 		default:
-				input = "-i " + s
+			args.input = "-i " + s
 		}
 	} else if streams.Get(s) != nil {
 		s = "rtsp://localhost:" + rtsp.Port + "/" + s
 		switch {
-			case queryVideo && !queryAudio:
+		case args.video > 0 && args.audio == 0:
 			s += "?video"
-			case queryAudio && !queryVideo:
+		case args.audio > 0 && args.video == 0:
 			s += "?audio"
 		}
-			input = strings.Replace(tpl["rtsp"], "{input}", s, 1)
+		args.input = strings.Replace(defaults["rtsp"], "{input}", s, 1)
 	} else if strings.HasPrefix(s, "device?") {
 		var err error
-			input, err = device.GetInput(s)
+		args.input, err = device.GetInput(s)
 		if err != nil {
-				return nil, err
+			return nil
 		}
 	} else {
-			input = strings.Replace(tpl["file"], "{input}", s, 1)
+		args.input = strings.Replace(defaults["file"], "{input}", s, 1)
 	}
 
-		if _, ok := query["async"]; ok {
-			input = "-use_wallclock_as_timestamps 1 -async 1 " + input
+	if query["async"] != nil {
+		args.input = "-use_wallclock_as_timestamps 1 -async 1 " + args.input
 	}
 
-		s = cmd + input
-
+	// Parse query params:
+	//   1. `width`/`height` params
+	//   2. `rotate` param
+	//   3. `video` params (support multiple)
+	//   4. `audio` params (support multiple)
+	//   5. `hardware` param
 	if query != nil {
 		// 1. Process raw params for FFmpeg
 		for _, raw := range query["raw"] {
-				s += " " + raw
+			args.AddCodec(raw)
 		}
 
 		// 2. Process video filters (resize and rotation)
-			var filters []string
-
 		if query["width"] != nil || query["height"] != nil {
 			filter := "scale="
 			if query["width"] != nil {
@@ -146,84 +181,60 @@ func Init() {
 			} else {
 				filter += "-1"
 			}
-				filters = append(filters, filter)
+			args.AddFilter(filter)
 		}
 
 		if query["rotate"] != nil {
+			var filter string
 			switch query["rotate"][0] {
 			case "90":
-					filters = append(filters, "transpose=1") // 90 degrees clockwise
+				filter = "transpose=1" // 90 degrees clockwise
 			case "180":
-					filters = append(filters, "transpose=1,transpose=1")
+				filter = "transpose=1,transpose=1"
 			case "-90", "270":
-					filters = append(filters, "transpose=2") // 90 degrees counterclockwise
+				filter = "transpose=2" // 90 degrees counterclockwise
 			}
+			if filter != "" {
+				args.AddFilter(filter)
 			}
-
-			if filters != nil {
-				s += " -vf " + strings.Join(filters, ",")
 		}
 
 		// 3. Process video codecs
-			switch len(query["video"]) {
-			case 0:
-				s += " -vn"
-			case 1:
-				if len(query["audio"]) > 1 {
-					s += " -map 0:v:0?"
-				}
+		if args.video > 0 {
 			for _, video := range query["video"] {
-					if video == "copy" {
-						s += " -c:v copy"
+				if video != "copy" {
+					args.AddCodec(defaults[video])
 				} else {
-						s += " " + tpl[video]
+					args.AddCodec("-c:v copy")
 				}
 			}
-			default:
-				for i, video := range query["video"] {
-					if video == "copy" {
-						s += " -map 0:v:0? -c:v:" + strconv.Itoa(i) + " copy"
 		} else {
-						s += " -map 0:v:0? " + strings.ReplaceAll(tpl[video], ":v ", ":v:"+strconv.Itoa(i)+" ")
-					}
-				}
+			args.AddCodec("-vn")
 		}
 
 		// 4. Process audio codecs
-			switch len(query["audio"]) {
-			case 0:
-				s += " -an"
-			case 1:
-				if len(query["video"]) > 1 {
-					s += " -map 0:a:0?"
-				}
+		if args.audio > 0 {
 			for _, audio := range query["audio"] {
-					if audio == "copy" {
-						s += " -c:a copy"
+				if audio != "copy" {
+					args.AddCodec(defaults[audio])
 				} else {
-						s += " " + tpl[audio]
-					}
-				}
-			default:
-				for i, audio := range query["audio"] {
-					if audio == "copy" {
-						s += " -map 0:a:0? -c:a:" + strconv.Itoa(i) + " copy"
-					} else {
-						s += " -map 0:a:0? " + strings.ReplaceAll(tpl[audio], ":a ", ":a:"+strconv.Itoa(i)+" ")
-					}
+					args.AddCodec("-c:a copy")
 				}
 			}
 		} else {
-			s += " -c copy"
+			args.AddCodec("-an")
 		}
 
-		s += " " + tpl["output"]
+		if query["hardware"] != nil {
+			MakeHardware(args, query["hardware"][0])
+		}
+	}
 
-		return exec.Handle(s)
-	})
+	if args.codecs == nil {
+		args.AddCodec("-c copy")
+	}
 
-	device.Bin = cfg.Mod["bin"]
-	device.Init()
+	return args
 }
 
 func parseQuery(s string) map[string][]string {
@@ -238,3 +249,76 @@ func parseQuery(s string) map[string][]string {
 	}
 	return query
 }
+
+type Args struct {
+	bin     string   // ffmpeg
+	global  string   // -hide_banner -v error
+	input   string   // -re -stream_loop -1 -i /media/bunny.mp4
+	codecs  []string // -c:v libx264 -g:v 30 -preset:v ultrafast -tune:v zerolatency
+	filters []string // scale=1920:1080
+	output  string   // -f rtsp {output}
+
+	video, audio int // count of video and audio params
+}
+
+func (a *Args) AddCodec(codec string) {
+	a.codecs = append(a.codecs, codec)
+}
+
+func (a *Args) AddFilter(filter string) {
+	a.filters = append(a.filters, filter)
+}
+
+func (a *Args) InsertFilter(filter string) {
+	a.filters = append([]string{filter}, a.filters...)
+}
+
+func (a *Args) String() string {
+	b := bytes.NewBuffer(make([]byte, 0, 512))
+
+	b.WriteString(a.bin)
+
+	if a.global != "" {
+		b.WriteByte(' ')
+		b.WriteString(a.global)
+	}
+
+	b.WriteByte(' ')
+	b.WriteString(a.input)
+
+	multimode := a.video > 1 || a.audio > 1
+	var iv, ia int
+
+	for _, codec := range a.codecs {
+		// support multiple video and/or audio codecs
+		if multimode && len(codec) >= 5 {
+			switch codec[:5] {
+			case "-c:v ":
+				codec = "-map 0:v:0? " + strings.ReplaceAll(codec, ":v ", ":v:"+strconv.Itoa(iv)+" ")
+				iv++
+			case "-c:a ":
+				codec = "-map 0:a:0? " + strings.ReplaceAll(codec, ":a ", ":a:"+strconv.Itoa(ia)+" ")
+				ia++
+			}
+		}
+
+		b.WriteByte(' ')
+		b.WriteString(codec)
+	}
+
+	if a.filters != nil {
+		for i, filter := range a.filters {
+			if i == 0 {
+				b.WriteString(" -vf ")
+			} else {
+				b.WriteByte(',')
+			}
+			b.WriteString(filter)
+		}
+	}
+
+	b.WriteByte(' ')
+	b.WriteString(a.output)
+
+	return b.String()
+}

cmd/ffmpeg/hardware.go

@@ -0,0 +1,112 @@
+package ffmpeg
+
+import (
+	"github.com/rs/zerolog/log"
+	"os/exec"
+	"strings"
+)
+
+const (
+	EngineSoftware     = "software"
+	EngineVAAPI        = "vaapi"        // Intel iGPU and AMD GPU
+	EngineV4L2         = "v4l2"         // Raspberry Pi 3 and 4
+	EngineCUDA         = "cuda"         // NVidia on Windows and Linux
+	EngineDXVA2        = "dxva2"        // Intel on Windows
+	EngineVideoToolbox = "videotoolbox" // macOS
+)
+
+var cache = map[string]string{}
+
+// MakeHardware converts software FFmpeg args to hardware args
+// empty engine for autoselect
+func MakeHardware(args *Args, engine string) {
+	for i, codec := range args.codecs {
+		if len(codec) < 12 {
+			continue // skip short line (-c:v libx264...)
+		}
+
+		// get current codec name
+		name := cut(codec, ' ', 1)
+		switch name {
+		case "libx264":
+			name = "h264"
+		case "libx265":
+			name = "h265"
+		case "mjpeg":
+		default:
+			continue // skip unsupported codec
+		}
+
+		// temporary disable probe for H265 and MJPEG
+		if engine == "" && name == "h264" {
+			if engine = cache[name]; engine == "" {
+				engine = ProbeHardware(name)
+				cache[name] = engine
+			}
+		}
+
+		switch engine {
+		case EngineVAAPI:
+			args.input = "-hwaccel vaapi -hwaccel_output_format vaapi " + args.input
+			args.codecs[i] = defaults[name+"/"+engine]
+
+			for i, filter := range args.filters {
+				if strings.HasPrefix(filter, "scale=") {
+					args.filters[i] = "scale_vaapi=" + filter[6:]
+				}
+			}
+
+			// fix if input doesn't support hwaccel, do nothing when support
+			args.InsertFilter("format=vaapi|nv12,hwupload")
+
+		case EngineCUDA:
+			args.input = "-hwaccel cuda -hwaccel_output_format cuda -extra_hw_frames 2 " + args.input
+			args.codecs[i] = defaults[name+"/"+engine]
+
+			for i, filter := range args.filters {
+				if strings.HasPrefix(filter, "scale=") {
+					args.filters[i] = "scale_cuda=" + filter[6:]
+				}
+			}
+
+		case EngineDXVA2:
+			args.input = "-hwaccel dxva2 -hwaccel_output_format dxva2_vld " + args.input
+			args.codecs[i] = defaults[name+"/"+engine]
+
+			for i, filter := range args.filters {
+				if strings.HasPrefix(filter, "scale=") {
+					args.filters[i] = "scale_qsv=" + filter[6:]
+				}
+			}
+
+			args.InsertFilter("hwmap=derive_device=qsv,format=qsv")
+
+		case EngineVideoToolbox:
+			args.input = "-hwaccel videotoolbox -hwaccel_output_format videotoolbox_vld " + args.input
+			args.codecs[i] = defaults[name+"/"+engine]
+
+		case EngineV4L2:
+			args.codecs[i] = defaults[name+"/"+engine]
+		}
+	}
+}
+
+func run(arg ...string) bool {
+	err := exec.Command(defaults["bin"], arg...).Run()
+	log.Printf("%v %v", arg, err)
+	return err == nil
+}
+
+func cut(s string, sep byte, pos int) string {
+	for n := 0; n < pos; n++ {
+		if i := strings.IndexByte(s, sep); i > 0 {
+			s = s[i+1:]
+		} else {
+			return ""
+		}
+	}
+	if i := strings.IndexByte(s, sep); i > 0 {
+		return s[:i]
+	}
+	return s
+}

cmd/ffmpeg/hardware_darwin.go

@@ -0,0 +1,21 @@
+package ffmpeg
+
+func ProbeHardware(name string) string {
+	switch name {
+	case "h264":
+		if run(
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "h264_videotoolbox", "-f", "null", "-") {
+			return EngineVideoToolbox
+		}
+
+	case "h265":
+		if run(
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "hevc_videotoolbox", "-f", "null", "-") {
+			return EngineVideoToolbox
+		}
+	}
+
+	return EngineSoftware
+}

cmd/ffmpeg/hardware_linux.go

@@ -0,0 +1,67 @@
+package ffmpeg
+
+import (
+	"runtime"
+)
+
+func ProbeHardware(name string) string {
+	if runtime.GOARCH == "arm64" || runtime.GOARCH == "arm" {
+		switch name {
+		case "h264":
+			if run(
+				"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+				"-c", "h264_v4l2m2m", "-f", "null", "-") {
+				return EngineV4L2
+			}
+
+		case "h265":
+			if run(
+				"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+				"-c", "hevc_v4l2m2m", "-f", "null", "-") {
+				return EngineV4L2
+			}
+		}
+
+		return EngineSoftware
+	}
+
+	switch name {
+	case "h264":
+		if run("-init_hw_device", "cuda",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "h264_nvenc", "-f", "null", "-") {
+			return EngineCUDA
+		}
+
+		if run("-init_hw_device", "vaapi",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-vf", "format=nv12,hwupload",
+			"-c", "h264_vaapi", "-f", "null", "-") {
+			return EngineVAAPI
+		}
+
+	case "h265":
+		if run("-init_hw_device", "cuda",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "hevc_nvenc", "-f", "null", "-") {
+			return EngineCUDA
+		}
+
+		if run("-init_hw_device", "vaapi",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-vf", "format=nv12,hwupload",
+			"-c", "hevc_vaapi", "-f", "null", "-") {
+			return EngineVAAPI
+		}
+
+	case "mjpeg":
+		if run("-init_hw_device", "vaapi",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-vf", "format=nv12,hwupload",
+			"-c", "mjpeg_vaapi", "-f", "null", "-") {
+			return EngineVAAPI
+		}
+	}
+
+	return EngineSoftware
+}

cmd/ffmpeg/hardware_windows.go

@@ -0,0 +1,40 @@
+package ffmpeg
+
+func ProbeHardware(name string) string {
+	switch name {
+	case "h264":
+		if run("-init_hw_device", "cuda",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "h264_nvenc", "-f", "null", "-") {
+			return EngineCUDA
+		}
+
+		if run("-init_hw_device", "dxva2",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "h264_qsv", "-f", "null", "-") {
+			return EngineDXVA2
+		}
+
+	case "h265":
+		if run("-init_hw_device", "cuda",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "hevc_nvenc", "-f", "null", "-") {
+			return EngineCUDA
+		}
+
+		if run("-init_hw_device", "dxva2",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "hevc_qsv", "-f", "null", "-") {
+			return EngineDXVA2
+		}
+
+	case "mjpeg":
+		if run("-init_hw_device", "dxva2",
+			"-f", "lavfi", "-i", "testsrc2", "-t", "1",
+			"-c", "mjpeg_qsv", "-f", "null", "-") {
+			return EngineDXVA2
+		}
+	}
+
+	return EngineSoftware
+}