FFmpeg: Fix Intel Quick Sync Video (QSV) hardware transcoding #4382

Signed-off-by: Michael Mayer <michael@photoprism.app>
2025-09-26 21:01:58 +08:00 · 2025-03-28 16:31:13 +01:00
parent a24b385560
commit a76bbba2a6
6 changed files with 273 additions and 19 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,7 @@ venv
 .venv
 .env
 .tmp
+.nv
 .eslintcache
 /tmp/
 /test/
--- a/7
+++ b/7
@@ -389,6 +389,13 @@ docker-build:
 docker-nvidia: docker-nvidia-up
 docker-nvidia-up:
 	docker compose -f compose.nvidia.yaml up
+docker-nvidia-build:
+	docker compose -f compose.nvidia.yaml up
+docker-intel: docker-intel-up
+docker-intel-up:
+	docker compose -f compose.intel.yaml up
+docker-intel-build:
+	docker compose -f compose.intel.yaml build
 docker-local-up:
 	$(DOCKER_COMPOSE) -f compose.local.yaml up --force-recreate
 docker-local-down:
--- a/compose.intel.yaml
+++ b/compose.intel.yaml
@@ -0,0 +1,168 @@
+services:
+  ## PhotoPrism (Development Environment for Intel QSV hardware transcoding)
+  photoprism:
+    build: .
+    image: photoprism/photoprism:develop
+    depends_on:
+      - mariadb
+      - dummy-webdav
+      - dummy-oidc
+    stop_grace_period: 10s
+    security_opt:
+      - seccomp:unconfined
+      - apparmor:unconfined
+    ## Expose HTTP and debug ports
+    ports:
+      - "2342:2342"   # Default HTTP port (host:container)
+      - "2443:2443"   # Default TLS port (host:container)
+      - "2343:2343"   # Acceptance Test HTTP port (host:container)
+      - "40000:40000" # Go Debugger (host:container)
+    shm_size: "2gb"
+    ## Set links and labels for use with Traefik reverse proxy
+    links:
+      - "traefik:localssl.dev"
+      - "traefik:app.localssl.dev"
+      - "traefik:keycloak.localssl.dev"
+      - "traefik:dummy-oidc.localssl.dev"
+      - "traefik:dummy-webdav.localssl.dev"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.photoprism.loadbalancer.server.port=2342"
+      - "traefik.http.services.photoprism.loadbalancer.server.scheme=http"
+      - "traefik.http.routers.photoprism.entrypoints=websecure"
+      - "traefik.http.routers.photoprism.rule=Host(`localssl.dev`) || HostRegexp(`^.+\\.localssl\\.dev`)"
+      - "traefik.http.routers.photoprism.priority=2"
+      - "traefik.http.routers.photoprism.tls.domains[0].main=localssl.dev"
+      - "traefik.http.routers.photoprism.tls.domains[0].sans=*.localssl.dev"
+      - "traefik.http.routers.photoprism.tls=true"
+    ## Configure development environment
+    environment:
+      ## Run as a non-root user after initialization (supported: 0, 33, 50-99, 500-600, and 900-1200):
+      PHOTOPRISM_UID: ${UID:-1000}                         # user id, should match your host user id
+      PHOTOPRISM_GID: ${GID:-1000}                         # group id
+      ## Access Management:
+      PHOTOPRISM_ADMIN_USER: "admin"                       # admin login username
+      PHOTOPRISM_ADMIN_PASSWORD: "photoprism"              # initial admin password (8-72 characters)
+      PHOTOPRISM_AUTH_MODE: "password"                     # authentication mode (public, password)
+      PHOTOPRISM_REGISTER_URI: "https://keycloak.localssl.dev/admin/"
+      PHOTOPRISM_PASSWORD_RESET_URI: "https://keycloak.localssl.dev/realms/master/login-actions/reset-credentials"
+      PHOTOPRISM_USAGE_INFO: "true"
+      PHOTOPRISM_FILES_QUOTA: "100"
+      ## OpenID Connect (pre-configured for local tests):
+      ## see https://keycloak.localssl.dev/realms/master/.well-known/openid-configuration
+      PHOTOPRISM_OIDC_URI: "https://keycloak.localssl.dev/realms/master"
+      PHOTOPRISM_OIDC_CLIENT: "photoprism-develop"
+      PHOTOPRISM_OIDC_SECRET: "9d8351a0-ca01-4556-9c37-85eb634869b9"
+      PHOTOPRISM_OIDC_PROVIDER: "Keycloak"
+      PHOTOPRISM_OIDC_REGISTER: "true"
+      PHOTOPRISM_OIDC_WEBDAV: "true"
+      PHOTOPRISM_DISABLE_OIDC: "false"
+      ## LDAP Authentication (pre-configured for local tests):
+      PHOTOPRISM_LDAP_URI: "ldap://dummy-ldap:389"
+      PHOTOPRISM_LDAP_INSECURE: "true"
+      PHOTOPRISM_LDAP_SYNC: "true"
+      PHOTOPRISM_LDAP_BIND: "simple"
+      PHOTOPRISM_LDAP_BIND_DN: "cn"
+      PHOTOPRISM_LDAP_BASE_DN: "dc=localssl,dc=dev"
+      PHOTOPRISM_LDAP_ROLE: ""
+      PHOTOPRISM_LDAP_ROLE_DN: "ou=photoprism-*,ou=groups,dc=localssl,dc=dev"
+      PHOTOPRISM_LDAP_WEBDAV_DN: "ou=photoprism-webdav,ou=groups,dc=localssl,dc=dev"
+      ## HTTPS/TLS Options:
+      ## see https://docs.photoprism.app/getting-started/using-https/
+      PHOTOPRISM_DISABLE_TLS: "true"
+      PHOTOPRISM_DEFAULT_TLS: "true"
+      ## Site Information:
+      PHOTOPRISM_SITE_URL: "https://app.localssl.dev/"    # server URL in the format "http(s)://domain.name(:port)/(path)"
+      PHOTOPRISM_SITE_CAPTION: "AI-Powered Photos App"
+      PHOTOPRISM_SITE_DESCRIPTION: "Tags and finds pictures without getting in your way!"
+      PHOTOPRISM_SITE_AUTHOR: "@photoprism_app"
+      PHOTOPRISM_DEBUG: "true"
+      PHOTOPRISM_READONLY: "false"
+      PHOTOPRISM_EXPERIMENTAL: "true"
+      PHOTOPRISM_HTTP_MODE: "debug"
+      PHOTOPRISM_HTTP_HOST: "0.0.0.0"
+      PHOTOPRISM_HTTP_PORT: 2342
+      PHOTOPRISM_HTTP_COMPRESSION: "gzip"                  # improves transfer speed and bandwidth utilization (none or gzip)
+      PHOTOPRISM_DATABASE_DRIVER: "mysql"
+      PHOTOPRISM_DATABASE_SERVER: "mariadb:4001"
+      PHOTOPRISM_DATABASE_NAME: "photoprism"
+      PHOTOPRISM_DATABASE_USER: "root"
+      PHOTOPRISM_DATABASE_PASSWORD: "photoprism"
+      PHOTOPRISM_TEST_DRIVER: "sqlite"
+      # PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true&timeout=15s"
+      PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets"
+      PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage"
+      PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals"
+      PHOTOPRISM_ORIGINALS_LIMIT: 128000             # sets originals file size limit to 128 GB
+      PHOTOPRISM_IMPORT_PATH: "/go/src/github.com/photoprism/photoprism/storage/import"
+      PHOTOPRISM_DISABLE_CHOWN: "false"              # disables updating storage permissions via chmod and chown on startup
+      PHOTOPRISM_DISABLE_BACKUPS: "false"            # disables backing up albums and photo metadata to YAML files
+      PHOTOPRISM_DISABLE_WEBDAV: "false"             # disables built-in WebDAV server
+      PHOTOPRISM_DISABLE_SETTINGS: "false"           # disables settings UI and API
+      PHOTOPRISM_DISABLE_PLACES: "false"             # disables reverse geocoding and maps
+      PHOTOPRISM_DISABLE_EXIFTOOL: "false"           # disables creating JSON metadata sidecar files with ExifTool
+      PHOTOPRISM_DISABLE_TENSORFLOW: "false"         # disables all features depending on TensorFlow
+      PHOTOPRISM_DISABLE_RAW: "false"                # disables indexing and conversion of RAW images
+      PHOTOPRISM_RAW_PRESETS: "false"                # enables applying user presets when converting RAW images (reduces performance)
+      PHOTOPRISM_DETECT_NSFW: "false"                # automatically flags photos as private that MAY be offensive (requires TensorFlow)
+      PHOTOPRISM_UPLOAD_NSFW: "false"                # allows uploads that MAY be offensive (no effect without TensorFlow)
+      PHOTOPRISM_THUMB_LIBRARY: "auto"               # image processing library to be used for generating thumbnails (auto, imaging, vips)
+      PHOTOPRISM_THUMB_FILTER: "auto"                # downscaling filter (imaging best to worst: blackman, lanczos, cubic, linear, nearest)
+      PHOTOPRISM_THUMB_UNCACHED: "true"              # enables on-demand thumbnail rendering (high memory and cpu usage)
+      TF_CPP_MIN_LOG_LEVEL: 0                        # show TensorFlow log messages for development
+      ## Intel Quick Sync Video (QSV) (https://docs.photoprism.app/getting-started/advanced/transcoding/#intel-quick-sync):
+      PHOTOPRISM_FFMPEG_ENCODER: "intel"             # H.264/AVC encoder (software, intel, nvidia, apple, raspberry, or vaapi)
+      PHOTOPRISM_FFMPEG_SIZE: "1920"                 # video size limit in pixels (720-7680) (default: 3840)
+      PHOTOPRISM_FFMPEG_BITRATE: "50"                # video bitrate limit in Mbit/s (default: 50)
+      ## Run/install on first startup (options: update https gpu ffmpeg tensorflow davfs clitools clean):
+      PHOTOPRISM_INIT: "https intel tensorflow"
+    ## Share hardware devices with FFmpeg for hardware video transcoding:
+    devices:
+      - "/dev/dri:/dev/dri"
+    working_dir: "/go/src/github.com/photoprism/photoprism"
+    volumes:
+      - ".:/go/src/github.com/photoprism/photoprism"
+      - "./storage:/photoprism"
+      - "go-mod:/go/pkg/mod"
+
+  mariadb:
+    extends:
+      file: ./compose.yaml
+      service: mariadb
+  traefik:
+    extends:
+      file: ./compose.yaml
+      service: traefik
+  dummy-webdav:
+    extends:
+      file: ./compose.yaml
+      service: dummy-webdav
+  dummy-oidc:
+    extends:
+      file: ./compose.yaml
+      service: dummy-oidc
+  dummy-ldap:
+    extends:
+      file: ./compose.yaml
+      service: dummy-ldap
+  keycloak:
+    extends:
+      file: ./compose.yaml
+      service: keycloak
+  prometheus:
+    extends:
+      file: ./compose.yaml
+      service: prometheus
+
+## Create named volume for Go module cache
+volumes:
+  go-mod:
+    driver: local
+  mariadb:
+    driver: local
+
+## Create shared "photoprism-develop" network for connecting with services in other compose.yaml files
+networks:
+  default:
+    name: photoprism
+    driver: bridge
--- a/internal/ffmpeg/intel/avc.go
+++ b/internal/ffmpeg/intel/avc.go
@@ -15,7 +15,6 @@ func TranscodeToAvcCmd(srcName, destName string, opt encode.Options) *exec.Cmd {
 		"-strict", "-2",
 		"-hwaccel", "qsv",
 		"-hwaccel_output_format", "qsv",
-		"-qsv_device", "/dev/dri/renderD128",
 		"-i", srcName,
 		"-c:a", "aac",
 		"-vf", opt.VideoFilter(encode.FormatQSV),
--- a/internal/ffmpeg/testdata/30fps.mov
+++ b/internal/ffmpeg/testdata/30fps.mov
--- a/internal/ffmpeg/transcode_cmd_test.go
+++ b/internal/ffmpeg/transcode_cmd_test.go
@@ -1,6 +1,7 @@
 package ffmpeg

 import (
+	"os"
 	"strings"
 	"testing"

@@ -11,6 +12,8 @@ import (
 )

 func TestTranscodeCmd(t *testing.T) {
+	ffmpegBin := "/usr/bin/ffmpeg"
+
 	t.Run("NoSource", func(t *testing.T) {
 		opt := encode.NewVideoOptions("", encode.IntelAvc, 1500, "50M", "", "")
 		_, _, err := TranscodeCmd("", "", opt)
@@ -34,7 +37,7 @@ func TestTranscodeCmd(t *testing.T) {
 		assert.Contains(t, r.String(), "bin/ffmpeg -y -strict -2 -i VID123.gif -pix_fmt yuv420p -vf scale='trunc(iw/2)*2:trunc(ih/2)*2' -f mp4 -movflags +faststart VID123.gif.avc")
 	})
 	t.Run("VP9toAVC", func(t *testing.T) {
-		opt := encode.NewVideoOptions("/usr/bin/ffmpeg", encode.SoftwareAvc, 1500, "50M", "", "")
+		opt := encode.NewVideoOptions(ffmpegBin, encode.SoftwareAvc, 1500, "50M", "", "")

 		srcName := fs.Abs("./testdata/25fps.vp9")
 		destName := fs.Abs("./testdata/25fps.avc")
@@ -55,7 +58,7 @@ func TestTranscodeCmd(t *testing.T) {
 		RunCommandTest(t, opt.Encoder, srcName, destName, cmd, true)
 	})
 	t.Run("Vaapi", func(t *testing.T) {
-		opt := encode.NewVideoOptions("/usr/bin/ffmpeg", encode.VaapiAvc, 1500, "50M", "", "")
+		opt := encode.NewVideoOptions(ffmpegBin, encode.VaapiAvc, 1500, "50M", "", "")

 		srcName := fs.Abs("./testdata/25fps.vp9")
 		destName := fs.Abs("./testdata/25fps.vaapi.avc")
@@ -72,18 +75,104 @@ func TestTranscodeCmd(t *testing.T) {

 		assert.Equal(t, "/usr/bin/ffmpeg -y -strict -2 -hwaccel vaapi -i SRC -c:a aac -vf scale='if(gte(iw,ih), min(1500, iw), -2):if(gte(iw,ih), -2, min(1500, ih))',format=nv12,hwupload -c:v h264_vaapi -map 0:v:0 -map 0:a:0? -r 30 -b:v 50M -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart DEST", cmdStr)

-		// Running the generated command to test vaapi transcoding requires a compatible device.
-		// RunCommandTest(t, encoder, srcName, destName, cmd, true)
+		// This transcoding test requires a supported hardware device that is properly configured:
+		if os.Getenv("PHOTOPRISM_FFMPEG_ENCODER") == "vaapi" {
+			RunCommandTest(t, encode.VaapiAvc, srcName, destName, cmd, true)
+		}
 	})
-	t.Run("QSV", func(t *testing.T) {
-		opt := encode.NewVideoOptions("", encode.IntelAvc, 1500, "50M", "", "")
-		r, _, err := TranscodeCmd("VID123.mov", "VID123.mov.avc", opt)
+	t.Run("IntelHvc", func(t *testing.T) {
+		opt := encode.NewVideoOptions(ffmpegBin, encode.IntelAvc, 1500, "50M", "", "")
+
+		// QuickTime MOV container with HVC1 (HEVC) codec.
+		srcName := fs.Abs("./testdata/30fps.mov")
+		destName := fs.Abs("./testdata/30fps.intel.avc")
+
+		cmd, _, err := TranscodeCmd(srcName, destName, opt)

 		if err != nil {
 			t.Fatal(err)
 		}

-		assert.Contains(t, r.String(), "/bin/ffmpeg -y -strict -2 -hwaccel qsv -hwaccel_output_format qsv -qsv_device /dev/dri/renderD128 -i VID123.mov -c:a aac -vf scale_qsv=w='if(gte(iw,ih), min(1500, iw), -1)':h='if(gte(iw,ih), -1, min(1500, ih))':format=nv12 -c:v h264_qsv -map 0:v:0 -map 0:a:0? -r 30 -b:v 50M -bitrate 50M -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart VID123.mov.avc")
+		cmdStr := cmd.String()
+		cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
+		cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
+
+		assert.Equal(t, "/usr/bin/ffmpeg -y -strict -2 -hwaccel qsv -hwaccel_output_format qsv -i SRC -c:a aac -vf scale_qsv=w='if(gte(iw,ih), min(1500, iw), -1)':h='if(gte(iw,ih), -1, min(1500, ih))':format=nv12 -c:v h264_qsv -map 0:v:0 -map 0:a:0? -r 30 -b:v 50M -bitrate 50M -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart DEST", cmdStr)
+
+		// This transcoding test requires a supported hardware device that is properly configured:
+		if os.Getenv("PHOTOPRISM_FFMPEG_ENCODER") == "intel" {
+			RunCommandTest(t, encode.IntelAvc, srcName, destName, cmd, true)
+		}
+	})
+	t.Run("IntelVp9", func(t *testing.T) {
+		opt := encode.NewVideoOptions(ffmpegBin, encode.IntelAvc, 1500, "50M", "", "")
+
+		srcName := fs.Abs("./testdata/25fps.vp9")
+		destName := fs.Abs("./testdata/25fps.intel.avc")
+
+		cmd, _, err := TranscodeCmd(srcName, destName, opt)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cmdStr := cmd.String()
+		cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
+		cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
+
+		assert.Equal(t, "/usr/bin/ffmpeg -y -strict -2 -hwaccel qsv -hwaccel_output_format qsv -i SRC -c:a aac -vf scale_qsv=w='if(gte(iw,ih), min(1500, iw), -1)':h='if(gte(iw,ih), -1, min(1500, ih))':format=nv12 -c:v h264_qsv -map 0:v:0 -map 0:a:0? -r 30 -b:v 50M -bitrate 50M -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart DEST", cmdStr)
+
+		// This transcoding test requires a supported hardware device that is properly configured:
+		if os.Getenv("PHOTOPRISM_FFMPEG_ENCODER") == "intel" {
+			RunCommandTest(t, encode.IntelAvc, srcName, destName, cmd, true)
+		}
+	})
+	t.Run("NvidiaHvc", func(t *testing.T) {
+		opt := encode.NewVideoOptions(ffmpegBin, encode.NvidiaAvc, 1500, "50M", "", "")
+
+		// QuickTime MOV container with HVC1 (HEVC) codec.
+		srcName := fs.Abs("./testdata/30fps.mov")
+		destName := fs.Abs("./testdata/30fps.nvidia.avc")
+
+		cmd, _, err := TranscodeCmd(srcName, destName, opt)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cmdStr := cmd.String()
+		cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
+		cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
+
+		assert.Equal(t, "/usr/bin/ffmpeg -y -strict -2 -hwaccel auto -i SRC -pix_fmt yuv420p -c:v h264_nvenc -map 0:v:0 -map 0:a:0? -c:a aac -preset 15 -pixel_format yuv420p -gpu any -vf scale='if(gte(iw,ih), min(1500, iw), -2):if(gte(iw,ih), -2, min(1500, ih))',format=yuv420p -rc:v constqp -cq 0 -tune 2 -r 30 -b:v 50M -profile:v 1 -level:v auto -coder:v 1 -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart DEST", cmdStr)
+
+		// This transcoding test requires a supported hardware device that is properly configured:
+		if os.Getenv("PHOTOPRISM_FFMPEG_ENCODER") == "nvidia" {
+			RunCommandTest(t, encode.NvidiaAvc, srcName, destName, cmd, true)
+		}
+	})
+	t.Run("NvidiaVp9", func(t *testing.T) {
+		opt := encode.NewVideoOptions(ffmpegBin, encode.NvidiaAvc, 1500, "50M", "", "")
+
+		srcName := fs.Abs("./testdata/25fps.vp9")
+		destName := fs.Abs("./testdata/25fps.nvidia.avc")
+
+		cmd, _, err := TranscodeCmd(srcName, destName, opt)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cmdStr := cmd.String()
+		cmdStr = strings.Replace(cmdStr, srcName, "SRC", 1)
+		cmdStr = strings.Replace(cmdStr, destName, "DEST", 1)
+
+		assert.Equal(t, "/usr/bin/ffmpeg -y -strict -2 -hwaccel auto -i SRC -pix_fmt yuv420p -c:v h264_nvenc -map 0:v:0 -map 0:a:0? -c:a aac -preset 15 -pixel_format yuv420p -gpu any -vf scale='if(gte(iw,ih), min(1500, iw), -2):if(gte(iw,ih), -2, min(1500, ih))',format=yuv420p -rc:v constqp -cq 0 -tune 2 -r 30 -b:v 50M -profile:v 1 -level:v auto -coder:v 1 -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart DEST", cmdStr)
+
+		// This transcoding test requires a supported hardware device that is properly configured:
+		if os.Getenv("PHOTOPRISM_FFMPEG_ENCODER") == "nvidia" {
+			RunCommandTest(t, encode.NvidiaAvc, srcName, destName, cmd, true)
+		}
 	})
 	t.Run("Apple", func(t *testing.T) {
 		opt := encode.NewVideoOptions("", encode.AppleAvc, 1500, "50M", "", "")
@@ -95,16 +184,6 @@ func TestTranscodeCmd(t *testing.T) {

 		assert.Contains(t, r.String(), "bin/ffmpeg -y -strict -2 -i VID123.mov -c:v h264_videotoolbox -map 0:v:0 -map 0:a:0? -c:a aac -vf scale='if(gte(iw,ih), min(1500, iw), -2):if(gte(iw,ih), -2, min(1500, ih))',format=yuv420p -profile high -level 51 -r 30 -b:v 50M -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart VID123.mov.avc")
 	})
-	t.Run("Nvidia", func(t *testing.T) {
-		opt := encode.NewVideoOptions("", encode.NvidiaAvc, 1500, "50M", "", "")
-		r, _, err := TranscodeCmd("VID123.mov", "VID123.mov.avc", opt)
-
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		assert.Contains(t, r.String(), "bin/ffmpeg -y -strict -2 -hwaccel auto -i VID123.mov -pix_fmt yuv420p -c:v h264_nvenc -map 0:v:0 -map 0:a:0? -c:a aac -preset 15 -pixel_format yuv420p -gpu any -vf scale='if(gte(iw,ih), min(1500, iw), -2):if(gte(iw,ih), -2, min(1500, ih))',format=yuv420p -rc:v constqp -cq 0 -tune 2 -r 30 -b:v 50M -profile:v 1 -level:v auto -coder:v 1 -f mp4 -movflags frag_keyframe+empty_moov+default_base_moof+faststart VID123.mov.avc")
-	})
 	t.Run("Video4Linux", func(t *testing.T) {
 		opt := encode.NewVideoOptions("", encode.V4LAvc, 1500, "50M", "", "")
 		r, _, err := TranscodeCmd("VID123.mov", "VID123.mov.avc", opt)