Add libva specific CodecParam

examples/codecparam/main.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pion/webrtc/v2"
 
 	_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder
-	"github.com/pion/mediadevices/pkg/codec/vpx"    // This is required to register VP8/VP9 video encoder
+	"github.com/pion/mediadevices/pkg/codec/vaapi"  // This is required to register VP8/VP9 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
@@ -63,19 +63,20 @@ func main() {
 			c.Enabled = true
 			c.Width = 640
 			c.Height = 480
-			c.BitRate = 400000 // 400kbps
+			c.FrameRate = 30
 
 			// Load default parameters.
-			cp, err := vpx.NewVP8Param()
+			cp, err := vaapi.NewVP8Param()
 			if err != nil {
 				panic(err)
 			}
 			fmt.Printf("default codec parameters: %+v\n", cp)
 
-			// Set parameters to avoid bitrate overshoot as possible.
-			cp.RateControlEndUsage = vpx.RateControlCBR
-			cp.RateControlOvershootPercent = 1
-			cp.RateControlUndershootPercent = 95
+			// This example is using libva's hardware accelerated codec.
+			// Set encoder parameters to prohibit bitrate overshoot as possible.
+			cp.RateControlMode = vaapi.RateControlVBR
+			cp.RateControl.BitsPerSecond = 400000
+			cp.RateControl.TargetPercentage = 95
 			c.CodecParams = cp
 		},
 	})

pkg/codec/vaapi/framerate.go

@@ -0,0 +1,31 @@
+package vaapi
+
+import (
+	"time"
+)
+
+type framerateDetector struct {
+	cnt   uint64
+	stamp time.Time
+	rate  uint32
+}
+
+func newFramerateDetector(initialRate uint32) *framerateDetector {
+	return &framerateDetector{
+		rate: initialRate,
+	}
+}
+
+func (f *framerateDetector) Calc() uint32 {
+	if f.cnt%16 == 0 {
+		now := time.Now()
+		interval := now.Sub(f.stamp)
+		if !f.stamp.IsZero() {
+			f.rate = uint32(interval.Nanoseconds()/(16*1000000))<<16 | 1000
+			// denominator << 16 | numerator
+		}
+		f.stamp = now
+	}
+	f.cnt++
+	return f.rate
+}

pkg/codec/vaapi/params.go

@@ -0,0 +1,54 @@
+package vaapi
+
+// ParamVP8 stores VP8 encoding parameters.
+type ParamVP8 struct {
+	Sequence        SequenceParamVP8
+	RateControlMode RateControlMode
+	RateControl     RateControlParam
+}
+
+// SequenceParamVP8 represents VAEncSequenceParameterBufferVP8 and other parameter buffers.
+type SequenceParamVP8 struct {
+	ErrorResilient  bool
+	ClampQindexHigh uint
+	ClampQindexLow  uint
+}
+
+// ParamVP9 represents VAEncSequenceParameterBufferVP9 and other parameter buffers.
+type ParamVP9 struct {
+	RateControlMode RateControlMode
+	RateControl     RateControlParam
+}
+
+// RateControlParam represents VAEncMiscParameterRateControl.
+type RateControlParam struct {
+	// BitsPerSecond is a maximum bit-rate.
+	// This parameter overwrites prop.Codec.BitRate.
+	BitsPerSecond uint
+	// TargetPercentage is a target bit-rate relative to BitsPerSecond.
+	TargetPercentage uint
+	// WindowSize is a rate control window size in milliseconds.
+	WindowSize uint
+	InitialQP  uint
+	MinQP      uint
+	MaxQP      uint
+}
+
+// RateControlMode represents rate control mode.
+// Note that supported mode depends on the codec and acceleration hardware.
+type RateControlMode uint
+
+// List of the RateControlMode.
+const (
+	RateControlCBR            RateControlMode = 0x00000002
+	RateControlVBR            RateControlMode = 0x00000004
+	RateControlVCM            RateControlMode = 0x00000008
+	RateControlCQP            RateControlMode = 0x00000010
+	RateControlVBRConstrained RateControlMode = 0x00000020
+	RateControlICQ            RateControlMode = 0x00000040
+	RateControlMB             RateControlMode = 0x00000080
+	RateControlCFS            RateControlMode = 0x00000100
+	RateControlParallel       RateControlMode = 0x00000200
+	RateControlQVBR           RateControlMode = 0x00000400
+	RateControlAVBR           RateControlMode = 0x00000800
+)

pkg/codec/vaapi/vp8.go

@@ -28,6 +28,12 @@ package vaapi
 // void setRefreshLastFlagVP8(VAEncPictureParameterBufferVP8 *p, uint32_t f) {
 //   p->pic_flags.bits.refresh_last = f;
 // }
+// void setRefreshGoldenFrameFlagVP8(VAEncPictureParameterBufferVP8 *p, uint32_t f) {
+//   p->pic_flags.bits.refresh_golden_frame = f;
+// }
+// void setRefreshAlternateFrameFlagVP8(VAEncPictureParameterBufferVP8 *p, uint32_t f) {
+//   p->pic_flags.bits.refresh_alternate_frame = f;
+// }
 // void setCopyBufferToGoldenFlagVP8(VAEncPictureParameterBufferVP8 *p, uint32_t f) {
 //   p->pic_flags.bits.copy_buffer_to_golden = f;
 // }
@@ -81,6 +87,8 @@ type encoderVP8 struct {
 	frameCnt int
 	prop     prop.Media
 
+	rate *framerateDetector
+
 	mu     sync.Mutex
 	closed bool
 }
@@ -89,6 +97,25 @@ func init() {
 	codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder))
 }
 
+// NewVP8Param returns default parameters of VP8 codec.
+func NewVP8Param() (ParamVP8, error) {
+	return ParamVP8{
+		Sequence: SequenceParamVP8{
+			ClampQindexLow:  9,
+			ClampQindexHigh: 127,
+		},
+		RateControlMode: RateControlVBR,
+		RateControl: RateControlParam{
+			BitsPerSecond:    400000,
+			TargetPercentage: 80,
+			WindowSize:       1500,
+			InitialQP:        60,
+			MinQP:            9,
+			MaxQP:            127,
+		},
+	}, nil
+}
+
 // NewVP8Encoder creates new VP8 encoder
 func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	if p.Width%16 != 0 || p.Width == 0 {
@@ -104,15 +131,28 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 		p.FrameRate = 30
 	}
 
+	var cp ParamVP8
+	switch params := p.Codec.CodecParams.(type) {
+	case nil:
+		cp, _ = NewVP8Param()
+		cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
+	case ParamVP8:
+		cp = params
+	default:
+		return nil, errors.New("unsupported CodecParams type")
+	}
+
 	// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp8enc.c
 	e := &encoderVP8{
 		r:    video.ToI420(r),
 		prop: p,
+		rate: newFramerateDetector(uint32(p.FrameRate)),
 		seqParam: C.VAEncSequenceParameterBufferVP8{
 			frame_width:     C.uint(p.Width),
 			frame_height:    C.uint(p.Height),
-			bits_per_second: C.uint(p.BitRate),
+			bits_per_second: C.uint(cp.RateControl.BitsPerSecond),
 			intra_period:    C.uint(p.KeyFrameInterval),
+			kf_max_dist:     C.uint(p.KeyFrameInterval),
 			reference_frames: [4]C.VASurfaceID{
 				C.VA_INVALID_ID,
 				C.VA_INVALID_ID,
@@ -125,8 +165,8 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 			ref_gf_frame:        C.VA_INVALID_SURFACE,
 			ref_arf_frame:       C.VA_INVALID_SURFACE,
 			reconstructed_frame: C.VA_INVALID_SURFACE,
-			clamp_qindex_low:    9,
-			clamp_qindex_high:   127,
+			clamp_qindex_low:    C.uint8_t(cp.Sequence.ClampQindexLow),
+			clamp_qindex_high:   C.uint8_t(cp.Sequence.ClampQindexHigh),
 			loop_filter_level: [4]C.int8_t{
 				19, 19, 19, 19,
 			},
@@ -144,8 +184,10 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 				_type: C.VAEncMiscParameterTypeHRD,
 			},
 			data: C.VAEncMiscParameterHRD{
-				initial_buffer_fullness: C.uint(float32(p.BitRate) * 1.5 / 2),
-				buffer_size:             C.uint(float32(p.BitRate) * 1.5),
+				initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond *
+					cp.RateControl.WindowSize / 2000),
+				buffer_size: C.uint(cp.RateControl.BitsPerSecond *
+					cp.RateControl.WindowSize / 1000),
 			},
 		},
 		frParam: frParam{
@@ -161,11 +203,12 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 				_type: C.VAEncMiscParameterTypeRateControl,
 			},
 			data: C.VAEncMiscParameterRateControl{
-				window_size:       1500,
-				initial_qp:        60,
-				min_qp:            9,
-				bits_per_second:   C.uint(float32(p.BitRate) * 1.25),
-				target_percentage: C.uint(100.0 / 1.25),
+				window_size:       C.uint(cp.RateControl.WindowSize),
+				initial_qp:        C.uint(cp.RateControl.InitialQP),
+				min_qp:            C.uint(cp.RateControl.MinQP),
+				max_qp:            C.uint(cp.RateControl.MaxQP),
+				bits_per_second:   C.uint(cp.RateControl.BitsPerSecond),
+				target_percentage: C.uint(cp.RateControl.TargetPercentage),
 			},
 		},
 	}
@@ -185,7 +228,7 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	}
 
 	var numEntrypoints C.int
-	entrypoints := make([]C.VAEntrypoint, 5)
+	entrypoints := make([]C.VAEntrypoint, int(C.vaMaxNumEntrypoints(e.display)))
 
 	if s := C.vaQueryConfigEntrypoints(
 		e.display,
@@ -221,11 +264,11 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
 		return nil, errors.New("the hardware encoder doesn't support YUV420")
 	}
-	if (confAttrs[1].value & C.VA_RC_VBR) == 0 {
-		return nil, errors.New("the hardware encoder doesn't support VBR mode")
+	if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 {
+		return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
 	}
 	confAttrs[0].value = C.VA_RT_FORMAT_YUV420
-	confAttrs[1].value = C.VA_RC_VBR
+	confAttrs[1].value = C.uint(cp.RateControlMode)
 
 	if s := C.vaCreateConfig(
 		e.display,
@@ -291,18 +334,24 @@ func (e *encoderVP8) Read(p []byte) (int, error) {
 	kf := e.frameCnt%e.prop.KeyFrameInterval == 0
 	e.frameCnt++
 
+	e.frParam.data.framerate = C.uint(e.rate.Calc())
+
 	if kf {
 		// Key frame
 		C.setForceKFFlagVP8(&e.picParam, 1)
 		C.setFrameTypeFlagVP8(&e.picParam, 0)
-		C.setRefreshLastFlagVP8(&e.picParam, 0)
+		C.setRefreshLastFlagVP8(&e.picParam, 1)
+		C.setRefreshGoldenFrameFlagVP8(&e.picParam, 1)
 		C.setCopyBufferToGoldenFlagVP8(&e.picParam, 0)
+		C.setRefreshAlternateFrameFlagVP8(&e.picParam, 1)
 		C.setCopyBufferToAlternateFlagVP8(&e.picParam, 0)
 	} else {
 		C.setForceKFFlagVP8(&e.picParam, 0)
 		C.setFrameTypeFlagVP8(&e.picParam, 1)
 		C.setRefreshLastFlagVP8(&e.picParam, 1)
+		C.setRefreshGoldenFrameFlagVP8(&e.picParam, 0)
 		C.setCopyBufferToGoldenFlagVP8(&e.picParam, 1)
+		C.setRefreshAlternateFrameFlagVP8(&e.picParam, 0)
 		C.setCopyBufferToAlternateFlagVP8(&e.picParam, 2)
 	}
 	if e.picParam.reconstructed_frame == C.VA_INVALID_SURFACE {

pkg/codec/vaapi/vp9.go

@@ -91,6 +91,8 @@ type encoderVP9 struct {
 	frameCnt int
 	prop     prop.Media
 
+	rate *framerateDetector
+
 	mu     sync.Mutex
 	closed bool
 }
@@ -99,6 +101,21 @@ func init() {
 	codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder))
 }
 
+// NewVP8Param returns default parameters of VP9 codec.
+func NewVP9Param() (ParamVP9, error) {
+	return ParamVP9{
+		RateControlMode: RateControlVBR,
+		RateControl: RateControlParam{
+			BitsPerSecond:    400000,
+			TargetPercentage: 80,
+			WindowSize:       1500,
+			InitialQP:        60,
+			MinQP:            9,
+			MaxQP:            127,
+		},
+	}, nil
+}
+
 // NewVP9Encoder creates new VP9 encoder
 func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	if p.Width%16 != 0 || p.Width == 0 {
@@ -114,14 +131,26 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 		p.FrameRate = 30
 	}
 
+	var cp ParamVP9
+	switch params := p.Codec.CodecParams.(type) {
+	case nil:
+		cp, _ = NewVP9Param()
+		cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
+	case ParamVP9:
+		cp = params
+	default:
+		return nil, errors.New("unsupported CodecParams type")
+	}
+
 	// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp9enc.c
 	e := &encoderVP9{
 		r:    video.ToI420(r),
 		prop: p,
+		rate: newFramerateDetector(uint32(p.FrameRate)),
 		seqParam: C.VAEncSequenceParameterBufferVP9{
 			max_frame_width:  8192,
 			max_frame_height: 8192,
-			bits_per_second:  C.uint(p.BitRate),
+			bits_per_second:  C.uint(cp.RateControl.BitsPerSecond),
 			intra_period:     C.uint(p.KeyFrameInterval),
 			kf_min_dist:      1,
 			kf_max_dist:      C.uint(p.KeyFrameInterval),
@@ -159,8 +188,10 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 				_type: C.VAEncMiscParameterTypeHRD,
 			},
 			data: C.VAEncMiscParameterHRD{
-				initial_buffer_fullness: C.uint(float32(p.BitRate) * 1.5 / 2),
-				buffer_size:             C.uint(float32(p.BitRate) * 1.5),
+				initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond *
+					cp.RateControl.WindowSize / 2000),
+				buffer_size: C.uint(cp.RateControl.BitsPerSecond *
+					cp.RateControl.WindowSize / 1000),
 			},
 		},
 		frParam: frParam{
@@ -176,11 +207,12 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 				_type: C.VAEncMiscParameterTypeRateControl,
 			},
 			data: C.VAEncMiscParameterRateControl{
-				window_size:       1500,
-				initial_qp:        60,
-				min_qp:            9,
-				bits_per_second:   C.uint(float32(p.BitRate) * 1.25),
-				target_percentage: C.uint(100.0 / 1.25),
+				window_size:       C.uint(cp.RateControl.WindowSize),
+				initial_qp:        C.uint(cp.RateControl.InitialQP),
+				min_qp:            C.uint(cp.RateControl.MinQP),
+				max_qp:            C.uint(cp.RateControl.MaxQP),
+				bits_per_second:   C.uint(cp.RateControl.BitsPerSecond),
+				target_percentage: C.uint(cp.RateControl.TargetPercentage),
 			},
 		},
 	}
@@ -199,7 +231,7 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	}
 
 	var numEntrypoints C.int
-	entrypoints := make([]C.VAEntrypoint, 5)
+	entrypoints := make([]C.VAEntrypoint, int(C.vaMaxNumEntrypoints(e.display)))
 
 	if s := C.vaQueryConfigEntrypoints(
 		e.display,
@@ -235,11 +267,11 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
 	if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
 		return nil, errors.New("the hardware encoder doesn't support YUV420")
 	}
-	if (confAttrs[1].value & C.VA_RC_VBR) == 0 {
-		return nil, errors.New("the hardware encoder doesn't support VBR mode")
+	if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 {
+		return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
 	}
 	confAttrs[0].value = C.VA_RT_FORMAT_YUV420
-	confAttrs[1].value = C.VA_RC_VBR
+	confAttrs[1].value = C.uint(cp.RateControlMode)
 
 	if s := C.vaCreateConfig(
 		e.display,
@@ -305,6 +337,8 @@ func (e *encoderVP9) Read(p []byte) (int, error) {
 	kf := e.frameCnt%e.prop.KeyFrameInterval == 0
 	e.frameCnt++
 
+	e.frParam.data.framerate = C.uint(e.rate.Calc())
+
 	if kf {
 		C.setForceKFFlag9(&e.picParam, 1)
 		C.setFrameTypeFlagVP9(&e.picParam, 0)