Redesign codec

Resolves https://github.com/pion/mediadevices/issues/114

* Remove codec registrar
* Completely redesign how codec is being discovered, tuned, and built
* Update examples
* Update unit tests
This commit is contained in:
Lukas Herman
2020-03-14 21:59:32 -04:00
parent 475f8cc458
commit c9b90fb233
22 changed files with 518 additions and 377 deletions

View File

@@ -5,18 +5,16 @@ import (
"github.com/pion/mediadevices" "github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal" "github.com/pion/mediadevices/examples/internal/signal"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/frame"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder "github.com/pion/mediadevices/pkg/codec/vaapi" // This is required to register VP8/VP9 video encoder
"github.com/pion/mediadevices/pkg/codec/vaapi" // This is required to register VP8/VP9 video encoder
// Note: If you don't have a camera or microphone or your adapters are not supported, // Note: If you don't have a camera or your adapters are not supported,
// you can always swap your adapters with our dummy adapters below. // you can always swap your adapters with our dummy adapters below.
// _ "github.com/pion/mediadevices/pkg/driver/videotest" // _ "github.com/pion/mediadevices/pkg/driver/videotest"
// _ "github.com/pion/mediadevices/pkg/driver/audiotest" _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
) )
func main() { func main() {
@@ -52,13 +50,7 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection) md := mediadevices.NewMediaDevices(peerConnection)
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Audio: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.Opus
c.Enabled = true
c.BitRate = 32000 // 32kbps
},
Video: func(c *mediadevices.MediaTrackConstraints) { Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.VP8
c.FrameFormat = frame.FormatYUY2 c.FrameFormat = frame.FormatYUY2
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
@@ -77,7 +69,7 @@ func main() {
cp.RateControlMode = vaapi.RateControlVBR cp.RateControlMode = vaapi.RateControlVBR
cp.RateControl.BitsPerSecond = 400000 cp.RateControl.BitsPerSecond = 400000
cp.RateControl.TargetPercentage = 95 cp.RateControl.TargetPercentage = 95
c.CodecParams = cp c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&cp}
}, },
}) })
if err != nil { if err != nil {

View File

@@ -6,9 +6,9 @@ import (
"os" "os"
"github.com/pion/mediadevices" "github.com/pion/mediadevices"
_ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder "github.com/pion/mediadevices/pkg/codec"
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/frame"
"github.com/pion/rtp" "github.com/pion/rtp"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
@@ -16,8 +16,7 @@ import (
) )
const ( const (
videoCodecName = webrtc.VP8 mtu = 1000
mtu = 1000
) )
func main() { func main() {
@@ -41,14 +40,19 @@ func main() {
), ),
) )
_, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) { Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatYUY2 c.FrameFormat = frame.FormatYUY2
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
c.Height = 480 c.Height = 480
c.BitRate = 100000 // 100kbps c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
}, },
}) })
if err != nil { if err != nil {

View File

@@ -5,18 +5,13 @@ import (
"github.com/pion/mediadevices" "github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal" "github.com/pion/mediadevices/examples/internal/signal"
_ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder "github.com/pion/mediadevices/pkg/codec"
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder _ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
) )
const (
videoCodecName = webrtc.VP8
)
func main() { func main() {
config := webrtc.Configuration{ config := webrtc.Configuration{
ICEServers: []webrtc.ICEServer{ ICEServers: []webrtc.ICEServer{
@@ -49,12 +44,17 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection) md := mediadevices.NewMediaDevices(peerConnection)
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{ s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) { Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.Enabled = true c.Enabled = true
c.BitRate = 100000 // 100kbps
c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
}, },
}) })
if err != nil { if err != nil {

View File

@@ -5,16 +5,18 @@ import (
"github.com/pion/mediadevices" "github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal" "github.com/pion/mediadevices/examples/internal/signal"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/frame"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder // This is required to register opus audio encoder
"github.com/pion/mediadevices/pkg/codec/opus"
// If you don't like vpx, you can also use x264 by importing as below // If you don't like vpx, you can also use x264 by importing as below
// _ "github.com/pion/mediadevices/pkg/codec/x264" // This is required to register h264 video encoder // _ "github.com/pion/mediadevices/pkg/codec/x264" // This is required to register h264 video encoder
// or you can also use openh264 for alternative h264 implementation // or you can also use openh264 for alternative h264 implementation
// _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder // _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
// Note: If you don't have a camera or microphone or your adapters are not supported, // Note: If you don't have a camera or microphone or your adapters are not supported,
// you can always swap your adapters with our dummy adapters below. // you can always swap your adapters with our dummy adapters below.
@@ -60,19 +62,29 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection) md := mediadevices.NewMediaDevices(peerConnection)
opusParams, err := opus.NewParams()
if err != nil {
panic(err)
}
opusParams.BitRate = 32000 // 32kbps
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Audio: func(c *mediadevices.MediaTrackConstraints) { Audio: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.Opus
c.Enabled = true c.Enabled = true
c.BitRate = 32000 // 32kbps c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
}, },
Video: func(c *mediadevices.MediaTrackConstraints) { Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatYUY2 c.FrameFormat = frame.FormatYUY2
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
c.Height = 480 c.Height = 480
c.BitRate = 100000 // 100kbps c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
}, },
}) })
if err != nil { if err != nil {

View File

@@ -7,17 +7,14 @@ import (
"github.com/disintegration/imaging" "github.com/disintegration/imaging"
"github.com/pion/mediadevices" "github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal" "github.com/pion/mediadevices/examples/internal/signal"
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder "github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
) )
const (
videoCodecName = webrtc.VP8
)
func rotate180(r video.Reader) video.Reader { func rotate180(r video.Reader) video.Reader {
return video.ReaderFunc(func() (img image.Image, err error) { return video.ReaderFunc(func() (img image.Image, err error) {
img, err = r.Read() img, err = r.Read()
@@ -62,15 +59,20 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection) md := mediadevices.NewMediaDevices(peerConnection)
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) { Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420 c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
c.Height = 480 c.Height = 480
c.BitRate = 100000 // 100kbps
c.VideoTransform = rotate180 c.VideoTransform = rotate180
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
}, },
}) })
if err != nil { if err != nil {

View File

@@ -18,23 +18,18 @@ import (
) )
func TestGetUserMedia(t *testing.T) { func TestGetUserMedia(t *testing.T) {
codec.Register("MockVideo", codec.VideoEncoderBuilder(func(r video.Reader, p prop.Media) (io.ReadCloser, error) { videoParams := mockParams{
if p.BitRate == 0 { BaseParams: codec.BaseParams{
// This is a dummy error to test the failure condition. BitRate: 100000,
return nil, errors.New("wrong codec parameter") },
} name: "MockVideo",
return &mockVideoCodec{ }
r: r, audioParams := mockParams{
closed: make(chan struct{}), BaseParams: codec.BaseParams{
}, nil BitRate: 32000,
})) },
codec.Register("MockAudio", codec.AudioEncoderBuilder(func(r audio.Reader, p prop.Media) (io.ReadCloser, error) { name: "MockAudio",
return &mockAudioCodec{ }
r: r,
closed: make(chan struct{}),
}, nil
}))
md := NewMediaDevicesFromCodecs( md := NewMediaDevicesFromCodecs(
map[webrtc.RTPCodecType][]*webrtc.RTPCodec{ map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{ webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
@@ -54,30 +49,31 @@ func TestGetUserMedia(t *testing.T) {
) )
constraints := MediaStreamConstraints{ constraints := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) { Video: func(c *MediaTrackConstraints) {
c.CodecName = "MockVideo"
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
c.Height = 480 c.Height = 480
c.BitRate = 100000 params := videoParams
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
}, },
Audio: func(c *MediaTrackConstraints) { Audio: func(c *MediaTrackConstraints) {
c.CodecName = "MockAudio"
c.Enabled = true c.Enabled = true
c.BitRate = 32000 params := audioParams
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
}, },
} }
constraintsWrong := MediaStreamConstraints{ constraintsWrong := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) { Video: func(c *MediaTrackConstraints) {
c.CodecName = "MockVideo"
c.Enabled = true c.Enabled = true
c.Width = 640 c.Width = 640
c.Height = 480 c.Height = 480
c.BitRate = 0 params := videoParams
params.BitRate = 0
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
}, },
Audio: func(c *MediaTrackConstraints) { Audio: func(c *MediaTrackConstraints) {
c.CodecName = "MockAudio"
c.Enabled = true c.Enabled = true
c.BitRate = 32000 params := audioParams
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
}, },
} }
@@ -156,6 +152,33 @@ func (t *mockTrack) Kind() webrtc.RTPCodecType {
return t.codec.Type return t.codec.Type
} }
type mockParams struct {
codec.BaseParams
name string
}
func (params *mockParams) Name() string {
return params.name
}
func (params *mockParams) BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if params.BitRate == 0 {
// This is a dummy error to test the failure condition.
return nil, errors.New("wrong codec parameter")
}
return &mockVideoCodec{
r: r,
closed: make(chan struct{}),
}, nil
}
func (params *mockParams) BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
return &mockAudioCodec{
r: r,
closed: make(chan struct{}),
}, nil
}
type mockVideoCodec struct { type mockVideoCodec struct {
r video.Reader r video.Reader
closed chan struct{} closed chan struct{}

View File

@@ -1,6 +1,7 @@
package mediadevices package mediadevices
import ( import (
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
@@ -15,6 +16,18 @@ type MediaStreamConstraints struct {
type MediaTrackConstraints struct { type MediaTrackConstraints struct {
prop.Media prop.Media
Enabled bool Enabled bool
// VideoEncoderBuilders are codec builders that are used for encoding the video
// and later being used for sending the appropriate RTP payload type.
//
// If one encoder builder fails to build the codec, the next builder will be used,
// repeating until a codec builds. If no builders build successfully, an error is returned.
VideoEncoderBuilders []codec.VideoEncoderBuilder
// AudioEncoderBuilders are codec builders that are used for encoding the audio
// and later being used for sending the appropriate RTP payload type.
//
// If one encoder builder fails to build the codec, the next builder will be used,
// repeating until a codec builds. If no builders build successfully, an error is returned.
AudioEncoderBuilders []codec.AudioEncoderBuilder
// VideoTransform will be used to transform the video that's coming from the driver. // VideoTransform will be used to transform the video that's coming from the driver.
// So, basically it'll look like following: driver -> VideoTransform -> codec // So, basically it'll look like following: driver -> VideoTransform -> codec
VideoTransform video.TransformFunc VideoTransform video.TransformFunc

View File

@@ -8,5 +8,35 @@ import (
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
) )
type VideoEncoderBuilder func(r video.Reader, p prop.Media) (io.ReadCloser, error) // AudioEncoderBuilder is the interface that wraps basic operations that are
type AudioEncoderBuilder func(r audio.Reader, p prop.Media) (io.ReadCloser, error) // necessary to build the audio encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type AudioEncoderBuilder interface {
// Name represents the codec name
Name() string
// BuildAudioEncoder builds audio encoder by given media params and audio input
BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error)
}
// VideoEncoderBuilder is the interface that wraps basic operations that are
// necessary to build the video encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type VideoEncoderBuilder interface {
// Name represents the codec name
Name() string
// BuildVideoEncoder builds video encoder by given media params and video input
BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error)
}
// BaseParams represents an codec's encoding properties
type BaseParams struct {
// Target bitrate in bps.
BitRate int
// Expected interval of the keyframes in frames.
KeyFrameInterval int
}

View File

@@ -9,19 +9,15 @@ package openh264
import "C" import "C"
import ( import (
"errors"
"fmt" "fmt"
"image" "image"
"io" "io"
"sync" "sync"
"unsafe" "unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
) )
type encoder struct { type encoder struct {
@@ -33,26 +29,16 @@ type encoder struct {
closed bool closed bool
} }
var _ codec.VideoEncoderBuilder = codec.VideoEncoderBuilder(NewEncoder) func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
if params.BitRate == 0 {
func init() { params.BitRate = 100000
codec.Register(webrtc.H264, codec.VideoEncoderBuilder(NewEncoder))
}
func NewEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.BitRate == 0 {
p.BitRate = 100000
}
if p.CodecParams != nil {
return nil, errors.New("unsupported CodecParams type")
} }
var rv C.int var rv C.int
cEncoder := C.enc_new(C.EncoderOptions{ cEncoder := C.enc_new(C.EncoderOptions{
width: C.int(p.Width), width: C.int(p.Width),
height: C.int(p.Height), height: C.int(p.Height),
target_bitrate: C.int(p.BitRate), target_bitrate: C.int(params.BitRate),
max_fps: C.float(p.FrameRate), max_fps: C.float(p.FrameRate),
}, &rv) }, &rv)
if err := errResult(rv); err != nil { if err := errResult(rv); err != nil {

View File

@@ -0,0 +1,34 @@
package openh264
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores libopenh264 specific encoding parameters.
type Params struct {
codec.BaseParams
}
// NewParams returns default openh264 codec specific parameters.
func NewParams() (Params, error) {
return Params{
BaseParams: codec.BaseParams{
BitRate: 100000,
},
}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.H264
}
// BuildVideoEncoder builds openh264 encoder with given params
func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

View File

@@ -1,7 +1,6 @@
package opus package opus
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"math" "math"
@@ -9,10 +8,8 @@ import (
"unsafe" "unsafe"
"github.com/lherman-cs/opus" "github.com/lherman-cs/opus"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
) )
type encoder struct { type encoder struct {
@@ -24,13 +21,8 @@ type encoder struct {
var latencies = []float64{5, 10, 20, 40, 60} var latencies = []float64{5, 10, 20, 40, 60}
var _ io.ReadCloser = &encoder{} var _ io.ReadCloser = &encoder{}
var _ codec.AudioEncoderBuilder = codec.AudioEncoderBuilder(NewEncoder)
func init() { func newEncoder(r audio.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
codec.Register(webrtc.Opus, codec.AudioEncoderBuilder(NewEncoder))
}
func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
if p.SampleRate == 0 { if p.SampleRate == 0 {
return nil, fmt.Errorf("opus: inProp.SampleRate is required") return nil, fmt.Errorf("opus: inProp.SampleRate is required")
} }
@@ -39,12 +31,8 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
p.Latency = 20 p.Latency = 20
} }
if p.BitRate == 0 { if params.BitRate == 0 {
p.BitRate = 32000 params.BitRate = 32000
}
if p.CodecParams != nil {
return nil, errors.New("unsupported CodecParams type")
} }
// Select the nearest supported latency // Select the nearest supported latency
@@ -69,7 +57,7 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := engine.SetBitrate(p.BitRate); err != nil { if err := engine.SetBitrate(params.BitRate); err != nil {
return nil, err return nil, err
} }

30
pkg/codec/opus/params.go Normal file
View File

@@ -0,0 +1,30 @@
package opus
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores opus specific encoding parameters.
type Params struct {
codec.BaseParams
}
// NewParams returns default opus codec specific parameters.
func NewParams() (Params, error) {
return Params{}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.Opus
}
// BuildVideoEncoder builds x264 encoder with given params
func (p *Params) BuildAudioEncoder(r audio.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

View File

@@ -1,42 +0,0 @@
package codec
import (
"fmt"
"io"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
var (
videoEncoders = make(map[string]VideoEncoderBuilder)
audioEncoders = make(map[string]AudioEncoderBuilder)
)
func Register(name string, builder interface{}) {
switch b := builder.(type) {
case VideoEncoderBuilder:
videoEncoders[name] = b
case AudioEncoderBuilder:
audioEncoders[name] = b
}
}
func BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
b, ok := videoEncoders[p.CodecName]
if !ok {
return nil, fmt.Errorf("codec: can't find %s video encoder", p.CodecName)
}
return b(r, p)
}
func BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
b, ok := audioEncoders[p.CodecName]
if !ok {
return nil, fmt.Errorf("codec: can't find %s audio encoder", p.CodecName)
}
return b(r, p)
}

View File

@@ -1,7 +1,17 @@
package vaapi package vaapi
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// ParamVP8 stores VP8 encoding parameters. // ParamVP8 stores VP8 encoding parameters.
type ParamVP8 struct { type ParamVP8 struct {
codec.BaseParams
Sequence SequenceParamVP8 Sequence SequenceParamVP8
RateControlMode RateControlMode RateControlMode RateControlMode
RateControl RateControlParam RateControl RateControlParam
@@ -14,8 +24,38 @@ type SequenceParamVP8 struct {
ClampQindexLow uint ClampQindexLow uint
} }
// NewVP8Param returns default parameters of VP8 codec.
func NewVP8Param() (ParamVP8, error) {
return ParamVP8{
Sequence: SequenceParamVP8{
ClampQindexLow: 9,
ClampQindexHigh: 127,
},
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// Name represents the codec name
func (p *ParamVP8) Name() string {
return webrtc.VP8
}
// BuildVideoEncoder builds VP8 encoder with given params
func (p *ParamVP8) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newVP8Encoder(r, property, *p)
}
// ParamVP9 represents VAEncSequenceParameterBufferVP9 and other parameter buffers. // ParamVP9 represents VAEncSequenceParameterBufferVP9 and other parameter buffers.
type ParamVP9 struct { type ParamVP9 struct {
codec.BaseParams
RateControlMode RateControlMode RateControlMode RateControlMode
RateControl RateControlParam RateControl RateControlParam
} }
@@ -52,3 +92,28 @@ const (
RateControlQVBR RateControlMode = 0x00000400 RateControlQVBR RateControlMode = 0x00000400
RateControlAVBR RateControlMode = 0x00000800 RateControlAVBR RateControlMode = 0x00000800
) )
// NewVP9Param returns default parameters of VP9 codec.
func NewVP9Param() (ParamVP9, error) {
return ParamVP9{
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// Name represents the codec name
func (p *ParamVP9) Name() string {
return webrtc.VP9
}
// BuildVideoEncoder builds VP9 encoder with given params
func (p *ParamVP9) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newVP9Encoder(r, property, *p)
}

View File

@@ -50,12 +50,9 @@ import (
"sync" "sync"
"unsafe" "unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
) )
const ( const (
@@ -86,6 +83,7 @@ type encoderVP8 struct {
frameCnt int frameCnt int
prop prop.Media prop prop.Media
params ParamVP8
rate *framerateDetector rate *framerateDetector
@@ -93,66 +91,37 @@ type encoderVP8 struct {
closed bool closed bool
} }
func init() { // newVP8Encoder creates new VP8 encoder
codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder)) func newVP8Encoder(r video.Reader, p prop.Media, params ParamVP8) (io.ReadCloser, error) {
}
// NewVP8Param returns default parameters of VP8 codec.
func NewVP8Param() (ParamVP8, error) {
return ParamVP8{
Sequence: SequenceParamVP8{
ClampQindexLow: 9,
ClampQindexHigh: 127,
},
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// NewVP8Encoder creates new VP8 encoder
func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.Width%16 != 0 || p.Width == 0 { if p.Width%16 != 0 || p.Width == 0 {
return nil, errors.New("width must be 16*n") return nil, errors.New("width must be 16*n")
} }
if p.Height%16 != 0 || p.Height == 0 { if p.Height%16 != 0 || p.Height == 0 {
return nil, errors.New("height must be 16*n") return nil, errors.New("height must be 16*n")
} }
if p.KeyFrameInterval == 0 { if params.KeyFrameInterval == 0 {
p.KeyFrameInterval = 30 params.KeyFrameInterval = 30
} }
if p.FrameRate == 0 { if p.FrameRate == 0 {
p.FrameRate = 30 p.FrameRate = 30
} }
var cp ParamVP8 if params.RateControl.BitsPerSecond == 0 {
switch params := p.Codec.CodecParams.(type) { params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5)
case nil:
cp, _ = NewVP8Param()
cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
case ParamVP8:
cp = params
default:
return nil, errors.New("unsupported CodecParams type")
} }
// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp8enc.c // Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp8enc.c
e := &encoderVP8{ e := &encoderVP8{
r: video.ToI420(r), r: video.ToI420(r),
prop: p, prop: p,
rate: newFramerateDetector(uint32(p.FrameRate)), params: params,
rate: newFramerateDetector(uint32(p.FrameRate)),
seqParam: C.VAEncSequenceParameterBufferVP8{ seqParam: C.VAEncSequenceParameterBufferVP8{
frame_width: C.uint(p.Width), frame_width: C.uint(p.Width),
frame_height: C.uint(p.Height), frame_height: C.uint(p.Height),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond), bits_per_second: C.uint(params.RateControl.BitsPerSecond),
intra_period: C.uint(p.KeyFrameInterval), intra_period: C.uint(params.KeyFrameInterval),
kf_max_dist: C.uint(p.KeyFrameInterval), kf_max_dist: C.uint(params.KeyFrameInterval),
reference_frames: [4]C.VASurfaceID{ reference_frames: [4]C.VASurfaceID{
C.VA_INVALID_ID, C.VA_INVALID_ID,
C.VA_INVALID_ID, C.VA_INVALID_ID,
@@ -165,8 +134,8 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
ref_gf_frame: C.VA_INVALID_SURFACE, ref_gf_frame: C.VA_INVALID_SURFACE,
ref_arf_frame: C.VA_INVALID_SURFACE, ref_arf_frame: C.VA_INVALID_SURFACE,
reconstructed_frame: C.VA_INVALID_SURFACE, reconstructed_frame: C.VA_INVALID_SURFACE,
clamp_qindex_low: C.uint8_t(cp.Sequence.ClampQindexLow), clamp_qindex_low: C.uint8_t(params.Sequence.ClampQindexLow),
clamp_qindex_high: C.uint8_t(cp.Sequence.ClampQindexHigh), clamp_qindex_high: C.uint8_t(params.Sequence.ClampQindexHigh),
loop_filter_level: [4]C.int8_t{ loop_filter_level: [4]C.int8_t{
19, 19, 19, 19, 19, 19, 19, 19,
}, },
@@ -184,10 +153,10 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeHRD, _type: C.VAEncMiscParameterTypeHRD,
}, },
data: C.VAEncMiscParameterHRD{ data: C.VAEncMiscParameterHRD{
initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond * initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 2000), params.RateControl.WindowSize / 2000),
buffer_size: C.uint(cp.RateControl.BitsPerSecond * buffer_size: C.uint(params.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 1000), params.RateControl.WindowSize / 1000),
}, },
}, },
frParam: frParam{ frParam: frParam{
@@ -203,12 +172,12 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeRateControl, _type: C.VAEncMiscParameterTypeRateControl,
}, },
data: C.VAEncMiscParameterRateControl{ data: C.VAEncMiscParameterRateControl{
window_size: C.uint(cp.RateControl.WindowSize), window_size: C.uint(params.RateControl.WindowSize),
initial_qp: C.uint(cp.RateControl.InitialQP), initial_qp: C.uint(params.RateControl.InitialQP),
min_qp: C.uint(cp.RateControl.MinQP), min_qp: C.uint(params.RateControl.MinQP),
max_qp: C.uint(cp.RateControl.MaxQP), max_qp: C.uint(params.RateControl.MaxQP),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond), bits_per_second: C.uint(params.RateControl.BitsPerSecond),
target_percentage: C.uint(cp.RateControl.TargetPercentage), target_percentage: C.uint(params.RateControl.TargetPercentage),
}, },
}, },
} }
@@ -264,11 +233,11 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 { if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
return nil, errors.New("the hardware encoder doesn't support YUV420") return nil, errors.New("the hardware encoder doesn't support YUV420")
} }
if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 { if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 {
return nil, errors.New("the hardware encoder doesn't support specified rate control mode") return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
} }
confAttrs[0].value = C.VA_RT_FORMAT_YUV420 confAttrs[0].value = C.VA_RT_FORMAT_YUV420
confAttrs[1].value = C.uint(cp.RateControlMode) confAttrs[1].value = C.uint(params.RateControlMode)
if s := C.vaCreateConfig( if s := C.vaCreateConfig(
e.display, e.display,
@@ -331,7 +300,7 @@ func (e *encoderVP8) Read(p []byte) (int, error) {
} }
yuvImg := img.(*image.YCbCr) yuvImg := img.(*image.YCbCr)
kf := e.frameCnt%e.prop.KeyFrameInterval == 0 kf := e.frameCnt%e.params.KeyFrameInterval == 0
e.frameCnt++ e.frameCnt++
e.frParam.data.framerate = C.uint(e.rate.Calc()) e.frParam.data.framerate = C.uint(e.rate.Calc())

View File

@@ -45,12 +45,9 @@ import (
"sync" "sync"
"unsafe" "unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
) )
const ( const (
@@ -90,6 +87,7 @@ type encoderVP9 struct {
frameCnt int frameCnt int
prop prop.Media prop prop.Media
params ParamVP9
rate *framerateDetector rate *framerateDetector
@@ -97,63 +95,38 @@ type encoderVP9 struct {
closed bool closed bool
} }
func init() { // newVP9Encoder creates new VP9 encoder
codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder)) func newVP9Encoder(r video.Reader, p prop.Media, params ParamVP9) (io.ReadCloser, error) {
}
// NewVP8Param returns default parameters of VP9 codec.
func NewVP9Param() (ParamVP9, error) {
return ParamVP9{
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// NewVP9Encoder creates new VP9 encoder
func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.Width%16 != 0 || p.Width == 0 { if p.Width%16 != 0 || p.Width == 0 {
return nil, errors.New("width must be 16*n") return nil, errors.New("width must be 16*n")
} }
if p.Height%16 != 0 || p.Height == 0 { if p.Height%16 != 0 || p.Height == 0 {
return nil, errors.New("height must be 16*n") return nil, errors.New("height must be 16*n")
} }
if p.KeyFrameInterval == 0 { if params.KeyFrameInterval == 0 {
p.KeyFrameInterval = 30 params.KeyFrameInterval = 30
} }
if p.FrameRate == 0 { if p.FrameRate == 0 {
p.FrameRate = 30 p.FrameRate = 30
} }
var cp ParamVP9 if params.RateControl.BitsPerSecond == 0 {
switch params := p.Codec.CodecParams.(type) { params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5)
case nil:
cp, _ = NewVP9Param()
cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
case ParamVP9:
cp = params
default:
return nil, errors.New("unsupported CodecParams type")
} }
// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp9enc.c // Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp9enc.c
e := &encoderVP9{ e := &encoderVP9{
r: video.ToI420(r), r: video.ToI420(r),
prop: p, prop: p,
rate: newFramerateDetector(uint32(p.FrameRate)), params: params,
rate: newFramerateDetector(uint32(p.FrameRate)),
seqParam: C.VAEncSequenceParameterBufferVP9{ seqParam: C.VAEncSequenceParameterBufferVP9{
max_frame_width: 8192, max_frame_width: 8192,
max_frame_height: 8192, max_frame_height: 8192,
bits_per_second: C.uint(cp.RateControl.BitsPerSecond), bits_per_second: C.uint(params.RateControl.BitsPerSecond),
intra_period: C.uint(p.KeyFrameInterval), intra_period: C.uint(params.KeyFrameInterval),
kf_min_dist: 1, kf_min_dist: 1,
kf_max_dist: C.uint(p.KeyFrameInterval), kf_max_dist: C.uint(params.KeyFrameInterval),
}, },
picParam: C.VAEncPictureParameterBufferVP9{ picParam: C.VAEncPictureParameterBufferVP9{
reference_frames: [8]C.VASurfaceID{ reference_frames: [8]C.VASurfaceID{
@@ -188,10 +161,10 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeHRD, _type: C.VAEncMiscParameterTypeHRD,
}, },
data: C.VAEncMiscParameterHRD{ data: C.VAEncMiscParameterHRD{
initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond * initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 2000), params.RateControl.WindowSize / 2000),
buffer_size: C.uint(cp.RateControl.BitsPerSecond * buffer_size: C.uint(params.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 1000), params.RateControl.WindowSize / 1000),
}, },
}, },
frParam: frParam{ frParam: frParam{
@@ -207,12 +180,12 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeRateControl, _type: C.VAEncMiscParameterTypeRateControl,
}, },
data: C.VAEncMiscParameterRateControl{ data: C.VAEncMiscParameterRateControl{
window_size: C.uint(cp.RateControl.WindowSize), window_size: C.uint(params.RateControl.WindowSize),
initial_qp: C.uint(cp.RateControl.InitialQP), initial_qp: C.uint(params.RateControl.InitialQP),
min_qp: C.uint(cp.RateControl.MinQP), min_qp: C.uint(params.RateControl.MinQP),
max_qp: C.uint(cp.RateControl.MaxQP), max_qp: C.uint(params.RateControl.MaxQP),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond), bits_per_second: C.uint(params.RateControl.BitsPerSecond),
target_percentage: C.uint(cp.RateControl.TargetPercentage), target_percentage: C.uint(params.RateControl.TargetPercentage),
}, },
}, },
} }
@@ -267,11 +240,11 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 { if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
return nil, errors.New("the hardware encoder doesn't support YUV420") return nil, errors.New("the hardware encoder doesn't support YUV420")
} }
if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 { if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 {
return nil, errors.New("the hardware encoder doesn't support specified rate control mode") return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
} }
confAttrs[0].value = C.VA_RT_FORMAT_YUV420 confAttrs[0].value = C.VA_RT_FORMAT_YUV420
confAttrs[1].value = C.uint(cp.RateControlMode) confAttrs[1].value = C.uint(params.RateControlMode)
if s := C.vaCreateConfig( if s := C.vaCreateConfig(
e.display, e.display,
@@ -334,7 +307,7 @@ func (e *encoderVP9) Read(p []byte) (int, error) {
} }
yuvImg := img.(*image.YCbCr) yuvImg := img.(*image.YCbCr)
kf := e.frameCnt%e.prop.KeyFrameInterval == 0 kf := e.frameCnt%e.params.KeyFrameInterval == 0
e.frameCnt++ e.frameCnt++
e.frParam.data.framerate = C.uint(e.rate.Calc()) e.frParam.data.framerate = C.uint(e.rate.Calc())

View File

@@ -1,8 +1,13 @@
package vpx package vpx
import (
"github.com/pion/mediadevices/pkg/codec"
)
// Params stores libvpx specific encoding parameters. // Params stores libvpx specific encoding parameters.
// Value range is codec (VP8/VP9) specific. // Value range is codec (VP8/VP9) specific.
type Params struct { type Params struct {
codec.BaseParams
RateControlEndUsage RateControlMode RateControlEndUsage RateControlMode
RateControlUndershootPercent uint RateControlUndershootPercent uint
RateControlOvershootPercent uint RateControlOvershootPercent uint

View File

@@ -55,7 +55,6 @@ import (
"time" "time"
"unsafe" "unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
@@ -78,28 +77,61 @@ type encoder struct {
closed bool closed bool
} }
func init() { // VP8Params is codec specific paramaters
codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder)) type VP8Params struct {
codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder)) Params
} }
// NewVP8Encoder creates new VP8 encoder // NewVP8Params returns default VP8 codec specific parameters.
func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { func NewVP8Params() (VP8Params, error) {
return newEncoder(r, p, C.ifaceVP8()) p, err := newParams(C.ifaceVP8())
if err != nil {
return VP8Params{}, err
}
return VP8Params{
Params: p,
}, nil
} }
// NewVP9Encoder creates new VP9 encoder // Name represents the codec name
func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { func (p *VP8Params) Name() string {
return newEncoder(r, p, C.ifaceVP9()) return webrtc.VP8
} }
// NewVP8Param returns default VP8 codec specific parameters. // BuildVideoEncoder builds VP8 encoder with given params
func NewVP8Param() (Params, error) { return newParam(C.ifaceVP8()) } func (p *VP8Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, p.Params, C.ifaceVP8())
}
// NewVP9Param returns default VP9 codec specific parameters. // VP9Params is codec specific paramaters
func NewVP9Param() (Params, error) { return newParam(C.ifaceVP9()) } type VP9Params struct {
Params
}
func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) { // NewVP9Params returns default VP9 codec specific parameters.
func NewVP9Params() (VP9Params, error) {
p, err := newParams(C.ifaceVP9())
if err != nil {
return VP9Params{}, err
}
return VP9Params{
Params: p,
}, nil
}
// Name represents the codec name
func (p *VP9Params) Name() string {
return webrtc.VP9
}
// BuildVideoEncoder builds VP9 encoder with given params
func (p *VP9Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, p.Params, C.ifaceVP9())
}
func newParams(codecIface *C.vpx_codec_iface_t) (Params, error) {
cfg := &C.vpx_codec_enc_cfg_t{} cfg := &C.vpx_codec_enc_cfg_t{}
if ec := C.vpx_codec_enc_config_default(codecIface, cfg, 0); ec != 0 { if ec := C.vpx_codec_enc_config_default(codecIface, cfg, 0); ec != 0 {
return Params{}, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec) return Params{}, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec)
@@ -113,13 +145,13 @@ func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) {
}, nil }, nil
} }
func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) { func newEncoder(r video.Reader, p prop.Media, params Params, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) {
if p.BitRate == 0 { if params.BitRate == 0 {
p.BitRate = 100000 params.BitRate = 100000
} }
if p.KeyFrameInterval == 0 { if params.KeyFrameInterval == 0 {
p.KeyFrameInterval = 60 params.KeyFrameInterval = 60
} }
cfg := &C.vpx_codec_enc_cfg_t{} cfg := &C.vpx_codec_enc_cfg_t{}
@@ -127,24 +159,18 @@ func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) (
return nil, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec) return nil, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec)
} }
switch cp := p.CodecParams.(type) { cfg.rc_end_usage = uint32(params.RateControlEndUsage)
case nil: cfg.rc_undershoot_pct = C.uint(params.RateControlUndershootPercent)
case Params: cfg.rc_overshoot_pct = C.uint(params.RateControlOvershootPercent)
cfg.rc_end_usage = uint32(cp.RateControlEndUsage) cfg.rc_min_quantizer = C.uint(params.RateControlMinQuantizer)
cfg.rc_undershoot_pct = C.uint(cp.RateControlUndershootPercent) cfg.rc_max_quantizer = C.uint(params.RateControlMaxQuantizer)
cfg.rc_overshoot_pct = C.uint(cp.RateControlOvershootPercent)
cfg.rc_min_quantizer = C.uint(cp.RateControlMinQuantizer)
cfg.rc_max_quantizer = C.uint(cp.RateControlMaxQuantizer)
default:
return nil, errors.New("unsupported CodecParams type")
}
cfg.g_w = C.uint(p.Width) cfg.g_w = C.uint(p.Width)
cfg.g_h = C.uint(p.Height) cfg.g_h = C.uint(p.Height)
cfg.g_timebase.num = 1 cfg.g_timebase.num = 1
cfg.g_timebase.den = 1000 cfg.g_timebase.den = 1000
cfg.rc_target_bitrate = C.uint(p.BitRate) / 1000 cfg.rc_target_bitrate = C.uint(params.BitRate) / 1000
cfg.kf_max_dist = C.uint(p.KeyFrameInterval) cfg.kf_max_dist = C.uint(params.KeyFrameInterval)
cfg.rc_resize_allowed = 0 cfg.rc_resize_allowed = 0
cfg.g_pass = C.VPX_RC_ONE_PASS cfg.g_pass = C.VPX_RC_ONE_PASS

View File

@@ -1,7 +1,18 @@
package x264 package x264
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores libx264 specific encoding parameters. // Params stores libx264 specific encoding parameters.
type Params struct { type Params struct {
codec.BaseParams
// Faster preset has lower CPU usage but lower quality // Faster preset has lower CPU usage but lower quality
Preset Preset Preset Preset
} }
@@ -21,3 +32,22 @@ const (
PresetVeryslow PresetVeryslow
PresetPlacebo PresetPlacebo
) )
// NewParams returns default x264 codec specific parameters.
func NewParams() (Params, error) {
return Params{
BaseParams: codec.BaseParams{
KeyFrameInterval: 60,
},
}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.H264
}
// BuildVideoEncoder builds x264 encoder with given params
func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

View File

@@ -7,18 +7,15 @@ package x264
// #include "bridge.h" // #include "bridge.h"
import "C" import "C"
import ( import (
"errors"
"fmt" "fmt"
"image" "image"
"io" "io"
"sync" "sync"
"unsafe" "unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
) )
type encoder struct { type encoder struct {
@@ -64,37 +61,24 @@ var (
errEncode = fmt.Errorf("failed to encode") errEncode = fmt.Errorf("failed to encode")
) )
func init() { func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
codec.Register(webrtc.H264, codec.VideoEncoderBuilder(newEncoder)) if params.KeyFrameInterval == 0 {
} params.KeyFrameInterval = 60
func newEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.KeyFrameInterval == 0 {
p.KeyFrameInterval = 60
}
var preset Preset
switch cp := p.CodecParams.(type) {
case nil:
case Params:
preset = cp.Preset
default:
return nil, errors.New("unsupported CodecParams type")
} }
param := C.x264_param_t{ param := C.x264_param_t{
i_csp: C.X264_CSP_I420, i_csp: C.X264_CSP_I420,
i_width: C.int(p.Width), i_width: C.int(p.Width),
i_height: C.int(p.Height), i_height: C.int(p.Height),
i_keyint_max: C.int(p.KeyFrameInterval), i_keyint_max: C.int(params.KeyFrameInterval),
} }
param.rc.i_bitrate = C.int(p.BitRate) param.rc.i_bitrate = C.int(params.BitRate)
param.rc.i_vbv_max_bitrate = param.rc.i_bitrate param.rc.i_vbv_max_bitrate = param.rc.i_bitrate
param.rc.i_vbv_buffer_size = param.rc.i_vbv_max_bitrate * 2 param.rc.i_vbv_buffer_size = param.rc.i_vbv_max_bitrate * 2
var rc C.int var rc C.int
// cPreset will be freed in C.enc_new // cPreset will be freed in C.enc_new
cPreset := C.CString(fmt.Sprint(preset)) cPreset := C.CString(fmt.Sprint(params.Preset))
engine := C.enc_new(param, cPreset, &rc) engine := C.enc_new(param, cPreset, &rc)
if err := errFromC(rc); err != nil { if err := errFromC(rc); err != nil {
return nil, err return nil, err

View File

@@ -14,7 +14,6 @@ type Media struct {
DeviceID string DeviceID string
Video Video
Audio Audio
Codec
} }
// Merge merges all the field values from o to p, except zero values. // Merge merges all the field values from o to p, except zero values.
@@ -111,17 +110,3 @@ type Audio struct {
SampleRate int SampleRate int
SampleSize int SampleSize int
} }
// Codec represents an codec's encoding properties
type Codec struct {
CodecName string
// Target bitrate in bps.
BitRate int
// Expected interval of the keyframes in frames.
KeyFrameInterval int
// Library specific parameter struct defined by each codec package
CodecParams interface{}
}

124
track.go
View File

@@ -6,7 +6,6 @@ import (
"math/rand" "math/rand"
"sync" "sync"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/driver" "github.com/pion/mediadevices/pkg/driver"
mio "github.com/pion/mediadevices/pkg/io" mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2"
@@ -42,16 +41,9 @@ type track struct {
endOnce sync.Once endOnce sync.Once
} }
func newTrack(codecs []*webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver, codecName string) (*track, error) { func newTrack(selectedCodec *webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver) (*track, error) {
var selectedCodec *webrtc.RTPCodec
for _, c := range codecs {
if c.Name == codecName {
selectedCodec = c
break
}
}
if selectedCodec == nil { if selectedCodec == nil {
return nil, fmt.Errorf("track: %s is not registered in media engine", codecName) panic("codec is required")
} }
t, err := trackGenerator( t, err := trackGenerator(
@@ -116,13 +108,7 @@ type videoTrack struct {
var _ Tracker = &videoTrack{} var _ Tracker = &videoTrack{}
func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*videoTrack, error) { func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*videoTrack, error) {
codecName := constraints.CodecName err := d.Open()
t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeVideo], opts.trackGenerator, d, codecName)
if err != nil {
return nil, err
}
err = d.Open()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -137,21 +123,47 @@ func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints Media
r = constraints.VideoTransform(r) r = constraints.VideoTransform(r)
} }
encoder, err := codec.BuildVideoEncoder(r, constraints.Media) var vt *videoTrack
if err != nil { rtpCodecs := opts.codecs[webrtc.RTPCodecTypeVideo]
_ = d.Close() for _, codecBuilder := range constraints.VideoEncoderBuilders {
return nil, err var matchedRTPCodec *webrtc.RTPCodec
for _, rtpCodec := range rtpCodecs {
if rtpCodec.Name == codecBuilder.Name() {
matchedRTPCodec = rtpCodec
break
}
}
if matchedRTPCodec == nil {
continue
}
t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d)
if err != nil {
continue
}
encoder, err := codecBuilder.BuildVideoEncoder(r, constraints.Media)
if err != nil {
continue
}
vt = &videoTrack{
track: t,
d: d,
constraints: constraints,
encoder: encoder,
}
break
} }
vt := videoTrack{ if vt == nil {
track: t, d.Close()
d: d, return nil, fmt.Errorf("failed to find a matching video codec")
constraints: constraints,
encoder: encoder,
} }
go vt.start() go vt.start()
return &vt, nil return vt, nil
} }
func (vt *videoTrack) start() { func (vt *videoTrack) start() {
@@ -192,41 +204,61 @@ type audioTrack struct {
var _ Tracker = &audioTrack{} var _ Tracker = &audioTrack{}
func newAudioTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*audioTrack, error) { func newAudioTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*audioTrack, error) {
codecName := constraints.CodecName err := d.Open()
t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeAudio], opts.trackGenerator, d, codecName)
if err != nil {
return nil, err
}
err = d.Open()
if err != nil { if err != nil {
return nil, err return nil, err
} }
ar := d.(driver.AudioRecorder) ar := d.(driver.AudioRecorder)
reader, err := ar.AudioRecord(constraints.Media) r, err := ar.AudioRecord(constraints.Media)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if constraints.AudioTransform != nil { if constraints.AudioTransform != nil {
reader = constraints.AudioTransform(reader) r = constraints.AudioTransform(r)
} }
encoder, err := codec.BuildAudioEncoder(reader, constraints.Media) var at *audioTrack
if err != nil { rtpCodecs := opts.codecs[webrtc.RTPCodecTypeAudio]
_ = d.Close() for _, codecBuilder := range constraints.AudioEncoderBuilders {
return nil, err var matchedRTPCodec *webrtc.RTPCodec
for _, rtpCodec := range rtpCodecs {
if rtpCodec.Name == codecBuilder.Name() {
matchedRTPCodec = rtpCodec
break
}
}
if matchedRTPCodec == nil {
continue
}
t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d)
if err != nil {
continue
}
encoder, err := codecBuilder.BuildAudioEncoder(r, constraints.Media)
if err != nil {
continue
}
at = &audioTrack{
track: t,
d: d,
constraints: constraints,
encoder: encoder,
}
} }
at := audioTrack{ if at == nil {
track: t, d.Close()
d: d, return nil, fmt.Errorf("failed to find a matching audio codec")
constraints: constraints,
encoder: encoder,
} }
go at.start() go at.start()
return &at, nil return at, nil
} }
func (t *audioTrack) start() { func (t *audioTrack) start() {