diff --git a/examples/codecparam/main.go b/examples/codecparam/main.go index c254079..b57a16e 100644 --- a/examples/codecparam/main.go +++ b/examples/codecparam/main.go @@ -5,18 +5,16 @@ import ( "github.com/pion/mediadevices" "github.com/pion/mediadevices/examples/internal/signal" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/webrtc/v2" - _ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder - "github.com/pion/mediadevices/pkg/codec/vaapi" // This is required to register VP8/VP9 video encoder + "github.com/pion/mediadevices/pkg/codec/vaapi" // This is required to register VP8/VP9 video encoder - // Note: If you don't have a camera or microphone or your adapters are not supported, + // Note: If you don't have a camera or your adapters are not supported, // you can always swap your adapters with our dummy adapters below. // _ "github.com/pion/mediadevices/pkg/driver/videotest" - // _ "github.com/pion/mediadevices/pkg/driver/audiotest" - _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter - _ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter + _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter ) func main() { @@ -52,13 +50,7 @@ func main() { md := mediadevices.NewMediaDevices(peerConnection) s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ - Audio: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = webrtc.Opus - c.Enabled = true - c.BitRate = 32000 // 32kbps - }, Video: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = webrtc.VP8 c.FrameFormat = frame.FormatYUY2 c.Enabled = true c.Width = 640 @@ -77,7 +69,7 @@ func main() { cp.RateControlMode = vaapi.RateControlVBR cp.RateControl.BitsPerSecond = 400000 cp.RateControl.TargetPercentage = 95 - c.CodecParams = cp + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&cp} }, }) if err != nil { diff --git a/examples/rtp-send/main.go b/examples/rtp-send/main.go index 8a820cb..da71720 100644 --- a/examples/rtp-send/main.go +++ b/examples/rtp-send/main.go @@ -6,9 +6,9 @@ import ( "os" "github.com/pion/mediadevices" - _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder - _ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder - _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder + _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter "github.com/pion/mediadevices/pkg/frame" "github.com/pion/rtp" "github.com/pion/webrtc/v2" @@ -16,8 +16,7 @@ import ( ) const ( - videoCodecName = webrtc.VP8 - mtu = 1000 + mtu = 1000 ) func main() { @@ -41,14 +40,19 @@ func main() { ), ) - _, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ + vp8Params, err := vpx.NewVP8Params() + if err != nil { + panic(err) + } + vp8Params.BitRate = 100000 // 100kbps + + _, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{ Video: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = videoCodecName c.FrameFormat = frame.FormatYUY2 c.Enabled = true c.Width = 640 c.Height = 480 - c.BitRate = 100000 // 100kbps + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params} }, }) if err != nil { diff --git a/examples/screenshare/main.go b/examples/screenshare/main.go index c19ac12..7fdda32 100644 --- a/examples/screenshare/main.go +++ b/examples/screenshare/main.go @@ -5,18 +5,13 @@ import ( "github.com/pion/mediadevices" "github.com/pion/mediadevices/examples/internal/signal" - _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder - _ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder - _ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder - _ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder + _ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/webrtc/v2" ) -const ( - videoCodecName = webrtc.VP8 -) - func main() { config := webrtc.Configuration{ ICEServers: []webrtc.ICEServer{ @@ -49,12 +44,17 @@ func main() { md := mediadevices.NewMediaDevices(peerConnection) + vp8Params, err := vpx.NewVP8Params() + if err != nil { + panic(err) + } + vp8Params.BitRate = 100000 // 100kbps + s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{ Video: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = videoCodecName c.Enabled = true - c.BitRate = 100000 // 100kbps c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params} }, }) if err != nil { diff --git a/examples/simple/main.go b/examples/simple/main.go index fea2490..a2d50e1 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -5,16 +5,18 @@ import ( "github.com/pion/mediadevices" "github.com/pion/mediadevices/examples/internal/signal" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/webrtc/v2" - _ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder + // This is required to register opus audio encoder + "github.com/pion/mediadevices/pkg/codec/opus" // If you don't like vpx, you can also use x264 by importing as below // _ "github.com/pion/mediadevices/pkg/codec/x264" // This is required to register h264 video encoder // or you can also use openh264 for alternative h264 implementation // _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder - _ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder + "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder // Note: If you don't have a camera or microphone or your adapters are not supported, // you can always swap your adapters with our dummy adapters below. @@ -60,19 +62,29 @@ func main() { md := mediadevices.NewMediaDevices(peerConnection) + opusParams, err := opus.NewParams() + if err != nil { + panic(err) + } + opusParams.BitRate = 32000 // 32kbps + + vp8Params, err := vpx.NewVP8Params() + if err != nil { + panic(err) + } + vp8Params.BitRate = 100000 // 100kbps + s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ Audio: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = webrtc.Opus c.Enabled = true - c.BitRate = 32000 // 32kbps + c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams} }, Video: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = videoCodecName c.FrameFormat = frame.FormatYUY2 c.Enabled = true c.Width = 640 c.Height = 480 - c.BitRate = 100000 // 100kbps + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params} }, }) if err != nil { diff --git a/examples/transform/main.go b/examples/transform/main.go index 36183b6..d150487 100644 --- a/examples/transform/main.go +++ b/examples/transform/main.go @@ -7,17 +7,14 @@ import ( "github.com/disintegration/imaging" "github.com/pion/mediadevices" "github.com/pion/mediadevices/examples/internal/signal" - _ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder _ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/webrtc/v2" ) -const ( - videoCodecName = webrtc.VP8 -) - func rotate180(r video.Reader) video.Reader { return video.ReaderFunc(func() (img image.Image, err error) { img, err = r.Read() @@ -62,15 +59,20 @@ func main() { md := mediadevices.NewMediaDevices(peerConnection) + vp8Params, err := vpx.NewVP8Params() + if err != nil { + panic(err) + } + vp8Params.BitRate = 100000 // 100kbps + s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ Video: func(c *mediadevices.MediaTrackConstraints) { - c.CodecName = videoCodecName c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420 c.Enabled = true c.Width = 640 c.Height = 480 - c.BitRate = 100000 // 100kbps c.VideoTransform = rotate180 + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params} }, }) if err != nil { diff --git a/mediadevices_test.go b/mediadevices_test.go index 8ef1239..6bc18d1 100644 --- a/mediadevices_test.go +++ b/mediadevices_test.go @@ -18,23 +18,18 @@ import ( ) func TestGetUserMedia(t *testing.T) { - codec.Register("MockVideo", codec.VideoEncoderBuilder(func(r video.Reader, p prop.Media) (io.ReadCloser, error) { - if p.BitRate == 0 { - // This is a dummy error to test the failure condition. - return nil, errors.New("wrong codec parameter") - } - return &mockVideoCodec{ - r: r, - closed: make(chan struct{}), - }, nil - })) - codec.Register("MockAudio", codec.AudioEncoderBuilder(func(r audio.Reader, p prop.Media) (io.ReadCloser, error) { - return &mockAudioCodec{ - r: r, - closed: make(chan struct{}), - }, nil - })) - + videoParams := mockParams{ + BaseParams: codec.BaseParams{ + BitRate: 100000, + }, + name: "MockVideo", + } + audioParams := mockParams{ + BaseParams: codec.BaseParams{ + BitRate: 32000, + }, + name: "MockAudio", + } md := NewMediaDevicesFromCodecs( map[webrtc.RTPCodecType][]*webrtc.RTPCodec{ webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{ @@ -54,30 +49,31 @@ func TestGetUserMedia(t *testing.T) { ) constraints := MediaStreamConstraints{ Video: func(c *MediaTrackConstraints) { - c.CodecName = "MockVideo" c.Enabled = true c.Width = 640 c.Height = 480 - c.BitRate = 100000 + params := videoParams + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{¶ms} }, Audio: func(c *MediaTrackConstraints) { - c.CodecName = "MockAudio" c.Enabled = true - c.BitRate = 32000 + params := audioParams + c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{¶ms} }, } constraintsWrong := MediaStreamConstraints{ Video: func(c *MediaTrackConstraints) { - c.CodecName = "MockVideo" c.Enabled = true c.Width = 640 c.Height = 480 - c.BitRate = 0 + params := videoParams + params.BitRate = 0 + c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{¶ms} }, Audio: func(c *MediaTrackConstraints) { - c.CodecName = "MockAudio" c.Enabled = true - c.BitRate = 32000 + params := audioParams + c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{¶ms} }, } @@ -156,6 +152,33 @@ func (t *mockTrack) Kind() webrtc.RTPCodecType { return t.codec.Type } +type mockParams struct { + codec.BaseParams + name string +} + +func (params *mockParams) Name() string { + return params.name +} + +func (params *mockParams) BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { + if params.BitRate == 0 { + // This is a dummy error to test the failure condition. + return nil, errors.New("wrong codec parameter") + } + return &mockVideoCodec{ + r: r, + closed: make(chan struct{}), + }, nil +} + +func (params *mockParams) BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) { + return &mockAudioCodec{ + r: r, + closed: make(chan struct{}), + }, nil +} + type mockVideoCodec struct { r video.Reader closed chan struct{} diff --git a/mediastreamconstraints.go b/mediastreamconstraints.go index a91be4b..c01c911 100644 --- a/mediastreamconstraints.go +++ b/mediastreamconstraints.go @@ -1,6 +1,7 @@ package mediadevices import ( + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" @@ -15,6 +16,18 @@ type MediaStreamConstraints struct { type MediaTrackConstraints struct { prop.Media Enabled bool + // VideoEncoderBuilders are codec builders that are used for encoding the video + // and later being used for sending the appropriate RTP payload type. + // + // If one encoder builder fails to build the codec, the next builder will be used, + // repeating until a codec builds. If no builders build successfully, an error is returned. + VideoEncoderBuilders []codec.VideoEncoderBuilder + // AudioEncoderBuilders are codec builders that are used for encoding the audio + // and later being used for sending the appropriate RTP payload type. + // + // If one encoder builder fails to build the codec, the next builder will be used, + // repeating until a codec builds. If no builders build successfully, an error is returned. + AudioEncoderBuilders []codec.AudioEncoderBuilder // VideoTransform will be used to transform the video that's coming from the driver. // So, basically it'll look like following: driver -> VideoTransform -> codec VideoTransform video.TransformFunc diff --git a/pkg/codec/codec.go b/pkg/codec/codec.go index 0d83f8f..252a873 100644 --- a/pkg/codec/codec.go +++ b/pkg/codec/codec.go @@ -8,5 +8,35 @@ import ( "github.com/pion/mediadevices/pkg/prop" ) -type VideoEncoderBuilder func(r video.Reader, p prop.Media) (io.ReadCloser, error) -type AudioEncoderBuilder func(r audio.Reader, p prop.Media) (io.ReadCloser, error) +// AudioEncoderBuilder is the interface that wraps basic operations that are +// necessary to build the audio encoder. +// +// This interface is for codec implementors to provide codec specific params, +// but still giving generality for the users. +type AudioEncoderBuilder interface { + // Name represents the codec name + Name() string + // BuildAudioEncoder builds audio encoder by given media params and audio input + BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) +} + +// VideoEncoderBuilder is the interface that wraps basic operations that are +// necessary to build the video encoder. +// +// This interface is for codec implementors to provide codec specific params, +// but still giving generality for the users. +type VideoEncoderBuilder interface { + // Name represents the codec name + Name() string + // BuildVideoEncoder builds video encoder by given media params and video input + BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) +} + +// BaseParams represents an codec's encoding properties +type BaseParams struct { + // Target bitrate in bps. + BitRate int + + // Expected interval of the keyframes in frames. + KeyFrameInterval int +} diff --git a/pkg/codec/openh264/openh264.go b/pkg/codec/openh264/openh264.go index 6c9ee83..69dd3c3 100644 --- a/pkg/codec/openh264/openh264.go +++ b/pkg/codec/openh264/openh264.go @@ -9,19 +9,15 @@ package openh264 import "C" import ( - "errors" "fmt" "image" "io" "sync" "unsafe" - "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" - - "github.com/pion/webrtc/v2" ) type encoder struct { @@ -33,26 +29,16 @@ type encoder struct { closed bool } -var _ codec.VideoEncoderBuilder = codec.VideoEncoderBuilder(NewEncoder) - -func init() { - codec.Register(webrtc.H264, codec.VideoEncoderBuilder(NewEncoder)) -} - -func NewEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { - if p.BitRate == 0 { - p.BitRate = 100000 - } - - if p.CodecParams != nil { - return nil, errors.New("unsupported CodecParams type") +func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) { + if params.BitRate == 0 { + params.BitRate = 100000 } var rv C.int cEncoder := C.enc_new(C.EncoderOptions{ width: C.int(p.Width), height: C.int(p.Height), - target_bitrate: C.int(p.BitRate), + target_bitrate: C.int(params.BitRate), max_fps: C.float(p.FrameRate), }, &rv) if err := errResult(rv); err != nil { diff --git a/pkg/codec/openh264/params.go b/pkg/codec/openh264/params.go new file mode 100644 index 0000000..0a2e962 --- /dev/null +++ b/pkg/codec/openh264/params.go @@ -0,0 +1,34 @@ +package openh264 + +import ( + "io" + + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" + "github.com/pion/webrtc/v2" +) + +// Params stores libopenh264 specific encoding parameters. +type Params struct { + codec.BaseParams +} + +// NewParams returns default openh264 codec specific parameters. +func NewParams() (Params, error) { + return Params{ + BaseParams: codec.BaseParams{ + BitRate: 100000, + }, + }, nil +} + +// Name represents the codec name +func (p *Params) Name() string { + return webrtc.H264 +} + +// BuildVideoEncoder builds openh264 encoder with given params +func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newEncoder(r, property, *p) +} diff --git a/pkg/codec/opus/opus.go b/pkg/codec/opus/opus.go index fdacd94..5dd1b01 100644 --- a/pkg/codec/opus/opus.go +++ b/pkg/codec/opus/opus.go @@ -1,7 +1,6 @@ package opus import ( - "errors" "fmt" "io" "math" @@ -9,10 +8,8 @@ import ( "unsafe" "github.com/lherman-cs/opus" - "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/prop" - "github.com/pion/webrtc/v2" ) type encoder struct { @@ -24,13 +21,8 @@ type encoder struct { var latencies = []float64{5, 10, 20, 40, 60} var _ io.ReadCloser = &encoder{} -var _ codec.AudioEncoderBuilder = codec.AudioEncoderBuilder(NewEncoder) -func init() { - codec.Register(webrtc.Opus, codec.AudioEncoderBuilder(NewEncoder)) -} - -func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) { +func newEncoder(r audio.Reader, p prop.Media, params Params) (io.ReadCloser, error) { if p.SampleRate == 0 { return nil, fmt.Errorf("opus: inProp.SampleRate is required") } @@ -39,12 +31,8 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) { p.Latency = 20 } - if p.BitRate == 0 { - p.BitRate = 32000 - } - - if p.CodecParams != nil { - return nil, errors.New("unsupported CodecParams type") + if params.BitRate == 0 { + params.BitRate = 32000 } // Select the nearest supported latency @@ -69,7 +57,7 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) { if err != nil { return nil, err } - if err := engine.SetBitrate(p.BitRate); err != nil { + if err := engine.SetBitrate(params.BitRate); err != nil { return nil, err } diff --git a/pkg/codec/opus/params.go b/pkg/codec/opus/params.go new file mode 100644 index 0000000..fad3440 --- /dev/null +++ b/pkg/codec/opus/params.go @@ -0,0 +1,30 @@ +package opus + +import ( + "io" + + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/io/audio" + "github.com/pion/mediadevices/pkg/prop" + "github.com/pion/webrtc/v2" +) + +// Params stores opus specific encoding parameters. +type Params struct { + codec.BaseParams +} + +// NewParams returns default opus codec specific parameters. +func NewParams() (Params, error) { + return Params{}, nil +} + +// Name represents the codec name +func (p *Params) Name() string { + return webrtc.Opus +} + +// BuildVideoEncoder builds x264 encoder with given params +func (p *Params) BuildAudioEncoder(r audio.Reader, property prop.Media) (io.ReadCloser, error) { + return newEncoder(r, property, *p) +} diff --git a/pkg/codec/registrar.go b/pkg/codec/registrar.go deleted file mode 100644 index eb05321..0000000 --- a/pkg/codec/registrar.go +++ /dev/null @@ -1,42 +0,0 @@ -package codec - -import ( - "fmt" - "io" - - "github.com/pion/mediadevices/pkg/io/audio" - "github.com/pion/mediadevices/pkg/io/video" - "github.com/pion/mediadevices/pkg/prop" -) - -var ( - videoEncoders = make(map[string]VideoEncoderBuilder) - audioEncoders = make(map[string]AudioEncoderBuilder) -) - -func Register(name string, builder interface{}) { - switch b := builder.(type) { - case VideoEncoderBuilder: - videoEncoders[name] = b - case AudioEncoderBuilder: - audioEncoders[name] = b - } -} - -func BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { - b, ok := videoEncoders[p.CodecName] - if !ok { - return nil, fmt.Errorf("codec: can't find %s video encoder", p.CodecName) - } - - return b(r, p) -} - -func BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) { - b, ok := audioEncoders[p.CodecName] - if !ok { - return nil, fmt.Errorf("codec: can't find %s audio encoder", p.CodecName) - } - - return b(r, p) -} diff --git a/pkg/codec/vaapi/params.go b/pkg/codec/vaapi/params.go index 77cf91a..d8425a8 100644 --- a/pkg/codec/vaapi/params.go +++ b/pkg/codec/vaapi/params.go @@ -1,7 +1,17 @@ package vaapi +import ( + "io" + + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" + "github.com/pion/webrtc/v2" +) + // ParamVP8 stores VP8 encoding parameters. type ParamVP8 struct { + codec.BaseParams Sequence SequenceParamVP8 RateControlMode RateControlMode RateControl RateControlParam @@ -14,8 +24,38 @@ type SequenceParamVP8 struct { ClampQindexLow uint } +// NewVP8Param returns default parameters of VP8 codec. +func NewVP8Param() (ParamVP8, error) { + return ParamVP8{ + Sequence: SequenceParamVP8{ + ClampQindexLow: 9, + ClampQindexHigh: 127, + }, + RateControlMode: RateControlVBR, + RateControl: RateControlParam{ + BitsPerSecond: 400000, + TargetPercentage: 80, + WindowSize: 1500, + InitialQP: 60, + MinQP: 9, + MaxQP: 127, + }, + }, nil +} + +// Name represents the codec name +func (p *ParamVP8) Name() string { + return webrtc.VP8 +} + +// BuildVideoEncoder builds VP8 encoder with given params +func (p *ParamVP8) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newVP8Encoder(r, property, *p) +} + // ParamVP9 represents VAEncSequenceParameterBufferVP9 and other parameter buffers. type ParamVP9 struct { + codec.BaseParams RateControlMode RateControlMode RateControl RateControlParam } @@ -52,3 +92,28 @@ const ( RateControlQVBR RateControlMode = 0x00000400 RateControlAVBR RateControlMode = 0x00000800 ) + +// NewVP9Param returns default parameters of VP9 codec. +func NewVP9Param() (ParamVP9, error) { + return ParamVP9{ + RateControlMode: RateControlVBR, + RateControl: RateControlParam{ + BitsPerSecond: 400000, + TargetPercentage: 80, + WindowSize: 1500, + InitialQP: 60, + MinQP: 9, + MaxQP: 127, + }, + }, nil +} + +// Name represents the codec name +func (p *ParamVP9) Name() string { + return webrtc.VP9 +} + +// BuildVideoEncoder builds VP9 encoder with given params +func (p *ParamVP9) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newVP9Encoder(r, property, *p) +} diff --git a/pkg/codec/vaapi/vp8.go b/pkg/codec/vaapi/vp8.go index 625447a..76d4c9b 100644 --- a/pkg/codec/vaapi/vp8.go +++ b/pkg/codec/vaapi/vp8.go @@ -50,12 +50,9 @@ import ( "sync" "unsafe" - "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" - - "github.com/pion/webrtc/v2" ) const ( @@ -86,6 +83,7 @@ type encoderVP8 struct { frameCnt int prop prop.Media + params ParamVP8 rate *framerateDetector @@ -93,66 +91,37 @@ type encoderVP8 struct { closed bool } -func init() { - codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder)) -} - -// NewVP8Param returns default parameters of VP8 codec. -func NewVP8Param() (ParamVP8, error) { - return ParamVP8{ - Sequence: SequenceParamVP8{ - ClampQindexLow: 9, - ClampQindexHigh: 127, - }, - RateControlMode: RateControlVBR, - RateControl: RateControlParam{ - BitsPerSecond: 400000, - TargetPercentage: 80, - WindowSize: 1500, - InitialQP: 60, - MinQP: 9, - MaxQP: 127, - }, - }, nil -} - -// NewVP8Encoder creates new VP8 encoder -func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { +// newVP8Encoder creates new VP8 encoder +func newVP8Encoder(r video.Reader, p prop.Media, params ParamVP8) (io.ReadCloser, error) { if p.Width%16 != 0 || p.Width == 0 { return nil, errors.New("width must be 16*n") } if p.Height%16 != 0 || p.Height == 0 { return nil, errors.New("height must be 16*n") } - if p.KeyFrameInterval == 0 { - p.KeyFrameInterval = 30 + if params.KeyFrameInterval == 0 { + params.KeyFrameInterval = 30 } if p.FrameRate == 0 { p.FrameRate = 30 } - var cp ParamVP8 - switch params := p.Codec.CodecParams.(type) { - case nil: - cp, _ = NewVP8Param() - cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5) - case ParamVP8: - cp = params - default: - return nil, errors.New("unsupported CodecParams type") + if params.RateControl.BitsPerSecond == 0 { + params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5) } // Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp8enc.c e := &encoderVP8{ - r: video.ToI420(r), - prop: p, - rate: newFramerateDetector(uint32(p.FrameRate)), + r: video.ToI420(r), + prop: p, + params: params, + rate: newFramerateDetector(uint32(p.FrameRate)), seqParam: C.VAEncSequenceParameterBufferVP8{ frame_width: C.uint(p.Width), frame_height: C.uint(p.Height), - bits_per_second: C.uint(cp.RateControl.BitsPerSecond), - intra_period: C.uint(p.KeyFrameInterval), - kf_max_dist: C.uint(p.KeyFrameInterval), + bits_per_second: C.uint(params.RateControl.BitsPerSecond), + intra_period: C.uint(params.KeyFrameInterval), + kf_max_dist: C.uint(params.KeyFrameInterval), reference_frames: [4]C.VASurfaceID{ C.VA_INVALID_ID, C.VA_INVALID_ID, @@ -165,8 +134,8 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { ref_gf_frame: C.VA_INVALID_SURFACE, ref_arf_frame: C.VA_INVALID_SURFACE, reconstructed_frame: C.VA_INVALID_SURFACE, - clamp_qindex_low: C.uint8_t(cp.Sequence.ClampQindexLow), - clamp_qindex_high: C.uint8_t(cp.Sequence.ClampQindexHigh), + clamp_qindex_low: C.uint8_t(params.Sequence.ClampQindexLow), + clamp_qindex_high: C.uint8_t(params.Sequence.ClampQindexHigh), loop_filter_level: [4]C.int8_t{ 19, 19, 19, 19, }, @@ -184,10 +153,10 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { _type: C.VAEncMiscParameterTypeHRD, }, data: C.VAEncMiscParameterHRD{ - initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond * - cp.RateControl.WindowSize / 2000), - buffer_size: C.uint(cp.RateControl.BitsPerSecond * - cp.RateControl.WindowSize / 1000), + initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond * + params.RateControl.WindowSize / 2000), + buffer_size: C.uint(params.RateControl.BitsPerSecond * + params.RateControl.WindowSize / 1000), }, }, frParam: frParam{ @@ -203,12 +172,12 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { _type: C.VAEncMiscParameterTypeRateControl, }, data: C.VAEncMiscParameterRateControl{ - window_size: C.uint(cp.RateControl.WindowSize), - initial_qp: C.uint(cp.RateControl.InitialQP), - min_qp: C.uint(cp.RateControl.MinQP), - max_qp: C.uint(cp.RateControl.MaxQP), - bits_per_second: C.uint(cp.RateControl.BitsPerSecond), - target_percentage: C.uint(cp.RateControl.TargetPercentage), + window_size: C.uint(params.RateControl.WindowSize), + initial_qp: C.uint(params.RateControl.InitialQP), + min_qp: C.uint(params.RateControl.MinQP), + max_qp: C.uint(params.RateControl.MaxQP), + bits_per_second: C.uint(params.RateControl.BitsPerSecond), + target_percentage: C.uint(params.RateControl.TargetPercentage), }, }, } @@ -264,11 +233,11 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 { return nil, errors.New("the hardware encoder doesn't support YUV420") } - if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 { + if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 { return nil, errors.New("the hardware encoder doesn't support specified rate control mode") } confAttrs[0].value = C.VA_RT_FORMAT_YUV420 - confAttrs[1].value = C.uint(cp.RateControlMode) + confAttrs[1].value = C.uint(params.RateControlMode) if s := C.vaCreateConfig( e.display, @@ -331,7 +300,7 @@ func (e *encoderVP8) Read(p []byte) (int, error) { } yuvImg := img.(*image.YCbCr) - kf := e.frameCnt%e.prop.KeyFrameInterval == 0 + kf := e.frameCnt%e.params.KeyFrameInterval == 0 e.frameCnt++ e.frParam.data.framerate = C.uint(e.rate.Calc()) diff --git a/pkg/codec/vaapi/vp9.go b/pkg/codec/vaapi/vp9.go index 82e916f..1362bd3 100644 --- a/pkg/codec/vaapi/vp9.go +++ b/pkg/codec/vaapi/vp9.go @@ -45,12 +45,9 @@ import ( "sync" "unsafe" - "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" - - "github.com/pion/webrtc/v2" ) const ( @@ -90,6 +87,7 @@ type encoderVP9 struct { frameCnt int prop prop.Media + params ParamVP9 rate *framerateDetector @@ -97,63 +95,38 @@ type encoderVP9 struct { closed bool } -func init() { - codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder)) -} - -// NewVP8Param returns default parameters of VP9 codec. -func NewVP9Param() (ParamVP9, error) { - return ParamVP9{ - RateControlMode: RateControlVBR, - RateControl: RateControlParam{ - BitsPerSecond: 400000, - TargetPercentage: 80, - WindowSize: 1500, - InitialQP: 60, - MinQP: 9, - MaxQP: 127, - }, - }, nil -} - -// NewVP9Encoder creates new VP9 encoder -func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { +// newVP9Encoder creates new VP9 encoder +func newVP9Encoder(r video.Reader, p prop.Media, params ParamVP9) (io.ReadCloser, error) { if p.Width%16 != 0 || p.Width == 0 { return nil, errors.New("width must be 16*n") } if p.Height%16 != 0 || p.Height == 0 { return nil, errors.New("height must be 16*n") } - if p.KeyFrameInterval == 0 { - p.KeyFrameInterval = 30 + if params.KeyFrameInterval == 0 { + params.KeyFrameInterval = 30 } if p.FrameRate == 0 { p.FrameRate = 30 } - var cp ParamVP9 - switch params := p.Codec.CodecParams.(type) { - case nil: - cp, _ = NewVP9Param() - cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5) - case ParamVP9: - cp = params - default: - return nil, errors.New("unsupported CodecParams type") + if params.RateControl.BitsPerSecond == 0 { + params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5) } // Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp9enc.c e := &encoderVP9{ - r: video.ToI420(r), - prop: p, - rate: newFramerateDetector(uint32(p.FrameRate)), + r: video.ToI420(r), + prop: p, + params: params, + rate: newFramerateDetector(uint32(p.FrameRate)), seqParam: C.VAEncSequenceParameterBufferVP9{ max_frame_width: 8192, max_frame_height: 8192, - bits_per_second: C.uint(cp.RateControl.BitsPerSecond), - intra_period: C.uint(p.KeyFrameInterval), + bits_per_second: C.uint(params.RateControl.BitsPerSecond), + intra_period: C.uint(params.KeyFrameInterval), kf_min_dist: 1, - kf_max_dist: C.uint(p.KeyFrameInterval), + kf_max_dist: C.uint(params.KeyFrameInterval), }, picParam: C.VAEncPictureParameterBufferVP9{ reference_frames: [8]C.VASurfaceID{ @@ -188,10 +161,10 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { _type: C.VAEncMiscParameterTypeHRD, }, data: C.VAEncMiscParameterHRD{ - initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond * - cp.RateControl.WindowSize / 2000), - buffer_size: C.uint(cp.RateControl.BitsPerSecond * - cp.RateControl.WindowSize / 1000), + initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond * + params.RateControl.WindowSize / 2000), + buffer_size: C.uint(params.RateControl.BitsPerSecond * + params.RateControl.WindowSize / 1000), }, }, frParam: frParam{ @@ -207,12 +180,12 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { _type: C.VAEncMiscParameterTypeRateControl, }, data: C.VAEncMiscParameterRateControl{ - window_size: C.uint(cp.RateControl.WindowSize), - initial_qp: C.uint(cp.RateControl.InitialQP), - min_qp: C.uint(cp.RateControl.MinQP), - max_qp: C.uint(cp.RateControl.MaxQP), - bits_per_second: C.uint(cp.RateControl.BitsPerSecond), - target_percentage: C.uint(cp.RateControl.TargetPercentage), + window_size: C.uint(params.RateControl.WindowSize), + initial_qp: C.uint(params.RateControl.InitialQP), + min_qp: C.uint(params.RateControl.MinQP), + max_qp: C.uint(params.RateControl.MaxQP), + bits_per_second: C.uint(params.RateControl.BitsPerSecond), + target_percentage: C.uint(params.RateControl.TargetPercentage), }, }, } @@ -267,11 +240,11 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 { return nil, errors.New("the hardware encoder doesn't support YUV420") } - if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 { + if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 { return nil, errors.New("the hardware encoder doesn't support specified rate control mode") } confAttrs[0].value = C.VA_RT_FORMAT_YUV420 - confAttrs[1].value = C.uint(cp.RateControlMode) + confAttrs[1].value = C.uint(params.RateControlMode) if s := C.vaCreateConfig( e.display, @@ -334,7 +307,7 @@ func (e *encoderVP9) Read(p []byte) (int, error) { } yuvImg := img.(*image.YCbCr) - kf := e.frameCnt%e.prop.KeyFrameInterval == 0 + kf := e.frameCnt%e.params.KeyFrameInterval == 0 e.frameCnt++ e.frParam.data.framerate = C.uint(e.rate.Calc()) diff --git a/pkg/codec/vpx/params.go b/pkg/codec/vpx/params.go index d68268f..2409879 100644 --- a/pkg/codec/vpx/params.go +++ b/pkg/codec/vpx/params.go @@ -1,8 +1,13 @@ package vpx +import ( + "github.com/pion/mediadevices/pkg/codec" +) + // Params stores libvpx specific encoding parameters. // Value range is codec (VP8/VP9) specific. type Params struct { + codec.BaseParams RateControlEndUsage RateControlMode RateControlUndershootPercent uint RateControlOvershootPercent uint diff --git a/pkg/codec/vpx/vpx.go b/pkg/codec/vpx/vpx.go index 9163492..5c9b275 100644 --- a/pkg/codec/vpx/vpx.go +++ b/pkg/codec/vpx/vpx.go @@ -55,7 +55,6 @@ import ( "time" "unsafe" - "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" @@ -78,28 +77,61 @@ type encoder struct { closed bool } -func init() { - codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder)) - codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder)) +// VP8Params is codec specific paramaters +type VP8Params struct { + Params } -// NewVP8Encoder creates new VP8 encoder -func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { - return newEncoder(r, p, C.ifaceVP8()) +// NewVP8Params returns default VP8 codec specific parameters. +func NewVP8Params() (VP8Params, error) { + p, err := newParams(C.ifaceVP8()) + if err != nil { + return VP8Params{}, err + } + + return VP8Params{ + Params: p, + }, nil } -// NewVP9Encoder creates new VP9 encoder -func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { - return newEncoder(r, p, C.ifaceVP9()) +// Name represents the codec name +func (p *VP8Params) Name() string { + return webrtc.VP8 } -// NewVP8Param returns default VP8 codec specific parameters. -func NewVP8Param() (Params, error) { return newParam(C.ifaceVP8()) } +// BuildVideoEncoder builds VP8 encoder with given params +func (p *VP8Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newEncoder(r, property, p.Params, C.ifaceVP8()) +} -// NewVP9Param returns default VP9 codec specific parameters. -func NewVP9Param() (Params, error) { return newParam(C.ifaceVP9()) } +// VP9Params is codec specific paramaters +type VP9Params struct { + Params +} -func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) { +// NewVP9Params returns default VP9 codec specific parameters. +func NewVP9Params() (VP9Params, error) { + p, err := newParams(C.ifaceVP9()) + if err != nil { + return VP9Params{}, err + } + + return VP9Params{ + Params: p, + }, nil +} + +// Name represents the codec name +func (p *VP9Params) Name() string { + return webrtc.VP9 +} + +// BuildVideoEncoder builds VP9 encoder with given params +func (p *VP9Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newEncoder(r, property, p.Params, C.ifaceVP9()) +} + +func newParams(codecIface *C.vpx_codec_iface_t) (Params, error) { cfg := &C.vpx_codec_enc_cfg_t{} if ec := C.vpx_codec_enc_config_default(codecIface, cfg, 0); ec != 0 { return Params{}, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec) @@ -113,13 +145,13 @@ func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) { }, nil } -func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) { - if p.BitRate == 0 { - p.BitRate = 100000 +func newEncoder(r video.Reader, p prop.Media, params Params, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) { + if params.BitRate == 0 { + params.BitRate = 100000 } - if p.KeyFrameInterval == 0 { - p.KeyFrameInterval = 60 + if params.KeyFrameInterval == 0 { + params.KeyFrameInterval = 60 } cfg := &C.vpx_codec_enc_cfg_t{} @@ -127,24 +159,18 @@ func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) ( return nil, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec) } - switch cp := p.CodecParams.(type) { - case nil: - case Params: - cfg.rc_end_usage = uint32(cp.RateControlEndUsage) - cfg.rc_undershoot_pct = C.uint(cp.RateControlUndershootPercent) - cfg.rc_overshoot_pct = C.uint(cp.RateControlOvershootPercent) - cfg.rc_min_quantizer = C.uint(cp.RateControlMinQuantizer) - cfg.rc_max_quantizer = C.uint(cp.RateControlMaxQuantizer) - default: - return nil, errors.New("unsupported CodecParams type") - } + cfg.rc_end_usage = uint32(params.RateControlEndUsage) + cfg.rc_undershoot_pct = C.uint(params.RateControlUndershootPercent) + cfg.rc_overshoot_pct = C.uint(params.RateControlOvershootPercent) + cfg.rc_min_quantizer = C.uint(params.RateControlMinQuantizer) + cfg.rc_max_quantizer = C.uint(params.RateControlMaxQuantizer) cfg.g_w = C.uint(p.Width) cfg.g_h = C.uint(p.Height) cfg.g_timebase.num = 1 cfg.g_timebase.den = 1000 - cfg.rc_target_bitrate = C.uint(p.BitRate) / 1000 - cfg.kf_max_dist = C.uint(p.KeyFrameInterval) + cfg.rc_target_bitrate = C.uint(params.BitRate) / 1000 + cfg.kf_max_dist = C.uint(params.KeyFrameInterval) cfg.rc_resize_allowed = 0 cfg.g_pass = C.VPX_RC_ONE_PASS diff --git a/pkg/codec/x264/params.go b/pkg/codec/x264/params.go index 9354cb1..153734f 100644 --- a/pkg/codec/x264/params.go +++ b/pkg/codec/x264/params.go @@ -1,7 +1,18 @@ package x264 +import ( + "io" + + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" + "github.com/pion/webrtc/v2" +) + // Params stores libx264 specific encoding parameters. type Params struct { + codec.BaseParams + // Faster preset has lower CPU usage but lower quality Preset Preset } @@ -21,3 +32,22 @@ const ( PresetVeryslow PresetPlacebo ) + +// NewParams returns default x264 codec specific parameters. +func NewParams() (Params, error) { + return Params{ + BaseParams: codec.BaseParams{ + KeyFrameInterval: 60, + }, + }, nil +} + +// Name represents the codec name +func (p *Params) Name() string { + return webrtc.H264 +} + +// BuildVideoEncoder builds x264 encoder with given params +func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) { + return newEncoder(r, property, *p) +} diff --git a/pkg/codec/x264/x264.go b/pkg/codec/x264/x264.go index c0231e2..f65242e 100644 --- a/pkg/codec/x264/x264.go +++ b/pkg/codec/x264/x264.go @@ -7,18 +7,15 @@ package x264 // #include "bridge.h" import "C" import ( - "errors" "fmt" "image" "io" "sync" "unsafe" - "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" - "github.com/pion/webrtc/v2" ) type encoder struct { @@ -64,37 +61,24 @@ var ( errEncode = fmt.Errorf("failed to encode") ) -func init() { - codec.Register(webrtc.H264, codec.VideoEncoderBuilder(newEncoder)) -} - -func newEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) { - if p.KeyFrameInterval == 0 { - p.KeyFrameInterval = 60 - } - - var preset Preset - switch cp := p.CodecParams.(type) { - case nil: - case Params: - preset = cp.Preset - default: - return nil, errors.New("unsupported CodecParams type") +func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) { + if params.KeyFrameInterval == 0 { + params.KeyFrameInterval = 60 } param := C.x264_param_t{ i_csp: C.X264_CSP_I420, i_width: C.int(p.Width), i_height: C.int(p.Height), - i_keyint_max: C.int(p.KeyFrameInterval), + i_keyint_max: C.int(params.KeyFrameInterval), } - param.rc.i_bitrate = C.int(p.BitRate) + param.rc.i_bitrate = C.int(params.BitRate) param.rc.i_vbv_max_bitrate = param.rc.i_bitrate param.rc.i_vbv_buffer_size = param.rc.i_vbv_max_bitrate * 2 var rc C.int // cPreset will be freed in C.enc_new - cPreset := C.CString(fmt.Sprint(preset)) + cPreset := C.CString(fmt.Sprint(params.Preset)) engine := C.enc_new(param, cPreset, &rc) if err := errFromC(rc); err != nil { return nil, err diff --git a/pkg/prop/prop.go b/pkg/prop/prop.go index 5b4bccc..80bf1b5 100644 --- a/pkg/prop/prop.go +++ b/pkg/prop/prop.go @@ -14,7 +14,6 @@ type Media struct { DeviceID string Video Audio - Codec } // Merge merges all the field values from o to p, except zero values. @@ -111,17 +110,3 @@ type Audio struct { SampleRate int SampleSize int } - -// Codec represents an codec's encoding properties -type Codec struct { - CodecName string - - // Target bitrate in bps. - BitRate int - - // Expected interval of the keyframes in frames. - KeyFrameInterval int - - // Library specific parameter struct defined by each codec package - CodecParams interface{} -} diff --git a/track.go b/track.go index 8b49642..d3abf27 100644 --- a/track.go +++ b/track.go @@ -6,7 +6,6 @@ import ( "math/rand" "sync" - "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/driver" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/webrtc/v2" @@ -42,16 +41,9 @@ type track struct { endOnce sync.Once } -func newTrack(codecs []*webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver, codecName string) (*track, error) { - var selectedCodec *webrtc.RTPCodec - for _, c := range codecs { - if c.Name == codecName { - selectedCodec = c - break - } - } +func newTrack(selectedCodec *webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver) (*track, error) { if selectedCodec == nil { - return nil, fmt.Errorf("track: %s is not registered in media engine", codecName) + panic("codec is required") } t, err := trackGenerator( @@ -116,13 +108,7 @@ type videoTrack struct { var _ Tracker = &videoTrack{} func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*videoTrack, error) { - codecName := constraints.CodecName - t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeVideo], opts.trackGenerator, d, codecName) - if err != nil { - return nil, err - } - - err = d.Open() + err := d.Open() if err != nil { return nil, err } @@ -137,21 +123,47 @@ func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints Media r = constraints.VideoTransform(r) } - encoder, err := codec.BuildVideoEncoder(r, constraints.Media) - if err != nil { - _ = d.Close() - return nil, err + var vt *videoTrack + rtpCodecs := opts.codecs[webrtc.RTPCodecTypeVideo] + for _, codecBuilder := range constraints.VideoEncoderBuilders { + var matchedRTPCodec *webrtc.RTPCodec + for _, rtpCodec := range rtpCodecs { + if rtpCodec.Name == codecBuilder.Name() { + matchedRTPCodec = rtpCodec + break + } + } + + if matchedRTPCodec == nil { + continue + } + + t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d) + if err != nil { + continue + } + + encoder, err := codecBuilder.BuildVideoEncoder(r, constraints.Media) + if err != nil { + continue + } + + vt = &videoTrack{ + track: t, + d: d, + constraints: constraints, + encoder: encoder, + } + break } - vt := videoTrack{ - track: t, - d: d, - constraints: constraints, - encoder: encoder, + if vt == nil { + d.Close() + return nil, fmt.Errorf("failed to find a matching video codec") } go vt.start() - return &vt, nil + return vt, nil } func (vt *videoTrack) start() { @@ -192,41 +204,61 @@ type audioTrack struct { var _ Tracker = &audioTrack{} func newAudioTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*audioTrack, error) { - codecName := constraints.CodecName - t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeAudio], opts.trackGenerator, d, codecName) - if err != nil { - return nil, err - } - - err = d.Open() + err := d.Open() if err != nil { return nil, err } ar := d.(driver.AudioRecorder) - reader, err := ar.AudioRecord(constraints.Media) + r, err := ar.AudioRecord(constraints.Media) if err != nil { return nil, err } if constraints.AudioTransform != nil { - reader = constraints.AudioTransform(reader) + r = constraints.AudioTransform(r) } - encoder, err := codec.BuildAudioEncoder(reader, constraints.Media) - if err != nil { - _ = d.Close() - return nil, err + var at *audioTrack + rtpCodecs := opts.codecs[webrtc.RTPCodecTypeAudio] + for _, codecBuilder := range constraints.AudioEncoderBuilders { + var matchedRTPCodec *webrtc.RTPCodec + for _, rtpCodec := range rtpCodecs { + if rtpCodec.Name == codecBuilder.Name() { + matchedRTPCodec = rtpCodec + break + } + } + + if matchedRTPCodec == nil { + continue + } + + t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d) + if err != nil { + continue + } + + encoder, err := codecBuilder.BuildAudioEncoder(r, constraints.Media) + if err != nil { + continue + } + + at = &audioTrack{ + track: t, + d: d, + constraints: constraints, + encoder: encoder, + } } - at := audioTrack{ - track: t, - d: d, - constraints: constraints, - encoder: encoder, + if at == nil { + d.Close() + return nil, fmt.Errorf("failed to find a matching audio codec") } + go at.start() - return &at, nil + return at, nil } func (t *audioTrack) start() {