package/mediadevices
1
0
Fork 0
mirror of https://github.com/pion/mediadevices.git synced 2025-10-04 16:22:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Redesign codec

Browse Source 
Resolves https://github.com/pion/mediadevices/issues/114

* Remove codec registrar
* Completely redesign how codec is being discovered, tuned, and built
* Update examples
* Update unit tests
This commit is contained in:
Lukas Herman
2020-03-14 21:59:32 -04:00
parent 475f8cc458
commit c9b90fb233
22 changed files with 518 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/codecparam/main.go
View File

@@ -5,18 +5,16 @@ import (
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/webrtc/v2"
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder
"github.com/pion/mediadevices/pkg/codec/vaapi" // This is required to register VP8/VP9 video encoder
// Note: If you don't have a camera or microphone or your adapters are not supported,
// Note: If you don't have a camera or your adapters are not supported,
// you can always swap your adapters with our dummy adapters below.
// _ "github.com/pion/mediadevices/pkg/driver/videotest"
// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
)
func main() {
@@ -52,13 +50,7 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection)
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Audio: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.Opus
c.Enabled = true
c.BitRate = 32000 // 32kbps
},
Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.VP8
c.FrameFormat = frame.FormatYUY2
c.Enabled = true
c.Width = 640
@@ -77,7 +69,7 @@ func main() {
cp.RateControlMode = vaapi.RateControlVBR
cp.RateControl.BitsPerSecond = 400000
cp.RateControl.TargetPercentage = 95
c.CodecParams = cp
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&cp}
},
})
if err != nil {

16
examples/rtp-send/main.go
View File

@@ -6,8 +6,8 @@ import (
"os"
"github.com/pion/mediadevices"
_ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/rtp"
@@ -16,7 +16,6 @@ import (
)
const (
videoCodecName = webrtc.VP8
mtu = 1000
)
@@ -41,14 +40,19 @@ func main() {
),
)
_, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatYUY2
c.Enabled = true
c.Width = 640
c.Height = 480
c.BitRate = 100000 // 100kbps
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})
if err != nil {

18
examples/screenshare/main.go
View File

@@ -5,18 +5,13 @@ import (
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal"
_ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/webrtc/v2"
)
const (
videoCodecName = webrtc.VP8
)
func main() {
config := webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
@@ -49,12 +44,17 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection)
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.Enabled = true
c.BitRate = 100000 // 100kbps
c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})
if err != nil {

24
examples/simple/main.go
View File

@@ -5,16 +5,18 @@ import (
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/webrtc/v2"
_ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder
// This is required to register opus audio encoder
"github.com/pion/mediadevices/pkg/codec/opus"
// If you don't like vpx, you can also use x264 by importing as below
// _ "github.com/pion/mediadevices/pkg/codec/x264" // This is required to register h264 video encoder
// or you can also use openh264 for alternative h264 implementation
// _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
// Note: If you don't have a camera or microphone or your adapters are not supported,
// you can always swap your adapters with our dummy adapters below.
@@ -60,19 +62,29 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection)
opusParams, err := opus.NewParams()
if err != nil {
panic(err)
}
opusParams.BitRate = 32000 // 32kbps
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Audio: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = webrtc.Opus
c.Enabled = true
c.BitRate = 32000 // 32kbps
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
},
Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatYUY2
c.Enabled = true
c.Width = 640
c.Height = 480
c.BitRate = 100000 // 100kbps
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})
if err != nil {

16
examples/transform/main.go
View File

@@ -7,17 +7,14 @@ import (
"github.com/disintegration/imaging"
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/examples/internal/signal"
_ "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to register VP8/VP9 video encoder
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/webrtc/v2"
)
const (
videoCodecName = webrtc.VP8
)
func rotate180(r video.Reader) video.Reader {
return video.ReaderFunc(func() (img image.Image, err error) {
img, err = r.Read()
@@ -62,15 +59,20 @@ func main() {
md := mediadevices.NewMediaDevices(peerConnection)
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 100000 // 100kbps
s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
Video: func(c *mediadevices.MediaTrackConstraints) {
c.CodecName = videoCodecName
c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420
c.Enabled = true
c.Width = 640
c.Height = 480
c.BitRate = 100000 // 100kbps
c.VideoTransform = rotate180
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
},
})
if err != nil {

71
mediadevices_test.go
View File

@@ -18,23 +18,18 @@ import (
)
func TestGetUserMedia(t *testing.T) {
codec.Register("MockVideo", codec.VideoEncoderBuilder(func(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.BitRate == 0 {
// This is a dummy error to test the failure condition.
return nil, errors.New("wrong codec parameter")
videoParams := mockParams{
BaseParams: codec.BaseParams{
BitRate: 100000,
},
name: "MockVideo",
}
audioParams := mockParams{
BaseParams: codec.BaseParams{
BitRate: 32000,
},
name: "MockAudio",
}
return &mockVideoCodec{
r: r,
closed: make(chan struct{}),
}, nil
}))
codec.Register("MockAudio", codec.AudioEncoderBuilder(func(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
return &mockAudioCodec{
r: r,
closed: make(chan struct{}),
}, nil
}))
md := NewMediaDevicesFromCodecs(
map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
@@ -54,30 +49,31 @@ func TestGetUserMedia(t *testing.T) {
)
constraints := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) {
c.CodecName = "MockVideo"
c.Enabled = true
c.Width = 640
c.Height = 480
c.BitRate = 100000
params := videoParams
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
},
Audio: func(c *MediaTrackConstraints) {
c.CodecName = "MockAudio"
c.Enabled = true
c.BitRate = 32000
params := audioParams
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
},
}
constraintsWrong := MediaStreamConstraints{
Video: func(c *MediaTrackConstraints) {
c.CodecName = "MockVideo"
c.Enabled = true
c.Width = 640
c.Height = 480
c.BitRate = 0
params := videoParams
params.BitRate = 0
c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&params}
},
Audio: func(c *MediaTrackConstraints) {
c.CodecName = "MockAudio"
c.Enabled = true
c.BitRate = 32000
params := audioParams
c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
},
}
@@ -156,6 +152,33 @@ func (t *mockTrack) Kind() webrtc.RTPCodecType {
return t.codec.Type
}
type mockParams struct {
codec.BaseParams
name string
}
func (params *mockParams) Name() string {
return params.name
}
func (params *mockParams) BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if params.BitRate == 0 {
// This is a dummy error to test the failure condition.
return nil, errors.New("wrong codec parameter")
}
return &mockVideoCodec{
r: r,
closed: make(chan struct{}),
}, nil
}
func (params *mockParams) BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
return &mockAudioCodec{
r: r,
closed: make(chan struct{}),
}, nil
}
type mockVideoCodec struct {
r video.Reader
closed chan struct{}

13
mediastreamconstraints.go
View File

@@ -1,6 +1,7 @@
package mediadevices
import (
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
@@ -15,6 +16,18 @@ type MediaStreamConstraints struct {
type MediaTrackConstraints struct {
prop.Media
Enabled bool
// VideoEncoderBuilders are codec builders that are used for encoding the video
// and later being used for sending the appropriate RTP payload type.
//
// If one encoder builder fails to build the codec, the next builder will be used,
// repeating until a codec builds. If no builders build successfully, an error is returned.
VideoEncoderBuilders []codec.VideoEncoderBuilder
// AudioEncoderBuilders are codec builders that are used for encoding the audio
// and later being used for sending the appropriate RTP payload type.
//
// If one encoder builder fails to build the codec, the next builder will be used,
// repeating until a codec builds. If no builders build successfully, an error is returned.
AudioEncoderBuilders []codec.AudioEncoderBuilder
// VideoTransform will be used to transform the video that's coming from the driver.
// So, basically it'll look like following: driver -> VideoTransform -> codec
VideoTransform video.TransformFunc

34
pkg/codec/codec.go
View File

@@ -8,5 +8,35 @@ import (
"github.com/pion/mediadevices/pkg/prop"
)
type VideoEncoderBuilder func(r video.Reader, p prop.Media) (io.ReadCloser, error)
type AudioEncoderBuilder func(r audio.Reader, p prop.Media) (io.ReadCloser, error)
// AudioEncoderBuilder is the interface that wraps basic operations that are
// necessary to build the audio encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type AudioEncoderBuilder interface {
// Name represents the codec name
Name() string
// BuildAudioEncoder builds audio encoder by given media params and audio input
BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error)
}
// VideoEncoderBuilder is the interface that wraps basic operations that are
// necessary to build the video encoder.
//
// This interface is for codec implementors to provide codec specific params,
// but still giving generality for the users.
type VideoEncoderBuilder interface {
// Name represents the codec name
Name() string
// BuildVideoEncoder builds video encoder by given media params and video input
BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error)
}
// BaseParams represents an codec's encoding properties
type BaseParams struct {
// Target bitrate in bps.
BitRate int
// Expected interval of the keyframes in frames.
KeyFrameInterval int
}

22
pkg/codec/openh264/openh264.go
View File

@@ -9,19 +9,15 @@ package openh264
import "C"
import (
"errors"
"fmt"
"image"
"io"
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
type encoder struct {
@@ -33,26 +29,16 @@ type encoder struct {
closed bool
}
var _ codec.VideoEncoderBuilder = codec.VideoEncoderBuilder(NewEncoder)
func init() {
codec.Register(webrtc.H264, codec.VideoEncoderBuilder(NewEncoder))
}
func NewEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.BitRate == 0 {
p.BitRate = 100000
}
if p.CodecParams != nil {
return nil, errors.New("unsupported CodecParams type")
func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
if params.BitRate == 0 {
params.BitRate = 100000
}
var rv C.int
cEncoder := C.enc_new(C.EncoderOptions{
width: C.int(p.Width),
height: C.int(p.Height),
target_bitrate: C.int(p.BitRate),
target_bitrate: C.int(params.BitRate),
max_fps: C.float(p.FrameRate),
}, &rv)
if err := errResult(rv); err != nil {

34
pkg/codec/openh264/params.go Normal file
View File

@@ -0,0 +1,34 @@
package openh264
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores libopenh264 specific encoding parameters.
type Params struct {
codec.BaseParams
}
// NewParams returns default openh264 codec specific parameters.
func NewParams() (Params, error) {
return Params{
BaseParams: codec.BaseParams{
BitRate: 100000,
},
}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.H264
}
// BuildVideoEncoder builds openh264 encoder with given params
func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

20
pkg/codec/opus/opus.go
View File

@@ -1,7 +1,6 @@
package opus
import (
"errors"
"fmt"
"io"
"math"
@@ -9,10 +8,8 @@ import (
"unsafe"
"github.com/lherman-cs/opus"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
type encoder struct {
@@ -24,13 +21,8 @@ type encoder struct {
var latencies = []float64{5, 10, 20, 40, 60}
var _ io.ReadCloser = &encoder{}
var _ codec.AudioEncoderBuilder = codec.AudioEncoderBuilder(NewEncoder)
func init() {
codec.Register(webrtc.Opus, codec.AudioEncoderBuilder(NewEncoder))
}
func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
func newEncoder(r audio.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
if p.SampleRate == 0 {
return nil, fmt.Errorf("opus: inProp.SampleRate is required")
}
@@ -39,12 +31,8 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
p.Latency = 20
}
if p.BitRate == 0 {
p.BitRate = 32000
}
if p.CodecParams != nil {
return nil, errors.New("unsupported CodecParams type")
if params.BitRate == 0 {
params.BitRate = 32000
}
// Select the nearest supported latency
@@ -69,7 +57,7 @@ func NewEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
if err != nil {
return nil, err
}
if err := engine.SetBitrate(p.BitRate); err != nil {
if err := engine.SetBitrate(params.BitRate); err != nil {
return nil, err
}

30
pkg/codec/opus/params.go Normal file
View File

@@ -0,0 +1,30 @@
package opus
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores opus specific encoding parameters.
type Params struct {
codec.BaseParams
}
// NewParams returns default opus codec specific parameters.
func NewParams() (Params, error) {
return Params{}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.Opus
}
// BuildVideoEncoder builds x264 encoder with given params
func (p *Params) BuildAudioEncoder(r audio.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

42
pkg/codec/registrar.go
View File

@@ -1,42 +0,0 @@
package codec
import (
"fmt"
"io"
"github.com/pion/mediadevices/pkg/io/audio"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
var (
videoEncoders = make(map[string]VideoEncoderBuilder)
audioEncoders = make(map[string]AudioEncoderBuilder)
)
func Register(name string, builder interface{}) {
switch b := builder.(type) {
case VideoEncoderBuilder:
videoEncoders[name] = b
case AudioEncoderBuilder:
audioEncoders[name] = b
}
}
func BuildVideoEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
b, ok := videoEncoders[p.CodecName]
if !ok {
return nil, fmt.Errorf("codec: can't find %s video encoder", p.CodecName)
}
return b(r, p)
}
func BuildAudioEncoder(r audio.Reader, p prop.Media) (io.ReadCloser, error) {
b, ok := audioEncoders[p.CodecName]
if !ok {
return nil, fmt.Errorf("codec: can't find %s audio encoder", p.CodecName)
}
return b(r, p)
}

65
pkg/codec/vaapi/params.go
View File

@@ -1,7 +1,17 @@
package vaapi
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// ParamVP8 stores VP8 encoding parameters.
type ParamVP8 struct {
codec.BaseParams
Sequence SequenceParamVP8
RateControlMode RateControlMode
RateControl RateControlParam
@@ -14,8 +24,38 @@ type SequenceParamVP8 struct {
ClampQindexLow uint
}
// NewVP8Param returns default parameters of VP8 codec.
func NewVP8Param() (ParamVP8, error) {
return ParamVP8{
Sequence: SequenceParamVP8{
ClampQindexLow: 9,
ClampQindexHigh: 127,
},
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// Name represents the codec name
func (p *ParamVP8) Name() string {
return webrtc.VP8
}
// BuildVideoEncoder builds VP8 encoder with given params
func (p *ParamVP8) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newVP8Encoder(r, property, *p)
}
// ParamVP9 represents VAEncSequenceParameterBufferVP9 and other parameter buffers.
type ParamVP9 struct {
codec.BaseParams
RateControlMode RateControlMode
RateControl RateControlParam
}
@@ -52,3 +92,28 @@ const (
RateControlQVBR RateControlMode = 0x00000400
RateControlAVBR RateControlMode = 0x00000800
)
// NewVP9Param returns default parameters of VP9 codec.
func NewVP9Param() (ParamVP9, error) {
return ParamVP9{
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// Name represents the codec name
func (p *ParamVP9) Name() string {
return webrtc.VP9
}
// BuildVideoEncoder builds VP9 encoder with given params
func (p *ParamVP9) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newVP9Encoder(r, property, *p)
}

83
pkg/codec/vaapi/vp8.go
View File

@@ -50,12 +50,9 @@ import (
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
const (
@@ -86,6 +83,7 @@ type encoderVP8 struct {
frameCnt int
prop prop.Media
params ParamVP8
rate *framerateDetector
@@ -93,66 +91,37 @@ type encoderVP8 struct {
closed bool
}
func init() {
codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder))
}
// NewVP8Param returns default parameters of VP8 codec.
func NewVP8Param() (ParamVP8, error) {
return ParamVP8{
Sequence: SequenceParamVP8{
ClampQindexLow: 9,
ClampQindexHigh: 127,
},
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// NewVP8Encoder creates new VP8 encoder
func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
// newVP8Encoder creates new VP8 encoder
func newVP8Encoder(r video.Reader, p prop.Media, params ParamVP8) (io.ReadCloser, error) {
if p.Width%16 != 0 || p.Width == 0 {
return nil, errors.New("width must be 16*n")
}
if p.Height%16 != 0 || p.Height == 0 {
return nil, errors.New("height must be 16*n")
}
if p.KeyFrameInterval == 0 {
p.KeyFrameInterval = 30
if params.KeyFrameInterval == 0 {
params.KeyFrameInterval = 30
}
if p.FrameRate == 0 {
p.FrameRate = 30
}
var cp ParamVP8
switch params := p.Codec.CodecParams.(type) {
case nil:
cp, _ = NewVP8Param()
cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
case ParamVP8:
cp = params
default:
return nil, errors.New("unsupported CodecParams type")
if params.RateControl.BitsPerSecond == 0 {
params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5)
}
// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp8enc.c
e := &encoderVP8{
r: video.ToI420(r),
prop: p,
params: params,
rate: newFramerateDetector(uint32(p.FrameRate)),
seqParam: C.VAEncSequenceParameterBufferVP8{
frame_width: C.uint(p.Width),
frame_height: C.uint(p.Height),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond),
intra_period: C.uint(p.KeyFrameInterval),
kf_max_dist: C.uint(p.KeyFrameInterval),
bits_per_second: C.uint(params.RateControl.BitsPerSecond),
intra_period: C.uint(params.KeyFrameInterval),
kf_max_dist: C.uint(params.KeyFrameInterval),
reference_frames: [4]C.VASurfaceID{
C.VA_INVALID_ID,
C.VA_INVALID_ID,
@@ -165,8 +134,8 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
ref_gf_frame: C.VA_INVALID_SURFACE,
ref_arf_frame: C.VA_INVALID_SURFACE,
reconstructed_frame: C.VA_INVALID_SURFACE,
clamp_qindex_low: C.uint8_t(cp.Sequence.ClampQindexLow),
clamp_qindex_high: C.uint8_t(cp.Sequence.ClampQindexHigh),
clamp_qindex_low: C.uint8_t(params.Sequence.ClampQindexLow),
clamp_qindex_high: C.uint8_t(params.Sequence.ClampQindexHigh),
loop_filter_level: [4]C.int8_t{
19, 19, 19, 19,
},
@@ -184,10 +153,10 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeHRD,
},
data: C.VAEncMiscParameterHRD{
initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 2000),
buffer_size: C.uint(cp.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 1000),
initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond *
params.RateControl.WindowSize / 2000),
buffer_size: C.uint(params.RateControl.BitsPerSecond *
params.RateControl.WindowSize / 1000),
},
},
frParam: frParam{
@@ -203,12 +172,12 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeRateControl,
},
data: C.VAEncMiscParameterRateControl{
window_size: C.uint(cp.RateControl.WindowSize),
initial_qp: C.uint(cp.RateControl.InitialQP),
min_qp: C.uint(cp.RateControl.MinQP),
max_qp: C.uint(cp.RateControl.MaxQP),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond),
target_percentage: C.uint(cp.RateControl.TargetPercentage),
window_size: C.uint(params.RateControl.WindowSize),
initial_qp: C.uint(params.RateControl.InitialQP),
min_qp: C.uint(params.RateControl.MinQP),
max_qp: C.uint(params.RateControl.MaxQP),
bits_per_second: C.uint(params.RateControl.BitsPerSecond),
target_percentage: C.uint(params.RateControl.TargetPercentage),
},
},
}
@@ -264,11 +233,11 @@ func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
return nil, errors.New("the hardware encoder doesn't support YUV420")
}
if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 {
if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 {
return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
}
confAttrs[0].value = C.VA_RT_FORMAT_YUV420
confAttrs[1].value = C.uint(cp.RateControlMode)
confAttrs[1].value = C.uint(params.RateControlMode)
if s := C.vaCreateConfig(
e.display,
@@ -331,7 +300,7 @@ func (e *encoderVP8) Read(p []byte) (int, error) {
}
yuvImg := img.(*image.YCbCr)
kf := e.frameCnt%e.prop.KeyFrameInterval == 0
kf := e.frameCnt%e.params.KeyFrameInterval == 0
e.frameCnt++
e.frParam.data.framerate = C.uint(e.rate.Calc())

75
pkg/codec/vaapi/vp9.go
View File

@@ -45,12 +45,9 @@ import (
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
const (
@@ -90,6 +87,7 @@ type encoderVP9 struct {
frameCnt int
prop prop.Media
params ParamVP9
rate *framerateDetector
@@ -97,63 +95,38 @@ type encoderVP9 struct {
closed bool
}
func init() {
codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder))
}
// NewVP8Param returns default parameters of VP9 codec.
func NewVP9Param() (ParamVP9, error) {
return ParamVP9{
RateControlMode: RateControlVBR,
RateControl: RateControlParam{
BitsPerSecond: 400000,
TargetPercentage: 80,
WindowSize: 1500,
InitialQP: 60,
MinQP: 9,
MaxQP: 127,
},
}, nil
}
// NewVP9Encoder creates new VP9 encoder
func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
// newVP9Encoder creates new VP9 encoder
func newVP9Encoder(r video.Reader, p prop.Media, params ParamVP9) (io.ReadCloser, error) {
if p.Width%16 != 0 || p.Width == 0 {
return nil, errors.New("width must be 16*n")
}
if p.Height%16 != 0 || p.Height == 0 {
return nil, errors.New("height must be 16*n")
}
if p.KeyFrameInterval == 0 {
p.KeyFrameInterval = 30
if params.KeyFrameInterval == 0 {
params.KeyFrameInterval = 30
}
if p.FrameRate == 0 {
p.FrameRate = 30
}
var cp ParamVP9
switch params := p.Codec.CodecParams.(type) {
case nil:
cp, _ = NewVP9Param()
cp.RateControl.BitsPerSecond = uint(float32(p.BitRate) * 1.5)
case ParamVP9:
cp = params
default:
return nil, errors.New("unsupported CodecParams type")
if params.RateControl.BitsPerSecond == 0 {
params.RateControl.BitsPerSecond = uint(float32(params.BitRate) * 1.5)
}
// Parameters are from https://github.com/intel/libva-utils/blob/master/encode/vp9enc.c
e := &encoderVP9{
r: video.ToI420(r),
prop: p,
params: params,
rate: newFramerateDetector(uint32(p.FrameRate)),
seqParam: C.VAEncSequenceParameterBufferVP9{
max_frame_width: 8192,
max_frame_height: 8192,
bits_per_second: C.uint(cp.RateControl.BitsPerSecond),
intra_period: C.uint(p.KeyFrameInterval),
bits_per_second: C.uint(params.RateControl.BitsPerSecond),
intra_period: C.uint(params.KeyFrameInterval),
kf_min_dist: 1,
kf_max_dist: C.uint(p.KeyFrameInterval),
kf_max_dist: C.uint(params.KeyFrameInterval),
},
picParam: C.VAEncPictureParameterBufferVP9{
reference_frames: [8]C.VASurfaceID{
@@ -188,10 +161,10 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeHRD,
},
data: C.VAEncMiscParameterHRD{
initial_buffer_fullness: C.uint(cp.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 2000),
buffer_size: C.uint(cp.RateControl.BitsPerSecond *
cp.RateControl.WindowSize / 1000),
initial_buffer_fullness: C.uint(params.RateControl.BitsPerSecond *
params.RateControl.WindowSize / 2000),
buffer_size: C.uint(params.RateControl.BitsPerSecond *
params.RateControl.WindowSize / 1000),
},
},
frParam: frParam{
@@ -207,12 +180,12 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
_type: C.VAEncMiscParameterTypeRateControl,
},
data: C.VAEncMiscParameterRateControl{
window_size: C.uint(cp.RateControl.WindowSize),
initial_qp: C.uint(cp.RateControl.InitialQP),
min_qp: C.uint(cp.RateControl.MinQP),
max_qp: C.uint(cp.RateControl.MaxQP),
bits_per_second: C.uint(cp.RateControl.BitsPerSecond),
target_percentage: C.uint(cp.RateControl.TargetPercentage),
window_size: C.uint(params.RateControl.WindowSize),
initial_qp: C.uint(params.RateControl.InitialQP),
min_qp: C.uint(params.RateControl.MinQP),
max_qp: C.uint(params.RateControl.MaxQP),
bits_per_second: C.uint(params.RateControl.BitsPerSecond),
target_percentage: C.uint(params.RateControl.TargetPercentage),
},
},
}
@@ -267,11 +240,11 @@ func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if (confAttrs[0].value & C.VA_RT_FORMAT_YUV420) == 0 {
return nil, errors.New("the hardware encoder doesn't support YUV420")
}
if (confAttrs[1].value & C.uint(cp.RateControlMode)) == 0 {
if (confAttrs[1].value & C.uint(params.RateControlMode)) == 0 {
return nil, errors.New("the hardware encoder doesn't support specified rate control mode")
}
confAttrs[0].value = C.VA_RT_FORMAT_YUV420
confAttrs[1].value = C.uint(cp.RateControlMode)
confAttrs[1].value = C.uint(params.RateControlMode)
if s := C.vaCreateConfig(
e.display,
@@ -334,7 +307,7 @@ func (e *encoderVP9) Read(p []byte) (int, error) {
}
yuvImg := img.(*image.YCbCr)
kf := e.frameCnt%e.prop.KeyFrameInterval == 0
kf := e.frameCnt%e.params.KeyFrameInterval == 0
e.frameCnt++
e.frParam.data.framerate = C.uint(e.rate.Calc())

5
pkg/codec/vpx/params.go
View File

@@ -1,8 +1,13 @@
package vpx
import (
"github.com/pion/mediadevices/pkg/codec"
)
// Params stores libvpx specific encoding parameters.
// Value range is codec (VP8/VP9) specific.
type Params struct {
codec.BaseParams
RateControlEndUsage RateControlMode
RateControlUndershootPercent uint
RateControlOvershootPercent uint

92
pkg/codec/vpx/vpx.go
View File

@@ -55,7 +55,6 @@ import (
"time"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
@@ -78,28 +77,61 @@ type encoder struct {
closed bool
}
func init() {
codec.Register(webrtc.VP8, codec.VideoEncoderBuilder(NewVP8Encoder))
codec.Register(webrtc.VP9, codec.VideoEncoderBuilder(NewVP9Encoder))
// VP8Params is codec specific paramaters
type VP8Params struct {
Params
}
// NewVP8Encoder creates new VP8 encoder
func NewVP8Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
return newEncoder(r, p, C.ifaceVP8())
// NewVP8Params returns default VP8 codec specific parameters.
func NewVP8Params() (VP8Params, error) {
p, err := newParams(C.ifaceVP8())
if err != nil {
return VP8Params{}, err
}
return VP8Params{
Params: p,
}, nil
}
// NewVP9Encoder creates new VP9 encoder
func NewVP9Encoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
return newEncoder(r, p, C.ifaceVP9())
// Name represents the codec name
func (p *VP8Params) Name() string {
return webrtc.VP8
}
// NewVP8Param returns default VP8 codec specific parameters.
func NewVP8Param() (Params, error) { return newParam(C.ifaceVP8()) }
// BuildVideoEncoder builds VP8 encoder with given params
func (p *VP8Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, p.Params, C.ifaceVP8())
}
// NewVP9Param returns default VP9 codec specific parameters.
func NewVP9Param() (Params, error) { return newParam(C.ifaceVP9()) }
// VP9Params is codec specific paramaters
type VP9Params struct {
Params
}
func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) {
// NewVP9Params returns default VP9 codec specific parameters.
func NewVP9Params() (VP9Params, error) {
p, err := newParams(C.ifaceVP9())
if err != nil {
return VP9Params{}, err
}
return VP9Params{
Params: p,
}, nil
}
// Name represents the codec name
func (p *VP9Params) Name() string {
return webrtc.VP9
}
// BuildVideoEncoder builds VP9 encoder with given params
func (p *VP9Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, p.Params, C.ifaceVP9())
}
func newParams(codecIface *C.vpx_codec_iface_t) (Params, error) {
cfg := &C.vpx_codec_enc_cfg_t{}
if ec := C.vpx_codec_enc_config_default(codecIface, cfg, 0); ec != 0 {
return Params{}, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec)
@@ -113,13 +145,13 @@ func newParam(codecIface *C.vpx_codec_iface_t) (Params, error) {
}, nil
}
func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) {
if p.BitRate == 0 {
p.BitRate = 100000
func newEncoder(r video.Reader, p prop.Media, params Params, codecIface *C.vpx_codec_iface_t) (io.ReadCloser, error) {
if params.BitRate == 0 {
params.BitRate = 100000
}
if p.KeyFrameInterval == 0 {
p.KeyFrameInterval = 60
if params.KeyFrameInterval == 0 {
params.KeyFrameInterval = 60
}
cfg := &C.vpx_codec_enc_cfg_t{}
@@ -127,24 +159,18 @@ func newEncoder(r video.Reader, p prop.Media, codecIface *C.vpx_codec_iface_t) (
return nil, fmt.Errorf("vpx_codec_enc_config_default failed (%d)", ec)
}
switch cp := p.CodecParams.(type) {
case nil:
case Params:
cfg.rc_end_usage = uint32(cp.RateControlEndUsage)
cfg.rc_undershoot_pct = C.uint(cp.RateControlUndershootPercent)
cfg.rc_overshoot_pct = C.uint(cp.RateControlOvershootPercent)
cfg.rc_min_quantizer = C.uint(cp.RateControlMinQuantizer)
cfg.rc_max_quantizer = C.uint(cp.RateControlMaxQuantizer)
default:
return nil, errors.New("unsupported CodecParams type")
}
cfg.rc_end_usage = uint32(params.RateControlEndUsage)
cfg.rc_undershoot_pct = C.uint(params.RateControlUndershootPercent)
cfg.rc_overshoot_pct = C.uint(params.RateControlOvershootPercent)
cfg.rc_min_quantizer = C.uint(params.RateControlMinQuantizer)
cfg.rc_max_quantizer = C.uint(params.RateControlMaxQuantizer)
cfg.g_w = C.uint(p.Width)
cfg.g_h = C.uint(p.Height)
cfg.g_timebase.num = 1
cfg.g_timebase.den = 1000
cfg.rc_target_bitrate = C.uint(p.BitRate) / 1000
cfg.kf_max_dist = C.uint(p.KeyFrameInterval)
cfg.rc_target_bitrate = C.uint(params.BitRate) / 1000
cfg.kf_max_dist = C.uint(params.KeyFrameInterval)
cfg.rc_resize_allowed = 0
cfg.g_pass = C.VPX_RC_ONE_PASS

30
pkg/codec/x264/params.go
View File

@@ -1,7 +1,18 @@
package x264
import (
"io"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
// Params stores libx264 specific encoding parameters.
type Params struct {
codec.BaseParams
// Faster preset has lower CPU usage but lower quality
Preset Preset
}
@@ -21,3 +32,22 @@ const (
PresetVeryslow
PresetPlacebo
)
// NewParams returns default x264 codec specific parameters.
func NewParams() (Params, error) {
return Params{
BaseParams: codec.BaseParams{
KeyFrameInterval: 60,
},
}, nil
}
// Name represents the codec name
func (p *Params) Name() string {
return webrtc.H264
}
// BuildVideoEncoder builds x264 encoder with given params
func (p *Params) BuildVideoEncoder(r video.Reader, property prop.Media) (io.ReadCloser, error) {
return newEncoder(r, property, *p)
}

28
pkg/codec/x264/x264.go
View File

@@ -7,18 +7,15 @@ package x264
// #include "bridge.h"
import "C"
import (
"errors"
"fmt"
"image"
"io"
"sync"
"unsafe"
"github.com/pion/mediadevices/pkg/codec"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
"github.com/pion/webrtc/v2"
)
type encoder struct {
@@ -64,37 +61,24 @@ var (
errEncode = fmt.Errorf("failed to encode")
)
func init() {
codec.Register(webrtc.H264, codec.VideoEncoderBuilder(newEncoder))
}
func newEncoder(r video.Reader, p prop.Media) (io.ReadCloser, error) {
if p.KeyFrameInterval == 0 {
p.KeyFrameInterval = 60
}
var preset Preset
switch cp := p.CodecParams.(type) {
case nil:
case Params:
preset = cp.Preset
default:
return nil, errors.New("unsupported CodecParams type")
func newEncoder(r video.Reader, p prop.Media, params Params) (io.ReadCloser, error) {
if params.KeyFrameInterval == 0 {
params.KeyFrameInterval = 60
}
param := C.x264_param_t{
i_csp: C.X264_CSP_I420,
i_width: C.int(p.Width),
i_height: C.int(p.Height),
i_keyint_max: C.int(p.KeyFrameInterval),
i_keyint_max: C.int(params.KeyFrameInterval),
}
param.rc.i_bitrate = C.int(p.BitRate)
param.rc.i_bitrate = C.int(params.BitRate)
param.rc.i_vbv_max_bitrate = param.rc.i_bitrate
param.rc.i_vbv_buffer_size = param.rc.i_vbv_max_bitrate * 2
var rc C.int
// cPreset will be freed in C.enc_new
cPreset := C.CString(fmt.Sprint(preset))
cPreset := C.CString(fmt.Sprint(params.Preset))
engine := C.enc_new(param, cPreset, &rc)
if err := errFromC(rc); err != nil {
return nil, err

15
pkg/prop/prop.go
View File

@@ -14,7 +14,6 @@ type Media struct {
DeviceID string
Video
Audio
Codec
}
// Merge merges all the field values from o to p, except zero values.
@@ -111,17 +110,3 @@ type Audio struct {
SampleRate int
SampleSize int
}
// Codec represents an codec's encoding properties
type Codec struct {
CodecName string
// Target bitrate in bps.
BitRate int
// Expected interval of the keyframes in frames.
KeyFrameInterval int
// Library specific parameter struct defined by each codec package
CodecParams interface{}
}

106
track.go
View File

@@ -6,7 +6,6 @@ import (
"math/rand"
"sync"
"github.com/pion/mediadevices/pkg/codec"
"github.com/pion/mediadevices/pkg/driver"
mio "github.com/pion/mediadevices/pkg/io"
"github.com/pion/webrtc/v2"
@@ -42,16 +41,9 @@ type track struct {
endOnce sync.Once
}
func newTrack(codecs []*webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver, codecName string) (*track, error) {
var selectedCodec *webrtc.RTPCodec
for _, c := range codecs {
if c.Name == codecName {
selectedCodec = c
break
}
}
func newTrack(selectedCodec *webrtc.RTPCodec, trackGenerator TrackGenerator, d driver.Driver) (*track, error) {
if selectedCodec == nil {
return nil, fmt.Errorf("track: %s is not registered in media engine", codecName)
panic("codec is required")
}
t, err := trackGenerator(
@@ -116,13 +108,7 @@ type videoTrack struct {
var _ Tracker = &videoTrack{}
func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*videoTrack, error) {
codecName := constraints.CodecName
t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeVideo], opts.trackGenerator, d, codecName)
if err != nil {
return nil, err
}
err = d.Open()
err := d.Open()
if err != nil {
return nil, err
}
@@ -137,21 +123,47 @@ func newVideoTrack(opts *MediaDevicesOptions, d driver.Driver, constraints Media
r = constraints.VideoTransform(r)
}
encoder, err := codec.BuildVideoEncoder(r, constraints.Media)
if err != nil {
_ = d.Close()
return nil, err
var vt *videoTrack
rtpCodecs := opts.codecs[webrtc.RTPCodecTypeVideo]
for _, codecBuilder := range constraints.VideoEncoderBuilders {
var matchedRTPCodec *webrtc.RTPCodec
for _, rtpCodec := range rtpCodecs {
if rtpCodec.Name == codecBuilder.Name() {
matchedRTPCodec = rtpCodec
break
}
}
vt := videoTrack{
if matchedRTPCodec == nil {
continue
}
t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d)
if err != nil {
continue
}
encoder, err := codecBuilder.BuildVideoEncoder(r, constraints.Media)
if err != nil {
continue
}
vt = &videoTrack{
track: t,
d: d,
constraints: constraints,
encoder: encoder,
}
break
}
if vt == nil {
d.Close()
return nil, fmt.Errorf("failed to find a matching video codec")
}
go vt.start()
return &vt, nil
return vt, nil
}
func (vt *videoTrack) start() {
@@ -192,41 +204,61 @@ type audioTrack struct {
var _ Tracker = &audioTrack{}
func newAudioTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*audioTrack, error) {
codecName := constraints.CodecName
t, err := newTrack(opts.codecs[webrtc.RTPCodecTypeAudio], opts.trackGenerator, d, codecName)
if err != nil {
return nil, err
}
err = d.Open()
err := d.Open()
if err != nil {
return nil, err
}
ar := d.(driver.AudioRecorder)
reader, err := ar.AudioRecord(constraints.Media)
r, err := ar.AudioRecord(constraints.Media)
if err != nil {
return nil, err
}
if constraints.AudioTransform != nil {
reader = constraints.AudioTransform(reader)
r = constraints.AudioTransform(r)
}
encoder, err := codec.BuildAudioEncoder(reader, constraints.Media)
var at *audioTrack
rtpCodecs := opts.codecs[webrtc.RTPCodecTypeAudio]
for _, codecBuilder := range constraints.AudioEncoderBuilders {
var matchedRTPCodec *webrtc.RTPCodec
for _, rtpCodec := range rtpCodecs {
if rtpCodec.Name == codecBuilder.Name() {
matchedRTPCodec = rtpCodec
break
}
}
if matchedRTPCodec == nil {
continue
}
t, err := newTrack(matchedRTPCodec, opts.trackGenerator, d)
if err != nil {
_ = d.Close()
return nil, err
continue
}
at := audioTrack{
encoder, err := codecBuilder.BuildAudioEncoder(r, constraints.Media)
if err != nil {
continue
}
at = &audioTrack{
track: t,
d: d,
constraints: constraints,
encoder: encoder,
}
}
if at == nil {
d.Close()
return nil, fmt.Errorf("failed to find a matching audio codec")
}
go at.start()
return &at, nil
return at, nil
}
func (t *audioTrack) start() {