From aece2b94c61af90810c78c72166093e8f312c7de Mon Sep 17 00:00:00 2001 From: Lukas Herman Date: Wed, 5 Feb 2020 22:47:57 -0800 Subject: [PATCH] Refractor, unify some APIs to be more DRY --- examples/simple/main.go | 23 +++--- mediadevices.go | 144 +++++++++++++++------------------ mediastreamconstraints.go | 70 ++-------------- pkg/codec/codec.go | 5 +- pkg/codec/openh264/openh264.go | 11 +-- pkg/codec/opus/opus.go | 33 +++----- pkg/codec/registrar.go | 9 ++- pkg/driver/camera_linux.go | 56 +++++-------- pkg/driver/const.go | 8 -- pkg/driver/driver.go | 51 ++---------- pkg/driver/manager.go | 38 ++++----- pkg/driver/microphone_linux.go | 87 +++++++++----------- pkg/driver/state.go | 32 +++----- pkg/driver/state_test.go | 26 ++++++ pkg/driver/wrapper.go | 82 +++++++------------ pkg/io/audio/property.go | 16 ---- pkg/io/video/property.go | 16 ---- pkg/prop/prop.go | 74 +++++++++++++++++ track.go | 70 +++++++++------- 19 files changed, 362 insertions(+), 489 deletions(-) delete mode 100644 pkg/driver/const.go create mode 100644 pkg/driver/state_test.go delete mode 100644 pkg/io/audio/property.go delete mode 100644 pkg/io/video/property.go create mode 100644 pkg/prop/prop.go diff --git a/examples/simple/main.go b/examples/simple/main.go index 94d4206..ba36f66 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -7,7 +7,6 @@ import ( "github.com/pion/mediadevices/examples/internal/signal" _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder _ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder - "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/webrtc/v2" ) @@ -32,20 +31,18 @@ func main() { fmt.Printf("Connection State has changed %s \n", connectionState.String()) }) - mediaDevices := mediadevices.NewMediaDevices(peerConnection) + md := mediadevices.NewMediaDevices(peerConnection) - s, err := mediaDevices.GetUserMedia(mediadevices.MediaStreamConstraints{ - Audio: mediadevices.AudioTrackConstraints{ - Enabled: true, - Codec: webrtc.Opus, + s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ + Audio: func(c *mediadevices.MediaTrackConstraints) { + c.Codec = webrtc.Opus + c.Enabled = true }, - Video: mediadevices.VideoTrackConstraints{ - Enabled: true, - Property: video.Property{ - Width: 800, // Optional. This is just an ideal value. - Height: 480, // Optional. This is just an ideal value. - }, - Codec: webrtc.H264, + Video: func(c *mediadevices.MediaTrackConstraints) { + c.Codec = webrtc.H264 + c.Enabled = true + c.Width = 800 + c.Height = 480 }, }) if err != nil { diff --git a/mediadevices.go b/mediadevices.go index 254fc8a..ac72b03 100644 --- a/mediadevices.go +++ b/mediadevices.go @@ -5,8 +5,7 @@ import ( "math" "github.com/pion/mediadevices/pkg/driver" - "github.com/pion/mediadevices/pkg/io/audio" - "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" "github.com/pion/webrtc/v2" ) @@ -33,8 +32,17 @@ func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaSt // TODO: It should return media stream based on constraints trackers := make([]Tracker, 0) - if constraints.Video.Enabled { - tracker, err := m.videoSelect(constraints.Video) + var videoConstraints, audioConstraints MediaTrackConstraints + if constraints.Video != nil { + constraints.Video(&videoConstraints) + } + + if constraints.Audio != nil { + constraints.Audio(&audioConstraints) + } + + if videoConstraints.Enabled { + tracker, err := m.selectVideo(videoConstraints) if err != nil { return nil, err } @@ -42,8 +50,8 @@ func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaSt trackers = append(trackers, tracker) } - if constraints.Audio.Enabled { - tracker, err := m.audioSelect(constraints.Audio) + if audioConstraints.Enabled { + tracker, err := m.selectAudio(audioConstraints) if err != nil { return nil, err } @@ -59,102 +67,76 @@ func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaSt return s, nil } -// videoSelect implements SelectSettings algorithm for video type. -// Reference: https://w3c.github.io/mediacapture-main/#dfn-selectsettings -func (m *mediaDevices) videoSelect(constraints VideoTrackConstraints) (Tracker, error) { - drivers := driver.GetManager().VideoDrivers() - - var bestDriver driver.VideoDriver - var bestProp video.AdvancedProperty - minFitnessDist := math.Inf(1) +func queryDriverProperties(filter driver.FilterFn) map[driver.Driver][]prop.Media { + var needToClose []driver.Driver + drivers := driver.GetManager().Query(filter) + m := make(map[driver.Driver][]prop.Media) for _, d := range drivers { - wasClosed := d.Status() == driver.StateClosed - - if wasClosed { + if d.Status() == driver.StateClosed { err := d.Open() if err != nil { - // Skip this driver if we failed to open because we can't get the settings + // Skip this driver if we failed to open because we can't get the properties continue } + needToClose = append(needToClose, d) } - vd := d.(driver.VideoDriver) - for _, prop := range vd.Properties() { - fitnessDist := constraints.fitnessDistance(prop) + m[d] = d.Properties() + } + for _, d := range needToClose { + // Since it was closed, we should close it to avoid a leak + d.Close() + } + + return m +} + +// select implements SelectSettings algorithm. +// Reference: https://w3c.github.io/mediacapture-main/#dfn-selectsettings +func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints) (driver.Driver, MediaTrackConstraints, error) { + var bestDriver driver.Driver + var bestProp prop.Media + minFitnessDist := math.Inf(1) + + driverProperties := queryDriverProperties(filter) + for d, props := range driverProperties { + for _, p := range props { + fitnessDist := constraints.Media.FitnessDistance(p) if fitnessDist < minFitnessDist { minFitnessDist = fitnessDist - bestDriver = vd - bestProp = prop + bestDriver = d + bestProp = p } } - - if wasClosed { - // Since it was closed, we should close it to avoid a leak - d.Close() - } } if bestDriver == nil { - return nil, fmt.Errorf("failed to find the best setting") + return nil, MediaTrackConstraints{}, fmt.Errorf("failed to find the best driver that fits the constraints") } - if bestDriver.Status() == driver.StateClosed { - err := bestDriver.Open() - if err != nil { - return nil, fmt.Errorf("failed in opening the best video driver") - } + bestConstraint := MediaTrackConstraints{ + Media: bestProp, + Enabled: true, + Codec: constraints.Codec, } - return newVideoTrack(m.pc, bestDriver, bestProp, constraints.Codec) + return bestDriver, bestConstraint, nil } -// audioSelect implements SelectSettings algorithm for audio type. -// Reference: https://w3c.github.io/mediacapture-main/#dfn-selectsettings -func (m *mediaDevices) audioSelect(constraints AudioTrackConstraints) (Tracker, error) { - drivers := driver.GetManager().AudioDrivers() - - var bestDriver driver.AudioDriver - var bestProp audio.AdvancedProperty - minFitnessDist := math.Inf(1) - - for _, d := range drivers { - wasClosed := d.Status() == driver.StateClosed - - if wasClosed { - err := d.Open() - if err != nil { - // Skip this driver if we failed to open because we can't get the settings - continue - } - } - - ad := d.(driver.AudioDriver) - for _, prop := range ad.Properties() { - fitnessDist := constraints.fitnessDistance(prop) - - if fitnessDist < minFitnessDist { - minFitnessDist = fitnessDist - bestDriver = ad - bestProp = prop - } - } - - if wasClosed { - // Since it was closed, we should close it to avoid a leak - d.Close() - } +func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker, error) { + d, c, err := selectBestDriver(driver.FilterAudioRecorder(), constraints) + if err != nil { + return nil, err } - if bestDriver == nil { - return nil, fmt.Errorf("failed to find the best setting") - } - - if bestDriver.Status() == driver.StateClosed { - err := bestDriver.Open() - if err != nil { - return nil, fmt.Errorf("failed in opening the best audio driver") - } - } - return newAudioTrack(m.pc, bestDriver, bestProp, constraints.Codec) + return newAudioTrack(m.pc, d, c) +} +func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker, error) { + d, c, err := selectBestDriver(driver.FilterVideoRecorder(), constraints) + if err != nil { + return nil, err + } + + return newVideoTrack(m.pc, d, c) } diff --git a/mediastreamconstraints.go b/mediastreamconstraints.go index da3ceed..793f548 100644 --- a/mediastreamconstraints.go +++ b/mediastreamconstraints.go @@ -1,75 +1,19 @@ package mediadevices import ( - "fmt" - "math" - "strconv" - - "github.com/pion/mediadevices/pkg/io/audio" - "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" ) type MediaStreamConstraints struct { - Audio AudioTrackConstraints - Video VideoTrackConstraints + Audio MediaOption + Video MediaOption } -type comparisons map[string]string - -func (c comparisons) Add(actual, ideal interface{}) { - c[fmt.Sprint(actual)] = fmt.Sprint(ideal) -} - -// fitnessDistance is an implementation for https://w3c.github.io/mediacapture-main/#dfn-fitness-distance -func (c comparisons) fitnessDistance() float64 { - var dist float64 - - for actual, ideal := range c { - if actual == ideal { - continue - } - - actual, err1 := strconv.ParseFloat(actual, 64) - ideal, err2 := strconv.ParseFloat(ideal, 64) - - switch { - // If both of the values are numeric, we need to normalize the values to get the distance - case err1 == nil && err2 == nil: - dist += math.Abs(actual-ideal) / math.Max(math.Abs(actual), math.Abs(ideal)) - // If both of the values are not numeric, the only comparison value is either 1 (matched) or 0 (not matched) - case err1 != nil && err2 != nil: - dist++ - // Comparing a numeric value with a non-numeric value is a an internal error, so panic. - default: - panic("fitnessDistance can't mix comparisons.") - } - } - - return dist -} - -type VideoTrackConstraints struct { - video.Property +// MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints +type MediaTrackConstraints struct { + prop.Media Enabled bool Codec string } -func (c *VideoTrackConstraints) fitnessDistance(prop video.AdvancedProperty) float64 { - cmps := comparisons{} - cmps.Add(prop.Width, c.Width) - cmps.Add(prop.Height, c.Height) - return cmps.fitnessDistance() -} - -type AudioTrackConstraints struct { - audio.Property - Enabled bool - Codec string -} - -func (c *AudioTrackConstraints) fitnessDistance(prop audio.AdvancedProperty) float64 { - cmps := comparisons{} - cmps.Add(prop.SampleRate, c.SampleRate) - cmps.Add(prop.Latency, c.Latency) - return cmps.fitnessDistance() -} +type MediaOption func(*MediaTrackConstraints) diff --git a/pkg/codec/codec.go b/pkg/codec/codec.go index 1999206..be600ae 100644 --- a/pkg/codec/codec.go +++ b/pkg/codec/codec.go @@ -5,7 +5,8 @@ import ( "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" ) -type VideoEncoderBuilder func(r video.Reader, prop video.AdvancedProperty) (io.ReadCloser, error) -type AudioEncoderBuilder func(r audio.Reader, inProp, outProp audio.AdvancedProperty) (io.ReadCloser, error) +type VideoEncoderBuilder func(r video.Reader, p prop.Video) (io.ReadCloser, error) +type AudioEncoderBuilder func(r audio.Reader, p prop.Audio) (io.ReadCloser, error) diff --git a/pkg/codec/openh264/openh264.go b/pkg/codec/openh264/openh264.go index d659388..f849762 100644 --- a/pkg/codec/openh264/openh264.go +++ b/pkg/codec/openh264/openh264.go @@ -18,6 +18,7 @@ import ( "github.com/pion/mediadevices/pkg/codec" mio "github.com/pion/mediadevices/pkg/io" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" "github.com/pion/webrtc/v2" ) @@ -34,12 +35,12 @@ func init() { codec.Register(webrtc.H264, codec.VideoEncoderBuilder(NewEncoder)) } -func NewEncoder(r video.Reader, prop video.AdvancedProperty) (io.ReadCloser, error) { +func NewEncoder(r video.Reader, p prop.Video) (io.ReadCloser, error) { cEncoder, err := C.enc_new(C.EncoderOptions{ - width: C.int(prop.Width), - height: C.int(prop.Height), - target_bitrate: C.int(prop.BitRate), - max_fps: C.float(prop.FrameRate), + width: C.int(p.Width), + height: C.int(p.Height), + target_bitrate: C.int(p.BitRate), + max_fps: C.float(p.FrameRate), }) if err != nil { // TODO: better error message diff --git a/pkg/codec/opus/opus.go b/pkg/codec/opus/opus.go index 573d646..e883196 100644 --- a/pkg/codec/opus/opus.go +++ b/pkg/codec/opus/opus.go @@ -7,9 +7,9 @@ import ( "reflect" "unsafe" - "github.com/faiface/beep" "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/io/audio" + "github.com/pion/mediadevices/pkg/prop" "github.com/pion/webrtc/v2" "gopkg.in/hraban/opus.v2" ) @@ -29,22 +29,18 @@ func init() { codec.Register(webrtc.Opus, codec.AudioEncoderBuilder(NewEncoder)) } -func NewEncoder(r audio.Reader, inProp, outProp audio.AdvancedProperty) (io.ReadCloser, error) { - if inProp.SampleRate == 0 { +func NewEncoder(r audio.Reader, p prop.Audio) (io.ReadCloser, error) { + if p.SampleRate == 0 { return nil, fmt.Errorf("opus: inProp.SampleRate is required") } - if outProp.SampleRate == 0 { - outProp.SampleRate = 48000 - } - - if inProp.Latency == 0 { - inProp.Latency = 20 + if p.Latency == 0 { + p.Latency = 20 } // Select the nearest supported latency var targetLatency float64 - latencyInMS := float64(inProp.Latency.Milliseconds()) + latencyInMS := float64(p.Latency.Milliseconds()) nearestDist := math.Inf(+1) for _, latency := range latencies { dist := math.Abs(latency - latencyInMS) @@ -59,20 +55,14 @@ func NewEncoder(r audio.Reader, inProp, outProp audio.AdvancedProperty) (io.Read // Since audio.Reader only supports stereo mode, channels is always 2 channels := 2 - engine, err := opus.NewEncoder(outProp.SampleRate, channels, opus.AppVoIP) + engine, err := opus.NewEncoder(p.SampleRate, channels, opus.AppVoIP) if err != nil { return nil, err } - inBuffSize := targetLatency * float64(outProp.SampleRate) / 1000 + inBuffSize := targetLatency * float64(p.SampleRate) / 1000 inBuff := make([][2]float32, int(inBuffSize)) - streamer := audio.ToBeep(r) - newSampleRate := beep.SampleRate(outProp.SampleRate) - oldSampleRate := beep.SampleRate(inProp.SampleRate) - streamer = beep.Resample(3, oldSampleRate, newSampleRate, streamer) - - reader := audio.FromBeep(streamer) - e := encoder{engine, inBuff, reader} + e := encoder{engine, inBuff, r} return &e, nil } @@ -91,16 +81,13 @@ func (e *encoder) Read(p []byte) (n int, err error) { // While the buffer is not full, keep reading so that we meet the latency requirement for curN < len(e.inBuff) { - n, err := e.reader.Read(e.inBuff[curN:]) + n, err = e.reader.Read(e.inBuff[curN:]) if err != nil { return 0, err } curN += n } - if err != nil { - return 0, err - } n, err = e.engine.EncodeFloat32(flatten(e.inBuff), p) if err != nil { diff --git a/pkg/codec/registrar.go b/pkg/codec/registrar.go index 513a3ab..a381417 100644 --- a/pkg/codec/registrar.go +++ b/pkg/codec/registrar.go @@ -6,6 +6,7 @@ import ( "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" ) var ( @@ -22,20 +23,20 @@ func Register(name string, builder interface{}) { } } -func BuildVideoEncoder(name string, r video.Reader, prop video.AdvancedProperty) (io.ReadCloser, error) { +func BuildVideoEncoder(name string, r video.Reader, p prop.Video) (io.ReadCloser, error) { b, ok := videoEncoders[name] if !ok { return nil, fmt.Errorf("codec: can't find %s video encoder", name) } - return b(r, prop) + return b(r, p) } -func BuildAudioEncoder(name string, r audio.Reader, inProp, outProp audio.AdvancedProperty) (io.ReadCloser, error) { +func BuildAudioEncoder(name string, r audio.Reader, p prop.Audio) (io.ReadCloser, error) { b, ok := audioEncoders[name] if !ok { return nil, fmt.Errorf("codec: can't find %s audio encoder", name) } - return b(r, inProp, outProp) + return b(r, p) } diff --git a/pkg/driver/camera_linux.go b/pkg/driver/camera_linux.go index 88c1fe3..eb05613 100644 --- a/pkg/driver/camera_linux.go +++ b/pkg/driver/camera_linux.go @@ -10,6 +10,7 @@ import ( "github.com/blackjack/webcam" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" ) // Camera implementation using v4l2 @@ -19,11 +20,8 @@ type camera struct { cam *webcam.Webcam formats map[webcam.PixelFormat]frame.Format reversedFormats map[frame.Format]webcam.PixelFormat - properties []video.AdvancedProperty } -var _ VideoAdapter = &camera{} - func init() { // TODO: Probably try to get more cameras // Get default camera @@ -57,41 +55,27 @@ func (c *camera) Open() error { return err } - properties := make([]video.AdvancedProperty, 0) - for format := range cam.GetSupportedFormats() { - for _, frameSize := range cam.GetSupportedFrameSizes(format) { - properties = append(properties, video.AdvancedProperty{ - Property: video.Property{ - Width: int(frameSize.MaxWidth), - Height: int(frameSize.MaxHeight), - }, - FrameFormat: c.formats[format], - }) - } - } - c.cam = cam - c.properties = properties return nil } func (c *camera) Close() error { - c.properties = nil if c.cam == nil { return nil } - return c.cam.StopStreaming() + c.cam.StopStreaming() + return nil } -func (c *camera) Start(prop video.AdvancedProperty) (video.Reader, error) { - decoder, err := frame.NewDecoder(prop.FrameFormat) +func (c *camera) VideoRecord(p prop.Media) (video.Reader, error) { + decoder, err := frame.NewDecoder(p.FrameFormat) if err != nil { return nil, err } - pf := c.reversedFormats[prop.FrameFormat] - _, _, _, err = c.cam.SetImageFormat(pf, uint32(prop.Width), uint32(prop.Height)) + pf := c.reversedFormats[p.FrameFormat] + _, _, _, err = c.cam.SetImageFormat(pf, uint32(p.Width), uint32(p.Height)) if err != nil { return nil, err } @@ -124,23 +108,25 @@ func (c *camera) Start(prop video.AdvancedProperty) (video.Reader, error) { continue } - return decoder.Decode(b, prop.Width, prop.Height) + return decoder.Decode(b, p.Width, p.Height) } }) return r, nil } -func (c *camera) Stop() error { - return c.cam.StopStreaming() -} - -func (c *camera) Info() Info { - return Info{ - DeviceType: Camera, +func (c *camera) Properties() []prop.Media { + properties := make([]prop.Media, 0) + for format := range c.cam.GetSupportedFormats() { + for _, frameSize := range c.cam.GetSupportedFrameSizes(format) { + properties = append(properties, prop.Media{ + Video: prop.Video{ + Width: int(frameSize.MaxWidth), + Height: int(frameSize.MaxHeight), + FrameFormat: c.formats[format], + }, + }) + } } -} - -func (c *camera) Properties() []video.AdvancedProperty { - return c.properties + return properties } diff --git a/pkg/driver/const.go b/pkg/driver/const.go deleted file mode 100644 index ecb5d51..0000000 --- a/pkg/driver/const.go +++ /dev/null @@ -1,8 +0,0 @@ -package driver - -type DeviceType string - -const ( - Camera DeviceType = "camera" - Microphone = "microphone" -) diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 8b790db..0ba38f5 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -3,46 +3,21 @@ package driver import ( "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" ) -type OpenCloser interface { - Open() error - Close() error +type VideoRecorder interface { + VideoRecord(p prop.Media) (r video.Reader, err error) } -type Infoer interface { - Info() Info -} - -type Info struct { - DeviceType DeviceType -} - -type VideoCapable interface { - Start(prop video.AdvancedProperty) (video.Reader, error) - Stop() error - Properties() []video.AdvancedProperty -} - -type AudioCapable interface { - Start(prop audio.AdvancedProperty) (audio.Reader, error) - Stop() error - Properties() []audio.AdvancedProperty +type AudioRecorder interface { + AudioRecord(p prop.Media) (r audio.Reader, err error) } type Adapter interface { - OpenCloser - Infoer -} - -type VideoAdapter interface { - Adapter - VideoCapable -} - -type AudioAdapter interface { - Adapter - AudioCapable + Open() error + Close() error + Properties() []prop.Media } type Driver interface { @@ -50,13 +25,3 @@ type Driver interface { ID() string Status() State } - -type VideoDriver interface { - Driver - VideoCapable -} - -type AudioDriver interface { - Driver - AudioCapable -} diff --git a/pkg/driver/manager.go b/pkg/driver/manager.go index ffd8b3a..db361ac 100644 --- a/pkg/driver/manager.go +++ b/pkg/driver/manager.go @@ -1,11 +1,25 @@ package driver -import "fmt" - // FilterFn is being used to decide if a driver should be included in the // query result. type FilterFn func(Driver) bool +// FilterVideoRecorder return a filter function to get a list of registered VideoRecorders +func FilterVideoRecorder() FilterFn { + return func(d Driver) bool { + _, ok := d.(VideoRecorder) + return ok + } +} + +// FilterAudioRecorder return a filter function to get a list of registered AudioRecorders +func FilterAudioRecorder() FilterFn { + return func(d Driver) bool { + _, ok := d.(AudioRecorder) + return ok + } +} + // Manager is a singleton to manage multiple drivers and their states type Manager struct { drivers map[string]Driver @@ -23,10 +37,6 @@ func GetManager() *Manager { // Register registers adapter to be discoverable by Query func (m *Manager) Register(a Adapter) error { d := wrapAdapter(a) - if d == nil { - return fmt.Errorf("adapter has to be either VideoAdapter/AudioAdapter") - } - m.drivers[d.ID()] = d return nil } @@ -42,19 +52,3 @@ func (m *Manager) Query(f FilterFn) []Driver { return results } - -// VideoDrivers gets a list of registered VideoDriver -func (m *Manager) VideoDrivers() []Driver { - return m.Query(func(d Driver) bool { - _, ok := d.(VideoDriver) - return ok - }) -} - -// AudioDrivers gets a list of registered AudioDriver -func (m *Manager) AudioDrivers() []Driver { - return m.Query(func(d Driver) bool { - _, ok := d.(AudioDriver) - return ok - }) -} diff --git a/pkg/driver/microphone_linux.go b/pkg/driver/microphone_linux.go index 20cfeff..e14f221 100644 --- a/pkg/driver/microphone_linux.go +++ b/pkg/driver/microphone_linux.go @@ -6,16 +6,14 @@ import ( "github.com/jfreymuth/pulse" "github.com/pion/mediadevices/pkg/io/audio" + "github.com/pion/mediadevices/pkg/prop" ) type microphone struct { c *pulse.Client - s *pulse.RecordStream samplesChan chan<- []float32 } -var _ AudioAdapter = µphone{} - func init() { GetManager().Register(µphone{}) } @@ -31,52 +29,30 @@ func (m *microphone) Open() error { } func (m *microphone) Close() error { - m.c.Close() - if m.s != nil { - m.s.Close() + if m.samplesChan != nil { + close(m.samplesChan) + m.samplesChan = nil } + m.c.Close() return nil } -func (m *microphone) Start(prop audio.AdvancedProperty) (audio.Reader, error) { +func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) { var options []pulse.RecordOption - if prop.ChannelCount == 1 { + if p.ChannelCount == 1 { options = append(options, pulse.RecordMono) } else { options = append(options, pulse.RecordStereo) } - latency := prop.Latency.Seconds() - options = append(options, pulse.RecordSampleRate(prop.SampleRate), pulse.RecordLatency(latency)) + latency := p.Latency.Seconds() + options = append(options, pulse.RecordSampleRate(p.SampleRate), pulse.RecordLatency(latency)) samplesChan := make(chan []float32, 1) var buff []float32 var bi int var more bool - reader := audio.ReaderFunc(func(samples [][2]float32) (n int, err error) { - for i := range samples { - // if we don't have anything left in buff, we'll wait until we receive - // more samples - if bi == len(buff) { - buff, more = <-samplesChan - if !more { - return i, io.EOF - } - bi = 0 - } - - samples[i][0] = buff[bi] - if prop.ChannelCount == 2 { - samples[i][1] = buff[bi+1] - bi++ - } - bi++ - } - - return len(samples), nil - }) - handler := func(b []float32) { samplesChan <- b } @@ -86,28 +62,39 @@ func (m *microphone) Start(prop audio.AdvancedProperty) (audio.Reader, error) { return nil, err } + reader := audio.ReaderFunc(func(samples [][2]float32) (n int, err error) { + for i := range samples { + // if we don't have anything left in buff, we'll wait until we receive + // more samples + if bi == len(buff) { + buff, more = <-samplesChan + if !more { + stream.Close() + return i, io.EOF + } + bi = 0 + } + + samples[i][0] = buff[bi] + if p.ChannelCount == 2 { + samples[i][1] = buff[bi+1] + bi++ + } + bi++ + } + + return len(samples), nil + }) + stream.Start() - m.s = stream m.samplesChan = samplesChan return reader, nil } -func (m *microphone) Stop() error { - close(m.samplesChan) - m.s.Stop() - return nil -} - -func (m *microphone) Info() Info { - return Info{ - DeviceType: Microphone, - } -} - -func (m *microphone) Properties() []audio.AdvancedProperty { +func (m *microphone) Properties() []prop.Media { // TODO: Get actual properties - monoProp := audio.AdvancedProperty{ - Property: audio.Property{ + monoProp := prop.Media{ + Audio: prop.Audio{ SampleRate: 48000, Latency: time.Millisecond * 20, ChannelCount: 1, @@ -117,5 +104,5 @@ func (m *microphone) Properties() []audio.AdvancedProperty { stereoProp := monoProp stereoProp.ChannelCount = 2 - return []audio.AdvancedProperty{monoProp, stereoProp} + return []prop.Media{monoProp, stereoProp} } diff --git a/pkg/driver/state.go b/pkg/driver/state.go index cc9c583..16af6e1 100644 --- a/pkg/driver/state.go +++ b/pkg/driver/state.go @@ -3,22 +3,19 @@ package driver import "fmt" // State represents driver's state -type State uint +type State string const ( // StateClosed means that the driver has not been opened. In this state, // all information related to the hardware are still unknown. For example, // if it's a video driver, the pixel format information is still unknown. - StateClosed State = iota + StateClosed State = "closed" // StateOpened means that the driver is already opened and information about // the hardware are already known and may be extracted from the driver. - StateOpened - // StateStarted means that the driver has been sending data. The caller + StateOpened = "opened" + // StateRunning means that the driver has been sending data. The caller // who started the driver may start reading data from the hardware. - StateStarted - // StateStopped means that the driver is no longer sending data. In this state, - // information about the hardware is still available. - StateStopped + StateRunning = "running" ) // Update updates current state, s, to next. If f fails to execute, @@ -28,8 +25,7 @@ func (s *State) Update(next State, f func() error) error { m := map[State]checkFunc{ StateOpened: s.toOpened, StateClosed: s.toClosed, - StateStarted: s.toStarted, - StateStopped: s.toStopped, + StateRunning: s.toRunning, } err := m[next]() @@ -55,21 +51,13 @@ func (s *State) toClosed() error { return nil } -func (s *State) toStarted() error { +func (s *State) toRunning() error { if *s == StateClosed { - return fmt.Errorf("invalid state: driver hasn't been opened") + return fmt.Errorf("invalid state: driver is closed") } - if *s == StateStarted { - return fmt.Errorf("invalid state: driver has been started") - } - - return nil -} - -func (s *State) toStopped() error { - if *s != StateStarted { - return fmt.Errorf("invalid state: driver hasn't been started") + if *s == StateRunning { + return fmt.Errorf("invalid state: driver is already running") } return nil diff --git a/pkg/driver/state_test.go b/pkg/driver/state_test.go new file mode 100644 index 0000000..0c35a67 --- /dev/null +++ b/pkg/driver/state_test.go @@ -0,0 +1,26 @@ +package driver + +import "testing" + +var noop = func() error { return nil } + +func TestUpdate1(t *testing.T) { + s := StateClosed + s.Update(StateOpened, noop) + + if s != StateOpened { + t.Fatalf("expected %s, got %s", StateOpened, s) + } + + s.Update(StateClosed, noop) + + if s != StateClosed { + t.Fatalf("expected %s, got %s", StateClosed, s) + } + + s.Update(StateOpened, noop) + + if s != StateOpened { + t.Fatalf("expected %s, got %s", StateOpened, s) + } +} diff --git a/pkg/driver/wrapper.go b/pkg/driver/wrapper.go index fff56e6..65b1107 100644 --- a/pkg/driver/wrapper.go +++ b/pkg/driver/wrapper.go @@ -3,32 +3,39 @@ package driver import ( "github.com/pion/mediadevices/pkg/io/audio" "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/mediadevices/pkg/prop" uuid "github.com/satori/go.uuid" ) func wrapAdapter(a Adapter) Driver { - var d Driver id := uuid.NewV4().String() - wrapper := adapterWrapper{Adapter: a, id: id} + d := &adapterWrapper{Adapter: a, id: id, state: StateClosed} switch v := a.(type) { - case VideoCapable: - d = &videoAdapterWrapper{ - adapterWrapper: &wrapper, - VideoCapable: v, - } - case AudioCapable: - d = &audioAdapterWrapper{ - adapterWrapper: &wrapper, - AudioCapable: v, - } + case VideoRecorder: + // Only expose Driver and VideoRecorder interfaces + d.VideoRecorder = v + r := &struct { + Driver + VideoRecorder + }{d, d} + return r + case AudioRecorder: + // Only expose Driver and AudioRecorder interfaces + d.AudioRecorder = v + return &struct { + Driver + AudioRecorder + }{d, d} + default: + panic("adapter has to be either VideoRecorder/AudioRecorder") } - - return d } type adapterWrapper struct { Adapter + VideoRecorder + AudioRecorder id string state State } @@ -49,53 +56,18 @@ func (w *adapterWrapper) Close() error { return w.state.Update(StateClosed, w.Adapter.Close) } -// TODO: Add state validation -type videoAdapterWrapper struct { - *adapterWrapper - VideoCapable -} - -func (w *videoAdapterWrapper) Start(prop video.AdvancedProperty) (r video.Reader, err error) { - w.state.Update(StateStarted, func() error { - r, err = w.VideoCapable.Start(prop) +func (w *adapterWrapper) VideoRecord(p prop.Media) (r video.Reader, err error) { + w.state.Update(StateRunning, func() error { + r, err = w.VideoRecorder.VideoRecord(p) return err }) return } -func (w *videoAdapterWrapper) Stop() error { - return w.state.Update(StateStopped, w.VideoCapable.Stop) -} - -func (w *videoAdapterWrapper) Properties() []video.AdvancedProperty { - if w.state == StateClosed { - return nil - } - - return w.VideoCapable.Properties() -} - -type audioAdapterWrapper struct { - *adapterWrapper - AudioCapable -} - -func (w *audioAdapterWrapper) Start(prop audio.AdvancedProperty) (r audio.Reader, err error) { - w.state.Update(StateStarted, func() error { - r, err = w.AudioCapable.Start(prop) +func (w *adapterWrapper) AudioRecord(p prop.Media) (r audio.Reader, err error) { + w.state.Update(StateRunning, func() error { + r, err = w.AudioRecorder.AudioRecord(p) return err }) return } - -func (w *audioAdapterWrapper) Stop() error { - return w.state.Update(StateStopped, w.AudioCapable.Stop) -} - -func (w *audioAdapterWrapper) Properties() []audio.AdvancedProperty { - if w.state == StateClosed { - return nil - } - - return w.AudioCapable.Properties() -} diff --git a/pkg/io/audio/property.go b/pkg/io/audio/property.go deleted file mode 100644 index bf692ee..0000000 --- a/pkg/io/audio/property.go +++ /dev/null @@ -1,16 +0,0 @@ -package audio - -import "time" - -// Property represents an audio's basic properties -type Property struct { - ChannelCount int - Latency time.Duration - SampleRate int - SampleSize int -} - -// AdvancedProperty represents an audio's advanced properties. -type AdvancedProperty struct { - Property -} diff --git a/pkg/io/video/property.go b/pkg/io/video/property.go deleted file mode 100644 index 0c33440..0000000 --- a/pkg/io/video/property.go +++ /dev/null @@ -1,16 +0,0 @@ -package video - -import "github.com/pion/mediadevices/pkg/frame" - -// Property represents a video's basic properties -type Property struct { - Width, Height int - FrameRate float32 -} - -// AdvancedProperty represents a video's advanced properties. -type AdvancedProperty struct { - Property - FrameFormat frame.Format - BitRate int -} diff --git a/pkg/prop/prop.go b/pkg/prop/prop.go new file mode 100644 index 0000000..7f495aa --- /dev/null +++ b/pkg/prop/prop.go @@ -0,0 +1,74 @@ +package prop + +import ( + "fmt" + "math" + "strconv" + "time" + + "github.com/pion/mediadevices/pkg/frame" +) + +type Media struct { + Video + Audio +} + +func (p *Media) FitnessDistance(o Media) float64 { + cmps := comparisons{} + cmps.add(p.Width, o.Width) + cmps.add(p.Height, o.Height) + cmps.add(p.SampleRate, o.SampleRate) + cmps.add(p.Latency, o.Latency) + return cmps.fitnessDistance() +} + +type comparisons map[string]string + +func (c comparisons) add(actual, ideal interface{}) { + c[fmt.Sprint(actual)] = fmt.Sprint(ideal) +} + +// fitnessDistance is an implementation for https://w3c.github.io/mediacapture-main/#dfn-fitness-distance +func (c comparisons) fitnessDistance() float64 { + var dist float64 + + for actual, ideal := range c { + if actual == ideal { + continue + } + + actual, err1 := strconv.ParseFloat(actual, 64) + ideal, err2 := strconv.ParseFloat(ideal, 64) + + switch { + // If both of the values are numeric, we need to normalize the values to get the distance + case err1 == nil && err2 == nil: + dist += math.Abs(actual-ideal) / math.Max(math.Abs(actual), math.Abs(ideal)) + // If both of the values are not numeric, the only comparison value is either 1 (matched) or 0 (not matched) + case err1 != nil && err2 != nil: + dist++ + // Comparing a numeric value with a non-numeric value is a an internal error, so panic. + default: + panic("fitnessDistance can't mix comparisons.") + } + } + + return dist +} + +// Video represents a video's properties +type Video struct { + Width, Height int + FrameRate float32 + FrameFormat frame.Format + BitRate int +} + +// Audio represents an audio's properties +type Audio struct { + ChannelCount int + Latency time.Duration + SampleRate int + SampleSize int +} diff --git a/track.go b/track.go index 655f90f..8922138 100644 --- a/track.go +++ b/track.go @@ -8,8 +8,6 @@ import ( "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/driver" mio "github.com/pion/mediadevices/pkg/io" - "github.com/pion/mediadevices/pkg/io/audio" - "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/webrtc/v2" "github.com/pion/webrtc/v2/pkg/media" ) @@ -30,9 +28,9 @@ type track struct { func newTrack(pc *webrtc.PeerConnection, d driver.Driver, codecName string) (*track, error) { var kind webrtc.RTPCodecType switch d.(type) { - case driver.VideoDriver: + case driver.VideoRecorder: kind = webrtc.RTPCodecTypeVideo - case driver.AudioDriver: + case driver.AudioRecorder: kind = webrtc.RTPCodecTypeAudio } @@ -66,36 +64,43 @@ func (t *track) Track() *webrtc.Track { type videoTrack struct { *track - d driver.VideoDriver - property video.AdvancedProperty - encoder io.ReadCloser + d driver.Driver + constraints MediaTrackConstraints + encoder io.ReadCloser } var _ Tracker = &videoTrack{} -func newVideoTrack(pc *webrtc.PeerConnection, d driver.VideoDriver, prop video.AdvancedProperty, codecName string) (*videoTrack, error) { +func newVideoTrack(pc *webrtc.PeerConnection, d driver.Driver, constraints MediaTrackConstraints) (*videoTrack, error) { + codecName := constraints.Codec t, err := newTrack(pc, d, codecName) if err != nil { return nil, err } - r, err := d.Start(prop) + err = d.Open() + if err != nil { + return nil, err + } + + vr := d.(driver.VideoRecorder) + r, err := vr.VideoRecord(constraints.Media) if err != nil { return nil, err } // TODO: Remove hardcoded bitrate - prop.BitRate = 100000 - encoder, err := codec.BuildVideoEncoder(codecName, r, prop) + constraints.BitRate = 100000 + encoder, err := codec.BuildVideoEncoder(codecName, r, constraints.Video) if err != nil { return nil, err } vt := videoTrack{ - track: t, - d: d, - property: prop, - encoder: encoder, + track: t, + d: d, + constraints: constraints, + encoder: encoder, } go vt.start() @@ -123,44 +128,47 @@ func (vt *videoTrack) start() { } func (vt *videoTrack) Stop() { - vt.d.Stop() + vt.d.Close() vt.encoder.Close() } type audioTrack struct { *track - d driver.AudioDriver - property audio.AdvancedProperty - encoder io.ReadCloser + d driver.Driver + constraints MediaTrackConstraints + encoder io.ReadCloser } var _ Tracker = &audioTrack{} -func newAudioTrack(pc *webrtc.PeerConnection, d driver.AudioDriver, prop audio.AdvancedProperty, codecName string) (*audioTrack, error) { +func newAudioTrack(pc *webrtc.PeerConnection, d driver.Driver, constraints MediaTrackConstraints) (*audioTrack, error) { + codecName := constraints.Codec t, err := newTrack(pc, d, codecName) if err != nil { return nil, err } - reader, err := d.Start(prop) + err = d.Open() if err != nil { return nil, err } - // TODO: Not sure how to decide inProp and outProp - inProp := prop - outProp := prop + ar := d.(driver.AudioRecorder) + reader, err := ar.AudioRecord(constraints.Media) + if err != nil { + return nil, err + } - encoder, err := codec.BuildAudioEncoder(codecName, reader, inProp, outProp) + encoder, err := codec.BuildAudioEncoder(codecName, reader, constraints.Audio) if err != nil { return nil, err } at := audioTrack{ - track: t, - d: d, - property: prop, - encoder: encoder, + track: t, + d: d, + constraints: constraints, + encoder: encoder, } go at.start() return &at, nil @@ -168,7 +176,7 @@ func newAudioTrack(pc *webrtc.PeerConnection, d driver.AudioDriver, prop audio.A func (t *audioTrack) start() { buff := make([]byte, 1024) - sampleSize := uint32(float64(t.property.SampleRate) * t.property.Latency.Seconds()) + sampleSize := uint32(float64(t.constraints.SampleRate) * t.constraints.Latency.Seconds()) for { n, err := t.encoder.Read(buff) if err != nil { @@ -184,6 +192,6 @@ func (t *audioTrack) start() { } func (t *audioTrack) Stop() { - t.d.Stop() + t.d.Close() t.encoder.Close() }