mirror of
https://github.com/pion/mediadevices.git
synced 2025-09-27 12:52:20 +08:00
Compare commits
46 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
5d5001d0b4 | ||
|
|
c734c53b00 | ||
|
|
92ac89a620 | ||
|
|
ce032411a7 | ||
|
|
3ea7120130 | ||
|
|
282d0a4fb4 | ||
|
|
356eee19ce | ||
|
|
02d0cd3f44 | ||
|
|
a8dbecff30 | ||
|
|
273457b370 | ||
|
|
9846a7eb67 | ||
|
|
b2e72af884 | ||
|
|
b8dd3811ac | ||
|
|
c1958b62a2 | ||
|
|
ea90f86abd | ||
|
|
716da16e4a | ||
|
|
1550a68003 | ||
|
|
d65170dfe3 | ||
|
|
4057524bf0 | ||
|
|
8dd84b269c | ||
|
|
a73b1922ed | ||
|
|
11aea3eb85 | ||
|
|
cd49cd9910 | ||
|
|
6900da9a5e | ||
|
|
0a1944dc77 | ||
|
|
c3100355e5 | ||
|
|
b35246730d | ||
|
|
0c61817369 | ||
|
|
2fe26ea1f7 | ||
|
|
9d98eb8aaf | ||
|
|
3ea35bebab | ||
|
|
83c08e6c5f | ||
|
|
2f17017450 | ||
|
|
7cbda134b0 | ||
|
|
115be126ec | ||
|
|
79dcb4f1af | ||
|
|
5db4007e73 | ||
|
|
77ebcecac6 | ||
|
|
a0d0949954 | ||
|
|
f396092609 | ||
|
|
ee6cf08c44 | ||
|
|
6a211aa19f | ||
|
|
b089610c27 | ||
|
|
1d34ec9c5d | ||
|
|
7bd3efc8b7 | ||
|
|
8396fd7aac |
273
README.md
273
README.md
@@ -13,74 +13,225 @@
|
||||
</p>
|
||||
<br>
|
||||
|
||||
|
||||
`mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
|
||||
|
||||
## Interfaces
|
||||
## Install
|
||||
|
||||
| Interface | Linux | Mac | Windows |
|
||||
| :--------: | :---: | :-: | :-----: |
|
||||
| Camera | ✔️ | ✔️ | ✔️ |
|
||||
| Microphone | ✔️ | ✖️ | ✔️ |
|
||||
| Screen | ✔️ | ✖️ | ✖️ |
|
||||
|
||||
### Camera
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [Video4Linux](https://en.wikipedia.org/wiki/Video4Linux) |
|
||||
| Mac | [AVFoundation](https://developer.apple.com/av-foundation/) |
|
||||
| Windows | [DirectShow](https://docs.microsoft.com/en-us/windows/win32/directshow) |
|
||||
|
||||
| Pixel Format | Linux | Mac | Windows |
|
||||
| :---------------------------------------------------: | :---: | :-: | :-----: |
|
||||
| [YUY2](https://www.fourcc.org/pixel-format/yuv-yuy2/) | ✔️ | ✖️ | ✔️ |
|
||||
| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) | ✔️ | ✔️ | ✖️ |
|
||||
| [I420](https://www.fourcc.org/pixel-format/yuv-i420/) | ✔️ | ✖️ | ✖️ |
|
||||
| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) | ✔️ | ✔️ | ✖️ |
|
||||
| [MJPEG](https://www.fourcc.org/mjpg/) | ✔️ | ✖️ | ✖️ |
|
||||
|
||||
### Microphone
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [PulseAudio](https://en.wikipedia.org/wiki/PulseAudio) |
|
||||
| Mac | N/A |
|
||||
| Windows | [waveIn](https://docs.microsoft.com/en-us/windows/win32/api/mmeapi/) |
|
||||
|
||||
### Screen casting
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [X11](https://en.wikipedia.org/wiki/X_Window_System) |
|
||||
| Mac | N/A |
|
||||
| Windows | N/A |
|
||||
|
||||
## Codecs
|
||||
|
||||
| Audio Codec | Library/Interface |
|
||||
| :---------: | :------------------------------------------------------: |
|
||||
| OPUS | [libopus](http://opus-codec.org/) |
|
||||
|
||||
| Video Codec | Library/Interface |
|
||||
| :---------: | :------------------------------------------------------: |
|
||||
| H.264 | [OpenH264](https://www.openh264.org/) |
|
||||
| VP8 | [libvpx](https://www.webmproject.org/code/) |
|
||||
| VP9 | [libvpx](https://www.webmproject.org/code/) |
|
||||
`go get -u github.com/pion/mediadevices`
|
||||
|
||||
## Usage
|
||||
|
||||
[Wiki](https://github.com/pion/mediadevices/wiki)
|
||||
The following snippet shows how to capture a camera stream and store a frame as a jpeg image:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"image/jpeg"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
|
||||
// This is required to register camera adapter
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera"
|
||||
// Note: If you don't have a camera or your adapters are not supported,
|
||||
// you can always swap your adapters with our dummy adapters below.
|
||||
// _ "github.com/pion/mediadevices/pkg/driver/videotest"
|
||||
)
|
||||
|
||||
func main() {
|
||||
stream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(constraint *mediadevices.MediaTrackConstraints) {
|
||||
// Query for ideal resolutions
|
||||
constraint.Width = prop.Int(600)
|
||||
constraint.Height = prop.Int(400)
|
||||
},
|
||||
})
|
||||
|
||||
// Since track can represent audio as well, we need to cast it to
|
||||
// *mediadevices.VideoTrack to get video specific functionalities
|
||||
track := stream.GetVideoTracks()[0]
|
||||
videoTrack := track.(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
// Create a new video reader to get the decoded frames. Release is used
|
||||
// to return the buffer to hold frame back to the source so that the buffer
|
||||
// can be reused for the next frames.
|
||||
videoReader := videoTrack.NewReader(false)
|
||||
frame, release, _ := videoReader.Read()
|
||||
defer release()
|
||||
|
||||
// Since frame is the standard image.Image, it's compatible with Go standard
|
||||
// library. For example, capturing the first frame and store it as a jpeg image.
|
||||
output, _ := os.Create("frame.jpg")
|
||||
jpeg.Encode(output, frame, nil)
|
||||
}
|
||||
|
||||
|
||||
```
|
||||
|
||||
## More Examples
|
||||
|
||||
* [Webrtc](/examples/webrtc) - Use Webrtc to create a realtime peer-to-peer video call
|
||||
* [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
|
||||
* [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
|
||||
* [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
|
||||
* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
|
||||
|
||||
## Available Media Inputs
|
||||
|
||||
| Input | Linux | Mac | Windows |
|
||||
| :--------: | :---: | :-: | :-----: |
|
||||
| Camera | ✔️ | ✔️ | ✔️ |
|
||||
| Microphone | ✔️ | ✔️ | ✔️ |
|
||||
| Screen | ✔️ | ✔️ | ✔️ |
|
||||
|
||||
By default, there's no media input registered. This decision was made to allow you to pay what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
|
||||
|
||||
```go
|
||||
import (
|
||||
...
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera"
|
||||
)
|
||||
```
|
||||
|
||||
## Available Codecs
|
||||
|
||||
In order to encode your video/audio, `mediadevices` needs to know what codecs that you want to use and their parameters. To do this, you need to import the associated packages for the codecs, and add them to the codec selector that you'll pass to `GetUserMedia`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
)
|
||||
|
||||
func main() {
|
||||
// configure codec specific parameters
|
||||
x264Params, _ := x264.NewParams()
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {},
|
||||
Codec: codecSelector, // let GetUsermedia know available codecs
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
Since `mediadevices` doesn't implement the video/audio codecs, it needs to call the codec libraries from the system through cgo. Therefore, you're required to install the codec libraries before you can use them in `mediadevices`. In the next section, it shows a list of available codecs, where the packages are defined (documentation linked), and installation instructions.
|
||||
|
||||
Note: we do not provide recommendations on choosing one codec or another as it is very complex and can be subjective.
|
||||
|
||||
### Video Codecs
|
||||
|
||||
#### x264
|
||||
A free software library and application for encoding video streams into the H.264/MPEG-4 AVC compression format.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/x264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/x264)
|
||||
* Installation:
|
||||
* Mac: `brew install x264`
|
||||
* Ubuntu: `apt install libx264-dev`
|
||||
|
||||
#### mmal
|
||||
A framework to enable H264 hardware encoding for Raspberry Pi or boards that use VideoCore GPUs.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/mmal](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/mmal)
|
||||
* Installation: no installation needed, mmal should come built in Raspberry Pi devices
|
||||
|
||||
#### openh264
|
||||
A codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/openh264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/openh264)
|
||||
* Installation: no installation needed, included as a static binary
|
||||
|
||||
#### vpx
|
||||
A free software video codec library from Google and the Alliance for Open Media that implements VP8/VP9 video coding formats.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/vpx](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vpx)
|
||||
* Installation:
|
||||
* Mac: `brew install libvpx`
|
||||
* Ubuntu: `apt install libvpx-dev`
|
||||
|
||||
#### vaapi
|
||||
An open source API that allows applications such as VLC media player or GStreamer to use hardware video acceleration capabilities (currently support VP8/VP9).
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/vaapi](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vaapi)
|
||||
* Installation:
|
||||
* Ubuntu: `apt install libva-dev`
|
||||
|
||||
|
||||
### Audio Codecs
|
||||
|
||||
#### opus
|
||||
A totally open, royalty-free, highly versatile audio codec.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/opus](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/opus)
|
||||
* Installation:
|
||||
* Mac: `brew install opus`
|
||||
* Ubuntu: `apt install libopus-dev`
|
||||
|
||||
## Benchmark
|
||||
|
||||
Result as of Nov 4, 2020 with Go 1.14 on a Raspberry pi 3, `mediadevices` can produce video, encode, send across network, and decode at **720p, 30 fps with < 500 ms latency**.
|
||||
|
||||
The test was taken by capturing a camera stream, decoding the raw frames, encoding the video stream with mmal, and sending the stream through Webrtc.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Failed to find the best driver that fits the constraints
|
||||
|
||||
`mediadevices` provides an automated driver discovery through `GetUserMedia` and `GetDisplayMedia`. The driver discover algorithm works something like:
|
||||
|
||||
1. Open all registered drivers
|
||||
2. Get all properties (property describes what a driver is capable of, e.g. resolution, frame rate, etc.) from opened drivers
|
||||
3. Find the best property that meets the criteria
|
||||
|
||||
So, when `mediadevices` returns `failed to find the best driver that fits the constraints` error, one of the following conditions might have occured:
|
||||
* Driver was not imported as a side effect in your program, e.g. `import _ github.com/pion/mediadevices/pkg/driver/camera`
|
||||
* Your constraint is too strict that there's no driver can fullfil your requirements. In this case, you can try to turn up the debug level by specifying the following environment variable: `export PION_LOG_DEBUG=all` to see what was too strict and tune that.
|
||||
* Your driver is not supported/implemented. In this case, you can either let us know (file an issue) and wait for the maintainers to implement it. Or, you can implement it yourself and register it through `RegisterDriverAdapter`
|
||||
|
||||
### Failed to find vpx/x264/mmal/opus codecs
|
||||
|
||||
Since `mediadevices` uses cgo to access video/audio codecs, it needs to find these libraries from the system. To accomplish this, [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) is used for library discovery.
|
||||
|
||||
If you see the following error message at compile time:
|
||||
```
|
||||
# pkg-config --cflags -- vpx
|
||||
Package vpx was not found in the pkg-config search path.
|
||||
Perhaps you should add the directory containing `vpx.pc'
|
||||
to the PKG_CONFIG_PATH environment variable
|
||||
No package 'vpx' found
|
||||
pkg-config: exit status 1
|
||||
```
|
||||
|
||||
There are 2 common problems:
|
||||
|
||||
* The required codec library is not installed (vpx in this example). In this case, please refer to the [available codecs](#available-codecs).
|
||||
* Pkg-config fails to find the `.pc` files for this codec ([reference](https://people.freedesktop.org/~dbn/pkg-config-guide.html#using)). In this case, you need to find where the codec library's `.pc` is stored, and let pkg-config knows with: `export PKG_CONFIG_PATH=/path/to/directory`.
|
||||
|
||||
|
||||
## Community
|
||||
Pion has an active community on the [Slack](https://pion.ly/slack).
|
||||
|
||||
Follow the [Pion Twitter](https://twitter.com/_pion) for project updates and important WebRTC news.
|
||||
|
||||
We are always looking to support **your projects**. Please reach out if you have something to build!
|
||||
If you need commercial support or don't want to use public methods you can contact us at [team@pion.ly](mailto:team@pion.ly)
|
||||
|
||||
## Contributing
|
||||
Check out the **[contributing wiki](https://github.com/pion/webrtc/wiki/Contributing)** to join the group of amazing people making this project possible:
|
||||
|
||||
- [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
|
||||
* [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
|
||||
* [Atsushi Watanabe](https://github.com/at-wat) - _VP8, Screencast, etc._
|
||||
|
||||
## Project Status
|
||||
|
||||
[](https://starchart.cc/pion/mediadevices)
|
||||
|
||||
## References
|
||||
|
||||
- https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs
|
||||
- https://tools.ietf.org/html/rfc7742
|
||||
## License
|
||||
MIT License - see [LICENSE](LICENSE) for full text
|
||||
|
||||
34
codec.go
34
codec.go
@@ -58,17 +58,16 @@ func (selector *CodecSelector) Populate(setting *webrtc.MediaEngine) {
|
||||
}
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectVideoCodec(wantCodecs []*webrtc.RTPCodec, reader video.Reader, inputProp prop.Media) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var selectedEncoder codec.VideoEncoderBuilder
|
||||
var encodedReader codec.ReadCloser
|
||||
var errReasons []string
|
||||
var err error
|
||||
|
||||
outer:
|
||||
for _, wantCodec := range wantCodecs {
|
||||
name := wantCodec.Name
|
||||
for _, wantCodec := range codecNames {
|
||||
for _, encoder := range selector.videoEncoders {
|
||||
if encoder.RTPCodec().Name == name {
|
||||
if encoder.RTPCodec().Name == wantCodec {
|
||||
encodedReader, err = encoder.BuildVideoEncoder(reader, inputProp)
|
||||
if err == nil {
|
||||
selectedEncoder = encoder
|
||||
@@ -87,17 +86,26 @@ outer:
|
||||
return encodedReader, selectedEncoder.RTPCodec(), nil
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectAudioCodec(wantCodecs []*webrtc.RTPCodec, reader audio.Reader, inputProp prop.Media) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var codecNames []string
|
||||
|
||||
for _, codec := range codecs {
|
||||
codecNames = append(codecNames, codec.Name)
|
||||
}
|
||||
|
||||
return selector.selectVideoCodecByNames(reader, inputProp, codecNames...)
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var selectedEncoder codec.AudioEncoderBuilder
|
||||
var encodedReader codec.ReadCloser
|
||||
var errReasons []string
|
||||
var err error
|
||||
|
||||
outer:
|
||||
for _, wantCodec := range wantCodecs {
|
||||
name := wantCodec.Name
|
||||
for _, wantCodec := range codecNames {
|
||||
for _, encoder := range selector.audioEncoders {
|
||||
if encoder.RTPCodec().Name == name {
|
||||
if encoder.RTPCodec().Name == wantCodec {
|
||||
encodedReader, err = encoder.BuildAudioEncoder(reader, inputProp)
|
||||
if err == nil {
|
||||
selectedEncoder = encoder
|
||||
@@ -115,3 +123,13 @@ outer:
|
||||
|
||||
return encodedReader, selectedEncoder.RTPCodec(), nil
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var codecNames []string
|
||||
|
||||
for _, codec := range codecs {
|
||||
codecNames = append(codecNames, codec.Name)
|
||||
}
|
||||
|
||||
return selector.selectAudioCodecByNames(reader, inputProp, codecNames...)
|
||||
}
|
||||
|
||||
38
examples/archive/README.md
Normal file
38
examples/archive/README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
## Instructions
|
||||
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
|
||||
### Download archive examplee
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Run archive example
|
||||
|
||||
Run `cd mediadevices/examples/archive && go build && ./archive recorded.h264`
|
||||
|
||||
To stop recording, press `Ctrl+c` or send a SIGINT signal.
|
||||
|
||||
### Playback recorded video
|
||||
|
||||
Install GStreamer and run:
|
||||
```
|
||||
gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
|
||||
```
|
||||
|
||||
Or run VLC media plyer:
|
||||
```
|
||||
vlc recorded.h264
|
||||
```
|
||||
|
||||
A video should start playing in your GStreamer or VLC window.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
|
||||
BIN
examples/archive/archive
Executable file
BIN
examples/archive/archive
Executable file
Binary file not shown.
82
examples/archive/main.go
Normal file
82
examples/archive/main.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/io/video"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s <path/to/file.h264>\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT)
|
||||
|
||||
x264Params, err := x264.NewParams()
|
||||
must(err)
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
Codec: codecSelector,
|
||||
})
|
||||
must(err)
|
||||
|
||||
videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
|
||||
return video.ReaderFunc(func() (img image.Image, release func(), err error) {
|
||||
// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
|
||||
// will finish its execution and the program will finish
|
||||
select {
|
||||
case <-sigs:
|
||||
return nil, func() {}, io.EOF
|
||||
default:
|
||||
}
|
||||
|
||||
return r.Read()
|
||||
})
|
||||
}))
|
||||
|
||||
reader, err := videoTrack.NewEncodedReader(x264Params.RTPCodec().Name)
|
||||
must(err)
|
||||
defer reader.Close()
|
||||
|
||||
out, err := os.Create(dest)
|
||||
must(err)
|
||||
|
||||
fmt.Println("Recording... Press Ctrl+c to stop")
|
||||
_, err = io.Copy(out, reader)
|
||||
must(err)
|
||||
fmt.Println("Your video has been recorded to", dest)
|
||||
}
|
||||
15
examples/facedetection/README.md
Normal file
15
examples/facedetection/README.md
Normal file
@@ -0,0 +1,15 @@
|
||||
## Instructions
|
||||
|
||||
### Download facedetection example
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Compile and Run facedetection
|
||||
|
||||
Run `cd mediadevices/examples/facedetection && go build && ./facedetection`
|
||||
|
||||
You should be able to see some loggings when it can see faces.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
BIN
examples/facedetection/facefinder
Normal file
BIN
examples/facedetection/facefinder
Normal file
Binary file not shown.
107
examples/facedetection/main.go
Normal file
107
examples/facedetection/main.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"image"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
pigo "github.com/esimov/pigo/core"
|
||||
"github.com/pion/mediadevices"
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
const (
|
||||
confidenceLevel = 5.0
|
||||
)
|
||||
|
||||
var (
|
||||
cascade []byte
|
||||
classifier *pigo.Pigo
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func detectFace(frame *image.YCbCr) bool {
|
||||
bounds := frame.Bounds()
|
||||
cascadeParams := pigo.CascadeParams{
|
||||
MinSize: 100,
|
||||
MaxSize: 600,
|
||||
ShiftFactor: 0.15,
|
||||
ScaleFactor: 1.1,
|
||||
ImageParams: pigo.ImageParams{
|
||||
Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
|
||||
Rows: bounds.Dy(),
|
||||
Cols: bounds.Dx(),
|
||||
Dim: bounds.Dx(),
|
||||
},
|
||||
}
|
||||
|
||||
// Run the classifier over the obtained leaf nodes and return the detection results.
|
||||
// The result contains quadruplets representing the row, column, scale and detection score.
|
||||
dets := classifier.RunCascade(cascadeParams, 0.0)
|
||||
|
||||
// Calculate the intersection over union (IoU) of two clusters.
|
||||
dets = classifier.ClusterDetections(dets, 0)
|
||||
|
||||
for _, det := range dets {
|
||||
if det.Q >= confidenceLevel {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
// prepare face detector
|
||||
var err error
|
||||
cascade, err = ioutil.ReadFile("facefinder")
|
||||
if err != nil {
|
||||
log.Fatalf("Error reading the cascade file: %s", err)
|
||||
}
|
||||
p := pigo.NewPigo()
|
||||
|
||||
// Unpack the binary file. This will return the number of cascade trees,
|
||||
// the tree depth, the threshold and the prediction from tree's leaf nodes.
|
||||
classifier, err = p.Unpack(cascade)
|
||||
if err != nil {
|
||||
log.Fatalf("Error unpacking the cascade file: %s", err)
|
||||
}
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
})
|
||||
must(err)
|
||||
|
||||
// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
|
||||
videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
videoReader := videoTrack.NewReader(false)
|
||||
// To save resources, we can simply use 4 fps to detect faces.
|
||||
ticker := time.NewTicker(time.Millisecond * 250)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
frame, release, err := videoReader.Read()
|
||||
must(err)
|
||||
|
||||
// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
|
||||
if detectFace(frame.(*image.YCbCr)) {
|
||||
log.Println("Detect a face")
|
||||
}
|
||||
|
||||
release()
|
||||
}
|
||||
}
|
||||
19
examples/http/README.md
Normal file
19
examples/http/README.md
Normal file
@@ -0,0 +1,19 @@
|
||||
## Instructions
|
||||
|
||||
### Download http example
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Compile and Run HTTP server
|
||||
|
||||
Run `cd mediadevices/examples/http && go build && ./http :1313`
|
||||
|
||||
|
||||
### Access the camera stream from the browser
|
||||
|
||||
Go to "http://localhost:1313"
|
||||
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
@@ -28,7 +29,13 @@ func must(err error) {
|
||||
}
|
||||
|
||||
func main() {
|
||||
s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s host:port\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(constraint *mediadevices.MediaTrackConstraints) {
|
||||
constraint.Width = prop.Int(600)
|
||||
constraint.Height = prop.Int(400)
|
||||
@@ -36,8 +43,9 @@ func main() {
|
||||
})
|
||||
must(err)
|
||||
|
||||
t := s.GetVideoTracks()[0]
|
||||
videoTrack := t.(*mediadevices.VideoTrack)
|
||||
track := mediaStream.GetVideoTracks()[0]
|
||||
videoTrack := track.(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
@@ -72,6 +80,6 @@ func main() {
|
||||
}
|
||||
})
|
||||
|
||||
fmt.Println("listening on http://localhost:1313")
|
||||
log.Println(http.ListenAndServe("localhost:1313", nil))
|
||||
fmt.Printf("listening on %s\n", dest)
|
||||
log.Println(http.ListenAndServe(dest, nil))
|
||||
}
|
||||
|
||||
38
examples/rtp/README.md
Normal file
38
examples/rtp/README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
## Instructions
|
||||
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
|
||||
### Download rtp example
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Listen RTP
|
||||
|
||||
Install GStreamer and run:
|
||||
```
|
||||
gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=H264 \
|
||||
! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
|
||||
```
|
||||
|
||||
Or run VLC media plyer:
|
||||
```
|
||||
vlc ./h264.sdp
|
||||
```
|
||||
|
||||
### Run rtp
|
||||
|
||||
Run `cd mediadevices/examples/archive && go build && ./rtp localhost:5000`
|
||||
|
||||
A video should start playing in your GStreamer or VLC window.
|
||||
It's not WebRTC, but pure RTP.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
|
||||
9
examples/rtp/h264.sdp
Normal file
9
examples/rtp/h264.sdp
Normal file
@@ -0,0 +1,9 @@
|
||||
v=0
|
||||
o=- 1234567890 1234567890 IN IP4 0.0.0.0
|
||||
s=RTP-Send Example
|
||||
i=Example
|
||||
c=IN IP4 0.0.0.0
|
||||
t=0 0
|
||||
a=recvonly
|
||||
m=video 5000 RTP/AVP 100
|
||||
a=rtpmap:100 H264/90000
|
||||
77
examples/rtp/main.go
Normal file
77
examples/rtp/main.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
const (
|
||||
mtu = 1000
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s host:port\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
x264Params, err := x264.NewParams()
|
||||
must(err)
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
Codec: codecSelector,
|
||||
})
|
||||
must(err)
|
||||
|
||||
videoTrack := mediaStream.GetVideoTracks()[0]
|
||||
defer videoTrack.Close()
|
||||
|
||||
rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().Name, mtu)
|
||||
must(err)
|
||||
|
||||
addr, err := net.ResolveUDPAddr("udp", dest)
|
||||
must(err)
|
||||
conn, err := net.DialUDP("udp", nil, addr)
|
||||
must(err)
|
||||
|
||||
buff := make([]byte, mtu)
|
||||
for {
|
||||
pkts, release, err := rtpReader.Read()
|
||||
must(err)
|
||||
|
||||
for _, pkt := range pkts {
|
||||
n, err := pkt.MarshalTo(buff)
|
||||
must(err)
|
||||
|
||||
_, err = conn.Write(buff[:n])
|
||||
must(err)
|
||||
}
|
||||
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -1,29 +1,42 @@
|
||||
## Instructions
|
||||
|
||||
### Download gstreamer-send
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
* [opus](https://github.com/pion/mediadevices#opus)
|
||||
|
||||
### Download webrtc example
|
||||
|
||||
```
|
||||
go get github.com/pion/mediadevices/examples/webrtc
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
#### Compile webrtc example
|
||||
|
||||
```
|
||||
cd mediadevices/examples/webrtc && go build
|
||||
```
|
||||
|
||||
### Open example page
|
||||
|
||||
[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
|
||||
|
||||
### Run webrtc with your browsers SessionDescription as stdin
|
||||
### Run the webrtc example with your browsers SessionDescription as stdin
|
||||
|
||||
In the jsfiddle the top textarea is your browser, copy that and:
|
||||
In the jsfiddle the top textarea is your browser, copy that, and store the session description in an environment variable, `export SDP=<put_the_sdp_here>`
|
||||
|
||||
#### Linux
|
||||
|
||||
Run `echo $BROWSER_SDP | webrtc`
|
||||
Run `echo $SDP | ./webrtc`
|
||||
|
||||
### Input webrtc's SessionDescription into your browser
|
||||
|
||||
Copy the text that `webrtc` just emitted and copy into second text area
|
||||
Copy the text that `./webrtc` just emitted and copy into second text area
|
||||
|
||||
### Hit 'Start Session' in jsfiddle, enjoy your video!
|
||||
|
||||
A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
|
||||
|
||||
Congrats, you have used pion-WebRTC! Now start building something cool
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/webrtc/v2"
|
||||
|
||||
// If you don't like vpx, you can also use x264 by importing as below
|
||||
// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
|
||||
// If you don't like x264, you can also use vpx by importing as below
|
||||
// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
|
||||
// or you can also use openh264 for alternative h264 implementation
|
||||
// "github.com/pion/mediadevices/pkg/codec/openh264"
|
||||
// or if you use a raspberry pi like, you can use mmal for using its hardware encoder
|
||||
// "github.com/pion/mediadevices/pkg/codec/mmal"
|
||||
"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use VP8/VP9 video encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use opus audio encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
|
||||
|
||||
// Note: If you don't have a camera or microphone or your adapters are not supported,
|
||||
// you can always swap your adapters with our dummy adapters below.
|
||||
@@ -26,10 +26,6 @@ import (
|
||||
_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
|
||||
)
|
||||
|
||||
const (
|
||||
videoCodecName = webrtc.VP8
|
||||
)
|
||||
|
||||
func main() {
|
||||
config := webrtc.Configuration{
|
||||
ICEServers: []webrtc.ICEServer{
|
||||
@@ -44,18 +40,18 @@ func main() {
|
||||
signal.Decode(signal.MustReadStdin(), &offer)
|
||||
|
||||
// Create a new RTCPeerConnection
|
||||
vp8Params, err := vpx.NewVP8Params()
|
||||
x264Params, err := x264.NewParams()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
vp8Params.BitRate = 300_000 // 300kbps
|
||||
x264Params.BitRate = 500_000 // 500kbps
|
||||
|
||||
opusParams, err := opus.NewParams()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&vp8Params),
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
mediadevices.WithAudioEncoders(&opusParams),
|
||||
)
|
||||
|
||||
|
||||
9
go.mod
9
go.mod
@@ -3,12 +3,17 @@ module github.com/pion/mediadevices
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd // indirect
|
||||
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
|
||||
github.com/gen2brain/malgo v0.10.25
|
||||
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
|
||||
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
|
||||
github.com/lherman-cs/opus v0.0.2
|
||||
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 // indirect
|
||||
github.com/pion/logging v0.2.2
|
||||
github.com/pion/rtp v1.6.1
|
||||
github.com/pion/webrtc/v2 v2.2.26
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
|
||||
)
|
||||
|
||||
19
go.sum
19
go.sum
@@ -1,3 +1,5 @@
|
||||
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd h1:u7K2oMFMd8APDV3fM1j2rO3U/XJf1g1qC3DDTKou8iM=
|
||||
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
|
||||
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
|
||||
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
|
||||
@@ -7,6 +9,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gen2brain/malgo v0.10.25 h1:VRiYTBmBeHTCXD0wCg7XyLi6lJJBqND/XVmSEyrGkGc=
|
||||
github.com/gen2brain/malgo v0.10.25/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
|
||||
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
|
||||
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
|
||||
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
@@ -15,8 +21,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
|
||||
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f h1:5hWo+DzJQSOBl6X+TDac0SPWffRonuRJ2///OYtYRT8=
|
||||
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -26,6 +32,8 @@ github.com/lherman-cs/opus v0.0.2 h1:fE9Du3NKXDBztqvoTd6P2y9eJ9vgIHahGK8yQostnhA
|
||||
github.com/lherman-cs/opus v0.0.2/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
|
||||
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
|
||||
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
|
||||
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 h1:4BxFx5XCtXc+nFtXDGDW+Uu5sPtsAbvPh6RObj3fG9o=
|
||||
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
|
||||
github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
|
||||
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -53,6 +61,8 @@ github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
|
||||
github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
|
||||
github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
|
||||
github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
|
||||
github.com/pion/rtp v1.6.1 h1:2Y2elcVBrahYnHKN2X7rMHX/r1R4TEBMP1LaVu/wNhk=
|
||||
github.com/pion/rtp v1.6.1/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
|
||||
github.com/pion/sctp v1.7.10 h1:o3p3/hZB5Cx12RMGyWmItevJtZ6o2cpuxaw6GOS4x+8=
|
||||
github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
|
||||
github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=
|
||||
@@ -109,8 +119,9 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
|
||||
BIN
img/demo.gif
BIN
img/demo.gif
Binary file not shown.
|
Before Width: | Height: | Size: 9.6 MiB |
14
ioreader.go
Normal file
14
ioreader.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package mediadevices
|
||||
|
||||
type encodedReadCloserImpl struct {
|
||||
readFn func([]byte) (int, error)
|
||||
closeFn func() error
|
||||
}
|
||||
|
||||
func (r *encodedReadCloserImpl) Read(b []byte) (int, error) {
|
||||
return r.readFn(b)
|
||||
}
|
||||
|
||||
func (r *encodedReadCloserImpl) Close() error {
|
||||
return r.closeFn()
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
// +build e2e
|
||||
|
||||
package mediadevices
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package mediadevices
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/pion/webrtc/v2"
|
||||
@@ -33,6 +34,14 @@ func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (track *mockMediaStreamTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestMediaStreamFilters(t *testing.T) {
|
||||
audioTracks := []Track{
|
||||
&mockMediaStreamTrack{AudioInput},
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
// Reference: https://github.com/raspberrypi/userland/tree/master/interface/mmal
|
||||
package mmal
|
||||
|
||||
// #cgo pkg-config: mmal
|
||||
// #cgo CFLAGS: -I/opt/vc/include
|
||||
// #cgo LDFLAGS: -L/opt/vc/lib -lmmal -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host -lvcsm -lvcos
|
||||
// #include "bridge.h"
|
||||
import "C"
|
||||
import (
|
||||
|
||||
@@ -237,10 +237,19 @@ func (e *encoder) Read() ([]byte, func(), error) {
|
||||
e.raw.d_w, e.raw.d_h = C.uint(width), C.uint(height)
|
||||
}
|
||||
|
||||
duration := t - e.tLastFrame
|
||||
// VPX doesn't allow 0 duration. If 0 is given, vpx_codec_encode will fail with VPX_CODEC_INVALID_PARAM.
|
||||
// 0 duration is possible because mediadevices first gets the frame meta data by reading from the source,
|
||||
// and consequently the codec will read the first frame from the buffer. This means the first frame won't
|
||||
// have a pause to the second frame, which means if the delay is <1 ms (vpx duration resolution), duration
|
||||
// is going to be 0.
|
||||
if duration == 0 {
|
||||
duration = 1
|
||||
}
|
||||
var flags int
|
||||
if ec := C.encode_wrapper(
|
||||
e.codec, e.raw,
|
||||
C.long(t-e.tStart), C.ulong(t-e.tLastFrame), C.long(flags), C.ulong(e.deadline),
|
||||
C.long(t-e.tStart), C.ulong(duration), C.long(flags), C.ulong(e.deadline),
|
||||
(*C.uchar)(&yuvImg.Y[0]), (*C.uchar)(&yuvImg.Cb[0]), (*C.uchar)(&yuvImg.Cr[0]),
|
||||
); ec != C.VPX_CODEC_OK {
|
||||
return nil, func() {}, fmt.Errorf("vpx_codec_encode failed (%d)", ec)
|
||||
|
||||
@@ -16,7 +16,7 @@ typedef struct Slice {
|
||||
|
||||
typedef struct Encoder {
|
||||
x264_t *h;
|
||||
x264_picture_t pic_in, pic_out;
|
||||
x264_picture_t pic_in;
|
||||
x264_param_t param;
|
||||
} Encoder;
|
||||
|
||||
@@ -47,20 +47,26 @@ Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
|
||||
e->param.b_repeat_headers = 1;
|
||||
e->param.b_annexb = 1;
|
||||
|
||||
if (x264_param_apply_profile(&e->param, "baseline") < 0) {
|
||||
if (x264_param_apply_profile(&e->param, "high") < 0) {
|
||||
*rc = ERR_APPLY_PROFILE;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
if (x264_picture_alloc(&e->pic_in, param.i_csp, param.i_width, param.i_height) < 0) {
|
||||
x264_picture_t pic_in;
|
||||
if (x264_picture_alloc(&pic_in, param.i_csp, param.i_width, param.i_height) < 0) {
|
||||
*rc = ERR_ALLOC_PICTURE;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
// FIXME: we use x264_picture_alloc to set the metadata only, we don't need the allocated memory
|
||||
// to store the frame. Since we free the frame memory here, we don't need to call
|
||||
// x264_picture_clean later.
|
||||
e->pic_in = pic_in;
|
||||
x264_picture_clean(&pic_in);
|
||||
|
||||
e->h = x264_encoder_open(&e->param);
|
||||
if (!e->h) {
|
||||
*rc = ERR_OPEN_ENGINE;
|
||||
x264_picture_clean(&e->pic_in);
|
||||
goto fail;
|
||||
}
|
||||
|
||||
@@ -75,24 +81,24 @@ Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
|
||||
x264_nal_t *nal;
|
||||
int i_nal;
|
||||
|
||||
x264_picture_t pic_out;
|
||||
e->pic_in.img.plane[0] = y;
|
||||
e->pic_in.img.plane[1] = cb;
|
||||
e->pic_in.img.plane[2] = cr;
|
||||
|
||||
int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &e->pic_out);
|
||||
int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
|
||||
Slice s = {.data_len = frame_size};
|
||||
if (frame_size <= 0) {
|
||||
*rc = ERR_ENCODE;
|
||||
return s;
|
||||
}
|
||||
|
||||
e->pic_in.i_pts++;
|
||||
// e->pic_in.i_pts++;
|
||||
s.data = nal->p_payload;
|
||||
return s;
|
||||
}
|
||||
|
||||
void enc_close(Encoder *e, int *rc) {
|
||||
x264_encoder_close(e->h);
|
||||
x264_picture_clean(&e->pic_in);
|
||||
free(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
const (
|
||||
maxEmptyFrameCount = 5
|
||||
prioritizedDevice = "video0"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -94,9 +95,14 @@ func init() {
|
||||
|
||||
discovered[reallink] = struct{}{}
|
||||
cam := newCamera(device)
|
||||
priority := driver.PriorityNormal
|
||||
if label == prioritizedDevice {
|
||||
priority = driver.PriorityHigh
|
||||
}
|
||||
driver.GetManager().Register(cam, driver.Info{
|
||||
Label: label,
|
||||
DeviceType: driver.Camera,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +1,200 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/gen2brain/malgo"
|
||||
"github.com/pion/mediadevices/internal/logging"
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDeviceIDLength = 20
|
||||
// TODO: should replace this with a more flexible approach
|
||||
sampleRateStep = 1000
|
||||
initialBufferSize = 1024
|
||||
)
|
||||
|
||||
var logger = logging.NewLogger("mediadevices/driver/microphone")
|
||||
var ctx *malgo.AllocatedContext
|
||||
var hostEndian binary.ByteOrder
|
||||
var (
|
||||
errUnsupportedFormat = errors.New("the provided audio format is not supported")
|
||||
)
|
||||
|
||||
type microphone struct {
|
||||
malgo.DeviceInfo
|
||||
chunkChan chan []byte
|
||||
}
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
ctx, err = malgo.InitContext(nil, malgo.ContextConfig{}, func(message string) {
|
||||
logger.Debugf("%v\n", message)
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
devices, err := ctx.Devices(malgo.Capture)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, device := range devices {
|
||||
info, err := ctx.DeviceInfo(malgo.Capture, device.ID, malgo.Shared)
|
||||
if err == nil {
|
||||
priority := driver.PriorityNormal
|
||||
if info.IsDefault > 0 {
|
||||
priority = driver.PriorityHigh
|
||||
}
|
||||
driver.GetManager().Register(newMicrophone(info), driver.Info{
|
||||
Label: device.ID.String(),
|
||||
DeviceType: driver.Microphone,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Decide which endian
|
||||
switch v := *(*uint16)(unsafe.Pointer(&([]byte{0x12, 0x34}[0]))); v {
|
||||
case 0x1234:
|
||||
hostEndian = binary.BigEndian
|
||||
case 0x3412:
|
||||
hostEndian = binary.LittleEndian
|
||||
default:
|
||||
panic(fmt.Sprintf("failed to determine host endianness: %x", v))
|
||||
}
|
||||
}
|
||||
|
||||
func newMicrophone(info malgo.DeviceInfo) *microphone {
|
||||
return µphone{
|
||||
DeviceInfo: info,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
m.chunkChan = make(chan []byte, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.chunkChan != nil {
|
||||
close(m.chunkChan)
|
||||
m.chunkChan = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
|
||||
var config malgo.DeviceConfig
|
||||
var callbacks malgo.DeviceCallbacks
|
||||
|
||||
decoder, err := wave.NewDecoder(&wave.RawFormat{
|
||||
SampleSize: inputProp.SampleSize,
|
||||
IsFloat: inputProp.IsFloat,
|
||||
Interleaved: inputProp.IsInterleaved,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.DeviceType = malgo.Capture
|
||||
config.PerformanceProfile = malgo.LowLatency
|
||||
config.Capture.Channels = uint32(inputProp.ChannelCount)
|
||||
config.SampleRate = uint32(inputProp.SampleRate)
|
||||
if inputProp.SampleSize == 4 && inputProp.IsFloat {
|
||||
config.Capture.Format = malgo.FormatF32
|
||||
} else if inputProp.SampleSize == 2 && !inputProp.IsFloat {
|
||||
config.Capture.Format = malgo.FormatS16
|
||||
} else {
|
||||
return nil, errUnsupportedFormat
|
||||
}
|
||||
|
||||
onRecvChunk := func(_, chunk []byte, framecount uint32) {
|
||||
m.chunkChan <- chunk
|
||||
}
|
||||
callbacks.Data = onRecvChunk
|
||||
|
||||
device, err := malgo.InitDevice(ctx.Context, config, callbacks)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = device.Start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var reader audio.Reader = audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
chunk, ok := <-m.chunkChan
|
||||
if !ok {
|
||||
device.Stop()
|
||||
device.Uninit()
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
decodedChunk, err := decoder.Decode(hostEndian, chunk, inputProp.ChannelCount)
|
||||
// FIXME: the decoder should also fill this information
|
||||
decodedChunk.(*wave.Float32Interleaved).Size.SamplingRate = inputProp.SampleRate
|
||||
return decodedChunk, func() {}, err
|
||||
})
|
||||
|
||||
// FIXME: The current audio detection and audio encoder can only work with a static latency. Since the latency from the driver
|
||||
// can fluctuate, we need to stabilize it. Maybe there's a better way for doing this?
|
||||
reader = audio.NewBuffer(int(inputProp.Latency.Seconds() * float64(inputProp.SampleRate)))(reader)
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
var supportedProps []prop.Media
|
||||
logger.Debug("Querying properties")
|
||||
|
||||
var isBigEndian bool
|
||||
// miniaudio only uses the host endian
|
||||
if hostEndian == binary.BigEndian {
|
||||
isBigEndian = true
|
||||
}
|
||||
|
||||
for ch := m.MinChannels; ch <= m.MaxChannels; ch++ {
|
||||
// FIXME: Currently support 48kHz only. We need to implement a resampler first.
|
||||
// for sampleRate := m.MinSampleRate; sampleRate <= m.MaxSampleRate; sampleRate += sampleRateStep {
|
||||
sampleRate := 48000
|
||||
for i := 0; i < int(m.FormatCount); i++ {
|
||||
format := m.Formats[i]
|
||||
|
||||
supportedProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
ChannelCount: int(ch),
|
||||
SampleRate: int(sampleRate),
|
||||
IsBigEndian: isBigEndian,
|
||||
// miniaudio only supports interleaved at the moment
|
||||
IsInterleaved: true,
|
||||
// FIXME: should change this to a less discrete value
|
||||
Latency: time.Millisecond * 20,
|
||||
},
|
||||
}
|
||||
|
||||
switch malgo.FormatType(format) {
|
||||
case malgo.FormatF32:
|
||||
supportedProp.SampleSize = 4
|
||||
supportedProp.IsFloat = true
|
||||
case malgo.FormatS16:
|
||||
supportedProp.SampleSize = 2
|
||||
supportedProp.IsFloat = false
|
||||
}
|
||||
|
||||
supportedProps = append(supportedProps, supportedProp)
|
||||
}
|
||||
// }
|
||||
}
|
||||
return supportedProps
|
||||
}
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/jfreymuth/pulse"
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
type microphone struct {
|
||||
c *pulse.Client
|
||||
id string
|
||||
samplesChan chan<- []int16
|
||||
}
|
||||
|
||||
func init() {
|
||||
pa, err := pulse.NewClient()
|
||||
if err != nil {
|
||||
// No pulseaudio
|
||||
return
|
||||
}
|
||||
defer pa.Close()
|
||||
sources, err := pa.ListSources()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defaultSource, err := pa.DefaultSource()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, source := range sources {
|
||||
priority := driver.PriorityNormal
|
||||
if defaultSource.ID() == source.ID() {
|
||||
priority = driver.PriorityHigh
|
||||
}
|
||||
driver.GetManager().Register(µphone{id: source.ID()}, driver.Info{
|
||||
Label: source.ID(),
|
||||
DeviceType: driver.Microphone,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
var err error
|
||||
m.c, err = pulse.NewClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.samplesChan != nil {
|
||||
close(m.samplesChan)
|
||||
m.samplesChan = nil
|
||||
}
|
||||
|
||||
m.c.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
|
||||
var options []pulse.RecordOption
|
||||
if p.ChannelCount == 1 {
|
||||
options = append(options, pulse.RecordMono)
|
||||
} else {
|
||||
options = append(options, pulse.RecordStereo)
|
||||
}
|
||||
latency := p.Latency.Seconds()
|
||||
|
||||
src, err := m.c.SourceByID(m.id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
options = append(options,
|
||||
pulse.RecordSampleRate(p.SampleRate),
|
||||
pulse.RecordLatency(latency),
|
||||
pulse.RecordSource(src),
|
||||
)
|
||||
|
||||
samplesChan := make(chan []int16, 1)
|
||||
|
||||
handler := func(b []int16) (int, error) {
|
||||
samplesChan <- b
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
stream, err := m.c.NewRecord(pulse.Int16Writer(handler), options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader := audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
buff, ok := <-samplesChan
|
||||
if !ok {
|
||||
stream.Close()
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
a := wave.NewInt16Interleaved(
|
||||
wave.ChunkInfo{
|
||||
Channels: p.ChannelCount,
|
||||
Len: len(buff) / p.ChannelCount,
|
||||
SamplingRate: p.SampleRate,
|
||||
},
|
||||
)
|
||||
copy(a.Data, buff)
|
||||
|
||||
return a, func() {}, nil
|
||||
})
|
||||
|
||||
stream.Start()
|
||||
m.samplesChan = samplesChan
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
// TODO: Get actual properties
|
||||
monoProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
SampleRate: 48000,
|
||||
Latency: time.Millisecond * 20,
|
||||
ChannelCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
stereoProp := monoProp
|
||||
stereoProp.ChannelCount = 2
|
||||
|
||||
return []prop.Media{monoProp, stereoProp}
|
||||
}
|
||||
@@ -1,348 +0,0 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"golang.org/x/sys/windows"
|
||||
"io"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
const (
|
||||
// bufferNumber * prop.Audio.Latency is the maximum blockable duration
|
||||
// to get data without dropping chunks.
|
||||
bufferNumber = 5
|
||||
)
|
||||
|
||||
// Windows APIs
|
||||
var (
|
||||
winmm = windows.NewLazySystemDLL("Winmm.dll")
|
||||
waveInOpen = winmm.NewProc("waveInOpen")
|
||||
waveInStart = winmm.NewProc("waveInStart")
|
||||
waveInStop = winmm.NewProc("waveInStop")
|
||||
waveInReset = winmm.NewProc("waveInReset")
|
||||
waveInClose = winmm.NewProc("waveInClose")
|
||||
waveInPrepareHeader = winmm.NewProc("waveInPrepareHeader")
|
||||
waveInAddBuffer = winmm.NewProc("waveInAddBuffer")
|
||||
waveInUnprepareHeader = winmm.NewProc("waveInUnprepareHeader")
|
||||
)
|
||||
|
||||
type buffer struct {
|
||||
waveHdr
|
||||
data []int16
|
||||
}
|
||||
|
||||
func newBuffer(samples int) *buffer {
|
||||
b := make([]int16, samples)
|
||||
return &buffer{
|
||||
waveHdr: waveHdr{
|
||||
// Sharing Go memory with Windows C API without reference.
|
||||
// Make sure that the lifetime of the buffer struct is longer
|
||||
// than the final access from cbWaveIn.
|
||||
lpData: uintptr(unsafe.Pointer(&b[0])),
|
||||
dwBufferLength: uint32(samples * 2),
|
||||
},
|
||||
data: b,
|
||||
}
|
||||
}
|
||||
|
||||
type microphone struct {
|
||||
hWaveIn windows.Pointer
|
||||
buf map[uintptr]*buffer
|
||||
chBuf chan *buffer
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
// TODO: enum devices
|
||||
driver.GetManager().Register(µphone{}, driver.Info{
|
||||
Label: "default",
|
||||
DeviceType: driver.Microphone,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
m.chBuf = make(chan *buffer)
|
||||
m.buf = make(map[uintptr]*buffer)
|
||||
m.closed = make(chan struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) cbWaveIn(hWaveIn windows.Pointer, uMsg uint, dwInstance, dwParam1, dwParam2 *int32) uintptr {
|
||||
switch uMsg {
|
||||
case MM_WIM_DATA:
|
||||
b := m.buf[uintptr(unsafe.Pointer(dwParam1))]
|
||||
m.chBuf <- b
|
||||
|
||||
case MM_WIM_OPEN:
|
||||
case MM_WIM_CLOSE:
|
||||
close(m.chBuf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.hWaveIn == nil {
|
||||
return nil
|
||||
}
|
||||
close(m.closed)
|
||||
|
||||
ret, _, _ := waveInStop.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
// All enqueued buffers are marked done by waveInReset.
|
||||
ret, _, _ = waveInReset.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, buf := range m.buf {
|
||||
// Detach buffers from waveIn API.
|
||||
ret, _, _ := waveInUnprepareHeader.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Now, it's ready to free the buffers.
|
||||
// As microphone struct still has reference to the buffers,
|
||||
// they will be GC-ed once microphone is reopened or unreferenced.
|
||||
|
||||
ret, _, _ = waveInClose.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
<-m.chBuf
|
||||
m.hWaveIn = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
|
||||
for i := 0; i < bufferNumber; i++ {
|
||||
b := newBuffer(
|
||||
int(uint64(p.Latency) * uint64(p.SampleRate) / uint64(time.Second)),
|
||||
)
|
||||
// Map the buffer by its data head address to restore access to the Go struct
|
||||
// in callback function. Don't resize the buffer after it.
|
||||
m.buf[uintptr(unsafe.Pointer(&b.waveHdr))] = b
|
||||
}
|
||||
|
||||
waveFmt := &waveFormatEx{
|
||||
wFormatTag: WAVE_FORMAT_PCM,
|
||||
nChannels: uint16(p.ChannelCount),
|
||||
nSamplesPerSec: uint32(p.SampleRate),
|
||||
nAvgBytesPerSec: uint32(p.SampleRate * p.ChannelCount * 2),
|
||||
nBlockAlign: uint16(p.ChannelCount * 2),
|
||||
wBitsPerSample: 16,
|
||||
}
|
||||
ret, _, _ := waveInOpen.Call(
|
||||
uintptr(unsafe.Pointer(&m.hWaveIn)),
|
||||
WAVE_MAPPER,
|
||||
uintptr(unsafe.Pointer(waveFmt)),
|
||||
windows.NewCallback(m.cbWaveIn),
|
||||
0,
|
||||
CALLBACK_FUNCTION,
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, buf := range m.buf {
|
||||
// Attach buffers to waveIn API.
|
||||
ret, _, _ := waveInPrepareHeader.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, buf := range m.buf {
|
||||
// Enqueue buffers.
|
||||
ret, _, _ := waveInAddBuffer.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
ret, _, _ = waveInStart.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// TODO: detect microphone device disconnection and return EOF
|
||||
|
||||
reader := audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
b, ok := <-m.chBuf
|
||||
if !ok {
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
select {
|
||||
case <-m.closed:
|
||||
default:
|
||||
// Re-enqueue used buffer.
|
||||
ret, _, _ := waveInAddBuffer.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&b.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(b.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, func() {}, err
|
||||
}
|
||||
}
|
||||
|
||||
a := wave.NewInt16Interleaved(
|
||||
wave.ChunkInfo{
|
||||
Channels: p.ChannelCount,
|
||||
Len: (int(b.waveHdr.dwBytesRecorded) / 2) / p.ChannelCount,
|
||||
SamplingRate: p.SampleRate,
|
||||
},
|
||||
)
|
||||
|
||||
j := 0
|
||||
for i := 0; i < a.Size.Len; i++ {
|
||||
for ch := 0; ch < a.Size.Channels; ch++ {
|
||||
a.SetInt16(i, ch, wave.Int16Sample(b.data[j]))
|
||||
j++
|
||||
}
|
||||
}
|
||||
|
||||
return a, func() {}, nil
|
||||
})
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
// TODO: Get actual properties
|
||||
monoProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
SampleRate: 48000,
|
||||
Latency: time.Millisecond * 20,
|
||||
ChannelCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
stereoProp := monoProp
|
||||
stereoProp.ChannelCount = 2
|
||||
|
||||
return []prop.Media{monoProp, stereoProp}
|
||||
}
|
||||
|
||||
// Windows API structures
|
||||
|
||||
type waveFormatEx struct {
|
||||
wFormatTag uint16
|
||||
nChannels uint16
|
||||
nSamplesPerSec uint32
|
||||
nAvgBytesPerSec uint32
|
||||
nBlockAlign uint16
|
||||
wBitsPerSample uint16
|
||||
cbSize uint16
|
||||
}
|
||||
|
||||
type waveHdr struct {
|
||||
lpData uintptr
|
||||
dwBufferLength uint32
|
||||
dwBytesRecorded uint32
|
||||
dwUser *uint32
|
||||
dwFlags uint32
|
||||
dwLoops uint32
|
||||
lpNext *waveHdr
|
||||
reserved *uint32
|
||||
}
|
||||
|
||||
// Windows consts
|
||||
|
||||
const (
|
||||
MMSYSERR_NOERROR = 0
|
||||
MMSYSERR_ERROR = 1
|
||||
MMSYSERR_BADDEVICEID = 2
|
||||
MMSYSERR_NOTENABLED = 3
|
||||
MMSYSERR_ALLOCATED = 4
|
||||
MMSYSERR_INVALHANDLE = 5
|
||||
MMSYSERR_NODRIVER = 6
|
||||
MMSYSERR_NOMEM = 7
|
||||
MMSYSERR_NOTSUPPORTED = 8
|
||||
MMSYSERR_BADERRNUM = 9
|
||||
MMSYSERR_INVALFLAG = 10
|
||||
MMSYSERR_INVALPARAM = 11
|
||||
MMSYSERR_HANDLEBUSY = 12
|
||||
MMSYSERR_INVALIDALIAS = 13
|
||||
MMSYSERR_BADDB = 14
|
||||
MMSYSERR_KEYNOTFOUND = 15
|
||||
MMSYSERR_READERROR = 16
|
||||
MMSYSERR_WRITEERROR = 17
|
||||
MMSYSERR_DELETEERROR = 18
|
||||
MMSYSERR_VALNOTFOUND = 19
|
||||
MMSYSERR_NODRIVERCB = 20
|
||||
|
||||
WAVERR_BADFORMAT = 32
|
||||
WAVERR_STILLPLAYING = 33
|
||||
WAVERR_UNPREPARED = 34
|
||||
WAVERR_SYNC = 35
|
||||
|
||||
WAVE_MAPPER = 0xFFFF
|
||||
WAVE_FORMAT_PCM = 1
|
||||
|
||||
CALLBACK_NULL = 0
|
||||
CALLBACK_WINDOW = 0x10000
|
||||
CALLBACK_TASK = 0x20000
|
||||
CALLBACK_FUNCTION = 0x30000
|
||||
CALLBACK_THREAD = CALLBACK_TASK
|
||||
CALLBACK_EVENT = 0x50000
|
||||
|
||||
MM_WIM_OPEN = 0x3BE
|
||||
MM_WIM_CLOSE = 0x3BF
|
||||
MM_WIM_DATA = 0x3C0
|
||||
)
|
||||
|
||||
var errWinmm = map[uintptr]error{
|
||||
MMSYSERR_NOERROR: nil,
|
||||
MMSYSERR_ERROR: errors.New("error"),
|
||||
MMSYSERR_BADDEVICEID: errors.New("bad device id"),
|
||||
MMSYSERR_NOTENABLED: errors.New("not enabled"),
|
||||
MMSYSERR_ALLOCATED: errors.New("already allocated"),
|
||||
MMSYSERR_INVALHANDLE: errors.New("invalid handler"),
|
||||
MMSYSERR_NODRIVER: errors.New("no driver"),
|
||||
MMSYSERR_NOMEM: errors.New("no memory"),
|
||||
MMSYSERR_NOTSUPPORTED: errors.New("not supported"),
|
||||
MMSYSERR_BADERRNUM: errors.New("band error number"),
|
||||
MMSYSERR_INVALFLAG: errors.New("invalid flag"),
|
||||
MMSYSERR_INVALPARAM: errors.New("invalid param"),
|
||||
MMSYSERR_HANDLEBUSY: errors.New("handle busy"),
|
||||
MMSYSERR_INVALIDALIAS: errors.New("invalid alias"),
|
||||
MMSYSERR_BADDB: errors.New("bad db"),
|
||||
MMSYSERR_KEYNOTFOUND: errors.New("key not found"),
|
||||
MMSYSERR_READERROR: errors.New("read error"),
|
||||
MMSYSERR_WRITEERROR: errors.New("write error"),
|
||||
MMSYSERR_DELETEERROR: errors.New("delete error"),
|
||||
MMSYSERR_VALNOTFOUND: errors.New("value not found"),
|
||||
MMSYSERR_NODRIVERCB: errors.New("no driver cb"),
|
||||
WAVERR_BADFORMAT: errors.New("bad format"),
|
||||
WAVERR_STILLPLAYING: errors.New("still playing"),
|
||||
WAVERR_UNPREPARED: errors.New("unprepared"),
|
||||
WAVERR_SYNC: errors.New("sync"),
|
||||
}
|
||||
@@ -1 +1,81 @@
|
||||
// +build !linux
|
||||
|
||||
package screen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
|
||||
"github.com/kbinani/screenshot"
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/io/video"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
type screen struct {
|
||||
displayIndex int
|
||||
doneCh chan struct{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
activeDisplays := screenshot.NumActiveDisplays()
|
||||
for i := 0; i < activeDisplays; i++ {
|
||||
priority := driver.PriorityNormal
|
||||
if i == 0 {
|
||||
priority = driver.PriorityHigh
|
||||
}
|
||||
|
||||
s := newScreen(i)
|
||||
driver.GetManager().Register(s, driver.Info{
|
||||
Label: fmt.Sprint(i),
|
||||
DeviceType: driver.Screen,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newScreen(displayIndex int) *screen {
|
||||
s := screen{
|
||||
displayIndex: displayIndex,
|
||||
}
|
||||
return &s
|
||||
}
|
||||
|
||||
func (s *screen) Open() error {
|
||||
s.doneCh = make(chan struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *screen) Close() error {
|
||||
close(s.doneCh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *screen) VideoRecord(selectedProp prop.Media) (video.Reader, error) {
|
||||
r := video.ReaderFunc(func() (img image.Image, release func(), err error) {
|
||||
select {
|
||||
case <-s.doneCh:
|
||||
return nil, nil, io.EOF
|
||||
default:
|
||||
}
|
||||
|
||||
img, err = screenshot.CaptureDisplay(s.displayIndex)
|
||||
release = func() {}
|
||||
return
|
||||
})
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (s *screen) Properties() []prop.Media {
|
||||
resolution := screenshot.GetDisplayBounds(s.displayIndex)
|
||||
supportedProp := prop.Media{
|
||||
Video: prop.Video{
|
||||
Width: resolution.Dx(),
|
||||
Height: resolution.Dy(),
|
||||
FrameFormat: frame.FormatRGBA,
|
||||
},
|
||||
}
|
||||
return []prop.Media{supportedProp}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,14 @@ import (
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (chunk wave.Audio, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,14 @@ package io
|
||||
// Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
|
||||
// to provide strong type.
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (data interface{}, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,14 @@ import (
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (img image.Image, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
21
rtpreader.go
Normal file
21
rtpreader.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package mediadevices
|
||||
|
||||
import "github.com/pion/rtp"
|
||||
|
||||
type RTPReadCloser interface {
|
||||
Read() (pkts []*rtp.Packet, release func(), err error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type rtpReadCloserImpl struct {
|
||||
readFn func() ([]*rtp.Packet, func(), error)
|
||||
closeFn func() error
|
||||
}
|
||||
|
||||
func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
|
||||
return r.readFn()
|
||||
}
|
||||
|
||||
func (r *rtpReadCloserImpl) Close() error {
|
||||
return r.closeFn()
|
||||
}
|
||||
24
sampler.go
24
sampler.go
@@ -3,34 +3,30 @@ package mediadevices
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/pion/webrtc/v2"
|
||||
"github.com/pion/webrtc/v2/pkg/media"
|
||||
)
|
||||
|
||||
type samplerFunc func(b []byte) error
|
||||
type samplerFunc func() uint32
|
||||
|
||||
// newVideoSampler creates a video sampler that uses the actual video frame rate and
|
||||
// the codec's clock rate to come up with a duration for each sample.
|
||||
func newVideoSampler(t *webrtc.Track) samplerFunc {
|
||||
clockRate := float64(t.Codec().ClockRate)
|
||||
func newVideoSampler(clockRate uint32) samplerFunc {
|
||||
clockRateFloat := float64(clockRate)
|
||||
lastTimestamp := time.Now()
|
||||
|
||||
return samplerFunc(func(b []byte) error {
|
||||
return samplerFunc(func() uint32 {
|
||||
now := time.Now()
|
||||
duration := now.Sub(lastTimestamp).Seconds()
|
||||
samples := uint32(math.Round(clockRate * duration))
|
||||
samples := uint32(math.Round(clockRateFloat * duration))
|
||||
lastTimestamp = now
|
||||
|
||||
return t.WriteSample(media.Sample{Data: b, Samples: samples})
|
||||
return samples
|
||||
})
|
||||
}
|
||||
|
||||
// newAudioSampler creates a audio sampler that uses a fixed latency and
|
||||
// the codec's clock rate to come up with a duration for each sample.
|
||||
func newAudioSampler(t *webrtc.Track, latency time.Duration) samplerFunc {
|
||||
samples := uint32(math.Round(float64(t.Codec().ClockRate) * latency.Seconds()))
|
||||
return samplerFunc(func(b []byte) error {
|
||||
return t.WriteSample(media.Sample{Data: b, Samples: samples})
|
||||
func newAudioSampler(clockRate uint32, latency time.Duration) samplerFunc {
|
||||
samples := uint32(math.Round(float64(clockRate) * latency.Seconds()))
|
||||
return samplerFunc(func() uint32 {
|
||||
return samples
|
||||
})
|
||||
}
|
||||
|
||||
151
track.go
151
track.go
@@ -3,6 +3,7 @@ package mediadevices
|
||||
import (
|
||||
"errors"
|
||||
"image"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sync"
|
||||
|
||||
@@ -11,7 +12,9 @@ import (
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/io/video"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/webrtc/v2"
|
||||
"github.com/pion/webrtc/v2/pkg/media"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -52,6 +55,11 @@ type Track interface {
|
||||
// Unbind is the clean up operation that should be called after Bind. Similar to Bind, unbind will
|
||||
// be called automatically in the future.
|
||||
Unbind(*webrtc.PeerConnection) error
|
||||
// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
|
||||
// the encoded data in RTP format with given mtu size.
|
||||
NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
|
||||
// NewEncodedReader creates a new Go standard io.ReadCloser that reads the encoded data in codecName format
|
||||
NewEncodedReader(codecName string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type baseTrack struct {
|
||||
@@ -109,7 +117,7 @@ func (track *baseTrack) onError(err error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.ReadCloser, selectedCodec *codec.RTPCodec, sampler func(*webrtc.Track) samplerFunc) (*webrtc.Track, error) {
|
||||
func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.ReadCloser, selectedCodec *codec.RTPCodec, sample samplerFunc) (*webrtc.Track, error) {
|
||||
track.mu.Lock()
|
||||
defer track.mu.Unlock()
|
||||
|
||||
@@ -118,7 +126,6 @@ func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.Read
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := sampler(webrtcTrack)
|
||||
signalCh := make(chan chan<- struct{})
|
||||
track.activePeerConnections[pc] = signalCh
|
||||
|
||||
@@ -147,7 +154,12 @@ func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.Read
|
||||
return
|
||||
}
|
||||
|
||||
if err := sample(buff); err != nil {
|
||||
sampleCount := sample()
|
||||
err = webrtcTrack.WriteSample(media.Sample{
|
||||
Data: buff,
|
||||
Samples: sampleCount,
|
||||
})
|
||||
if err != nil {
|
||||
track.onError(err)
|
||||
return
|
||||
}
|
||||
@@ -173,6 +185,31 @@ func (track *baseTrack) unbind(pc *webrtc.PeerConnection) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (track *baseTrack) newEncodedReader(reader codec.ReadCloser) (io.ReadCloser, error) {
|
||||
var encoded []byte
|
||||
release := func() {}
|
||||
return &encodedReadCloserImpl{
|
||||
readFn: func(b []byte) (int, error) {
|
||||
var err error
|
||||
|
||||
if len(encoded) == 0 {
|
||||
release()
|
||||
encoded, release, err = reader.Read()
|
||||
if err != nil {
|
||||
reader.Close()
|
||||
track.onError(err)
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
n := copy(b, encoded)
|
||||
encoded = encoded[n:]
|
||||
return n, nil
|
||||
},
|
||||
closeFn: reader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newTrackFromDriver(d driver.Driver, constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
|
||||
if err := d.Open(); err != nil {
|
||||
return nil, err
|
||||
@@ -242,18 +279,68 @@ func (track *VideoTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
|
||||
}
|
||||
|
||||
wantCodecs := pc.GetRegisteredRTPCodecs(webrtc.RTPCodecTypeVideo)
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodec(wantCodecs, reader, inputProp)
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodec(reader, inputProp, wantCodecs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.bind(pc, encodedReader, selectedCodec, newVideoSampler)
|
||||
return track.bind(pc, encodedReader, selectedCodec, newVideoSampler(selectedCodec.ClockRate))
|
||||
}
|
||||
|
||||
func (track *VideoTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return track.unbind(pc)
|
||||
}
|
||||
|
||||
func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentVideoProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := newVideoSampler(selectedCodec.ClockRate)
|
||||
|
||||
// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
|
||||
packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
|
||||
|
||||
return &rtpReadCloserImpl{
|
||||
readFn: func() ([]*rtp.Packet, func(), error) {
|
||||
encoded, release, err := encodedReader.Read()
|
||||
if err != nil {
|
||||
encodedReader.Close()
|
||||
track.onError(err)
|
||||
return nil, func() {}, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
samples := sample()
|
||||
pkts := packetizer.Packetize(encoded, samples)
|
||||
return pkts, release, err
|
||||
},
|
||||
closeFn: encodedReader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (track *VideoTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentVideoProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, _, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.newEncodedReader(encodedReader)
|
||||
}
|
||||
|
||||
// AudioTrack is a specific track type that contains audio source which allows multiple readers to access, and
|
||||
// manipulate.
|
||||
type AudioTrack struct {
|
||||
@@ -309,14 +396,64 @@ func (track *AudioTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
|
||||
}
|
||||
|
||||
wantCodecs := pc.GetRegisteredRTPCodecs(webrtc.RTPCodecTypeAudio)
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodec(wantCodecs, reader, inputProp)
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodec(reader, inputProp, wantCodecs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.bind(pc, encodedReader, selectedCodec, func(t *webrtc.Track) samplerFunc { return newAudioSampler(t, inputProp.Latency) })
|
||||
return track.bind(pc, encodedReader, selectedCodec, newAudioSampler(selectedCodec.ClockRate, inputProp.Latency))
|
||||
}
|
||||
|
||||
func (track *AudioTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return track.unbind(pc)
|
||||
}
|
||||
|
||||
func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentAudioProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := newAudioSampler(selectedCodec.ClockRate, inputProp.Latency)
|
||||
|
||||
// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
|
||||
packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
|
||||
|
||||
return &rtpReadCloserImpl{
|
||||
readFn: func() ([]*rtp.Packet, func(), error) {
|
||||
encoded, release, err := encodedReader.Read()
|
||||
if err != nil {
|
||||
encodedReader.Close()
|
||||
track.onError(err)
|
||||
return nil, func() {}, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
samples := sample()
|
||||
pkts := packetizer.Packetize(encoded, samples)
|
||||
return pkts, release, err
|
||||
},
|
||||
closeFn: encodedReader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (track *AudioTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentAudioProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, _, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.newEncodedReader(encodedReader)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user