mirror of
https://github.com/pion/mediadevices.git
synced 2025-09-27 12:52:20 +08:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
7df20d004e |
1
.gitattributes
vendored
1
.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
*.gif filter=lfs diff=lfs merge=lfs -text
|
||||
274
README.md
274
README.md
@@ -13,228 +13,74 @@
|
||||
</p>
|
||||
<br>
|
||||
|
||||
`mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
|
||||
|
||||
|
||||
|
||||
## Install
|
||||
## Interfaces
|
||||
|
||||
`go get -u github.com/pion/mediadevices`
|
||||
| Interface | Linux | Mac | Windows |
|
||||
| :--------: | :---: | :-: | :-----: |
|
||||
| Camera | ✔️ | ✔️ | ✔️ |
|
||||
| Microphone | ✔️ | ✖️ | ✔️ |
|
||||
| Screen | ✔️ | ✖️ | ✖️ |
|
||||
|
||||
### Camera
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [Video4Linux](https://en.wikipedia.org/wiki/Video4Linux) |
|
||||
| Mac | [AVFoundation](https://developer.apple.com/av-foundation/) |
|
||||
| Windows | [DirectShow](https://docs.microsoft.com/en-us/windows/win32/directshow) |
|
||||
|
||||
| Pixel Format | Linux | Mac | Windows |
|
||||
| :---------------------------------------------------: | :---: | :-: | :-----: |
|
||||
| [YUY2](https://www.fourcc.org/pixel-format/yuv-yuy2/) | ✔️ | ✖️ | ✔️ |
|
||||
| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) | ✔️ | ✔️ | ✖️ |
|
||||
| [I420](https://www.fourcc.org/pixel-format/yuv-i420/) | ✔️ | ✖️ | ✖️ |
|
||||
| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) | ✔️ | ✔️ | ✖️ |
|
||||
| [MJPEG](https://www.fourcc.org/mjpg/) | ✔️ | ✖️ | ✖️ |
|
||||
|
||||
### Microphone
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [PulseAudio](https://en.wikipedia.org/wiki/PulseAudio) |
|
||||
| Mac | N/A |
|
||||
| Windows | [waveIn](https://docs.microsoft.com/en-us/windows/win32/api/mmeapi/) |
|
||||
|
||||
### Screen casting
|
||||
|
||||
| OS | Library/Interface |
|
||||
| :-----: | :---------------------------------------------------------------------: |
|
||||
| Linux | [X11](https://en.wikipedia.org/wiki/X_Window_System) |
|
||||
| Mac | N/A |
|
||||
| Windows | N/A |
|
||||
|
||||
## Codecs
|
||||
|
||||
| Audio Codec | Library/Interface |
|
||||
| :---------: | :------------------------------------------------------: |
|
||||
| OPUS | [libopus](http://opus-codec.org/) |
|
||||
|
||||
| Video Codec | Library/Interface |
|
||||
| :---------: | :------------------------------------------------------: |
|
||||
| H.264 | [OpenH264](https://www.openh264.org/) |
|
||||
| VP8 | [libvpx](https://www.webmproject.org/code/) |
|
||||
| VP9 | [libvpx](https://www.webmproject.org/code/) |
|
||||
|
||||
## Usage
|
||||
|
||||
The following snippet shows how to capture a camera stream and store a frame as a jpeg image:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"image/jpeg"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
|
||||
// This is required to register camera adapter
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera"
|
||||
// Note: If you don't have a camera or your adapters are not supported,
|
||||
// you can always swap your adapters with our dummy adapters below.
|
||||
// _ "github.com/pion/mediadevices/pkg/driver/videotest"
|
||||
)
|
||||
|
||||
func main() {
|
||||
stream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(constraint *mediadevices.MediaTrackConstraints) {
|
||||
// Query for ideal resolutions
|
||||
constraint.Width = prop.Int(600)
|
||||
constraint.Height = prop.Int(400)
|
||||
},
|
||||
})
|
||||
|
||||
// Since track can represent audio as well, we need to cast it to
|
||||
// *mediadevices.VideoTrack to get video specific functionalities
|
||||
track := stream.GetVideoTracks()[0]
|
||||
videoTrack := track.(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
// Create a new video reader to get the decoded frames. Release is used
|
||||
// to return the buffer to hold frame back to the source so that the buffer
|
||||
// can be reused for the next frames.
|
||||
videoReader := videoTrack.NewReader(false)
|
||||
frame, release, _ := videoReader.Read()
|
||||
defer release()
|
||||
|
||||
// Since frame is the standard image.Image, it's compatible with Go standard
|
||||
// library. For example, capturing the first frame and store it as a jpeg image.
|
||||
output, _ := os.Create("frame.jpg")
|
||||
jpeg.Encode(output, frame, nil)
|
||||
}
|
||||
|
||||
|
||||
```
|
||||
|
||||
## More Examples
|
||||
|
||||
* [Webrtc](/examples/webrtc) - Use Webrtc to create a realtime peer-to-peer video call
|
||||
* [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
|
||||
* [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
|
||||
* [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
|
||||
* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
|
||||
|
||||
## Available Media Inputs
|
||||
|
||||
| Input | Linux | Mac | Windows |
|
||||
| :--------: | :---: | :-: | :-----: |
|
||||
| Camera | ✔️ | ✔️ | ✔️ |
|
||||
| Microphone | ✔️ | ✔️ | ✔️ |
|
||||
| Screen | ✔️ | ✖️ | ✖️ |
|
||||
|
||||
By default, there's no media input registered. This decision was made to allow you to pay what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
|
||||
|
||||
```go
|
||||
import (
|
||||
...
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera"
|
||||
)
|
||||
```
|
||||
|
||||
## Available Codecs
|
||||
|
||||
In order to encode your video/audio, `mediadevices` needs to know what codecs that you want to use and their parameters. To do this, you need to import the associated packages for the codecs, and add them to the codec selector that you'll pass to `GetUserMedia`:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
)
|
||||
|
||||
func main() {
|
||||
// configure codec specific parameters
|
||||
x264Params, _ := x264.NewParams()
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {},
|
||||
Codec: codecSelector, // let GetUsermedia know available codecs
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
Since `mediadevices` doesn't implement the video/audio codecs, it needs to call the codec libraries from the system through cgo. Therefore, you're required to install the codec libraries before you can use them in `mediadevices`. In the next section, it shows a list of available codecs, where the packages are defined (documentation linked), and installation instructions.
|
||||
|
||||
Note: we do not provide recommendations on choosing one codec or another as it is very complex and can be subjective.
|
||||
|
||||
### Video Codecs
|
||||
|
||||
#### x264
|
||||
A free software library and application for encoding video streams into the H.264/MPEG-4 AVC compression format.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/x264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/x264)
|
||||
* Installation:
|
||||
* Mac: `brew install x264`
|
||||
* Ubuntu: `apt install libx264-dev`
|
||||
|
||||
#### mmal
|
||||
A framework to enable H264 hardware encoding for Raspberry Pi or boards that use VideoCore GPUs.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/mmal](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/mmal)
|
||||
* Installation:
|
||||
* Raspbian: `export PKG_CONFIG_PATH=/opt/vc/lib/pkgconfig`
|
||||
|
||||
#### openh264
|
||||
A codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/openh264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/openh264)
|
||||
* Installation: no installation needed, included as a static binary
|
||||
|
||||
#### vpx
|
||||
A free software video codec library from Google and the Alliance for Open Media that implements VP8/VP9 video coding formats.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/vpx](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vpx)
|
||||
* Installation:
|
||||
* Mac: `brew install libvpx`
|
||||
* Ubuntu: `apt install libvpx-dev`
|
||||
|
||||
#### vaapi
|
||||
An open source API that allows applications such as VLC media player or GStreamer to use hardware video acceleration capabilities (currently support VP8/VP9).
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/vaapi](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vaapi)
|
||||
* Installation:
|
||||
* Ubuntu: `apt install libva-dev`
|
||||
|
||||
|
||||
### Audio Codecs
|
||||
|
||||
#### opus
|
||||
A totally open, royalty-free, highly versatile audio codec.
|
||||
|
||||
* Package: [github.com/pion/mediadevices/pkg/codec/opus](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/opus)
|
||||
* Installation:
|
||||
* Mac: `brew install opus`
|
||||
* Ubuntu: `apt install libopus-dev`
|
||||
|
||||
## Benchmark
|
||||
|
||||
Result as of Nov 4, 2020 with Go 1.14 on a Raspberry pi 3, `mediadevices` can produce video, encode, send across network, and decode at **720p, 30 fps with < 500 ms latency**.
|
||||
|
||||
The test was taken by capturing a camera stream, decoding the raw frames, encoding the video stream with mmal, and sending the stream through Webrtc.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Failed to find the best driver that fits the constraints
|
||||
|
||||
`mediadevices` provides an automated driver discovery through `GetUserMedia` and `GetDisplayMedia`. The driver discover algorithm works something like:
|
||||
|
||||
1. Open all registered drivers
|
||||
2. Get all properties (property describes what a driver is capable of, e.g. resolution, frame rate, etc.) from opened drivers
|
||||
3. Find the best property that meets the criteria
|
||||
|
||||
So, when `mediadevices` returns `failed to find the best driver that fits the constraints` error, one of the following conditions might have occured:
|
||||
* Driver was not imported as a side effect in your program, e.g. `import _ github.com/pion/mediadevices/pkg/driver/camera`
|
||||
* Your constraint is too strict that there's no driver can fullfil your requirements. In this case, you can try to turn up the debug level by specifying the following environment variable: `export PION_LOG_DEBUG=all` to see what was too strict and tune that.
|
||||
* Your driver is not supported/implemented. In this case, you can either let us know (file an issue) and wait for the maintainers to implement it. Or, you can implement it yourself and register it through `RegisterDriverAdapter`
|
||||
|
||||
### Failed to find vpx/x264/mmal/opus codecs
|
||||
|
||||
Since `mediadevices` uses cgo to access video/audio codecs, it needs to find these libraries from the system. To accomplish this, [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) is used for library discovery.
|
||||
|
||||
If you see the following error message at compile time:
|
||||
```
|
||||
# pkg-config --cflags -- vpx
|
||||
Package vpx was not found in the pkg-config search path.
|
||||
Perhaps you should add the directory containing `vpx.pc'
|
||||
to the PKG_CONFIG_PATH environment variable
|
||||
No package 'vpx' found
|
||||
pkg-config: exit status 1
|
||||
```
|
||||
|
||||
There are 2 common problems:
|
||||
|
||||
* The required codec library is not installed (vpx in this example). In this case, please refer to the [available codecs](#available-codecs).
|
||||
* Pkg-config fails to find the `.pc` files for this codec ([reference](https://people.freedesktop.org/~dbn/pkg-config-guide.html#using)). In this case, you need to find where the codec library's `.pc` is stored, and let pkg-config knows with: `export PKG_CONFIG_PATH=/path/to/directory`.
|
||||
|
||||
|
||||
## Community
|
||||
Pion has an active community on the [Slack](https://pion.ly/slack).
|
||||
|
||||
Follow the [Pion Twitter](https://twitter.com/_pion) for project updates and important WebRTC news.
|
||||
|
||||
We are always looking to support **your projects**. Please reach out if you have something to build!
|
||||
If you need commercial support or don't want to use public methods you can contact us at [team@pion.ly](mailto:team@pion.ly)
|
||||
[Wiki](https://github.com/pion/mediadevices/wiki)
|
||||
|
||||
## Contributing
|
||||
Check out the **[contributing wiki](https://github.com/pion/webrtc/wiki/Contributing)** to join the group of amazing people making this project possible:
|
||||
|
||||
* [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
|
||||
- [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
|
||||
* [Atsushi Watanabe](https://github.com/at-wat) - _VP8, Screencast, etc._
|
||||
|
||||
## License
|
||||
MIT License - see [LICENSE](LICENSE) for full text
|
||||
## Project Status
|
||||
|
||||
[](https://starchart.cc/pion/mediadevices)
|
||||
|
||||
## References
|
||||
|
||||
- https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs
|
||||
- https://tools.ietf.org/html/rfc7742
|
||||
|
||||
34
codec.go
34
codec.go
@@ -58,16 +58,17 @@ func (selector *CodecSelector) Populate(setting *webrtc.MediaEngine) {
|
||||
}
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectVideoCodecByNames(reader video.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
func (selector *CodecSelector) selectVideoCodec(wantCodecs []*webrtc.RTPCodec, reader video.Reader, inputProp prop.Media) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var selectedEncoder codec.VideoEncoderBuilder
|
||||
var encodedReader codec.ReadCloser
|
||||
var errReasons []string
|
||||
var err error
|
||||
|
||||
outer:
|
||||
for _, wantCodec := range codecNames {
|
||||
for _, wantCodec := range wantCodecs {
|
||||
name := wantCodec.Name
|
||||
for _, encoder := range selector.videoEncoders {
|
||||
if encoder.RTPCodec().Name == wantCodec {
|
||||
if encoder.RTPCodec().Name == name {
|
||||
encodedReader, err = encoder.BuildVideoEncoder(reader, inputProp)
|
||||
if err == nil {
|
||||
selectedEncoder = encoder
|
||||
@@ -86,26 +87,17 @@ outer:
|
||||
return encodedReader, selectedEncoder.RTPCodec(), nil
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectVideoCodec(reader video.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var codecNames []string
|
||||
|
||||
for _, codec := range codecs {
|
||||
codecNames = append(codecNames, codec.Name)
|
||||
}
|
||||
|
||||
return selector.selectVideoCodecByNames(reader, inputProp, codecNames...)
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectAudioCodecByNames(reader audio.Reader, inputProp prop.Media, codecNames ...string) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
func (selector *CodecSelector) selectAudioCodec(wantCodecs []*webrtc.RTPCodec, reader audio.Reader, inputProp prop.Media) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var selectedEncoder codec.AudioEncoderBuilder
|
||||
var encodedReader codec.ReadCloser
|
||||
var errReasons []string
|
||||
var err error
|
||||
|
||||
outer:
|
||||
for _, wantCodec := range codecNames {
|
||||
for _, wantCodec := range wantCodecs {
|
||||
name := wantCodec.Name
|
||||
for _, encoder := range selector.audioEncoders {
|
||||
if encoder.RTPCodec().Name == wantCodec {
|
||||
if encoder.RTPCodec().Name == name {
|
||||
encodedReader, err = encoder.BuildAudioEncoder(reader, inputProp)
|
||||
if err == nil {
|
||||
selectedEncoder = encoder
|
||||
@@ -123,13 +115,3 @@ outer:
|
||||
|
||||
return encodedReader, selectedEncoder.RTPCodec(), nil
|
||||
}
|
||||
|
||||
func (selector *CodecSelector) selectAudioCodec(reader audio.Reader, inputProp prop.Media, codecs ...*webrtc.RTPCodec) (codec.ReadCloser, *codec.RTPCodec, error) {
|
||||
var codecNames []string
|
||||
|
||||
for _, codec := range codecs {
|
||||
codecNames = append(codecNames, codec.Name)
|
||||
}
|
||||
|
||||
return selector.selectAudioCodecByNames(reader, inputProp, codecNames...)
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
## Instructions
|
||||
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
|
||||
### Download archive examplee
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Run archive example
|
||||
|
||||
Run `cd mediadevices/examples/archive && go build && ./archive recorded.h264`
|
||||
|
||||
### Playback recorded video
|
||||
|
||||
Install GStreamer and run:
|
||||
```
|
||||
gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
|
||||
```
|
||||
|
||||
Or run VLC media plyer:
|
||||
```
|
||||
vlc recorded.h264
|
||||
```
|
||||
|
||||
A video should start playing in your GStreamer or VLC window.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
|
||||
Binary file not shown.
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/io/video"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s <path/to/file.h264>\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT)
|
||||
|
||||
x264Params, err := x264.NewParams()
|
||||
must(err)
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
Codec: codecSelector,
|
||||
})
|
||||
must(err)
|
||||
|
||||
videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
|
||||
return video.ReaderFunc(func() (img image.Image, release func(), err error) {
|
||||
// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
|
||||
// will finish its execution and the program will finish
|
||||
select {
|
||||
case <-sigs:
|
||||
return nil, func() {}, io.EOF
|
||||
default:
|
||||
}
|
||||
|
||||
return r.Read()
|
||||
})
|
||||
}))
|
||||
|
||||
reader, err := videoTrack.NewEncodedReader(x264Params.RTPCodec().Name)
|
||||
must(err)
|
||||
defer reader.Close()
|
||||
|
||||
out, err := os.Create(dest)
|
||||
must(err)
|
||||
|
||||
fmt.Println("Recording... Press Ctrl+c to stop")
|
||||
_, err = io.Copy(out, reader)
|
||||
must(err)
|
||||
fmt.Println("Your video has been recorded to", dest)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
## Instructions
|
||||
|
||||
### Download facedetection example
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Compile and Run facedetection
|
||||
|
||||
Run `cd mediadevices/examples/facedetection && go build && ./facedetection`
|
||||
|
||||
You should be able to see some loggings when it can see faces.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
Binary file not shown.
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"image"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
pigo "github.com/esimov/pigo/core"
|
||||
"github.com/pion/mediadevices"
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
const (
|
||||
confidenceLevel = 5.0
|
||||
)
|
||||
|
||||
var (
|
||||
cascade []byte
|
||||
classifier *pigo.Pigo
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func detectFace(frame *image.YCbCr) bool {
|
||||
bounds := frame.Bounds()
|
||||
cascadeParams := pigo.CascadeParams{
|
||||
MinSize: 100,
|
||||
MaxSize: 600,
|
||||
ShiftFactor: 0.15,
|
||||
ScaleFactor: 1.1,
|
||||
ImageParams: pigo.ImageParams{
|
||||
Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
|
||||
Rows: bounds.Dy(),
|
||||
Cols: bounds.Dx(),
|
||||
Dim: bounds.Dx(),
|
||||
},
|
||||
}
|
||||
|
||||
// Run the classifier over the obtained leaf nodes and return the detection results.
|
||||
// The result contains quadruplets representing the row, column, scale and detection score.
|
||||
dets := classifier.RunCascade(cascadeParams, 0.0)
|
||||
|
||||
// Calculate the intersection over union (IoU) of two clusters.
|
||||
dets = classifier.ClusterDetections(dets, 0)
|
||||
|
||||
for _, det := range dets {
|
||||
if det.Q >= confidenceLevel {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
// prepare face detector
|
||||
var err error
|
||||
cascade, err = ioutil.ReadFile("facefinder")
|
||||
if err != nil {
|
||||
log.Fatalf("Error reading the cascade file: %s", err)
|
||||
}
|
||||
p := pigo.NewPigo()
|
||||
|
||||
// Unpack the binary file. This will return the number of cascade trees,
|
||||
// the tree depth, the threshold and the prediction from tree's leaf nodes.
|
||||
classifier, err = p.Unpack(cascade)
|
||||
if err != nil {
|
||||
log.Fatalf("Error unpacking the cascade file: %s", err)
|
||||
}
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
})
|
||||
must(err)
|
||||
|
||||
// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
|
||||
videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
|
||||
videoReader := videoTrack.NewReader(false)
|
||||
// To save resources, we can simply use 4 fps to detect faces.
|
||||
ticker := time.NewTicker(time.Millisecond * 250)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
frame, release, err := videoReader.Read()
|
||||
must(err)
|
||||
|
||||
// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
|
||||
if detectFace(frame.(*image.YCbCr)) {
|
||||
log.Println("Detect a face")
|
||||
}
|
||||
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
## Instructions
|
||||
|
||||
### Download http example
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
### Compile and Run HTTP server
|
||||
|
||||
Run `cd mediadevices/examples/http && go build && ./http :1313`
|
||||
|
||||
|
||||
### Access the camera stream from the browser
|
||||
|
||||
Go to "http://localhost:1313"
|
||||
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
@@ -29,13 +28,7 @@ func must(err error) {
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s host:port\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(constraint *mediadevices.MediaTrackConstraints) {
|
||||
constraint.Width = prop.Int(600)
|
||||
constraint.Height = prop.Int(400)
|
||||
@@ -43,9 +36,8 @@ func main() {
|
||||
})
|
||||
must(err)
|
||||
|
||||
track := mediaStream.GetVideoTracks()[0]
|
||||
videoTrack := track.(*mediadevices.VideoTrack)
|
||||
defer videoTrack.Close()
|
||||
t := s.GetVideoTracks()[0]
|
||||
videoTrack := t.(*mediadevices.VideoTrack)
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
@@ -80,6 +72,6 @@ func main() {
|
||||
}
|
||||
})
|
||||
|
||||
fmt.Printf("listening on %s\n", dest)
|
||||
log.Println(http.ListenAndServe(dest, nil))
|
||||
fmt.Println("listening on http://localhost:1313")
|
||||
log.Println(http.ListenAndServe("localhost:1313", nil))
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
## Instructions
|
||||
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
* [opus](https://github.com/pion/mediadevices#opus)
|
||||
|
||||
### Download rtpexample
|
||||
|
||||
```
|
||||
go get github.com/pion/mediadevices/examples/rtp
|
||||
```
|
||||
|
||||
### Listen RTP
|
||||
|
||||
Install GStreamer and run:
|
||||
```
|
||||
gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=H264 \
|
||||
! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
|
||||
```
|
||||
|
||||
Or run VLC media plyer:
|
||||
```
|
||||
vlc ./h264.sdp
|
||||
```
|
||||
|
||||
### Run rtp
|
||||
|
||||
Run `rtp localhost:5000`
|
||||
|
||||
A video should start playing in your GStreamer or VLC window.
|
||||
It's not WebRTC, but pure RTP.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
v=0
|
||||
o=- 1234567890 1234567890 IN IP4 0.0.0.0
|
||||
s=RTP-Send Example
|
||||
i=Example
|
||||
c=IN IP4 0.0.0.0
|
||||
t=0 0
|
||||
a=recvonly
|
||||
m=video 5000 RTP/AVP 100
|
||||
a=rtpmap:100 H264/90000
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
"github.com/pion/mediadevices"
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use H264 video encoder
|
||||
_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
)
|
||||
|
||||
const (
|
||||
mtu = 1000
|
||||
)
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Printf("usage: %s host:port\n", os.Args[0])
|
||||
return
|
||||
}
|
||||
dest := os.Args[1]
|
||||
|
||||
x264Params, err := x264.NewParams()
|
||||
must(err)
|
||||
x264Params.Preset = x264.PresetMedium
|
||||
x264Params.BitRate = 1_000_000 // 1mbps
|
||||
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
)
|
||||
|
||||
mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
|
||||
Video: func(c *mediadevices.MediaTrackConstraints) {
|
||||
c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
|
||||
c.Width = prop.Int(640)
|
||||
c.Height = prop.Int(480)
|
||||
},
|
||||
Codec: codecSelector,
|
||||
})
|
||||
must(err)
|
||||
|
||||
videoTrack := mediaStream.GetVideoTracks()[0]
|
||||
defer videoTrack.Close()
|
||||
|
||||
rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().Name, mtu)
|
||||
must(err)
|
||||
|
||||
addr, err := net.ResolveUDPAddr("udp", dest)
|
||||
must(err)
|
||||
conn, err := net.DialUDP("udp", nil, addr)
|
||||
must(err)
|
||||
|
||||
buff := make([]byte, mtu)
|
||||
for {
|
||||
pkts, release, err := rtpReader.Read()
|
||||
must(err)
|
||||
|
||||
for _, pkt := range pkts {
|
||||
n, err := pkt.MarshalTo(buff)
|
||||
must(err)
|
||||
|
||||
_, err = conn.Write(buff[:n])
|
||||
must(err)
|
||||
}
|
||||
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -1,42 +1,29 @@
|
||||
## Instructions
|
||||
|
||||
### Install required codecs
|
||||
|
||||
In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system.
|
||||
|
||||
Installation steps:
|
||||
|
||||
* [x264](https://github.com/pion/mediadevices#x264)
|
||||
* [opus](https://github.com/pion/mediadevices#opus)
|
||||
|
||||
### Download webrtc example
|
||||
### Download gstreamer-send
|
||||
|
||||
```
|
||||
git clone https://github.com/pion/mediadevices.git
|
||||
```
|
||||
|
||||
#### Compile webrtc example
|
||||
|
||||
```
|
||||
cd mediadevices/examples/webrtc && go build
|
||||
go get github.com/pion/mediadevices/examples/webrtc
|
||||
```
|
||||
|
||||
### Open example page
|
||||
|
||||
[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
|
||||
|
||||
### Run the webrtc example with your browsers SessionDescription as stdin
|
||||
### Run webrtc with your browsers SessionDescription as stdin
|
||||
|
||||
In the jsfiddle the top textarea is your browser, copy that, and store the session description in an environment variable, `export SDP=<put_the_sdp_here>`
|
||||
In the jsfiddle the top textarea is your browser, copy that and:
|
||||
|
||||
Run `echo $SDP | ./webrtc`
|
||||
#### Linux
|
||||
|
||||
Run `echo $BROWSER_SDP | webrtc`
|
||||
|
||||
### Input webrtc's SessionDescription into your browser
|
||||
|
||||
Copy the text that `./webrtc` just emitted and copy into second text area
|
||||
Copy the text that `webrtc` just emitted and copy into second text area
|
||||
|
||||
### Hit 'Start Session' in jsfiddle, enjoy your video!
|
||||
|
||||
A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
|
||||
|
||||
Congrats, you have used pion-MediaDevices! Now start building something cool
|
||||
Congrats, you have used pion-WebRTC! Now start building something cool
|
||||
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/webrtc/v2"
|
||||
|
||||
// If you don't like x264, you can also use vpx by importing as below
|
||||
// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
|
||||
// If you don't like vpx, you can also use x264 by importing as below
|
||||
// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
|
||||
// or you can also use openh264 for alternative h264 implementation
|
||||
// "github.com/pion/mediadevices/pkg/codec/openh264"
|
||||
// or if you use a raspberry pi like, you can use mmal for using its hardware encoder
|
||||
// "github.com/pion/mediadevices/pkg/codec/mmal"
|
||||
"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use opus audio encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/opus" // This is required to use VP8/VP9 video encoder
|
||||
"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
|
||||
|
||||
// Note: If you don't have a camera or microphone or your adapters are not supported,
|
||||
// you can always swap your adapters with our dummy adapters below.
|
||||
@@ -26,6 +26,10 @@ import (
|
||||
_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
|
||||
)
|
||||
|
||||
const (
|
||||
videoCodecName = webrtc.VP8
|
||||
)
|
||||
|
||||
func main() {
|
||||
config := webrtc.Configuration{
|
||||
ICEServers: []webrtc.ICEServer{
|
||||
@@ -40,18 +44,18 @@ func main() {
|
||||
signal.Decode(signal.MustReadStdin(), &offer)
|
||||
|
||||
// Create a new RTCPeerConnection
|
||||
x264Params, err := x264.NewParams()
|
||||
vp8Params, err := vpx.NewVP8Params()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
x264Params.BitRate = 500_000 // 500kbps
|
||||
vp8Params.BitRate = 300_000 // 300kbps
|
||||
|
||||
opusParams, err := opus.NewParams()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
codecSelector := mediadevices.NewCodecSelector(
|
||||
mediadevices.WithVideoEncoders(&x264Params),
|
||||
mediadevices.WithVideoEncoders(&vp8Params),
|
||||
mediadevices.WithAudioEncoders(&opusParams),
|
||||
)
|
||||
|
||||
|
||||
5
go.mod
5
go.mod
@@ -4,12 +4,11 @@ go 1.13
|
||||
|
||||
require (
|
||||
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
|
||||
github.com/gen2brain/malgo v0.10.19
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
|
||||
github.com/lherman-cs/opus v0.0.2
|
||||
github.com/pion/logging v0.2.2
|
||||
github.com/pion/rtp v1.6.0
|
||||
github.com/pion/webrtc/v2 v2.2.26
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@@ -7,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gen2brain/malgo v0.10.19 h1:IUVF6WdVV7Txt47Kx2ajz0rWQ0MU0zO+tbcKmhva7l8=
|
||||
github.com/gen2brain/malgo v0.10.19/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
|
||||
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
@@ -17,6 +15,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
|
||||
github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -109,8 +109,8 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
|
||||
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
|
||||
BIN
img/demo.gif
BIN
img/demo.gif
Binary file not shown.
|
Before Width: | Height: | Size: 133 B After Width: | Height: | Size: 9.6 MiB |
14
ioreader.go
14
ioreader.go
@@ -1,14 +0,0 @@
|
||||
package mediadevices
|
||||
|
||||
type encodedReadCloserImpl struct {
|
||||
readFn func([]byte) (int, error)
|
||||
closeFn func() error
|
||||
}
|
||||
|
||||
func (r *encodedReadCloserImpl) Read(b []byte) (int, error) {
|
||||
return r.readFn(b)
|
||||
}
|
||||
|
||||
func (r *encodedReadCloserImpl) Close() error {
|
||||
return r.closeFn()
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
// +build e2e
|
||||
|
||||
package mediadevices
|
||||
|
||||
import (
|
||||
"image"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/pion/mediadevices/pkg/codec/x264"
|
||||
"github.com/pion/mediadevices/pkg/frame"
|
||||
)
|
||||
|
||||
type mockVideoSource struct {
|
||||
width, height int
|
||||
pool sync.Pool
|
||||
decoder frame.Decoder
|
||||
}
|
||||
|
||||
func newMockVideoSource(width, height int) *mockVideoSource {
|
||||
decoder, err := frame.NewDecoder(frame.FormatYUY2)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return &mockVideoSource{
|
||||
width: width,
|
||||
height: height,
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
resolution := width * height
|
||||
return make([]byte, resolution*2)
|
||||
},
|
||||
},
|
||||
decoder: decoder,
|
||||
}
|
||||
}
|
||||
|
||||
func (source *mockVideoSource) ID() string { return "" }
|
||||
func (source *mockVideoSource) Close() error { return nil }
|
||||
func (source *mockVideoSource) Read() (image.Image, func(), error) {
|
||||
raw := source.pool.Get().([]byte)
|
||||
decoded, release, err := source.decoder.Decode(raw, source.width, source.height)
|
||||
source.pool.Put(raw)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return decoded, release, nil
|
||||
}
|
||||
|
||||
func BenchmarkEndToEnd(b *testing.B) {
|
||||
params, err := x264.NewParams()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
params.BitRate = 300_000
|
||||
|
||||
videoSource := newMockVideoSource(1920, 1080)
|
||||
track := NewVideoTrack(videoSource, nil).(*VideoTrack)
|
||||
defer track.Close()
|
||||
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentVideoProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
encodedReader, err := params.BuildVideoEncoder(reader, inputProp)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer encodedReader.Close()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, release, err := encodedReader.Read()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package mediadevices
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/pion/webrtc/v2"
|
||||
@@ -34,14 +33,6 @@ func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (track *mockMediaStreamTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestMediaStreamFilters(t *testing.T) {
|
||||
audioTracks := []Track{
|
||||
&mockMediaStreamTrack{AudioInput},
|
||||
|
||||
@@ -237,19 +237,10 @@ func (e *encoder) Read() ([]byte, func(), error) {
|
||||
e.raw.d_w, e.raw.d_h = C.uint(width), C.uint(height)
|
||||
}
|
||||
|
||||
duration := t - e.tLastFrame
|
||||
// VPX doesn't allow 0 duration. If 0 is given, vpx_codec_encode will fail with VPX_CODEC_INVALID_PARAM.
|
||||
// 0 duration is possible because mediadevices first gets the frame meta data by reading from the source,
|
||||
// and consequently the codec will read the first frame from the buffer. This means the first frame won't
|
||||
// have a pause to the second frame, which means if the delay is <1 ms (vpx duration resolution), duration
|
||||
// is going to be 0.
|
||||
if duration == 0 {
|
||||
duration = 1
|
||||
}
|
||||
var flags int
|
||||
if ec := C.encode_wrapper(
|
||||
e.codec, e.raw,
|
||||
C.long(t-e.tStart), C.ulong(duration), C.long(flags), C.ulong(e.deadline),
|
||||
C.long(t-e.tStart), C.ulong(t-e.tLastFrame), C.long(flags), C.ulong(e.deadline),
|
||||
(*C.uchar)(&yuvImg.Y[0]), (*C.uchar)(&yuvImg.Cb[0]), (*C.uchar)(&yuvImg.Cr[0]),
|
||||
); ec != C.VPX_CODEC_OK {
|
||||
return nil, func() {}, fmt.Errorf("vpx_codec_encode failed (%d)", ec)
|
||||
|
||||
@@ -47,7 +47,7 @@ Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
|
||||
e->param.b_repeat_headers = 1;
|
||||
e->param.b_annexb = 1;
|
||||
|
||||
if (x264_param_apply_profile(&e->param, "high") < 0) {
|
||||
if (x264_param_apply_profile(&e->param, "baseline") < 0) {
|
||||
*rc = ERR_APPLY_PROFILE;
|
||||
goto fail;
|
||||
}
|
||||
@@ -95,4 +95,4 @@ void enc_close(Encoder *e, int *rc) {
|
||||
x264_encoder_close(e->h);
|
||||
x264_picture_clean(&e->pic_in);
|
||||
free(e);
|
||||
}
|
||||
}
|
||||
@@ -1,204 +1 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/gen2brain/malgo"
|
||||
"github.com/pion/mediadevices/internal/logging"
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDeviceIDLength = 20
|
||||
// TODO: should replace this with a more flexible approach
|
||||
sampleRateStep = 1000
|
||||
initialBufferSize = 1024
|
||||
)
|
||||
|
||||
var logger = logging.NewLogger("mediadevices/driver/microphone")
|
||||
var ctx *malgo.AllocatedContext
|
||||
var hostEndian binary.ByteOrder
|
||||
var (
|
||||
errUnsupportedFormat = errors.New("the provided audio format is not supported")
|
||||
)
|
||||
|
||||
type microphone struct {
|
||||
malgo.DeviceInfo
|
||||
chunkChan chan []byte
|
||||
}
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
/*
|
||||
backends := []malgo.Backend{
|
||||
malgo.BackendPulseaudio,
|
||||
malgo.BackendAlsa,
|
||||
}
|
||||
*/
|
||||
ctx, err = malgo.InitContext(nil, malgo.ContextConfig{}, func(message string) {
|
||||
logger.Debugf("%v\n", message)
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
devices, err := ctx.Devices(malgo.Capture)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, device := range devices {
|
||||
// TODO: Detect default device and prioritize it
|
||||
driver.GetManager().Register(newMicrophone(device), driver.Info{
|
||||
Label: device.ID.String(),
|
||||
DeviceType: driver.Microphone,
|
||||
})
|
||||
}
|
||||
|
||||
// Decide which endian
|
||||
switch v := *(*uint16)(unsafe.Pointer(&([]byte{0x12, 0x34}[0]))); v {
|
||||
case 0x1234:
|
||||
hostEndian = binary.BigEndian
|
||||
case 0x3412:
|
||||
hostEndian = binary.LittleEndian
|
||||
default:
|
||||
panic(fmt.Sprintf("failed to determine host endianness: %x", v))
|
||||
}
|
||||
}
|
||||
|
||||
func newMicrophone(info malgo.DeviceInfo) *microphone {
|
||||
return µphone{
|
||||
DeviceInfo: info,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
m.chunkChan = make(chan []byte, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.chunkChan != nil {
|
||||
close(m.chunkChan)
|
||||
m.chunkChan = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(inputProp prop.Media) (audio.Reader, error) {
|
||||
var config malgo.DeviceConfig
|
||||
var callbacks malgo.DeviceCallbacks
|
||||
|
||||
decoder, err := wave.NewDecoder(&wave.RawFormat{
|
||||
SampleSize: inputProp.SampleSize,
|
||||
IsFloat: inputProp.IsFloat,
|
||||
Interleaved: inputProp.IsInterleaved,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.DeviceType = malgo.Capture
|
||||
config.PerformanceProfile = malgo.LowLatency
|
||||
config.Capture.Channels = uint32(inputProp.ChannelCount)
|
||||
config.SampleRate = uint32(inputProp.SampleRate)
|
||||
if inputProp.SampleSize == 4 && inputProp.IsFloat {
|
||||
config.Capture.Format = malgo.FormatF32
|
||||
} else if inputProp.SampleSize == 2 && !inputProp.IsFloat {
|
||||
config.Capture.Format = malgo.FormatS16
|
||||
} else {
|
||||
return nil, errUnsupportedFormat
|
||||
}
|
||||
|
||||
onRecvChunk := func(_, chunk []byte, framecount uint32) {
|
||||
m.chunkChan <- chunk
|
||||
}
|
||||
callbacks.Data = onRecvChunk
|
||||
|
||||
device, err := malgo.InitDevice(ctx.Context, config, callbacks)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = device.Start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
chunk, ok := <-m.chunkChan
|
||||
if !ok {
|
||||
device.Stop()
|
||||
device.Uninit()
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
decodedChunk, err := decoder.Decode(hostEndian, chunk, inputProp.ChannelCount)
|
||||
// FIXME: the decoder should also fill this information
|
||||
decodedChunk.(*wave.Float32Interleaved).Size.SamplingRate = inputProp.SampleRate
|
||||
return decodedChunk, func() {}, err
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
var supportedProps []prop.Media
|
||||
logger.Debug("Querying properties")
|
||||
|
||||
var isBigEndian bool
|
||||
// miniaudio only uses the host endian
|
||||
if hostEndian == binary.BigEndian {
|
||||
isBigEndian = true
|
||||
}
|
||||
|
||||
for ch := m.MinChannels; ch <= m.MaxChannels; ch++ {
|
||||
for sampleRate := m.MinSampleRate; sampleRate <= m.MaxSampleRate; sampleRate += sampleRateStep {
|
||||
for i := 0; i < int(m.FormatCount); i++ {
|
||||
format := m.Formats[i]
|
||||
|
||||
supportedProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
ChannelCount: int(ch),
|
||||
SampleRate: int(sampleRate),
|
||||
IsBigEndian: isBigEndian,
|
||||
// miniaudio only supports interleaved at the moment
|
||||
IsInterleaved: true,
|
||||
},
|
||||
}
|
||||
|
||||
switch malgo.FormatType(format) {
|
||||
case malgo.FormatF32:
|
||||
supportedProp.SampleSize = 4
|
||||
supportedProp.IsFloat = true
|
||||
case malgo.FormatS16:
|
||||
supportedProp.SampleSize = 2
|
||||
supportedProp.IsFloat = false
|
||||
}
|
||||
|
||||
supportedProps = append(supportedProps, supportedProp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: remove this hardcoded value. Malgo doesn't support "ma_context_get_device_info" API yet. The above iterations
|
||||
// will always return nothing as of now
|
||||
supportedProps = append(supportedProps, prop.Media{
|
||||
Audio: prop.Audio{
|
||||
Latency: time.Millisecond * 20,
|
||||
ChannelCount: 1,
|
||||
SampleRate: 48000,
|
||||
SampleSize: 4,
|
||||
IsFloat: true,
|
||||
IsBigEndian: isBigEndian,
|
||||
IsInterleaved: true,
|
||||
},
|
||||
})
|
||||
return supportedProps
|
||||
}
|
||||
|
||||
138
pkg/driver/microphone/microphone_linux.go
Normal file
138
pkg/driver/microphone/microphone_linux.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/jfreymuth/pulse"
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
type microphone struct {
|
||||
c *pulse.Client
|
||||
id string
|
||||
samplesChan chan<- []int16
|
||||
}
|
||||
|
||||
func init() {
|
||||
pa, err := pulse.NewClient()
|
||||
if err != nil {
|
||||
// No pulseaudio
|
||||
return
|
||||
}
|
||||
defer pa.Close()
|
||||
sources, err := pa.ListSources()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defaultSource, err := pa.DefaultSource()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, source := range sources {
|
||||
priority := driver.PriorityNormal
|
||||
if defaultSource.ID() == source.ID() {
|
||||
priority = driver.PriorityHigh
|
||||
}
|
||||
driver.GetManager().Register(µphone{id: source.ID()}, driver.Info{
|
||||
Label: source.ID(),
|
||||
DeviceType: driver.Microphone,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
var err error
|
||||
m.c, err = pulse.NewClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.samplesChan != nil {
|
||||
close(m.samplesChan)
|
||||
m.samplesChan = nil
|
||||
}
|
||||
|
||||
m.c.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
|
||||
var options []pulse.RecordOption
|
||||
if p.ChannelCount == 1 {
|
||||
options = append(options, pulse.RecordMono)
|
||||
} else {
|
||||
options = append(options, pulse.RecordStereo)
|
||||
}
|
||||
latency := p.Latency.Seconds()
|
||||
|
||||
src, err := m.c.SourceByID(m.id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
options = append(options,
|
||||
pulse.RecordSampleRate(p.SampleRate),
|
||||
pulse.RecordLatency(latency),
|
||||
pulse.RecordSource(src),
|
||||
)
|
||||
|
||||
samplesChan := make(chan []int16, 1)
|
||||
|
||||
handler := func(b []int16) (int, error) {
|
||||
samplesChan <- b
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
stream, err := m.c.NewRecord(pulse.Int16Writer(handler), options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader := audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
buff, ok := <-samplesChan
|
||||
if !ok {
|
||||
stream.Close()
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
a := wave.NewInt16Interleaved(
|
||||
wave.ChunkInfo{
|
||||
Channels: p.ChannelCount,
|
||||
Len: len(buff) / p.ChannelCount,
|
||||
SamplingRate: p.SampleRate,
|
||||
},
|
||||
)
|
||||
copy(a.Data, buff)
|
||||
|
||||
return a, func() {}, nil
|
||||
})
|
||||
|
||||
stream.Start()
|
||||
m.samplesChan = samplesChan
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
// TODO: Get actual properties
|
||||
monoProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
SampleRate: 48000,
|
||||
Latency: time.Millisecond * 20,
|
||||
ChannelCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
stereoProp := monoProp
|
||||
stereoProp.ChannelCount = 2
|
||||
|
||||
return []prop.Media{monoProp, stereoProp}
|
||||
}
|
||||
348
pkg/driver/microphone/microphone_windows.go
Normal file
348
pkg/driver/microphone/microphone_windows.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package microphone
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"golang.org/x/sys/windows"
|
||||
"io"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pion/mediadevices/pkg/driver"
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/prop"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
)
|
||||
|
||||
const (
|
||||
// bufferNumber * prop.Audio.Latency is the maximum blockable duration
|
||||
// to get data without dropping chunks.
|
||||
bufferNumber = 5
|
||||
)
|
||||
|
||||
// Windows APIs
|
||||
var (
|
||||
winmm = windows.NewLazySystemDLL("Winmm.dll")
|
||||
waveInOpen = winmm.NewProc("waveInOpen")
|
||||
waveInStart = winmm.NewProc("waveInStart")
|
||||
waveInStop = winmm.NewProc("waveInStop")
|
||||
waveInReset = winmm.NewProc("waveInReset")
|
||||
waveInClose = winmm.NewProc("waveInClose")
|
||||
waveInPrepareHeader = winmm.NewProc("waveInPrepareHeader")
|
||||
waveInAddBuffer = winmm.NewProc("waveInAddBuffer")
|
||||
waveInUnprepareHeader = winmm.NewProc("waveInUnprepareHeader")
|
||||
)
|
||||
|
||||
type buffer struct {
|
||||
waveHdr
|
||||
data []int16
|
||||
}
|
||||
|
||||
func newBuffer(samples int) *buffer {
|
||||
b := make([]int16, samples)
|
||||
return &buffer{
|
||||
waveHdr: waveHdr{
|
||||
// Sharing Go memory with Windows C API without reference.
|
||||
// Make sure that the lifetime of the buffer struct is longer
|
||||
// than the final access from cbWaveIn.
|
||||
lpData: uintptr(unsafe.Pointer(&b[0])),
|
||||
dwBufferLength: uint32(samples * 2),
|
||||
},
|
||||
data: b,
|
||||
}
|
||||
}
|
||||
|
||||
type microphone struct {
|
||||
hWaveIn windows.Pointer
|
||||
buf map[uintptr]*buffer
|
||||
chBuf chan *buffer
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func init() {
|
||||
// TODO: enum devices
|
||||
driver.GetManager().Register(µphone{}, driver.Info{
|
||||
Label: "default",
|
||||
DeviceType: driver.Microphone,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *microphone) Open() error {
|
||||
m.chBuf = make(chan *buffer)
|
||||
m.buf = make(map[uintptr]*buffer)
|
||||
m.closed = make(chan struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) cbWaveIn(hWaveIn windows.Pointer, uMsg uint, dwInstance, dwParam1, dwParam2 *int32) uintptr {
|
||||
switch uMsg {
|
||||
case MM_WIM_DATA:
|
||||
b := m.buf[uintptr(unsafe.Pointer(dwParam1))]
|
||||
m.chBuf <- b
|
||||
|
||||
case MM_WIM_OPEN:
|
||||
case MM_WIM_CLOSE:
|
||||
close(m.chBuf)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *microphone) Close() error {
|
||||
if m.hWaveIn == nil {
|
||||
return nil
|
||||
}
|
||||
close(m.closed)
|
||||
|
||||
ret, _, _ := waveInStop.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
// All enqueued buffers are marked done by waveInReset.
|
||||
ret, _, _ = waveInReset.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, buf := range m.buf {
|
||||
// Detach buffers from waveIn API.
|
||||
ret, _, _ := waveInUnprepareHeader.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Now, it's ready to free the buffers.
|
||||
// As microphone struct still has reference to the buffers,
|
||||
// they will be GC-ed once microphone is reopened or unreferenced.
|
||||
|
||||
ret, _, _ = waveInClose.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return err
|
||||
}
|
||||
<-m.chBuf
|
||||
m.hWaveIn = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *microphone) AudioRecord(p prop.Media) (audio.Reader, error) {
|
||||
for i := 0; i < bufferNumber; i++ {
|
||||
b := newBuffer(
|
||||
int(uint64(p.Latency) * uint64(p.SampleRate) / uint64(time.Second)),
|
||||
)
|
||||
// Map the buffer by its data head address to restore access to the Go struct
|
||||
// in callback function. Don't resize the buffer after it.
|
||||
m.buf[uintptr(unsafe.Pointer(&b.waveHdr))] = b
|
||||
}
|
||||
|
||||
waveFmt := &waveFormatEx{
|
||||
wFormatTag: WAVE_FORMAT_PCM,
|
||||
nChannels: uint16(p.ChannelCount),
|
||||
nSamplesPerSec: uint32(p.SampleRate),
|
||||
nAvgBytesPerSec: uint32(p.SampleRate * p.ChannelCount * 2),
|
||||
nBlockAlign: uint16(p.ChannelCount * 2),
|
||||
wBitsPerSample: 16,
|
||||
}
|
||||
ret, _, _ := waveInOpen.Call(
|
||||
uintptr(unsafe.Pointer(&m.hWaveIn)),
|
||||
WAVE_MAPPER,
|
||||
uintptr(unsafe.Pointer(waveFmt)),
|
||||
windows.NewCallback(m.cbWaveIn),
|
||||
0,
|
||||
CALLBACK_FUNCTION,
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, buf := range m.buf {
|
||||
// Attach buffers to waveIn API.
|
||||
ret, _, _ := waveInPrepareHeader.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, buf := range m.buf {
|
||||
// Enqueue buffers.
|
||||
ret, _, _ := waveInAddBuffer.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&buf.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(buf.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
ret, _, _ = waveInStart.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// TODO: detect microphone device disconnection and return EOF
|
||||
|
||||
reader := audio.ReaderFunc(func() (wave.Audio, func(), error) {
|
||||
b, ok := <-m.chBuf
|
||||
if !ok {
|
||||
return nil, func() {}, io.EOF
|
||||
}
|
||||
|
||||
select {
|
||||
case <-m.closed:
|
||||
default:
|
||||
// Re-enqueue used buffer.
|
||||
ret, _, _ := waveInAddBuffer.Call(
|
||||
uintptr(unsafe.Pointer(m.hWaveIn)),
|
||||
uintptr(unsafe.Pointer(&b.waveHdr)),
|
||||
uintptr(unsafe.Sizeof(b.waveHdr)),
|
||||
)
|
||||
if err := errWinmm[ret]; err != nil {
|
||||
return nil, func() {}, err
|
||||
}
|
||||
}
|
||||
|
||||
a := wave.NewInt16Interleaved(
|
||||
wave.ChunkInfo{
|
||||
Channels: p.ChannelCount,
|
||||
Len: (int(b.waveHdr.dwBytesRecorded) / 2) / p.ChannelCount,
|
||||
SamplingRate: p.SampleRate,
|
||||
},
|
||||
)
|
||||
|
||||
j := 0
|
||||
for i := 0; i < a.Size.Len; i++ {
|
||||
for ch := 0; ch < a.Size.Channels; ch++ {
|
||||
a.SetInt16(i, ch, wave.Int16Sample(b.data[j]))
|
||||
j++
|
||||
}
|
||||
}
|
||||
|
||||
return a, func() {}, nil
|
||||
})
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (m *microphone) Properties() []prop.Media {
|
||||
// TODO: Get actual properties
|
||||
monoProp := prop.Media{
|
||||
Audio: prop.Audio{
|
||||
SampleRate: 48000,
|
||||
Latency: time.Millisecond * 20,
|
||||
ChannelCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
stereoProp := monoProp
|
||||
stereoProp.ChannelCount = 2
|
||||
|
||||
return []prop.Media{monoProp, stereoProp}
|
||||
}
|
||||
|
||||
// Windows API structures
|
||||
|
||||
type waveFormatEx struct {
|
||||
wFormatTag uint16
|
||||
nChannels uint16
|
||||
nSamplesPerSec uint32
|
||||
nAvgBytesPerSec uint32
|
||||
nBlockAlign uint16
|
||||
wBitsPerSample uint16
|
||||
cbSize uint16
|
||||
}
|
||||
|
||||
type waveHdr struct {
|
||||
lpData uintptr
|
||||
dwBufferLength uint32
|
||||
dwBytesRecorded uint32
|
||||
dwUser *uint32
|
||||
dwFlags uint32
|
||||
dwLoops uint32
|
||||
lpNext *waveHdr
|
||||
reserved *uint32
|
||||
}
|
||||
|
||||
// Windows consts
|
||||
|
||||
const (
|
||||
MMSYSERR_NOERROR = 0
|
||||
MMSYSERR_ERROR = 1
|
||||
MMSYSERR_BADDEVICEID = 2
|
||||
MMSYSERR_NOTENABLED = 3
|
||||
MMSYSERR_ALLOCATED = 4
|
||||
MMSYSERR_INVALHANDLE = 5
|
||||
MMSYSERR_NODRIVER = 6
|
||||
MMSYSERR_NOMEM = 7
|
||||
MMSYSERR_NOTSUPPORTED = 8
|
||||
MMSYSERR_BADERRNUM = 9
|
||||
MMSYSERR_INVALFLAG = 10
|
||||
MMSYSERR_INVALPARAM = 11
|
||||
MMSYSERR_HANDLEBUSY = 12
|
||||
MMSYSERR_INVALIDALIAS = 13
|
||||
MMSYSERR_BADDB = 14
|
||||
MMSYSERR_KEYNOTFOUND = 15
|
||||
MMSYSERR_READERROR = 16
|
||||
MMSYSERR_WRITEERROR = 17
|
||||
MMSYSERR_DELETEERROR = 18
|
||||
MMSYSERR_VALNOTFOUND = 19
|
||||
MMSYSERR_NODRIVERCB = 20
|
||||
|
||||
WAVERR_BADFORMAT = 32
|
||||
WAVERR_STILLPLAYING = 33
|
||||
WAVERR_UNPREPARED = 34
|
||||
WAVERR_SYNC = 35
|
||||
|
||||
WAVE_MAPPER = 0xFFFF
|
||||
WAVE_FORMAT_PCM = 1
|
||||
|
||||
CALLBACK_NULL = 0
|
||||
CALLBACK_WINDOW = 0x10000
|
||||
CALLBACK_TASK = 0x20000
|
||||
CALLBACK_FUNCTION = 0x30000
|
||||
CALLBACK_THREAD = CALLBACK_TASK
|
||||
CALLBACK_EVENT = 0x50000
|
||||
|
||||
MM_WIM_OPEN = 0x3BE
|
||||
MM_WIM_CLOSE = 0x3BF
|
||||
MM_WIM_DATA = 0x3C0
|
||||
)
|
||||
|
||||
var errWinmm = map[uintptr]error{
|
||||
MMSYSERR_NOERROR: nil,
|
||||
MMSYSERR_ERROR: errors.New("error"),
|
||||
MMSYSERR_BADDEVICEID: errors.New("bad device id"),
|
||||
MMSYSERR_NOTENABLED: errors.New("not enabled"),
|
||||
MMSYSERR_ALLOCATED: errors.New("already allocated"),
|
||||
MMSYSERR_INVALHANDLE: errors.New("invalid handler"),
|
||||
MMSYSERR_NODRIVER: errors.New("no driver"),
|
||||
MMSYSERR_NOMEM: errors.New("no memory"),
|
||||
MMSYSERR_NOTSUPPORTED: errors.New("not supported"),
|
||||
MMSYSERR_BADERRNUM: errors.New("band error number"),
|
||||
MMSYSERR_INVALFLAG: errors.New("invalid flag"),
|
||||
MMSYSERR_INVALPARAM: errors.New("invalid param"),
|
||||
MMSYSERR_HANDLEBUSY: errors.New("handle busy"),
|
||||
MMSYSERR_INVALIDALIAS: errors.New("invalid alias"),
|
||||
MMSYSERR_BADDB: errors.New("bad db"),
|
||||
MMSYSERR_KEYNOTFOUND: errors.New("key not found"),
|
||||
MMSYSERR_READERROR: errors.New("read error"),
|
||||
MMSYSERR_WRITEERROR: errors.New("write error"),
|
||||
MMSYSERR_DELETEERROR: errors.New("delete error"),
|
||||
MMSYSERR_VALNOTFOUND: errors.New("value not found"),
|
||||
MMSYSERR_NODRIVERCB: errors.New("no driver cb"),
|
||||
WAVERR_BADFORMAT: errors.New("bad format"),
|
||||
WAVERR_STILLPLAYING: errors.New("still playing"),
|
||||
WAVERR_UNPREPARED: errors.New("unprepared"),
|
||||
WAVERR_SYNC: errors.New("sync"),
|
||||
}
|
||||
8
pkg/frame/buffer.go
Normal file
8
pkg/frame/buffer.go
Normal file
@@ -0,0 +1,8 @@
|
||||
package frame
|
||||
|
||||
import "image"
|
||||
|
||||
type buffer struct {
|
||||
image image.Image
|
||||
raw []uint8
|
||||
}
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"image/jpeg"
|
||||
)
|
||||
|
||||
func decodeMJPEG(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
img, err := jpeg.Decode(bytes.NewReader(frame))
|
||||
return img, func() {}, err
|
||||
func decodeMJPEG() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
img, err := jpeg.Decode(bytes.NewReader(frame))
|
||||
return img, func() {}, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,22 +5,22 @@ import (
|
||||
)
|
||||
|
||||
func NewDecoder(f Format) (Decoder, error) {
|
||||
var decoder decoderFunc
|
||||
var buildDecoder func() decoderFunc
|
||||
|
||||
switch f {
|
||||
case FormatI420:
|
||||
decoder = decodeI420
|
||||
buildDecoder = decodeI420
|
||||
case FormatNV21:
|
||||
decoder = decodeNV21
|
||||
buildDecoder = decodeNV21
|
||||
case FormatYUY2:
|
||||
decoder = decodeYUY2
|
||||
buildDecoder = decodeYUY2
|
||||
case FormatUYVY:
|
||||
decoder = decodeUYVY
|
||||
buildDecoder = decodeUYVY
|
||||
case FormatMJPEG:
|
||||
decoder = decodeMJPEG
|
||||
buildDecoder = decodeMJPEG
|
||||
default:
|
||||
return nil, fmt.Errorf("%s is not supported", f)
|
||||
}
|
||||
|
||||
return decoder, nil
|
||||
return buildDecoder(), nil
|
||||
}
|
||||
|
||||
@@ -5,47 +5,51 @@ import (
|
||||
"image"
|
||||
)
|
||||
|
||||
func decodeI420(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
cbi := yi + width*height/4
|
||||
cri := cbi + width*height/4
|
||||
func decodeI420() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
cbi := yi + width*height/4
|
||||
cri := cbi + width*height/4
|
||||
|
||||
if cri > len(frame) {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), cri)
|
||||
if cri > len(frame) {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), cri)
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: frame[:yi],
|
||||
YStride: width,
|
||||
Cb: frame[yi:cbi],
|
||||
Cr: frame[cbi:cri],
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio420,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: frame[:yi],
|
||||
YStride: width,
|
||||
Cb: frame[yi:cbi],
|
||||
Cr: frame[cbi:cri],
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio420,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
func decodeNV21(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi + width*height/2
|
||||
func decodeNV21() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi + width*height/2
|
||||
|
||||
if ci > len(frame) {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), ci)
|
||||
if ci > len(frame) {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), ci)
|
||||
}
|
||||
|
||||
var cb, cr []byte
|
||||
for i := yi; i < ci; i += 2 {
|
||||
cb = append(cb, frame[i])
|
||||
cr = append(cr, frame[i+1])
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: frame[:yi],
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio420,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
var cb, cr []byte
|
||||
for i := yi; i < ci; i += 2 {
|
||||
cb = append(cb, frame[i])
|
||||
cr = append(cr, frame[i+1])
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: frame[:yi],
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio420,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
@@ -12,66 +12,70 @@ import (
|
||||
// void decodeUYVYCGO(uint8_t* y, uint8_t* cb, uint8_t* cr, uint8_t* uyvy, int width, int height);
|
||||
import "C"
|
||||
|
||||
func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
func decodeYUY2() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
C.decodeYUY2CGO(
|
||||
(*C.uchar)(&y[0]),
|
||||
(*C.uchar)(&cb[0]),
|
||||
(*C.uchar)(&cr[0]),
|
||||
(*C.uchar)(&frame[0]),
|
||||
C.int(width), C.int(height),
|
||||
)
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
C.decodeYUY2CGO(
|
||||
(*C.uchar)(&y[0]),
|
||||
(*C.uchar)(&cb[0]),
|
||||
(*C.uchar)(&cr[0]),
|
||||
(*C.uchar)(&frame[0]),
|
||||
C.int(width), C.int(height),
|
||||
)
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
func decodeUYVY() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
C.decodeUYVYCGO(
|
||||
(*C.uchar)(&y[0]),
|
||||
(*C.uchar)(&cb[0]),
|
||||
(*C.uchar)(&cr[0]),
|
||||
(*C.uchar)(&frame[0]),
|
||||
C.int(width), C.int(height),
|
||||
)
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
C.decodeUYVYCGO(
|
||||
(*C.uchar)(&y[0]),
|
||||
(*C.uchar)(&cb[0]),
|
||||
(*C.uchar)(&cr[0]),
|
||||
(*C.uchar)(&frame[0]),
|
||||
C.int(width), C.int(height),
|
||||
)
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
@@ -5,74 +5,97 @@ package frame
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
func decodeYUY2() decoderFunc {
|
||||
pool := sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &buffer{
|
||||
image: &image.YCbCr{},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
buff := pool.Get().(*buffer)
|
||||
|
||||
fast := 0
|
||||
slow := 0
|
||||
for i := 0; i < fi; i += 4 {
|
||||
y[fast] = frame[i]
|
||||
cb[slow] = frame[i+1]
|
||||
y[fast+1] = frame[i+2]
|
||||
cr[slow] = frame[i+3]
|
||||
fast += 2
|
||||
slow++
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
|
||||
if len(frame) != fi {
|
||||
pool.Put(buff)
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
}
|
||||
|
||||
if len(buff.raw) < fi {
|
||||
need := fi - len(buff.raw)
|
||||
buff.raw = append(buff.raw, make([]uint8, need)...)
|
||||
}
|
||||
y := buff.raw[:yi:yi]
|
||||
cb := buff.raw[yi : yi+ci : yi+ci]
|
||||
cr := buff.raw[yi+ci : fi : fi]
|
||||
|
||||
fast := 0
|
||||
slow := 0
|
||||
for i := 0; i < fi; i += 4 {
|
||||
y[fast] = frame[i]
|
||||
cb[slow] = frame[i+1]
|
||||
y[fast+1] = frame[i+2]
|
||||
cr[slow] = frame[i+3]
|
||||
fast += 2
|
||||
slow++
|
||||
}
|
||||
|
||||
img := buff.image.(*image.YCbCr)
|
||||
img.Y = y
|
||||
img.YStride = width
|
||||
img.Cb = cb
|
||||
img.Cr = cr
|
||||
img.CStride = width / 2
|
||||
img.SubsampleRatio = image.YCbCrSubsampleRatio422
|
||||
img.Rect.Min.X = 0
|
||||
img.Rect.Min.Y = 0
|
||||
img.Rect.Max.X = width
|
||||
img.Rect.Max.Y = height
|
||||
return img, func() { pool.Put(buff) }, nil
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
func decodeUYVY() decoderFunc {
|
||||
return func(frame []byte, width, height int) (image.Image, func(), error) {
|
||||
yi := width * height
|
||||
ci := yi / 2
|
||||
fi := yi + 2*ci
|
||||
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
if len(frame) != fi {
|
||||
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
fast := 0
|
||||
slow := 0
|
||||
for i := 0; i < fi; i += 4 {
|
||||
cb[slow] = frame[i]
|
||||
y[fast] = frame[i+1]
|
||||
cr[slow] = frame[i+2]
|
||||
y[fast+1] = frame[i+3]
|
||||
fast += 2
|
||||
slow++
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
y := make([]byte, yi)
|
||||
cb := make([]byte, ci)
|
||||
cr := make([]byte, ci)
|
||||
|
||||
fast := 0
|
||||
slow := 0
|
||||
for i := 0; i < fi; i += 4 {
|
||||
cb[slow] = frame[i]
|
||||
y[fast] = frame[i+1]
|
||||
cr[slow] = frame[i+2]
|
||||
y[fast+1] = frame[i+3]
|
||||
fast += 2
|
||||
slow++
|
||||
}
|
||||
|
||||
return &image.YCbCr{
|
||||
Y: y,
|
||||
YStride: width,
|
||||
Cb: cb,
|
||||
Cr: cr,
|
||||
CStride: width / 2,
|
||||
SubsampleRatio: image.YCbCrSubsampleRatio422,
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}, func() {}, nil
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestDecodeYUY2(t *testing.T) {
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}
|
||||
|
||||
img, _, err := decodeYUY2(input, width, height)
|
||||
img, _, err := decodeYUY2()(input, width, height)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestDecodeUYVY(t *testing.T) {
|
||||
Rect: image.Rect(0, 0, width, height),
|
||||
}
|
||||
|
||||
img, _, err := decodeUYVY(input, width, height)
|
||||
img, _, err := decodeUYVY()(input, width, height)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,11 +76,13 @@ func BenchmarkDecodeYUY2(b *testing.B) {
|
||||
sz := sz
|
||||
b.Run(fmt.Sprintf("%dx%d", sz.width, sz.height), func(b *testing.B) {
|
||||
input := make([]byte, sz.width*sz.height*2)
|
||||
decode := decodeYUY2()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _, err := decodeYUY2(input, sz.width, sz.height)
|
||||
_, release, err := decode(input, sz.width, sz.height)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,14 +5,6 @@ import (
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (chunk wave.Audio, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,14 +3,6 @@ package io
|
||||
// Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
|
||||
// to provide strong type.
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (data interface{}, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,14 +5,6 @@ import (
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
// Read reads data from the source. The caller is responsible to release the memory that's associated
|
||||
// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
|
||||
// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
|
||||
// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
|
||||
// optimization. If release is not called, the source is forced to allocate a new memory, which also means
|
||||
// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
|
||||
// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
|
||||
// slower as the heap memory usage increases and more garbage to collect.
|
||||
Read() (img image.Image, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
21
rtpreader.go
21
rtpreader.go
@@ -1,21 +0,0 @@
|
||||
package mediadevices
|
||||
|
||||
import "github.com/pion/rtp"
|
||||
|
||||
type RTPReadCloser interface {
|
||||
Read() (pkts []*rtp.Packet, release func(), err error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type rtpReadCloserImpl struct {
|
||||
readFn func() ([]*rtp.Packet, func(), error)
|
||||
closeFn func() error
|
||||
}
|
||||
|
||||
func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
|
||||
return r.readFn()
|
||||
}
|
||||
|
||||
func (r *rtpReadCloserImpl) Close() error {
|
||||
return r.closeFn()
|
||||
}
|
||||
24
sampler.go
24
sampler.go
@@ -3,30 +3,34 @@ package mediadevices
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/pion/webrtc/v2"
|
||||
"github.com/pion/webrtc/v2/pkg/media"
|
||||
)
|
||||
|
||||
type samplerFunc func() uint32
|
||||
type samplerFunc func(b []byte) error
|
||||
|
||||
// newVideoSampler creates a video sampler that uses the actual video frame rate and
|
||||
// the codec's clock rate to come up with a duration for each sample.
|
||||
func newVideoSampler(clockRate uint32) samplerFunc {
|
||||
clockRateFloat := float64(clockRate)
|
||||
func newVideoSampler(t *webrtc.Track) samplerFunc {
|
||||
clockRate := float64(t.Codec().ClockRate)
|
||||
lastTimestamp := time.Now()
|
||||
|
||||
return samplerFunc(func() uint32 {
|
||||
return samplerFunc(func(b []byte) error {
|
||||
now := time.Now()
|
||||
duration := now.Sub(lastTimestamp).Seconds()
|
||||
samples := uint32(math.Round(clockRateFloat * duration))
|
||||
samples := uint32(math.Round(clockRate * duration))
|
||||
lastTimestamp = now
|
||||
return samples
|
||||
|
||||
return t.WriteSample(media.Sample{Data: b, Samples: samples})
|
||||
})
|
||||
}
|
||||
|
||||
// newAudioSampler creates a audio sampler that uses a fixed latency and
|
||||
// the codec's clock rate to come up with a duration for each sample.
|
||||
func newAudioSampler(clockRate uint32, latency time.Duration) samplerFunc {
|
||||
samples := uint32(math.Round(float64(clockRate) * latency.Seconds()))
|
||||
return samplerFunc(func() uint32 {
|
||||
return samples
|
||||
func newAudioSampler(t *webrtc.Track, latency time.Duration) samplerFunc {
|
||||
samples := uint32(math.Round(float64(t.Codec().ClockRate) * latency.Seconds()))
|
||||
return samplerFunc(func(b []byte) error {
|
||||
return t.WriteSample(media.Sample{Data: b, Samples: samples})
|
||||
})
|
||||
}
|
||||
|
||||
154
track.go
154
track.go
@@ -3,7 +3,6 @@ package mediadevices
|
||||
import (
|
||||
"errors"
|
||||
"image"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sync"
|
||||
|
||||
@@ -12,9 +11,7 @@ import (
|
||||
"github.com/pion/mediadevices/pkg/io/audio"
|
||||
"github.com/pion/mediadevices/pkg/io/video"
|
||||
"github.com/pion/mediadevices/pkg/wave"
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/webrtc/v2"
|
||||
"github.com/pion/webrtc/v2/pkg/media"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -55,11 +52,6 @@ type Track interface {
|
||||
// Unbind is the clean up operation that should be called after Bind. Similar to Bind, unbind will
|
||||
// be called automatically in the future.
|
||||
Unbind(*webrtc.PeerConnection) error
|
||||
// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
|
||||
// the encoded data in RTP format with given mtu size.
|
||||
NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
|
||||
// NewEncodedReader creates a new Go standard io.ReadCloser that reads the encoded data in codecName format
|
||||
NewEncodedReader(codecName string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type baseTrack struct {
|
||||
@@ -117,7 +109,7 @@ func (track *baseTrack) onError(err error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.ReadCloser, selectedCodec *codec.RTPCodec, sample samplerFunc) (*webrtc.Track, error) {
|
||||
func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.ReadCloser, selectedCodec *codec.RTPCodec, sampler func(*webrtc.Track) samplerFunc) (*webrtc.Track, error) {
|
||||
track.mu.Lock()
|
||||
defer track.mu.Unlock()
|
||||
|
||||
@@ -126,6 +118,7 @@ func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.Read
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := sampler(webrtcTrack)
|
||||
signalCh := make(chan chan<- struct{})
|
||||
track.activePeerConnections[pc] = signalCh
|
||||
|
||||
@@ -154,12 +147,7 @@ func (track *baseTrack) bind(pc *webrtc.PeerConnection, encodedReader codec.Read
|
||||
return
|
||||
}
|
||||
|
||||
sampleCount := sample()
|
||||
err = webrtcTrack.WriteSample(media.Sample{
|
||||
Data: buff,
|
||||
Samples: sampleCount,
|
||||
})
|
||||
if err != nil {
|
||||
if err := sample(buff); err != nil {
|
||||
track.onError(err)
|
||||
return
|
||||
}
|
||||
@@ -185,31 +173,6 @@ func (track *baseTrack) unbind(pc *webrtc.PeerConnection) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (track *baseTrack) newEncodedReader(reader codec.ReadCloser) (io.ReadCloser, error) {
|
||||
var encoded []byte
|
||||
release := func() {}
|
||||
return &encodedReadCloserImpl{
|
||||
readFn: func(b []byte) (int, error) {
|
||||
var err error
|
||||
|
||||
if len(encoded) == 0 {
|
||||
release()
|
||||
encoded, release, err = reader.Read()
|
||||
if err != nil {
|
||||
reader.Close()
|
||||
track.onError(err)
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
n := copy(b, encoded)
|
||||
encoded = encoded[n:]
|
||||
return n, nil
|
||||
},
|
||||
closeFn: reader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newTrackFromDriver(d driver.Driver, constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
|
||||
if err := d.Open(); err != nil {
|
||||
return nil, err
|
||||
@@ -279,68 +242,18 @@ func (track *VideoTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
|
||||
}
|
||||
|
||||
wantCodecs := pc.GetRegisteredRTPCodecs(webrtc.RTPCodecTypeVideo)
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodec(reader, inputProp, wantCodecs...)
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodec(wantCodecs, reader, inputProp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.bind(pc, encodedReader, selectedCodec, newVideoSampler(selectedCodec.ClockRate))
|
||||
return track.bind(pc, encodedReader, selectedCodec, newVideoSampler)
|
||||
}
|
||||
|
||||
func (track *VideoTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return track.unbind(pc)
|
||||
}
|
||||
|
||||
func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentVideoProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, selectedCodec, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := newVideoSampler(selectedCodec.ClockRate)
|
||||
|
||||
// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
|
||||
packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
|
||||
|
||||
return &rtpReadCloserImpl{
|
||||
readFn: func() ([]*rtp.Packet, func(), error) {
|
||||
encoded, release, err := encodedReader.Read()
|
||||
if err != nil {
|
||||
encodedReader.Close()
|
||||
track.onError(err)
|
||||
return nil, func() {}, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
samples := sample()
|
||||
pkts := packetizer.Packetize(encoded, samples)
|
||||
return pkts, release, err
|
||||
},
|
||||
closeFn: encodedReader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (track *VideoTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentVideoProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, _, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.newEncodedReader(encodedReader)
|
||||
}
|
||||
|
||||
// AudioTrack is a specific track type that contains audio source which allows multiple readers to access, and
|
||||
// manipulate.
|
||||
type AudioTrack struct {
|
||||
@@ -379,9 +292,6 @@ func newAudioTrackFromDriver(d driver.Driver, recorder driver.AudioRecorder, con
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// FIXME: The current audio detection and audio encoder can only work with a static latency. Since the latency from the driver
|
||||
// can fluctuate, we need to stabilize it. Maybe there's a better way for doing this?
|
||||
reader = audio.NewBuffer(int(constraints.selectedMedia.Latency.Seconds() * float64(constraints.selectedMedia.SampleRate)))(reader)
|
||||
return newAudioTrackFromReader(d, reader, selector), nil
|
||||
}
|
||||
|
||||
@@ -399,64 +309,14 @@ func (track *AudioTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
|
||||
}
|
||||
|
||||
wantCodecs := pc.GetRegisteredRTPCodecs(webrtc.RTPCodecTypeAudio)
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodec(reader, inputProp, wantCodecs...)
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodec(wantCodecs, reader, inputProp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.bind(pc, encodedReader, selectedCodec, newAudioSampler(selectedCodec.ClockRate, inputProp.Latency))
|
||||
return track.bind(pc, encodedReader, selectedCodec, func(t *webrtc.Track) samplerFunc { return newAudioSampler(t, inputProp.Latency) })
|
||||
}
|
||||
|
||||
func (track *AudioTrack) Unbind(pc *webrtc.PeerConnection) error {
|
||||
return track.unbind(pc)
|
||||
}
|
||||
|
||||
func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentAudioProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, selectedCodec, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sample := newAudioSampler(selectedCodec.ClockRate, inputProp.Latency)
|
||||
|
||||
// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
|
||||
packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
|
||||
|
||||
return &rtpReadCloserImpl{
|
||||
readFn: func() ([]*rtp.Packet, func(), error) {
|
||||
encoded, release, err := encodedReader.Read()
|
||||
if err != nil {
|
||||
encodedReader.Close()
|
||||
track.onError(err)
|
||||
return nil, func() {}, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
samples := sample()
|
||||
pkts := packetizer.Packetize(encoded, samples)
|
||||
return pkts, release, err
|
||||
},
|
||||
closeFn: encodedReader.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (track *AudioTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
|
||||
reader := track.NewReader(false)
|
||||
inputProp, err := detectCurrentAudioProp(track.Broadcaster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encodedReader, _, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return track.newEncodedReader(encodedReader)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user