Remove git LFS

Generated by Renovate Bot

README.md

@@ -13,74 +13,226 @@
 </p>
 <br>
 
-![](img/demo.gif)
+`mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
 
-## Interfaces
+## Install
 
-| Interface  | Linux | Mac | Windows |
-| :--------: | :---: | :-: | :-----: |
-|   Camera   |  ✔️   | ✔️  |   ✔️    |
-| Microphone |  ✔️   | ✖️  |   ✔️    |
-|   Screen   |  ✔️   | ✖️  |   ✖️    |
-
-### Camera
-
-|   OS    |                           Library/Interface                             |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |        [Video4Linux](https://en.wikipedia.org/wiki/Video4Linux)         |
-|   Mac   |       [AVFoundation](https://developer.apple.com/av-foundation/)        |
-| Windows | [DirectShow](https://docs.microsoft.com/en-us/windows/win32/directshow) |
-
-|                     Pixel Format                      | Linux | Mac | Windows |
-| :---------------------------------------------------: | :---: | :-: | :-----: |
-| [YUY2](https://www.fourcc.org/pixel-format/yuv-yuy2/) |  ✔️   | ✖️  |   ✔️    |
-| [UYVY](https://www.fourcc.org/pixel-format/yuv-uyvy/) |  ✔️   | ✔️  |   ✖️    |
-| [I420](https://www.fourcc.org/pixel-format/yuv-i420/) |  ✔️   | ✖️  |   ✖️    |
-| [NV21](https://www.fourcc.org/pixel-format/yuv-nv21/) |  ✔️   | ✔️  |   ✖️    |
-|         [MJPEG](https://www.fourcc.org/mjpg/)         |  ✔️   | ✖️  |   ✖️    |
-
-### Microphone
-
-|   OS    |                            Library/Interface                            |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |         [PulseAudio](https://en.wikipedia.org/wiki/PulseAudio)          |
-|   Mac   |                                   N/A                                   |
-| Windows |  [waveIn](https://docs.microsoft.com/en-us/windows/win32/api/mmeapi/)   |
-
-### Screen casting
-
-|   OS    |                            Library/Interface                            |
-| :-----: | :---------------------------------------------------------------------: |
-|  Linux  |          [X11](https://en.wikipedia.org/wiki/X_Window_System)           |
-|   Mac   |                                   N/A                                   |
-| Windows |                                   N/A                                   |
-
-## Codecs
-
-| Audio Codec |                    Library/Interface                     |
-| :---------: | :------------------------------------------------------: |
-|    OPUS     | [libopus](http://opus-codec.org/)                        |
-
-| Video Codec |                    Library/Interface                     |
-| :---------: | :------------------------------------------------------: |
-|    H.264    | [OpenH264](https://www.openh264.org/)                    |
-|     VP8     | [libvpx](https://www.webmproject.org/code/)              |
-|     VP9     | [libvpx](https://www.webmproject.org/code/)              |
+`go get -u github.com/pion/mediadevices`
 
 ## Usage
 
-[Wiki](https://github.com/pion/mediadevices/wiki)
+The following snippet shows how to capture a camera stream and store a frame as a jpeg image:
+
+```go
+package main
+
+import (
+	"image/jpeg"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/prop"
+
+	// This is required to register camera adapter
+	_ "github.com/pion/mediadevices/pkg/driver/camera" 
+	// Note: If you don't have a camera or your adapters are not supported,
+	//       you can always swap your adapters with our dummy adapters below.
+	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
+)
+
+func main() {
+	stream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(constraint *mediadevices.MediaTrackConstraints) {
+			// Query for ideal resolutions
+			constraint.Width = prop.Int(600)
+			constraint.Height = prop.Int(400)
+		},
+	})
+
+	// Since track can represent audio as well, we need to cast it to 
+	// *mediadevices.VideoTrack to get video specific functionalities
+	track := stream.GetVideoTracks()[0]
+	videoTrack := track.(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	// Create a new video reader to get the decoded frames. Release is used 
+	// to return the buffer to hold frame back to the source so that the buffer 
+	// can be reused for the next frames.
+	videoReader := videoTrack.NewReader(false)
+	frame, release, _ := videoReader.Read()
+	defer release()
+
+	// Since frame is the standard image.Image, it's compatible with Go standard 
+	// library. For example, capturing the first frame and store it as a jpeg image.
+	output, _ := os.Create("frame.jpg")
+	jpeg.Encode(output, frame, nil)
+}
+
+
+```
+
+## More Examples
+
+* [Webrtc](/examples/webrtc) - Use Webrtc to create a realtime peer-to-peer video call
+* [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
+* [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
+* [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
+* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
+
+## Available Media Inputs
+
+| Input  | Linux | Mac | Windows |
+| :--------: | :---: | :-: | :-----: |
+|   Camera   |  ✔️   | ✔️  |   ✔️    |
+| Microphone |  ✔️   | ✔️  |   ✔️    |
+|   Screen   |  ✔️   | ✖️  |   ✖️    |
+
+By default, there's no media input registered. This decision was made to allow you to pay what you need. Therefore, you need to import the associated packages for the media inputs. For example, if you want to use a camera, you need to import the camera package as a side effect:
+
+```go
+import (
+	...
+	_ "github.com/pion/mediadevices/pkg/driver/camera"
+)
+```
+
+## Available Codecs
+
+In order to encode your video/audio, `mediadevices` needs to know what codecs that you want to use and their parameters. To do this, you need to import the associated packages for the codecs, and add them to the codec selector that you'll pass to `GetUserMedia`:
+
+```go
+package main
+
+import (
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+)
+
+func main() {
+	// configure codec specific parameters
+	x264Params, _ := x264.NewParams()
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+	
+	mediaStream, _ := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {},
+		Codec: codecSelector, // let GetUsermedia know available codecs
+	})
+}
+```
+
+Since `mediadevices` doesn't implement the video/audio codecs, it needs to call the codec libraries from the system through cgo. Therefore, you're required to install the codec libraries before you can use them in `mediadevices`. In the next section, it shows a list of available codecs, where the packages are defined (documentation linked), and installation instructions.
+
+Note: we do not provide recommendations on choosing one codec or another as it is very complex and can be subjective.
+
+### Video Codecs
+
+#### x264
+A free software library and application for encoding video streams into the H.264/MPEG-4 AVC compression format.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/x264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/x264)
+* Installation:
+  * Mac: `brew install x264`
+  * Ubuntu: `apt install libx264-dev`
+  
+#### mmal
+A framework to enable H264 hardware encoding for Raspberry Pi or boards that use VideoCore GPUs.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/mmal](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/mmal)
+* Installation:
+  * Raspbian: `export PKG_CONFIG_PATH=/opt/vc/lib/pkgconfig`
+
+#### openh264
+A codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/openh264](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/openh264)
+* Installation: no installation needed, included as a static binary
+
+#### vpx
+A free software video codec library from Google and the Alliance for Open Media that implements VP8/VP9 video coding formats.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/vpx](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vpx)
+* Installation:
+  * Mac: `brew install libvpx`
+  * Ubuntu: `apt install libvpx-dev`
+  
+#### vaapi
+An open source API that allows applications such as VLC media player or GStreamer to use hardware video acceleration capabilities (currently support VP8/VP9).
+
+* Package: [github.com/pion/mediadevices/pkg/codec/vaapi](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/vaapi)
+* Installation:
+  * Ubuntu: `apt install libva-dev`
+
+
+### Audio Codecs
+
+#### opus
+A totally open, royalty-free, highly versatile audio codec.
+
+* Package: [github.com/pion/mediadevices/pkg/codec/opus](https://pkg.go.dev/github.com/pion/mediadevices/pkg/codec/opus)
+* Installation:
+  * Mac: `brew install opus`
+  * Ubuntu: `apt install libopus-dev`
+
+## Benchmark
+
+Result as of Nov 4, 2020 with Go 1.14 on a Raspberry pi 3, `mediadevices` can produce video, encode, send across network, and decode at **720p, 30 fps with < 500 ms latency**.  
+
+The test was taken by capturing a camera stream, decoding the raw frames, encoding the video stream with mmal, and sending the stream through Webrtc.
+
+## FAQ
+
+### Failed to find the best driver that fits the constraints
+
+`mediadevices` provides an automated driver discovery through `GetUserMedia` and `GetDisplayMedia`. The driver discover algorithm works something like:
+
+1. Open all registered drivers
+2. Get all properties (property describes what a driver is capable of, e.g. resolution, frame rate, etc.) from opened drivers
+3. Find the best property that meets the criteria
+
+So, when `mediadevices` returns `failed to find the best driver that fits the constraints` error, one of the following conditions might have occured:
+* Driver was not imported as a side effect in your program, e.g. `import _ github.com/pion/mediadevices/pkg/driver/camera`
+* Your constraint is too strict that there's no driver can fullfil your requirements. In this case, you can try to turn up the debug level by specifying the following environment variable: `export PION_LOG_DEBUG=all` to see what was too strict and tune that.
+* Your driver is not supported/implemented. In this case, you can either let us know (file an issue) and wait for the maintainers to implement it. Or, you can implement it yourself and register it through `RegisterDriverAdapter`
+
+### Failed to find vpx/x264/mmal/opus codecs
+
+Since `mediadevices` uses cgo to access video/audio codecs, it needs to find these libraries from the system. To accomplish this, [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) is used for library discovery.
+
+If you see the following error message at compile time:
+```
+# pkg-config --cflags  -- vpx
+Package vpx was not found in the pkg-config search path.
+Perhaps you should add the directory containing `vpx.pc'
+to the PKG_CONFIG_PATH environment variable
+No package 'vpx' found
+pkg-config: exit status 1
+```
+
+There are 2 common problems:
+
+* The required codec library is not installed (vpx in this example). In this case, please refer to the [available codecs](#available-codecs).
+* Pkg-config fails to find the `.pc` files for this codec ([reference](https://people.freedesktop.org/~dbn/pkg-config-guide.html#using)). In this case, you need to find where the codec library's `.pc` is stored, and let pkg-config knows with: `export PKG_CONFIG_PATH=/path/to/directory`.
+
+
+## Community
+Pion has an active community on the [Slack](https://pion.ly/slack).
+
+Follow the [Pion Twitter](https://twitter.com/_pion) for project updates and important WebRTC news.
+
+We are always looking to support **your projects**. Please reach out if you have something to build!
+If you need commercial support or don't want to use public methods you can contact us at [team@pion.ly](mailto:team@pion.ly)
 
 ## Contributing
+Check out the **[contributing wiki](https://github.com/pion/webrtc/wiki/Contributing)** to join the group of amazing people making this project possible:
 
-- [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
+* [Lukas Herman](https://github.com/lherman-cs) - _Original Author_
 * [Atsushi Watanabe](https://github.com/at-wat) - _VP8, Screencast, etc._
 
-## Project Status
-
-[![Stargazers over time](https://starchart.cc/pion/mediadevices.svg)](https://starchart.cc/pion/mediadevices)
-
-## References
-
-- https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs
-- https://tools.ietf.org/html/rfc7742
+## License
+MIT License - see [LICENSE](LICENSE) for full text

examples/archive/README.md

@@ -0,0 +1,38 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download archive examplee
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Run archive example
+
+Run `cd mediadevices/examples/archive && go build && ./archive recorded.h264`
+
+To stop recording, press `Ctrl+c` or send a SIGINT signal.
+
+### Playback recorded video
+
+Install GStreamer and run:
+```
+gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
+```
+
+Or run VLC media plyer:
+```
+vlc recorded.h264
+```
+
+A video should start playing in your GStreamer or VLC window.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/archive/main.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s <path/to/file.h264>\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT)
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
+		return video.ReaderFunc(func() (img image.Image, release func(), err error) {
+			// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
+			// will finish its execution and the program will finish
+			select {
+			case <-sigs:
+				return nil, func() {}, io.EOF
+			default:
+			}
+
+			return r.Read()
+		})
+	}))
+
+	reader, err := videoTrack.NewEncodedReader(x264Params.RTPCodec().Name)
+	must(err)
+	defer reader.Close()
+
+	out, err := os.Create(dest)
+	must(err)
+
+	fmt.Println("Recording... Press Ctrl+c to stop")
+	_, err = io.Copy(out, reader)
+	must(err)
+	fmt.Println("Your video has been recorded to", dest)
+}

examples/facedetection/README.md

@@ -0,0 +1,15 @@
+## Instructions
+
+### Download facedetection example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Compile and Run facedetection
+
+Run `cd mediadevices/examples/facedetection && go build && ./facedetection`
+
+You should be able to see some loggings when it can see faces.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/facedetection/main.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"image"
+	"io/ioutil"
+	"log"
+	"time"
+
+	pigo "github.com/esimov/pigo/core"
+	"github.com/pion/mediadevices"
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+const (
+	confidenceLevel = 5.0
+)
+
+var (
+	cascade    []byte
+	classifier *pigo.Pigo
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func detectFace(frame *image.YCbCr) bool {
+	bounds := frame.Bounds()
+	cascadeParams := pigo.CascadeParams{
+		MinSize:     100,
+		MaxSize:     600,
+		ShiftFactor: 0.15,
+		ScaleFactor: 1.1,
+		ImageParams: pigo.ImageParams{
+			Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
+			Rows:   bounds.Dy(),
+			Cols:   bounds.Dx(),
+			Dim:    bounds.Dx(),
+		},
+	}
+
+	// Run the classifier over the obtained leaf nodes and return the detection results.
+	// The result contains quadruplets representing the row, column, scale and detection score.
+	dets := classifier.RunCascade(cascadeParams, 0.0)
+
+	// Calculate the intersection over union (IoU) of two clusters.
+	dets = classifier.ClusterDetections(dets, 0)
+
+	for _, det := range dets {
+		if det.Q >= confidenceLevel {
+			return true
+		}
+	}
+
+	return false
+}
+
+func main() {
+	// prepare face detector
+	var err error
+	cascade, err = ioutil.ReadFile("facefinder")
+	if err != nil {
+		log.Fatalf("Error reading the cascade file: %s", err)
+	}
+	p := pigo.NewPigo()
+
+	// Unpack the binary file. This will return the number of cascade trees,
+	// the tree depth, the threshold and the prediction from tree's leaf nodes.
+	classifier, err = p.Unpack(cascade)
+	if err != nil {
+		log.Fatalf("Error unpacking the cascade file: %s", err)
+	}
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+	})
+	must(err)
+
+	// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoReader := videoTrack.NewReader(false)
+	// To save resources, we can simply use 4 fps to detect faces.
+	ticker := time.NewTicker(time.Millisecond * 250)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		frame, release, err := videoReader.Read()
+		must(err)
+
+		// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
+		if detectFace(frame.(*image.YCbCr)) {
+			log.Println("Detect a face")
+		}
+
+		release()
+	}
+}

examples/http/README.md

@@ -0,0 +1,19 @@
+## Instructions
+
+### Download http example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Compile and Run HTTP server
+
+Run `cd mediadevices/examples/http && go build && ./http :1313`
+
+
+### Access the camera stream from the browser
+
+Go to "http://localhost:1313"
+
+
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/http/main.go

@@ -11,6 +11,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
+	"os"
 
 	"github.com/pion/mediadevices"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -28,7 +29,13 @@ func must(err error) {
 }
 
 func main() {
-	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(constraint *mediadevices.MediaTrackConstraints) {
 			constraint.Width = prop.Int(600)
 			constraint.Height = prop.Int(400)
@@ -36,8 +43,9 @@ func main() {
 	})
 	must(err)
 
-	t := s.GetVideoTracks()[0]
-	videoTrack := t.(*mediadevices.VideoTrack)
+	track := mediaStream.GetVideoTracks()[0]
+	videoTrack := track.(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		var buf bytes.Buffer
@@ -72,6 +80,6 @@ func main() {
 		}
 	})
 
-	fmt.Println("listening on http://localhost:1313")
-	log.Println(http.ListenAndServe("localhost:1313", nil))
+	fmt.Printf("listening on %s\n", dest)
+	log.Println(http.ListenAndServe(dest, nil))
 }

examples/rtp/README.md

@@ -0,0 +1,38 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download rtp example
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Listen RTP
+
+Install GStreamer and run:
+```
+gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=H264 \
+    ! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
+```
+
+Or run VLC media plyer:
+```
+vlc ./h264.sdp
+```
+
+### Run rtp
+
+Run `cd mediadevices/examples/archive && go build && ./rtp localhost:5000`
+
+A video should start playing in your GStreamer or VLC window.
+It's not WebRTC, but pure RTP.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/rtp/h264.sdp

@@ -0,0 +1,9 @@
+v=0
+o=- 1234567890 1234567890 IN IP4 0.0.0.0
+s=RTP-Send Example
+i=Example
+c=IN IP4 0.0.0.0
+t=0 0
+a=recvonly
+m=video 5000 RTP/AVP 100
+a=rtpmap:100 H264/90000

examples/rtp/main.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+const (
+	mtu = 1000
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0]
+	defer videoTrack.Close()
+
+	rtpReader, err := videoTrack.NewRTPReader(x264Params.RTPCodec().Name, mtu)
+	must(err)
+
+	addr, err := net.ResolveUDPAddr("udp", dest)
+	must(err)
+	conn, err := net.DialUDP("udp", nil, addr)
+	must(err)
+
+	buff := make([]byte, mtu)
+	for {
+		pkts, release, err := rtpReader.Read()
+		must(err)
+
+		for _, pkt := range pkts {
+			n, err := pkt.MarshalTo(buff)
+			must(err)
+
+			_, err = conn.Write(buff[:n])
+			must(err)
+		}
+
+		release()
+	}
+}

examples/webrtc/README.md

@@ -1,29 +1,42 @@
 ## Instructions
 
-### Download gstreamer-send
+### Install required codecs
+
+In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+* [opus](https://github.com/pion/mediadevices#opus)
+
+### Download webrtc example
 
 ```
-go get github.com/pion/mediadevices/examples/webrtc
+git clone https://github.com/pion/mediadevices.git
+```
+
+#### Compile webrtc example
+
+```
+cd mediadevices/examples/webrtc && go build
 ```
 
 ### Open example page
 
 [jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
 
-### Run webrtc with your browsers SessionDescription as stdin
+### Run the webrtc example with your browsers SessionDescription as stdin
 
-In the jsfiddle the top textarea is your browser, copy that and:
+In the jsfiddle the top textarea is your browser, copy that, and store the session description in an environment variable, `export SDP=<put_the_sdp_here>`
 
-#### Linux
-
-Run `echo $BROWSER_SDP | webrtc`
+Run `echo $SDP | ./webrtc`
 
 ### Input webrtc's SessionDescription into your browser
 
-Copy the text that `webrtc` just emitted and copy into second text area
+Copy the text that `./webrtc` just emitted and copy into second text area
 
 ### Hit 'Start Session' in jsfiddle, enjoy your video!
 
 A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
 
-Congrats, you have used pion-WebRTC! Now start building something cool
+Congrats, you have used pion-MediaDevices! Now start building something cool

examples/webrtc/main.go

@@ -26,10 +26,6 @@ import (
 	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
 )
 
-const (
-	videoCodecName = webrtc.VP8
-)
-
 func main() {
 	config := webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{

go.mod

@@ -4,12 +4,12 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
+	github.com/gen2brain/malgo v0.10.24
 	github.com/lherman-cs/opus v0.0.2
 	github.com/pion/logging v0.2.2
-	github.com/pion/rtp v1.6.0
+	github.com/pion/rtp v1.6.1
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
+	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
 )

go.sum

@@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gen2brain/malgo v0.10.24 h1:q9TFP4lRYpK8UbH3XSa/SNnMwMLUZraRyZt2u+qKYxg=
+github.com/gen2brain/malgo v0.10.24/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -15,8 +17,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
-github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -53,6 +53,8 @@ github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
 github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
 github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
 github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
+github.com/pion/rtp v1.6.1 h1:2Y2elcVBrahYnHKN2X7rMHX/r1R4TEBMP1LaVu/wNhk=
+github.com/pion/rtp v1.6.1/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.7.10 h1:o3p3/hZB5Cx12RMGyWmItevJtZ6o2cpuxaw6GOS4x+8=
 github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=

ioreader.go

@@ -0,0 +1,14 @@
+package mediadevices
+
+type encodedReadCloserImpl struct {
+	readFn  func([]byte) (int, error)
+	closeFn func() error
+}
+
+func (r *encodedReadCloserImpl) Read(b []byte) (int, error) {
+	return r.readFn(b)
+}
+
+func (r *encodedReadCloserImpl) Close() error {
+	return r.closeFn()
+}

mediastream_test.go

@@ -1,6 +1,7 @@
 package mediadevices
 
 import (
+	"io"
 	"testing"
 
 	"github.com/pion/webrtc/v2"
@@ -33,6 +34,14 @@ func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return nil
 }
 
+func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	return nil, nil
+}
+
+func (track *mockMediaStreamTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	return nil, nil
+}
+
 func TestMediaStreamFilters(t *testing.T) {
 	audioTracks := []Track{
 		&mockMediaStreamTrack{AudioInput},

pkg/codec/vpx/vpx.go

@@ -237,10 +237,19 @@ func (e *encoder) Read() ([]byte, func(), error) {
 		e.raw.d_w, e.raw.d_h = C.uint(width), C.uint(height)
 	}
 
+	duration := t - e.tLastFrame
+	// VPX doesn't allow 0 duration. If 0 is given, vpx_codec_encode will fail with VPX_CODEC_INVALID_PARAM.
+	// 0 duration is possible because mediadevices first gets the frame meta data by reading from the source,
+	// and consequently the codec will read the first frame from the buffer. This means the first frame won't
+	// have a pause to the second frame, which means if the delay is <1 ms (vpx duration resolution), duration
+	// is going to be 0.
+	if duration == 0 {
+		duration = 1
+	}
 	var flags int
 	if ec := C.encode_wrapper(
 		e.codec, e.raw,
-		C.long(t-e.tStart), C.ulong(t-e.tLastFrame), C.long(flags), C.ulong(e.deadline),
+		C.long(t-e.tStart), C.ulong(duration), C.long(flags), C.ulong(e.deadline),
 		(*C.uchar)(&yuvImg.Y[0]), (*C.uchar)(&yuvImg.Cb[0]), (*C.uchar)(&yuvImg.Cr[0]),
 	); ec != C.VPX_CODEC_OK {
 		return nil, func() {}, fmt.Errorf("vpx_codec_encode failed (%d)", ec)

pkg/io/audio/audio.go

@@ -5,6 +5,14 @@ import (
 )
 
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (chunk wave.Audio, release func(), err error)
 }
 

pkg/io/reader.go

@@ -3,6 +3,14 @@ package io
 // Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
 // to provide strong type.
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (data interface{}, release func(), err error)
 }
 

pkg/io/video/video.go

@@ -5,6 +5,14 @@ import (
 )
 
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (img image.Image, release func(), err error)
 }
 

rtpreader.go

@@ -0,0 +1,21 @@
+package mediadevices
+
+import "github.com/pion/rtp"
+
+type RTPReadCloser interface {
+	Read() (pkts []*rtp.Packet, release func(), err error)
+	Close() error
+}
+
+type rtpReadCloserImpl struct {
+	readFn  func() ([]*rtp.Packet, func(), error)
+	closeFn func() error
+}
+
+func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
+	return r.readFn()
+}
+
+func (r *rtpReadCloserImpl) Close() error {
+	return r.closeFn()
+}

track.go

@@ -3,6 +3,7 @@ package mediadevices
 import (
 	"errors"
 	"image"
+	"io"
 	"math/rand"
 	"sync"
 
@@ -11,6 +12,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/wave"
+	"github.com/pion/rtp"
 	"github.com/pion/webrtc/v2"
 	"github.com/pion/webrtc/v2/pkg/media"
 )
@@ -53,6 +55,11 @@ type Track interface {
 	// Unbind is the clean up operation that should be called after Bind. Similar to Bind, unbind will
 	// be called automatically in the future.
 	Unbind(*webrtc.PeerConnection) error
+	// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
+	// the encoded data in RTP format with given mtu size.
+	NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
+	// NewEncodedReader creates a new Go standard io.ReadCloser that reads the encoded data in codecName format
+	NewEncodedReader(codecName string) (io.ReadCloser, error)
 }
 
 type baseTrack struct {
@@ -178,6 +185,31 @@ func (track *baseTrack) unbind(pc *webrtc.PeerConnection) error {
 	return nil
 }
 
+func (track *baseTrack) newEncodedReader(reader codec.ReadCloser) (io.ReadCloser, error) {
+	var encoded []byte
+	release := func() {}
+	return &encodedReadCloserImpl{
+		readFn: func(b []byte) (int, error) {
+			var err error
+
+			if len(encoded) == 0 {
+				release()
+				encoded, release, err = reader.Read()
+				if err != nil {
+					reader.Close()
+					track.onError(err)
+					return 0, err
+				}
+			}
+
+			n := copy(b, encoded)
+			encoded = encoded[n:]
+			return n, nil
+		},
+		closeFn: reader.Close,
+	}, nil
+}
+
 func newTrackFromDriver(d driver.Driver, constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
 	if err := d.Open(); err != nil {
 		return nil, err
@@ -259,6 +291,56 @@ func (track *VideoTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
 
+func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, selectedCodec, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	sample := newVideoSampler(selectedCodec.ClockRate)
+
+	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
+	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
+
+	return &rtpReadCloserImpl{
+		readFn: func() ([]*rtp.Packet, func(), error) {
+			encoded, release, err := encodedReader.Read()
+			if err != nil {
+				encodedReader.Close()
+				track.onError(err)
+				return nil, func() {}, err
+			}
+			defer release()
+
+			samples := sample()
+			pkts := packetizer.Packetize(encoded, samples)
+			return pkts, release, err
+		},
+		closeFn: encodedReader.Close,
+	}, nil
+}
+
+func (track *VideoTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, _, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	return track.newEncodedReader(encodedReader)
+}
+
 // AudioTrack is a specific track type that contains audio source which allows multiple readers to access, and
 // manipulate.
 type AudioTrack struct {
@@ -328,3 +410,53 @@ func (track *AudioTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
 func (track *AudioTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
+
+func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentAudioProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, selectedCodec, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	sample := newAudioSampler(selectedCodec.ClockRate, inputProp.Latency)
+
+	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
+	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
+
+	return &rtpReadCloserImpl{
+		readFn: func() ([]*rtp.Packet, func(), error) {
+			encoded, release, err := encodedReader.Read()
+			if err != nil {
+				encodedReader.Close()
+				track.onError(err)
+				return nil, func() {}, err
+			}
+			defer release()
+
+			samples := sample()
+			pkts := packetizer.Packetize(encoded, samples)
+			return pkts, release, err
+		},
+		closeFn: encodedReader.Close,
+	}, nil
+}
+
+func (track *AudioTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentAudioProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, _, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	return track.newEncodedReader(encodedReader)
+}