WIP

codec.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 // CodecSelector is a container of video and audio encoder builders, which later will be used

examples/facedetection/main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"image"
+	"io/ioutil"
+	"log"
+	"time"
+
+	pigo "github.com/esimov/pigo/core"
+	"github.com/pion/mediadevices"
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+const (
+	confidenceLevel = 5.0
+)
+
+var (
+	cascade    []byte
+	classifier *pigo.Pigo
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func detectFace(frame *image.YCbCr) bool {
+	bounds := frame.Bounds()
+	cascadeParams := pigo.CascadeParams{
+		MinSize:     100,
+		MaxSize:     600,
+		ShiftFactor: 0.15,
+		ScaleFactor: 1.1,
+		ImageParams: pigo.ImageParams{
+			Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
+			Rows:   bounds.Dy(),
+			Cols:   bounds.Dx(),
+			Dim:    bounds.Dx(),
+		},
+	}
+
+	// Run the classifier over the obtained leaf nodes and return the detection results.
+	// The result contains quadruplets representing the row, column, scale and detection score.
+	dets := classifier.RunCascade(cascadeParams, 0.0)
+
+	// Calculate the intersection over union (IoU) of two clusters.
+	dets = classifier.ClusterDetections(dets, 0)
+
+	for _, det := range dets {
+		if det.Q >= confidenceLevel {
+			return true
+		}
+	}
+
+	return false
+}
+
+func main() {
+	// prepare face detector
+	var err error
+	cascade, err = ioutil.ReadFile("facefinder")
+	if err != nil {
+		log.Fatalf("Error reading the cascade file: %s", err)
+	}
+	p := pigo.NewPigo()
+
+	// Unpack the binary file. This will return the number of cascade trees,
+	// the tree depth, the threshold and the prediction from tree's leaf nodes.
+	classifier, err = p.Unpack(cascade)
+	if err != nil {
+		log.Fatalf("Error unpacking the cascade file: %s", err)
+	}
+
+	devices := mediadevices.EnumerateDevices()
+	deviceID := ""
+
+	for _, device := range devices {
+		if device.Label == "video0" {
+			deviceID = device.DeviceID
+		}
+	}
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.DeviceID = prop.StringExact(deviceID)
+			c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+	})
+	must(err)
+
+	// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoReader := videoTrack.NewReader(false)
+	// To save resources, we can simply use 4 fps to detect faces.
+	ticker := time.NewTicker(time.Millisecond * 250)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		frame, release, err := videoReader.Read()
+		must(err)
+
+		// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
+		if detectFace(frame.(*image.YCbCr)) {
+			log.Println("Detect a face")
+		}
+
+		release()
+	}
+}

examples/rtp/README.md

@@ -0,0 +1,30 @@
+## Instructions
+
+### Download rtpexample
+
+```
+go get github.com/pion/mediadevices/examples/rtp
+```
+
+### Listen RTP
+
+Install GStreamer and run:
+```
+gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=VP8 \
+    ! rtpvp8depay ! vp8dec ! videoconvert ! autovideosink
+```
+
+Or run VLC media plyer:
+```
+vlc ./vp8.sdp
+```
+
+### Run rtp
+
+Run `rtp localhost:5000`
+
+A video should start playing in your GStreamer or VLC window.
+It's not WebRTC, but pure RTP.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/rtp/main.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+const (
+	mtu = 1000
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	vp8Params, err := vpx.NewVP8Params()
+	must(err)
+	vp8Params.BitRate = 100000 // 100kbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&vp8Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0]
+	defer videoTrack.Close()
+
+	rtpReader, err := videoTrack.NewRTPReader(vp8Params.RTPCodec().Name, mtu)
+	must(err)
+
+	addr, err := net.ResolveUDPAddr("udp", dest)
+	must(err)
+	conn, err := net.DialUDP("udp", nil, addr)
+	must(err)
+
+	buff := make([]byte, mtu)
+	for {
+		pkts, release, err := rtpReader.Read()
+		must(err)
+
+		for _, pkt := range pkts {
+			n, err := pkt.MarshalTo(buff)
+			must(err)
+
+			_, err = conn.Write(buff[:n])
+			must(err)
+		}
+
+		release()
+	}
+}

examples/rtp/vp8.sdp

@@ -0,0 +1,9 @@
+v=0
+o=- 1234567890 1234567890 IN IP4 0.0.0.0
+s=RTP-Send Example
+i=Example
+c=IN IP4 0.0.0.0
+t=0 0
+a=recvonly
+m=video 5000 RTP/AVP 100
+a=rtpmap:100 VP8/90000

examples/webrtc/main.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pion/mediadevices/examples/internal/signal"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 
 	// If you don't like x264, you can also use vpx by importing as below
 	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder

go.mod

@@ -4,12 +4,12 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
+	github.com/gen2brain/malgo v0.10.19
 	github.com/lherman-cs/opus v0.0.2
 	github.com/pion/logging v0.2.2
 	github.com/pion/rtp v1.6.0
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
+	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
 )

go.sum

@@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gen2brain/malgo v0.10.19 h1:IUVF6WdVV7Txt47Kx2ajz0rWQ0MU0zO+tbcKmhva7l8=
+github.com/gen2brain/malgo v0.10.19/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -15,8 +17,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
-github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

mediastream_test.go

@@ -33,6 +33,10 @@ func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return nil
 }
 
+func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	return nil, nil
+}
+
 func TestMediaStreamFilters(t *testing.T) {
 	audioTracks := []Track{
 		&mockMediaStreamTrack{AudioInput},

pkg/codec/codec.go

@@ -4,7 +4,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 // RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.

pkg/io/audio/audio.go

@@ -5,6 +5,14 @@ import (
 )
 
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (chunk wave.Audio, release func(), err error)
 }
 

pkg/io/reader.go

@@ -3,6 +3,14 @@ package io
 // Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
 // to provide strong type.
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (data interface{}, release func(), err error)
 }
 

pkg/io/video/video.go

@@ -5,6 +5,14 @@ import (
 )
 
 type Reader interface {
+	// Read reads data from the source. The caller is responsible to release the memory that's associated
+	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
+	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
+	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
+	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
+	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
+	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
+	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (img image.Image, release func(), err error)
 }
 

rtpreader.go

@@ -0,0 +1,21 @@
+package mediadevices
+
+import "github.com/pion/rtp"
+
+type RTPReadCloser interface {
+	Read() (pkts []*rtp.Packet, release func(), err error)
+	Close() error
+}
+
+type rtpReadCloserImpl struct {
+	readFn  func() ([]*rtp.Packet, func(), error)
+	closeFn func() error
+}
+
+func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
+	return r.readFn()
+}
+
+func (r *rtpReadCloserImpl) Close() error {
+	return r.closeFn()
+}

track.go

@@ -11,8 +11,9 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/wave"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2/pkg/media"
+	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v3/pkg/media"
 )
 
 var (
@@ -53,6 +54,9 @@ type Track interface {
 	// Unbind is the clean up operation that should be called after Bind. Similar to Bind, unbind will
 	// be called automatically in the future.
 	Unbind(*webrtc.PeerConnection) error
+	// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
+	// the encoded data in RTP format with given mtu size.
+	NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
 }
 
 type baseTrack struct {
@@ -259,6 +263,41 @@ func (track *VideoTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
 
+func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, selectedCodec, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	sample := newVideoSampler(selectedCodec.ClockRate)
+
+	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
+	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
+
+	return &rtpReadCloserImpl{
+		readFn: func() ([]*rtp.Packet, func(), error) {
+			encoded, release, err := encodedReader.Read()
+			if err != nil {
+				encodedReader.Close()
+				track.onError(err)
+				return nil, func() {}, err
+			}
+			defer release()
+
+			samples := sample()
+			pkts := packetizer.Packetize(encoded, samples)
+			return pkts, release, err
+		},
+		closeFn: encodedReader.Close,
+	}, nil
+}
+
 // AudioTrack is a specific track type that contains audio source which allows multiple readers to access, and
 // manipulate.
 type AudioTrack struct {
@@ -328,3 +367,38 @@ func (track *AudioTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
 func (track *AudioTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
+
+func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentAudioProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, selectedCodec, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	sample := newVideoSampler(selectedCodec.ClockRate)
+
+	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
+	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
+
+	return &rtpReadCloserImpl{
+		readFn: func() ([]*rtp.Packet, func(), error) {
+			encoded, release, err := encodedReader.Read()
+			if err != nil {
+				encodedReader.Close()
+				track.onError(err)
+				return nil, func() {}, err
+			}
+			defer release()
+
+			samples := sample()
+			pkts := packetizer.Packetize(encoded, samples)
+			return pkts, release, err
+		},
+		closeFn: encodedReader.Close,
+	}, nil
+}