Add video RTP reader

codec.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 )
 
 // CodecSelector is a container of video and audio encoder builders, which later will be used

examples/facedetection/main.go

@@ -1,117 +0,0 @@
-package main
-
-import (
-	"image"
-	"io/ioutil"
-	"log"
-	"time"
-
-	pigo "github.com/esimov/pigo/core"
-	"github.com/pion/mediadevices"
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/prop"
-)
-
-const (
-	confidenceLevel = 5.0
-)
-
-var (
-	cascade    []byte
-	classifier *pigo.Pigo
-)
-
-func must(err error) {
-	if err != nil {
-		panic(err)
-	}
-}
-
-func detectFace(frame *image.YCbCr) bool {
-	bounds := frame.Bounds()
-	cascadeParams := pigo.CascadeParams{
-		MinSize:     100,
-		MaxSize:     600,
-		ShiftFactor: 0.15,
-		ScaleFactor: 1.1,
-		ImageParams: pigo.ImageParams{
-			Pixels: frame.Y, // Y in YCbCr should be enough to detect faces
-			Rows:   bounds.Dy(),
-			Cols:   bounds.Dx(),
-			Dim:    bounds.Dx(),
-		},
-	}
-
-	// Run the classifier over the obtained leaf nodes and return the detection results.
-	// The result contains quadruplets representing the row, column, scale and detection score.
-	dets := classifier.RunCascade(cascadeParams, 0.0)
-
-	// Calculate the intersection over union (IoU) of two clusters.
-	dets = classifier.ClusterDetections(dets, 0)
-
-	for _, det := range dets {
-		if det.Q >= confidenceLevel {
-			return true
-		}
-	}
-
-	return false
-}
-
-func main() {
-	// prepare face detector
-	var err error
-	cascade, err = ioutil.ReadFile("facefinder")
-	if err != nil {
-		log.Fatalf("Error reading the cascade file: %s", err)
-	}
-	p := pigo.NewPigo()
-
-	// Unpack the binary file. This will return the number of cascade trees,
-	// the tree depth, the threshold and the prediction from tree's leaf nodes.
-	classifier, err = p.Unpack(cascade)
-	if err != nil {
-		log.Fatalf("Error unpacking the cascade file: %s", err)
-	}
-
-	devices := mediadevices.EnumerateDevices()
-	deviceID := ""
-
-	for _, device := range devices {
-		if device.Label == "video0" {
-			deviceID = device.DeviceID
-		}
-	}
-
-	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.DeviceID = prop.StringExact(deviceID)
-			c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-		},
-	})
-	must(err)
-
-	// since we're trying to access the raw data, we need to cast Track to its real type, *mediadevices.VideoTrack
-	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
-	defer videoTrack.Close()
-
-	videoReader := videoTrack.NewReader(false)
-	// To save resources, we can simply use 4 fps to detect faces.
-	ticker := time.NewTicker(time.Millisecond * 250)
-	defer ticker.Stop()
-
-	for range ticker.C {
-		frame, release, err := videoReader.Read()
-		must(err)
-
-		// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
-		if detectFace(frame.(*image.YCbCr)) {
-			log.Println("Detect a face")
-		}
-
-		release()
-	}
-}

examples/rtp/README.md

@@ -1,9 +1,9 @@
 ## Instructions
 
-### Download rtpexample
+### Download rtp-send example
 
 ```
-go get github.com/pion/mediadevices/examples/rtp
+go get github.com/pion/mediadevices/examples/rtp-send
 ```
 
 ### Listen RTP
@@ -19,9 +19,9 @@ Or run VLC media plyer:
 vlc ./vp8.sdp
 ```
 
-### Run rtp
+### Run rtp-send
 
-Run `rtp localhost:5000`
+Run `rtp-send localhost:5000`
 
 A video should start playing in your GStreamer or VLC window.
 It's not WebRTC, but pure RTP.

examples/rtp/main.go

@@ -47,7 +47,7 @@ func main() {
 	})
 	must(err)
 
-	videoTrack := mediaStream.GetVideoTracks()[0]
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
 	defer videoTrack.Close()
 
 	rtpReader, err := videoTrack.NewRTPReader(vp8Params.RTPCodec().Name, mtu)

examples/webrtc/main.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pion/mediadevices/examples/internal/signal"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 
 	// If you don't like x264, you can also use vpx by importing as below
 	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder

go.mod

@@ -4,12 +4,12 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/gen2brain/malgo v0.10.19
+	github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
 	github.com/lherman-cs/opus v0.0.2
 	github.com/pion/logging v0.2.2
 	github.com/pion/rtp v1.6.0
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
+	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
 )

go.sum

@@ -7,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gen2brain/malgo v0.10.19 h1:IUVF6WdVV7Txt47Kx2ajz0rWQ0MU0zO+tbcKmhva7l8=
-github.com/gen2brain/malgo v0.10.19/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -17,6 +15,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
+github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

mediastream_test.go

@@ -33,10 +33,6 @@ func (track *mockMediaStreamTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return nil
 }
 
-func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
-	return nil, nil
-}
-
 func TestMediaStreamFilters(t *testing.T) {
 	audioTracks := []Track{
 		&mockMediaStreamTrack{AudioInput},

pkg/codec/codec.go

@@ -4,7 +4,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 )
 
 // RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.

pkg/io/audio/audio.go

@@ -5,14 +5,6 @@ import (
 )
 
 type Reader interface {
-	// Read reads data from the source. The caller is responsible to release the memory that's associated
-	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
-	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
-	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
-	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
-	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
-	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
-	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (chunk wave.Audio, release func(), err error)
 }
 

pkg/io/reader.go

@@ -3,14 +3,6 @@ package io
 // Reader is a generic data reader. In the future, interface{} should be replaced by a generic type
 // to provide strong type.
 type Reader interface {
-	// Read reads data from the source. The caller is responsible to release the memory that's associated
-	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
-	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
-	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
-	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
-	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
-	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
-	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (data interface{}, release func(), err error)
 }
 

pkg/io/video/video.go

@@ -5,14 +5,6 @@ import (
 )
 
 type Reader interface {
-	// Read reads data from the source. The caller is responsible to release the memory that's associated
-	// with data by calling the given release function. When err is not nil, the caller MUST NOT call release
-	// as data is going to be nil (no memory was given). Otherwise, the caller SHOULD call release after
-	// using the data. The caller is NOT REQUIRED to call release, as this is only a part of memory management
-	// optimization. If release is not called, the source is forced to allocate a new memory, which also means
-	// there will be new allocations during streaming, and old unused memory will become garbage. As a consequence,
-	// these garbage will put a lot of pressure to the garbage collector and makes it to run more often and finish
-	// slower as the heap memory usage increases and more garbage to collect.
 	Read() (img image.Image, release func(), err error)
 }
 

rtpreader.go

@@ -2,20 +2,20 @@ package mediadevices
 
 import "github.com/pion/rtp"
 
-type RTPReadCloser interface {
+type RTPReader interface {
 	Read() (pkts []*rtp.Packet, release func(), err error)
 	Close() error
 }
 
-type rtpReadCloserImpl struct {
+type rtpReaderImpl struct {
 	readFn  func() ([]*rtp.Packet, func(), error)
 	closeFn func() error
 }
 
-func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) {
+func (r *rtpReaderImpl) Read() ([]*rtp.Packet, func(), error) {
 	return r.readFn()
 }
 
-func (r *rtpReadCloserImpl) Close() error {
+func (r *rtpReaderImpl) Close() error {
 	return r.closeFn()
 }

track.go

@@ -12,8 +12,8 @@ import (
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/wave"
 	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v3"
-	"github.com/pion/webrtc/v3/pkg/media"
+	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v2/pkg/media"
 )
 
 var (
@@ -54,9 +54,6 @@ type Track interface {
 	// Unbind is the clean up operation that should be called after Bind. Similar to Bind, unbind will
 	// be called automatically in the future.
 	Unbind(*webrtc.PeerConnection) error
-	// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
-	// the encoded data in RTP format with given mtu size.
-	NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
 }
 
 type baseTrack struct {
@@ -263,7 +260,7 @@ func (track *VideoTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
 
-func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
+func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReader, error) {
 	reader := track.NewReader(false)
 	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
 	if err != nil {
@@ -280,7 +277,7 @@ func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser,
 	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
 	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
 
-	return &rtpReadCloserImpl{
+	return &rtpReaderImpl{
 		readFn: func() ([]*rtp.Packet, func(), error) {
 			encoded, release, err := encodedReader.Read()
 			if err != nil {
@@ -367,38 +364,3 @@ func (track *AudioTrack) Bind(pc *webrtc.PeerConnection) (*webrtc.Track, error)
 func (track *AudioTrack) Unbind(pc *webrtc.PeerConnection) error {
 	return track.unbind(pc)
 }
-
-func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser, error) {
-	reader := track.NewReader(false)
-	inputProp, err := detectCurrentAudioProp(track.Broadcaster)
-	if err != nil {
-		return nil, err
-	}
-
-	encodedReader, selectedCodec, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
-	if err != nil {
-		return nil, err
-	}
-
-	sample := newVideoSampler(selectedCodec.ClockRate)
-
-	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
-	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
-
-	return &rtpReadCloserImpl{
-		readFn: func() ([]*rtp.Packet, func(), error) {
-			encoded, release, err := encodedReader.Read()
-			if err != nil {
-				encodedReader.Close()
-				track.onError(err)
-				return nil, func() {}, err
-			}
-			defer release()
-
-			samples := sample()
-			pkts := packetizer.Packetize(encoded, samples)
-			return pkts, release, err
-		},
-		closeFn: encodedReader.Close,
-	}, nil
-}