Use square instead

codec.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 )
 
 // CodecSelector is a container of video and audio encoder builders, which later will be used

examples/facedetection/main.go

@@ -1,20 +1,26 @@
 package main
 
 import (
+	"fmt"
 	"image"
 	"io/ioutil"
 	"log"
-	"time"
+	"net"
+	"os"
 
 	pigo "github.com/esimov/pigo/core"
 	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use h264 video encoder
 	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
 	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
 const (
-	confidenceLevel = 5.0
+	confidenceLevel = 9.5
+	mtu             = 1000
+	thickness       = 5
 )
 
 var (
@@ -28,7 +34,7 @@ func must(err error) {
 	}
 }
 
-func detectFace(frame *image.YCbCr) bool {
+func detectFaces(frame *image.YCbCr) []pigo.Detection {
 	bounds := frame.Bounds()
 	cascadeParams := pigo.CascadeParams{
 		MinSize:     100,
@@ -49,17 +55,77 @@ func detectFace(frame *image.YCbCr) bool {
 
 	// Calculate the intersection over union (IoU) of two clusters.
 	dets = classifier.ClusterDetections(dets, 0)
+	return dets
+}
 
-	for _, det := range dets {
+func drawRect(frame *image.YCbCr, x0, y0, size int) {
-		if det.Q >= confidenceLevel {
+	if x0 < 0 {
-			return true
+		x0 = 0
+	}
+
+	if y0 < 0 {
+		y0 = 0
+	}
+
+	width := frame.Bounds().Dx()
+	height := frame.Bounds().Dy()
+	x1 := x0 + size
+	y1 := y0 + size
+
+	if x1 >= width {
+		x1 = width - 1
+	}
+
+	if y1 >= height {
+		y1 = height - 1
+	}
+
+	convert := func(x, y int) int {
+		return y*width + x
+	}
+
+	for x := x0; x < x1; x++ {
+		for t := 0; t < thickness; t++ {
+			frame.Y[convert(x, y0+t)] = 0
+			frame.Y[convert(x, y1-t)] = 0
 		}
 	}
 
-	return false
+	for y := y0; y < y1; y++ {
+		for t := 0; t < thickness; t++ {
+			frame.Y[convert(x0+t, y)] = 0
+			frame.Y[convert(x1-t, y)] = 0
+		}
+	}
+}
+
+func detectFace(r video.Reader) video.Reader {
+	return video.ReaderFunc(func() (img image.Image, release func(), err error) {
+		img, release, err = r.Read()
+		if err != nil {
+			return
+		}
+
+		yuv := img.(*image.YCbCr)
+		dets := detectFaces(yuv)
+		for _, det := range dets {
+			if det.Q < confidenceLevel {
+				continue
+			}
+
+			drawRect(yuv, det.Col-det.Scale/2, det.Row-det.Scale/2, det.Scale)
+		}
+		return
+	})
 }
 
 func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s host:port\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
 	// prepare face detector
 	var err error
 	cascade, err = ioutil.ReadFile("facefinder")
@@ -75,22 +141,21 @@ func main() {
 		log.Fatalf("Error unpacking the cascade file: %s", err)
 	}
 
-	devices := mediadevices.EnumerateDevices()
+	vp8Params, err := vpx.NewVP8Params()
-	deviceID := ""
+	must(err)
+	vp8Params.BitRate = 1_000_000 // 100kbps
 
-	for _, device := range devices {
+	codecSelector := mediadevices.NewCodecSelector(
-		if device.Label == "video0" {
+		mediadevices.WithVideoEncoders(&vp8Params),
-			deviceID = device.DeviceID
+	)
-		}
-	}
 
 	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
 		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.DeviceID = prop.StringExact(deviceID)
 			c.FrameFormat = prop.FrameFormatExact(frame.FormatUYVY)
 			c.Width = prop.Int(640)
 			c.Height = prop.Int(480)
 		},
+		Codec: codecSelector,
 	})
 	must(err)
 
@@ -98,18 +163,27 @@ func main() {
 	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
 	defer videoTrack.Close()
 
-	videoReader := videoTrack.NewReader(false)
+	videoTrack.Transform(detectFace)
-	// To save resources, we can simply use 4 fps to detect faces.
-	ticker := time.NewTicker(time.Millisecond * 250)
-	defer ticker.Stop()
 
-	for range ticker.C {
+	rtpReader, err := videoTrack.NewRTPReader(vp8Params.RTPCodec().Name, mtu)
-		frame, release, err := videoReader.Read()
+	must(err)
+
+	addr, err := net.ResolveUDPAddr("udp", dest)
+	must(err)
+	conn, err := net.DialUDP("udp", nil, addr)
+	must(err)
+
+	buff := make([]byte, mtu)
+	for {
+		pkts, release, err := rtpReader.Read()
 		must(err)
 
-		// Since we asked the frame format to be exactly YUY2 in GetUserMedia, we can guarantee that it must be YCbCr
+		for _, pkt := range pkts {
-		if detectFace(frame.(*image.YCbCr)) {
+			n, err := pkt.MarshalTo(buff)
-			log.Println("Detect a face")
+			must(err)
+
+			_, err = conn.Write(buff[:n])
+			must(err)
 		}
 
 		release()

examples/webrtc/main.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pion/mediadevices/examples/internal/signal"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 
 	// If you don't like x264, you can also use vpx by importing as below
 	// "github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder

go.mod

@@ -4,12 +4,12 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/gen2brain/malgo v0.10.19
+	github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93
 	github.com/lherman-cs/opus v0.0.2
 	github.com/pion/logging v0.2.2
 	github.com/pion/rtp v1.6.0
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
+	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
 )

go.sum

@@ -7,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gen2brain/malgo v0.10.19 h1:IUVF6WdVV7Txt47Kx2ajz0rWQ0MU0zO+tbcKmhva7l8=
-github.com/gen2brain/malgo v0.10.19/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -17,6 +15,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93 h1:gDcaH96SZ7q1JU6hj0tSv8FiuqadFERU17lLxhphLa8=
+github.com/jfreymuth/pulse v0.0.0-20201014123913-1e525c426c93/go.mod h1:cpYspI6YljhkUf1WLXLLDmeaaPFc3CnGLjDZf9dZ4no=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

pkg/codec/codec.go

@@ -4,7 +4,7 @@ import (
 	"github.com/pion/mediadevices/pkg/io/audio"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
 )
 
 // RTPCodec wraps webrtc.RTPCodec. RTPCodec might extend webrtc.RTPCodec in the future.

track.go

@@ -12,8 +12,8 @@ import (
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/wave"
 	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v3"
+	"github.com/pion/webrtc/v2"
-	"github.com/pion/webrtc/v3/pkg/media"
+	"github.com/pion/webrtc/v2/pkg/media"
 )
 
 var (