WIP

* Decouple encoding and webrtc from GetUserMedia
* Redesign VideoTrack and AudioTrack
* Remove mediadevices constructor
* Remove examples with old design
* Add a new simple example
* Remove Enable from constraints
* Implicitly disable a media when nil

examples/facedetection/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download facedetection
-
-```
-go get github.com/pion/mediadevices/examples/facedetection
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/video) you should see two text-areas and a 'Start Session' button
-
-### Run facedetection with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | facedetection`
-
-### Input facedetection's SessionDescription into your browser
-
-Copy the text that `facedetection` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/facedetection/detector.go

@@ -1,118 +0,0 @@
-package main
-
-import (
-	"image"
-	"image/color"
-	"image/draw"
-	"io/ioutil"
-	"log"
-
-	"github.com/disintegration/imaging"
-	pigo "github.com/esimov/pigo/core"
-)
-
-var (
-	cascade    []byte
-	err        error
-	classifier *pigo.Pigo
-)
-
-func imgToGrayscale(img image.Image) []uint8 {
-	bounds := img.Bounds()
-	flatten := bounds.Dy() * bounds.Dx()
-	grayImg := make([]uint8, flatten)
-
-	i := 0
-	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
-		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			pix := img.At(x, y)
-			grayPix := color.GrayModel.Convert(pix).(color.Gray)
-			grayImg[i] = grayPix.Y
-			i++
-		}
-	}
-	return grayImg
-}
-
-// clusterDetection runs Pigo face detector core methods
-// and returns a cluster with the detected faces coordinates.
-func clusterDetection(img image.Image) []pigo.Detection {
-	grayscale := imgToGrayscale(img)
-	bounds := img.Bounds()
-	cParams := pigo.CascadeParams{
-		MinSize:     100,
-		MaxSize:     600,
-		ShiftFactor: 0.15,
-		ScaleFactor: 1.1,
-		ImageParams: pigo.ImageParams{
-			Pixels: grayscale,
-			Rows:   bounds.Dy(),
-			Cols:   bounds.Dx(),
-			Dim:    bounds.Dx(),
-		},
-	}
-
-	if len(cascade) == 0 {
-		cascade, err = ioutil.ReadFile("facefinder")
-		if err != nil {
-			log.Fatalf("Error reading the cascade file: %s", err)
-		}
-		p := pigo.NewPigo()
-
-		// Unpack the binary file. This will return the number of cascade trees,
-		// the tree depth, the threshold and the prediction from tree's leaf nodes.
-		classifier, err = p.Unpack(cascade)
-		if err != nil {
-			log.Fatalf("Error unpacking the cascade file: %s", err)
-		}
-	}
-
-	// Run the classifier over the obtained leaf nodes and return the detection results.
-	// The result contains quadruplets representing the row, column, scale and detection score.
-	dets := classifier.RunCascade(cParams, 0.0)
-
-	// Calculate the intersection over union (IoU) of two clusters.
-	dets = classifier.ClusterDetections(dets, 0)
-
-	return dets
-}
-
-func drawCircle(img draw.Image, x0, y0, r int, c color.Color) {
-	x, y, dx, dy := r-1, 0, 1, 1
-	err := dx - (r * 2)
-
-	for x > y {
-		img.Set(x0+x, y0+y, c)
-		img.Set(x0+y, y0+x, c)
-		img.Set(x0-y, y0+x, c)
-		img.Set(x0-x, y0+y, c)
-		img.Set(x0-x, y0-y, c)
-		img.Set(x0-y, y0-x, c)
-		img.Set(x0+y, y0-x, c)
-		img.Set(x0+x, y0-y, c)
-
-		if err <= 0 {
-			y++
-			err += dy
-			dy += 2
-		}
-		if err > 0 {
-			x--
-			dx += 2
-			err += dx - (r * 2)
-		}
-	}
-}
-
-func markFaces(img image.Image) image.Image {
-	nrgba := imaging.Clone(img)
-	dets := clusterDetection(img)
-	for _, det := range dets {
-		if det.Q < 5.0 {
-			continue
-		}
-
-		drawCircle(nrgba, det.Col, det.Row, det.Scale/2, color.Black)
-	}
-	return nrgba
-}

examples/facedetection/main.go

@@ -1,119 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"image"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
-)
-
-func markFacesTransformer(r video.Reader) video.Reader {
-	return video.ReaderFunc(func() (img image.Image, err error) {
-		img, err = r.Read()
-		if err != nil {
-			return
-		}
-
-		img = markFaces(img)
-		return
-	})
-}
-
-func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormatExact(frame.FormatI420) // most of the encoder accepts I420
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoTransform = markFacesTransformer
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
-}

examples/rtp-send/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download rtp-send example
-
-```
-go get github.com/pion/mediadevices/examples/rtp-send
-```
-
-### Listen RTP
-
-Install GStreamer and run:
-```
-gst-launch-1.0 udpsrc port=5000 caps=application/x-rtp,encode-name=VP8 \
-    ! rtpvp8depay ! vp8dec ! videoconvert ! autovideosink
-```
-
-Or run VLC media plyer:
-```
-vlc ./vp8.sdp
-```
-
-### Run rtp-send
-
-Run `rtp-send localhost:5000`
-
-A video should start playing in your GStreamer or VLC window.
-It's not WebRTC, but pure RTP.
-
-Congrats, you have used pion-MediaDevices! Now start building something cool

examples/rtp-send/main.go

@@ -1,120 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"net"
-	"os"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	"github.com/pion/mediadevices/pkg/frame"
-	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v2"
-	"github.com/pion/webrtc/v2/pkg/media"
-)
-
-const (
-	mtu = 1000
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Printf("usage: %s host:port\n", os.Args[0])
-		return
-	}
-
-	md := mediadevices.NewMediaDevicesFromCodecs(
-		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
-			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
-				webrtc.NewRTPVP8Codec(100, 90000),
-			},
-		},
-		mediadevices.WithTrackGenerator(
-			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
-				mediadevices.LocalTrack, error,
-			) {
-				return newTrack(codec, id, os.Args[1]), nil
-			},
-		),
-	)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	_, err = md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	select {}
-}
-
-type track struct {
-	codec      *webrtc.RTPCodec
-	packetizer rtp.Packetizer
-	id         string
-	conn       net.Conn
-}
-
-func newTrack(codec *webrtc.RTPCodec, id, dest string) *track {
-	addr, err := net.ResolveUDPAddr("udp", dest)
-	if err != nil {
-		panic(err)
-	}
-	conn, err := net.DialUDP("udp", nil, addr)
-	if err != nil {
-		panic(err)
-	}
-	return &track{
-		codec: codec,
-		packetizer: rtp.NewPacketizer(
-			mtu,
-			codec.PayloadType,
-			1,
-			codec.Payloader,
-			rtp.NewRandomSequencer(),
-			codec.ClockRate,
-		),
-		id:   id,
-		conn: conn,
-	}
-}
-
-func (t *track) WriteSample(s media.Sample) error {
-	buf := make([]byte, mtu)
-	pkts := t.packetizer.Packetize(s.Data, s.Samples)
-	for _, p := range pkts {
-		n, err := p.MarshalTo(buf)
-		if err != nil {
-			panic(err)
-		}
-		_, _ = t.conn.Write(buf[:n])
-	}
-	return nil
-}
-
-func (t *track) Codec() *webrtc.RTPCodec {
-	return t.codec
-}
-
-func (t *track) ID() string {
-	return t.id
-}
-
-func (t *track) Kind() webrtc.RTPCodecType {
-	return t.codec.Type
-}

examples/rtp-send/vp8.sdp

@@ -1,9 +0,0 @@
-v=0
-o=- 1234567890 1234567890 IN IP4 0.0.0.0
-s=RTP-Send Example
-i=Example
-c=IN IP4 0.0.0.0
-t=0 0
-a=recvonly
-m=video 5000 RTP/AVP 100
-a=rtpmap:100 VP8/90000

examples/screenshare/README.md

@@ -1,29 +0,0 @@
-## Instructions
-
-### Download screenshare
-
-```
-go get github.com/pion/mediadevices/examples/screenshare
-```
-
-### Open example page
-
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
-
-### Run screenshare with your browsers SessionDescription as stdin
-
-In the jsfiddle the top textarea is your browser, copy that and:
-
-#### Linux
-
-Run `echo $BROWSER_SDP | screenshare`
-
-### Input screenshare's SessionDescription into your browser
-
-Copy the text that `screenshare` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/screenshare/main.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/codec/vpx"       // This is required to use VP8/VP9 video encoder
-	_ "github.com/pion/mediadevices/pkg/driver/screen" // This is required to register screen capture adapter
-	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/webrtc/v2"
-)
-
-func main() {
-	config := webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{"stun:stun.l.google.com:19302"},
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetDisplayMedia(mediadevices.MediaStreamConstraints{
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.VideoTransform = video.Scale(-1, 360, nil) // Resize to 360p
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
-		},
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	for _, tracker := range s.GetTracks() {
-		t := tracker.Track()
-		tracker.OnEnded(func(err error) {
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
-				t.ID(), t.Label(), err)
-		})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Set the remote SessionDescription
-	err = peerConnection.SetRemoteDescription(offer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
-}

examples/simple/README.md

@@ -1,29 +1,17 @@
 ## Instructions
 
-### Download gstreamer-send
+### Download the example
 
 ```
 go get github.com/pion/mediadevices/examples/simple
 ```
 
-### Open example page
+### Run the sample
 
-[jsfiddle.net](https://jsfiddle.net/gh/get/library/pure/pion/mediadevices/tree/master/examples/internal/jsfiddle/audio-and-video) you should see two text-areas and a 'Start Session' button
+```
+simple
+```
 
-### Run simple with your browsers SessionDescription as stdin
+### View yourself in the browser
 
-In the jsfiddle the top textarea is your browser, copy that and:
+Open your browser and go to "http://localhost:1313"
-
-#### Linux
-
-Run `echo $BROWSER_SDP | simple`
-
-### Input simple's SessionDescription into your browser
-
-Copy the text that `simple` just emitted and copy into second text area
-
-### Hit 'Start Session' in jsfiddle, enjoy your video!
-
-A video should start playing in your browser above the input boxes, and will continue playing until you close the application.
-
-Congrats, you have used pion-WebRTC! Now start building something cool

examples/simple/main.go

@@ -2,131 +2,68 @@ package main
 
 import (
 	"fmt"
+	"image/jpeg"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
 
 	"github.com/pion/mediadevices"
-	"github.com/pion/mediadevices/examples/internal/signal"
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
-
-	// This is required to use opus audio encoder
-	"github.com/pion/mediadevices/pkg/codec/opus"
-
-	// If you don't like vpx, you can also use x264 by importing as below
-	// "github.com/pion/mediadevices/pkg/codec/x264" // This is required to use h264 video encoder
-	// or you can also use openh264 for alternative h264 implementation
-	// "github.com/pion/mediadevices/pkg/codec/openh264"
-	"github.com/pion/mediadevices/pkg/codec/vpx" // This is required to use VP8/VP9 video encoder
 
 	// Note: If you don't have a camera or microphone or your adapters are not supported,
 	//       you can always swap your adapters with our dummy adapters below.
 	// _ "github.com/pion/mediadevices/pkg/driver/videotest"
-	// _ "github.com/pion/mediadevices/pkg/driver/audiotest"
 	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
-	_ "github.com/pion/mediadevices/pkg/driver/microphone" // This is required to register microphone adapter
-)
-
-const (
-	videoCodecName = webrtc.VP8
 )
 
 func main() {
-	config := webrtc.Configuration{
+	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
-		ICEServers: []webrtc.ICEServer{
+		Video: func(constraint *mediadevices.MediaTrackConstraints) {
-			{
+			constraint.Width = prop.Int(600)
-				URLs: []string{"stun:stun.l.google.com:19302"},
+			constraint.Height = prop.Int(400)
-			},
-		},
-	}
-
-	// Wait for the offer to be pasted
-	offer := webrtc.SessionDescription{}
-	signal.Decode(signal.MustReadStdin(), &offer)
-
-	// Create a new RTCPeerConnection
-	mediaEngine := webrtc.MediaEngine{}
-	if err := mediaEngine.PopulateFromSDP(offer); err != nil {
-		panic(err)
-	}
-	api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
-	peerConnection, err := api.NewPeerConnection(config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Set the handler for ICE connection state
-	// This will notify you when the peer has connected/disconnected
-	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		fmt.Printf("Connection State has changed %s \n", connectionState.String())
-	})
-
-	md := mediadevices.NewMediaDevices(peerConnection)
-
-	opusParams, err := opus.NewParams()
-	if err != nil {
-		panic(err)
-	}
-	opusParams.BitRate = 32000 // 32kbps
-
-	vp8Params, err := vpx.NewVP8Params()
-	if err != nil {
-		panic(err)
-	}
-	vp8Params.BitRate = 100000 // 100kbps
-
-	s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Audio: func(c *mediadevices.MediaTrackConstraints) {
-			c.Enabled = true
-			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&opusParams}
-		},
-		Video: func(c *mediadevices.MediaTrackConstraints) {
-			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
-			c.Enabled = true
-			c.Width = prop.Int(640)
-			c.Height = prop.Int(480)
-			c.VideoEncoderBuilders = []codec.VideoEncoderBuilder{&vp8Params}
 		},
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	for _, tracker := range s.GetTracks() {
+	t := s.GetVideoTracks()[0]
-		t := tracker.Track()
+	defer t.Stop()
-		tracker.OnEnded(func(err error) {
+	videoTrack := t.(*mediadevices.VideoTrack)
-			fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n",
+
-				t.ID(), t.Label(), err)
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		videoReader := videoTrack.NewReader()
+		mimeWriter := multipart.NewWriter(w)
+
+		contentType := fmt.Sprintf("multipart/x-mixed-replace;boundary=%s", mimeWriter.Boundary())
+		w.Header().Add("Content-Type", contentType)
+
+		partHeader := make(textproto.MIMEHeader)
+		partHeader.Add("Content-Type", "image/jpeg")
+
+		for {
+			frame, err := videoReader.Read()
+			if err != nil {
+				if err == io.EOF {
+					return
+				}
+				panic(err)
+			}
+
+			partWriter, err := mimeWriter.CreatePart(partHeader)
+			if err != nil {
+				panic(err)
+			}
+
+			err = jpeg.Encode(partWriter, frame, nil)
+			if err != nil {
+				panic(err)
+			}
+		}
 	})
-		_, err = peerConnection.AddTransceiverFromTrack(t,
-			webrtc.RtpTransceiverInit{
-				Direction: webrtc.RTPTransceiverDirectionSendonly,
-			},
-		)
-		if err != nil {
-			panic(err)
-		}
-	}
 
-	// Set the remote SessionDescription
+	fmt.Println("listening on http://localhost:1313")
-	err = peerConnection.SetRemoteDescription(offer)
+	log.Println(http.ListenAndServe("localhost:1313", nil))
-	if err != nil {
-		panic(err)
-	}
-
-	// Create an answer
-	answer, err := peerConnection.CreateAnswer(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// Sets the LocalDescription, and starts our UDP listeners
-	err = peerConnection.SetLocalDescription(answer)
-	if err != nil {
-		panic(err)
-	}
-
-	// Output the answer in base64 so we can paste it in browser
-	fmt.Println(signal.Encode(answer))
-	select {}
 }

mediadevices.go

@@ -6,106 +6,37 @@ import (
 
 	"github.com/pion/mediadevices/pkg/driver"
 	"github.com/pion/mediadevices/pkg/prop"
-	"github.com/pion/webrtc/v2"
 )
 
 var errNotFound = fmt.Errorf("failed to find the best driver that fits the constraints")
 
-// MediaDevices is an interface that's defined on https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices
-type MediaDevices interface {
-	GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error)
-	EnumerateDevices() []MediaDeviceInfo
-}
-
-// NewMediaDevices creates MediaDevices interface that provides access to connected media input devices
-// like cameras and microphones, as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevices(pc *webrtc.PeerConnection, opts ...MediaDevicesOption) MediaDevices {
-	codecs := make(map[webrtc.RTPCodecType][]*webrtc.RTPCodec)
-	for _, kind := range []webrtc.RTPCodecType{
-		webrtc.RTPCodecTypeAudio,
-		webrtc.RTPCodecTypeVideo,
-	} {
-		codecs[kind] = pc.GetRegisteredRTPCodecs(kind)
-	}
-	return NewMediaDevicesFromCodecs(codecs, opts...)
-}
-
-// NewMediaDevicesFromCodecs creates MediaDevices interface from lists of the available codecs
-// that provides access to connected media input devices like cameras and microphones,
-// as well as screen sharing.
-// In essence, it lets you obtain access to any hardware source of media data.
-func NewMediaDevicesFromCodecs(codecs map[webrtc.RTPCodecType][]*webrtc.RTPCodec, opts ...MediaDevicesOption) MediaDevices {
-	mdo := MediaDevicesOptions{
-		codecs:         codecs,
-		trackGenerator: defaultTrackGenerator,
-	}
-	for _, o := range opts {
-		o(&mdo)
-	}
-	return &mediaDevices{
-		MediaDevicesOptions: mdo,
-	}
-}
-
-// TrackGenerator is a function to create new track.
-type TrackGenerator func(payloadType uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error)
-
-var defaultTrackGenerator = TrackGenerator(func(pt uint8, ssrc uint32, id, label string, codec *webrtc.RTPCodec) (LocalTrack, error) {
-	return webrtc.NewTrack(pt, ssrc, id, label, codec)
-})
-
-type mediaDevices struct {
-	MediaDevicesOptions
-}
-
-// MediaDevicesOptions stores parameters used by MediaDevices.
-type MediaDevicesOptions struct {
-	codecs         map[webrtc.RTPCodecType][]*webrtc.RTPCodec
-	trackGenerator TrackGenerator
-}
-
-// MediaDevicesOption is a type of MediaDevices functional option.
-type MediaDevicesOption func(*MediaDevicesOptions)
-
-// WithTrackGenerator specifies a TrackGenerator to use customized track.
-func WithTrackGenerator(gen TrackGenerator) MediaDevicesOption {
-	return func(o *MediaDevicesOptions) {
-		o.trackGenerator = gen
-	}
-}
-
 // GetDisplayMedia prompts the user to select and grant permission to capture the contents
 // of a display or portion thereof (such as a window) as a MediaStream.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia
-func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+func GetDisplayMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	trackers := make([]Tracker, 0)
+	tracks := make([]Track, 0)
 
-	cleanTrackers := func() {
+	cleanTracks := func() {
-		for _, t := range trackers {
+		for _, t := range tracks {
 			t.Stop()
 		}
 	}
 
-	var videoConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		constraints.Video(&videoConstraints)
+		var p MediaTrackConstraints
-	}
+		constraints.Video(&p)
-
+		track, err := selectScreen(p)
-	if videoConstraints.Enabled {
-		tracker, err := m.selectScreen(videoConstraints)
 		if err != nil {
-			cleanTrackers()
+			cleanTracks()
 			return nil, err
 		}
 
-		trackers = append(trackers, tracker)
+		tracks = append(tracks, track)
 	}
 
-	s, err := NewMediaStream(trackers...)
+	s, err := NewMediaStream(tracks...)
 	if err != nil {
-		cleanTrackers()
+		cleanTracks()
 		return nil, err
 	}
 
@@ -115,48 +46,42 @@ func (m *mediaDevices) GetDisplayMedia(constraints MediaStreamConstraints) (Medi
 // GetUserMedia prompts the user for permission to use a media input which produces a MediaStream
 // with tracks containing the requested types of media.
 // Reference: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
-func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
+func GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) {
-	// TODO: It should return media stream based on constraints
+	tracks := make([]Track, 0)
-	trackers := make([]Tracker, 0)
 
-	cleanTrackers := func() {
+	cleanTracks := func() {
-		for _, t := range trackers {
+		for _, t := range tracks {
 			t.Stop()
 		}
 	}
 
-	var videoConstraints, audioConstraints MediaTrackConstraints
 	if constraints.Video != nil {
-		constraints.Video(&videoConstraints)
+		var p MediaTrackConstraints
+		constraints.Video(&p)
+		track, err := selectVideo(p)
+		if err != nil {
+			cleanTracks()
+			return nil, err
+		}
+
+		tracks = append(tracks, track)
 	}
 
 	if constraints.Audio != nil {
-		constraints.Audio(&audioConstraints)
+		var p MediaTrackConstraints
-	}
+		constraints.Audio(&p)
-
+		track, err := selectAudio(p)
-	if videoConstraints.Enabled {
-		tracker, err := m.selectVideo(videoConstraints)
 		if err != nil {
-			cleanTrackers()
+			cleanTracks()
 			return nil, err
 		}
 
-		trackers = append(trackers, tracker)
+		tracks = append(tracks, track)
 	}
 
-	if audioConstraints.Enabled {
+	s, err := NewMediaStream(tracks...)
-		tracker, err := m.selectAudio(audioConstraints)
 	if err != nil {
-			cleanTrackers()
+		cleanTracks()
-			return nil, err
-		}
-
-		trackers = append(trackers, tracker)
-	}
-
-	s, err := NewMediaStream(trackers...)
-	if err != nil {
-		cleanTrackers()
 		return nil, err
 	}
 
@@ -223,7 +148,7 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints)
 	return bestDriver, constraints, nil
 }
 
-func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker, error) {
+func selectAudio(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterAudioRecorder()
 
 	d, c, err := selectBestDriver(typeFilter, constraints)
@@ -231,9 +156,10 @@ func (m *mediaDevices) selectAudio(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newAudioTrack(d, c)
 }
-func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker, error) {
+
+func selectVideo(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	notScreenFilter := driver.FilterNot(driver.FilterDeviceType(driver.Screen))
 	filter := driver.FilterAnd(typeFilter, notScreenFilter)
@@ -243,10 +169,10 @@ func (m *mediaDevices) selectVideo(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newVideoTrack(d, c)
 }
 
-func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker, error) {
+func selectScreen(constraints MediaTrackConstraints) (Track, error) {
 	typeFilter := driver.FilterVideoRecorder()
 	screenFilter := driver.FilterDeviceType(driver.Screen)
 	filter := driver.FilterAnd(typeFilter, screenFilter)
@@ -256,10 +182,10 @@ func (m *mediaDevices) selectScreen(constraints MediaTrackConstraints) (Tracker,
 		return nil, err
 	}
 
-	return newTrack(&m.MediaDevicesOptions, d, c)
+	return newVideoTrack(d, c)
 }
 
-func (m *mediaDevices) EnumerateDevices() []MediaDeviceInfo {
+func EnumerateDevices() []MediaDeviceInfo {
 	drivers := driver.GetManager().Query(
 		driver.FilterFn(func(driver.Driver) bool { return true }))
 	info := make([]MediaDeviceInfo, 0, len(drivers))

mediadevices_test.go

@@ -19,18 +19,25 @@ import (
 )
 
 func TestGetUserMedia(t *testing.T) {
-	videoParams := mockParams{
+	brokenVideoParams := mockParams{
-		BaseParams: codec.BaseParams{
-			BitRate: 100000,
-		},
 		name: "MockVideo",
 	}
+	videoParams := brokenVideoParams
+	videoParams.BitRate = 100000
 	audioParams := mockParams{
 		BaseParams: codec.BaseParams{
 			BitRate: 32000,
 		},
 		name: "MockAudio",
 	}
+	constraints := MediaStreamConstraints{
+		Video: func(p *prop.Media) {
+			p.Width = 640
+			p.Height = 480
+		},
+		Audio: func(p *prop.Media) {},
+	}
+
 	md := NewMediaDevicesFromCodecs(
 		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
 			webrtc.RTPCodecTypeVideo: {
@@ -47,7 +54,10 @@ func TestGetUserMedia(t *testing.T) {
 				return newMockTrack(codec, id), nil
 			},
 		),
+		WithVideoEncoders(&brokenVideoParams),
+		WithAudioEncoders(&audioParams),
 	)
+<<<<<<< HEAD
 	constraints := MediaStreamConstraints{
 		Video: func(c *MediaTrackConstraints) {
 			c.Enabled = true
@@ -77,13 +87,35 @@ func TestGetUserMedia(t *testing.T) {
 			c.AudioEncoderBuilders = []codec.AudioEncoderBuilder{&params}
 		},
 	}
+=======
+>>>>>>> ccd7985... Redesign GetUserMedia API
 
 	// GetUserMedia with broken parameters
-	ms, err := md.GetUserMedia(constraintsWrong)
+	ms, err := md.GetUserMedia(constraints)
 	if err == nil {
 		t.Fatal("Expected error, but got nil")
 	}
 
+	md = NewMediaDevicesFromCodecs(
+		map[webrtc.RTPCodecType][]*webrtc.RTPCodec{
+			webrtc.RTPCodecTypeVideo: []*webrtc.RTPCodec{
+				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeVideo, Name: "MockVideo", PayloadType: 1},
+			},
+			webrtc.RTPCodecTypeAudio: []*webrtc.RTPCodec{
+				&webrtc.RTPCodec{Type: webrtc.RTPCodecTypeAudio, Name: "MockAudio", PayloadType: 2},
+			},
+		},
+		WithTrackGenerator(
+			func(_ uint8, _ uint32, id, _ string, codec *webrtc.RTPCodec) (
+				LocalTrack, error,
+			) {
+				return newMockTrack(codec, id), nil
+			},
+		),
+		WithVideoEncoders(&videoParams),
+		WithAudioEncoders(&audioParams),
+	)
+
 	// GetUserMedia with correct parameters
 	ms, err = md.GetUserMedia(constraints)
 	if err != nil {

mediastream.go

@@ -9,19 +9,19 @@ import (
 // MediaStream is an interface that represents a collection of existing tracks.
 type MediaStream interface {
 	// GetAudioTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
-	GetAudioTracks() []Tracker
+	GetAudioTracks() []Track
 	// GetVideoTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
-	GetVideoTracks() []Tracker
+	GetVideoTracks() []Track
 	// GetTracks implements https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
-	GetTracks() []Tracker
+	GetTracks() []Track
 	// AddTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
-	AddTrack(t Tracker)
+	AddTrack(t Track)
 	// RemoveTrack implements https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
-	RemoveTrack(t Tracker)
+	RemoveTrack(t Track)
 }
 
 type mediaStream struct {
-	trackers map[string]Tracker
+	tracks map[string]Track
 	l      sync.RWMutex
 }
 
@@ -29,62 +29,62 @@ const rtpCodecTypeDefault webrtc.RTPCodecType = 0
 
 // NewMediaStream creates a MediaStream interface that's defined in
 // https://w3c.github.io/mediacapture-main/#dom-mediastream
-func NewMediaStream(trackers ...Tracker) (MediaStream, error) {
+func NewMediaStream(tracks ...Track) (MediaStream, error) {
-	m := mediaStream{trackers: make(map[string]Tracker)}
+	m := mediaStream{tracks: make(map[string]Track)}
 
-	for _, tracker := range trackers {
+	for _, track := range tracks {
-		id := tracker.LocalTrack().ID()
+		id := track.ID()
-		if _, ok := m.trackers[id]; !ok {
+		if _, ok := m.tracks[id]; !ok {
-			m.trackers[id] = tracker
+			m.tracks[id] = track
 		}
 	}
 
 	return &m, nil
 }
 
-func (m *mediaStream) GetAudioTracks() []Tracker {
+func (m *mediaStream) GetAudioTracks() []Track {
-	return m.queryTracks(webrtc.RTPCodecTypeAudio)
+	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindAudio })
 }
 
-func (m *mediaStream) GetVideoTracks() []Tracker {
+func (m *mediaStream) GetVideoTracks() []Track {
-	return m.queryTracks(webrtc.RTPCodecTypeVideo)
+	return m.queryTracks(func(t Track) bool { return t.Kind() == TrackKindVideo })
 }
 
-func (m *mediaStream) GetTracks() []Tracker {
+func (m *mediaStream) GetTracks() []Track {
-	return m.queryTracks(rtpCodecTypeDefault)
+	return m.queryTracks(func(t Track) bool { return true })
 }
 
 // queryTracks returns all tracks that are the same kind as t.
 // If t is 0, which is the default, queryTracks will return all the tracks.
-func (m *mediaStream) queryTracks(t webrtc.RTPCodecType) []Tracker {
+func (m *mediaStream) queryTracks(filter func(track Track) bool) []Track {
 	m.l.RLock()
 	defer m.l.RUnlock()
 
-	result := make([]Tracker, 0)
+	result := make([]Track, 0)
-	for _, tracker := range m.trackers {
+	for _, track := range m.tracks {
-		if tracker.LocalTrack().Kind() == t || t == rtpCodecTypeDefault {
+		if filter(track) {
-			result = append(result, tracker)
+			result = append(result, track)
 		}
 	}
 
 	return result
 }
 
-func (m *mediaStream) AddTrack(t Tracker) {
+func (m *mediaStream) AddTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	id := t.LocalTrack().ID()
+	id := t.ID()
-	if _, ok := m.trackers[id]; ok {
+	if _, ok := m.tracks[id]; ok {
 		return
 	}
 
-	m.trackers[id] = t
+	m.tracks[id] = t
 }
 
-func (m *mediaStream) RemoveTrack(t Tracker) {
+func (m *mediaStream) RemoveTrack(t Track) {
 	m.l.Lock()
 	defer m.l.Unlock()
 
-	delete(m.trackers, t.LocalTrack().ID())
+	delete(m.tracks, t.ID())
 }

mediastreamconstraints.go

@@ -1,9 +1,6 @@
 package mediadevices
 
 import (
-	"github.com/pion/mediadevices/pkg/codec"
-	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
 )
 
@@ -15,26 +12,6 @@ type MediaStreamConstraints struct {
 // MediaTrackConstraints represents https://w3c.github.io/mediacapture-main/#dom-mediatrackconstraints
 type MediaTrackConstraints struct {
 	prop.MediaConstraints
-	Enabled bool
-	// VideoEncoderBuilders are codec builders that are used for encoding the video
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	VideoEncoderBuilders []codec.VideoEncoderBuilder
-	// AudioEncoderBuilders are codec builders that are used for encoding the audio
-	// and later being used for sending the appropriate RTP payload type.
-	//
-	// If one encoder builder fails to build the codec, the next builder will be used,
-	// repeating until a codec builds. If no builders build successfully, an error is returned.
-	AudioEncoderBuilders []codec.AudioEncoderBuilder
-	// VideoTransform will be used to transform the video that's coming from the driver.
-	// So, basically it'll look like following: driver -> VideoTransform -> codec
-	VideoTransform video.TransformFunc
-	// AudioTransform will be used to transform the audio that's coming from the driver.
-	// So, basically it'll look like following: driver -> AudioTransform -> code
-	AudioTransform audio.TransformFunc
-
 	selectedMedia prop.Media
 }
 

rtp.go

@@ -0,0 +1,101 @@
+package mediadevices
+
+import (
+	"fmt"
+
+	"github.com/pion/mediadevices/pkg/codec"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v2"
+)
+
+type RTPTracker struct {
+	videoEncoders []codec.VideoEncoderBuilder
+	audioEncoders []codec.AudioEncoderBuilder
+}
+
+type RTPTrackerOption func(*RTPTracker)
+
+func WithVideoEncoders(codecs ...codec.VideoEncoderBuilder) func(*RTPTracker) {
+	return func(tracker *RTPTracker) {
+		tracker.videoEncoders = codecs
+	}
+}
+
+func WithAudioEncoders(codecs ...codec.AudioEncoderBuilder) func(*RTPTracker) {
+	return func(tracker *RTPTracker) {
+		tracker.audioEncoders = codecs
+	}
+}
+
+func NewRTPTracker(opts ...RTPTrackerOption) *RTPTracker {
+	var tracker RTPTracker
+
+	for _, opt := range opts {
+		opt(&tracker)
+	}
+
+	return &tracker
+}
+
+func (tracker *RTPTracker) Track(track Track) *RTPTrack {
+	rtpTrack := RTPTrack{
+		Track: track,
+	}
+
+	return &rtpTrack
+}
+
+type RTPTrack struct {
+	Track
+	tracker        *RTPTracker
+	currentEncoder codec.ReadCloser
+	currentParams  RTPParameters
+	lastProp       prop.Media
+}
+
+func (track *RTPTrack) SetParameters(params RTPParameters) error {
+	var err error
+
+	switch t := track.Track.(type) {
+	case *VideoTrack:
+		err = track.setParametersVideo(t, &params)
+	case *AudioTrack:
+		err = track.setParametersAudio(t, &params)
+	default:
+		err = fmt.Errorf("unsupported track type")
+	}
+
+	if err == nil {
+		track.currentParams = params
+	}
+
+	return err
+}
+
+func (track *RTPTrack) setParametersVideo(videoTrack *VideoTrack, params *RTPParameters) error {
+	if params.SelectedCodec.Type != webrtc.RTPCodecTypeVideo {
+		return fmt.Errorf("invalid selected RTP codec type. Expected video but got audio")
+	}
+
+	video.DetectChanges(interval time.Duration, onChange func(prop.Media))
+	return nil
+}
+
+func (track *RTPTrack) setParametersAudio(audioTrack *AudioTrack, params *RTPParameters) error {
+	return nil
+}
+
+func (track *RTPTrack) ReadRTP() (*rtp.Packet, error) {
+	if track.currentEncoder == nil {
+		return nil, fmt.Errorf("Encoder has not been specified. Please call SetParameters to specify.")
+	}
+
+	return nil, nil
+}
+
+func (track *RTPTrack) WriteRTCP(packet rtcp.Packet) error {
+	return nil
+}

sampler.go

@@ -1,35 +0,0 @@
-package mediadevices
-
-import (
-	"math"
-	"time"
-
-	"github.com/pion/webrtc/v2/pkg/media"
-)
-
-type samplerFunc func(b []byte) error
-
-// newVideoSampler creates a video sampler that uses the actual video frame rate and
-// the codec's clock rate to come up with a duration for each sample.
-func newVideoSampler(t LocalTrack) samplerFunc {
-	clockRate := float64(t.Codec().ClockRate)
-	lastTimestamp := time.Now()
-
-	return samplerFunc(func(b []byte) error {
-		now := time.Now()
-		duration := now.Sub(lastTimestamp).Seconds()
-		samples := uint32(math.Round(clockRate * duration))
-		lastTimestamp = now
-
-		return t.WriteSample(media.Sample{Data: b, Samples: samples})
-	})
-}
-
-// newAudioSampler creates a audio sampler that uses a fixed latency and
-// the codec's clock rate to come up with a duration for each sample.
-func newAudioSampler(t LocalTrack, latency time.Duration) samplerFunc {
-	samples := uint32(math.Round(float64(t.Codec().ClockRate) * latency.Seconds()))
-	return samplerFunc(func(b []byte) error {
-		return t.WriteSample(media.Sample{Data: b, Samples: samples})
-	})
-}

track.go

@@ -1,22 +1,29 @@
 package mediadevices
 
 import (
-	"errors"
+	"fmt"
-	"math/rand"
+	"image"
 	"sync"
 
-	"github.com/pion/mediadevices/pkg/codec"
 	"github.com/pion/mediadevices/pkg/driver"
-	mio "github.com/pion/mediadevices/pkg/io"
+	"github.com/pion/mediadevices/pkg/io/audio"
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/mediadevices/pkg/io/video"
-	"github.com/pion/webrtc/v2/pkg/media"
+	"github.com/pion/mediadevices/pkg/wave"
 )
 
-// Tracker is an interface that represent MediaStreamTrack
+// TrackKind represents content type of a track
+type TrackKind string
+
+const (
+	TrackKindVideo TrackKind = "video"
+	TrackKindAudio TrackKind = "audio"
+)
+
+// Track is an interface that represent MediaStreamTrack
 // Reference: https://w3c.github.io/mediacapture-main/#mediastreamtrack
-type Tracker interface {
+type Track interface {
-	Track() *webrtc.Track
+	ID() string
-	LocalTrack() LocalTrack
+	Kind() TrackKind
 	Stop()
 	// OnEnded registers a handler to receive an error from the media stream track.
 	// If the error is already occured before registering, the handler will be
@@ -24,18 +31,170 @@ type Tracker interface {
 	OnEnded(func(error))
 }
 
-type LocalTrack interface {
+// VideoTrack is a specialized track for video
-	WriteSample(s media.Sample) error
+type VideoTrack struct {
-	Codec() *webrtc.RTPCodec
+	baseTrack
-	ID() string
+	src         video.Reader
-	Kind() webrtc.RTPCodecType
+	transformed video.Reader
+	mux         sync.Mutex
+	frameCount  int
+	lastFrame   image.Image
+	lastErr     error
 }
 
-type track struct {
+func newVideoTrack(d driver.Driver, constraints MediaTrackConstraints) (*VideoTrack, error) {
-	localTrack LocalTrack
+	err := d.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	recorder, ok := d.(driver.VideoRecorder)
+	if !ok {
+		d.Close()
+		return nil, fmt.Errorf("driver is not an video recorder")
+	}
+
+	r, err := recorder.VideoRecord(constraints.selectedMedia)
+	if err != nil {
+		d.Close()
+		return nil, err
+	}
+
+	return &VideoTrack{
+		baseTrack:   newBaseTrack(d, constraints),
+		src:         r,
+		transformed: r,
+	}, nil
+}
+
+// Kind returns track's kind
+func (track *VideoTrack) Kind() TrackKind {
+	return TrackKindVideo
+}
+
+// NewReader returns a reader to read frames from the source. You may create multiple
+// readers and read from them in different goroutines.
+//
+// In the case of multiple readers, reading from the source will only get triggered
+// when the reader has the latest frame from the source
+func (track *VideoTrack) NewReader() video.Reader {
+	var curFrameCount int
+	return video.ReaderFunc(func() (img image.Image, err error) {
+		track.mux.Lock()
+		defer track.mux.Unlock()
+
+		if curFrameCount != track.frameCount {
+			img = copyFrame(img, track.lastFrame)
+			err = track.lastErr
+		} else {
+			img, err = track.transformed.Read()
+			track.lastFrame = img
+			track.lastErr = err
+			track.frameCount++
+			if err != nil {
+				track.onErrorHandler(err)
+			}
+		}
+
+		curFrameCount = track.frameCount
+		return
+	})
+}
+
+// TODO: implement copy in place
+func copyFrame(dst, src image.Image) image.Image { return src }
+
+// Transform transforms the underlying source. The transformation will reflect to
+// all readers
+func (track *VideoTrack) Transform(fns ...video.TransformFunc) {
+	track.mux.Lock()
+	defer track.mux.Unlock()
+	track.transformed = video.Merge(fns...)(track.src)
+}
+
+// AudioTrack is a specialized track for audio
+type AudioTrack struct {
+	baseTrack
+	src         audio.Reader
+	transformed audio.Reader
+	mux         sync.Mutex
+	chunkCount  int
+	lastChunks  wave.Audio
+	lastErr     error
+}
+
+func newAudioTrack(d driver.Driver, constraints MediaTrackConstraints) (*AudioTrack, error) {
+	err := d.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	recorder, ok := d.(driver.AudioRecorder)
+	if !ok {
+		d.Close()
+		return nil, fmt.Errorf("driver is not an audio recorder")
+	}
+
+	r, err := recorder.AudioRecord(constraints.selectedMedia)
+	if err != nil {
+		d.Close()
+		return nil, err
+	}
+
+	return &AudioTrack{
+		baseTrack:   newBaseTrack(d, constraints),
+		src:         r,
+		transformed: r,
+	}, nil
+}
+
+func (track *AudioTrack) Kind() TrackKind {
+	return TrackKindAudio
+}
+
+// NewReader returns a reader to read audio chunks from the source. You may create multiple
+// readers and read from them in different goroutines.
+//
+// In the case of multiple readers, reading from the source will only get triggered
+// when the reader has the latest chunk from the source
+func (track *AudioTrack) NewReader() audio.Reader {
+	var currChunkCount int
+	return audio.ReaderFunc(func() (chunks wave.Audio, err error) {
+		track.mux.Lock()
+		defer track.mux.Unlock()
+
+		if currChunkCount != track.chunkCount {
+			chunks = copyChunks(chunks, track.lastChunks)
+			err = track.lastErr
+		} else {
+			chunks, err = track.transformed.Read()
+			track.lastChunks = chunks
+			track.lastErr = err
+			track.chunkCount++
+			if err != nil {
+				track.onErrorHandler(err)
+			}
+		}
+
+		currChunkCount = track.chunkCount
+		return
+	})
+}
+
+// TODO: implement copy in place
+func copyChunks(dst, src wave.Audio) wave.Audio { return src }
+
+// Transform transforms the underlying source. The transformation will reflect to
+// all readers
+func (track *AudioTrack) Transform(fns ...audio.TransformFunc) {
+	track.mux.Lock()
+	defer track.mux.Unlock()
+	track.transformed = audio.Merge(fns...)(track.src)
+}
+
+type baseTrack struct {
 	d           driver.Driver
-	sample     samplerFunc
+	constraints MediaTrackConstraints
-	encoder    codec.ReadCloser
 
 	onErrorHandler func(error)
 	err            error
@@ -43,83 +202,17 @@ type track struct {
 	endOnce        sync.Once
 }
 
-func newTrack(opts *MediaDevicesOptions, d driver.Driver, constraints MediaTrackConstraints) (*track, error) {
+func newBaseTrack(d driver.Driver, constraints MediaTrackConstraints) baseTrack {
-	var encoderBuilders []encoderBuilder
+	return baseTrack{d: d, constraints: constraints}
-	var rtpCodecs []*webrtc.RTPCodec
+}
-	var buildSampler func(t LocalTrack) samplerFunc
-	var err error
 
-	err = d.Open()
+func (t *baseTrack) ID() string {
-	if err != nil {
+	return t.d.ID()
-		return nil, err
-	}
-
-	switch r := d.(type) {
-	case driver.VideoRecorder:
-		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeVideo]
-		buildSampler = newVideoSampler
-		encoderBuilders, err = newVideoEncoderBuilders(r, constraints)
-	case driver.AudioRecorder:
-		rtpCodecs = opts.codecs[webrtc.RTPCodecTypeAudio]
-		buildSampler = func(t LocalTrack) samplerFunc {
-			return newAudioSampler(t, constraints.selectedMedia.Latency)
-		}
-		encoderBuilders, err = newAudioEncoderBuilders(r, constraints)
-	default:
-		err = errors.New("newTrack: invalid driver type")
-	}
-
-	if err != nil {
-		d.Close()
-		return nil, err
-	}
-
-	for _, builder := range encoderBuilders {
-		var matchedRTPCodec *webrtc.RTPCodec
-		for _, rtpCodec := range rtpCodecs {
-			if rtpCodec.Name == builder.name {
-				matchedRTPCodec = rtpCodec
-				break
-			}
-		}
-
-		if matchedRTPCodec == nil {
-			continue
-		}
-
-		localTrack, err := opts.trackGenerator(
-			matchedRTPCodec.PayloadType,
-			rand.Uint32(),
-			d.ID(),
-			matchedRTPCodec.Type.String(),
-			matchedRTPCodec,
-		)
-		if err != nil {
-			continue
-		}
-
-		encoder, err := builder.build()
-		if err != nil {
-			continue
-		}
-
-		t := track{
-			localTrack: localTrack,
-			sample:     buildSampler(localTrack),
-			d:          d,
-			encoder:    encoder,
-		}
-		go t.start()
-		return &t, nil
-	}
-
-	d.Close()
-	return nil, errors.New("newTrack: failed to find a matching codec")
 }
 
 // OnEnded sets an error handler. When a track has been created and started, if an
 // error occurs, handler will get called with the error given to the parameter.
-func (t *track) OnEnded(handler func(error)) {
+func (t *baseTrack) OnEnded(handler func(error)) {
 	t.mu.Lock()
 	t.onErrorHandler = handler
 	err := t.err
@@ -134,7 +227,7 @@ func (t *track) OnEnded(handler func(error)) {
 }
 
 // onError is a callback when an error occurs
-func (t *track) onError(err error) {
+func (t *baseTrack) onError(err error) {
 	t.mu.Lock()
 	t.err = err
 	handler := t.onErrorHandler
@@ -147,92 +240,6 @@ func (t *track) onError(err error) {
 	}
 }
 
-// start starts the data flow from the driver all the way to the localTrack
+func (t *baseTrack) Stop() {
-func (t *track) start() {
-	var n int
-	var err error
-	buff := make([]byte, 1024)
-	for {
-		n, err = t.encoder.Read(buff)
-		if err != nil {
-			if e, ok := err.(*mio.InsufficientBufferError); ok {
-				buff = make([]byte, 2*e.RequiredSize)
-				continue
-			}
-
-			t.onError(err)
-			return
-		}
-
-		if err := t.sample(buff[:n]); err != nil {
-			t.onError(err)
-			return
-		}
-	}
-}
-
-// Stop stops the underlying driver and encoder
-func (t *track) Stop() {
 	t.d.Close()
-	t.encoder.Close()
-}
-
-func (t *track) Track() *webrtc.Track {
-	return t.localTrack.(*webrtc.Track)
-}
-
-func (t *track) LocalTrack() LocalTrack {
-	return t.localTrack
-}
-
-// encoderBuilder is a generic encoder builder that acts as a delegator for codec.VideoEncoderBuilder and
-// codec.AudioEncoderBuilder. The idea of having a delegator is to reduce redundant codes that are being
-// duplicated for managing video and audio.
-type encoderBuilder struct {
-	name  string
-	build func() (codec.ReadCloser, error)
-}
-
-// newVideoEncoderBuilders transforms video given by VideoRecorder with the video transformer that is passed through
-// constraints and create a list of generic encoder builders
-func newVideoEncoderBuilders(vr driver.VideoRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
-	r, err := vr.VideoRecord(constraints.selectedMedia)
-	if err != nil {
-		return nil, err
-	}
-
-	if constraints.VideoTransform != nil {
-		r = constraints.VideoTransform(r)
-	}
-
-	encoderBuilders := make([]encoderBuilder, len(constraints.VideoEncoderBuilders))
-	for i, b := range constraints.VideoEncoderBuilders {
-		encoderBuilders[i].name = b.Name()
-		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
-			return b.BuildVideoEncoder(r, constraints.selectedMedia)
-		}
-	}
-	return encoderBuilders, nil
-}
-
-// newAudioEncoderBuilders transforms audio given by AudioRecorder with the audio transformer that is passed through
-// constraints and create a list of generic encoder builders
-func newAudioEncoderBuilders(ar driver.AudioRecorder, constraints MediaTrackConstraints) ([]encoderBuilder, error) {
-	r, err := ar.AudioRecord(constraints.selectedMedia)
-	if err != nil {
-		return nil, err
-	}
-
-	if constraints.AudioTransform != nil {
-		r = constraints.AudioTransform(r)
-	}
-
-	encoderBuilders := make([]encoderBuilder, len(constraints.AudioEncoderBuilders))
-	for i, b := range constraints.AudioEncoderBuilders {
-		encoderBuilders[i].name = b.Name()
-		encoderBuilders[i].build = func() (codec.ReadCloser, error) {
-			return b.BuildAudioEncoder(r, constraints.selectedMedia)
-		}
-	}
-	return encoderBuilders, nil
 }

track_test.go

@@ -10,7 +10,7 @@ func TestOnEnded(t *testing.T) {
 	errExpected := errors.New("an error")
 
 	t.Run("ErrorAfterRegister", func(t *testing.T) {
-		tr := &track{}
+		tr := &baseTrack{}
 
 		called := make(chan error, 1)
 		tr.OnEnded(func(error) {
@@ -35,7 +35,7 @@ func TestOnEnded(t *testing.T) {
 	})
 
 	t.Run("ErrorBeforeRegister", func(t *testing.T) {
-		tr := &track{}
+		tr := &baseTrack{}
 
 		tr.onError(errExpected)
 

webrtc.go

@@ -0,0 +1,45 @@
+package mediadevices
+
+import (
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v2"
+)
+
+// == WebRTC v3 design ==
+
+// Reader is an interface to handle incoming RTP stream.
+type Reader interface {
+	ReadRTP() (*rtp.Packet, error)
+	WriteRTCP(rtcp.Packet) error
+}
+
+// TrackBase represents common MediaStreamTrack functionality of LocalTrack and RemoteTrack.
+type TrackBase interface {
+	ID() string
+}
+
+type LocalRTPTrack interface {
+	TrackBase
+	Reader
+
+	// SetParameters sets information about how the data is to be encoded.
+	// This will be called by PeerConnection according to the result of
+	// SDP based negotiation.
+	// It will be called via RTPSender.Parameters() by PeerConnection to
+	// tell the negotiated media codec information.
+	//
+	// This is pion's extension to process data without having encoder/decoder
+	// in webrtc package.
+	SetParameters(RTPParameters) error
+}
+
+// RTPParameters represents RTCRtpParameters which contains information about
+// how the RTC data is to be encoded/decoded.
+//
+// ref: https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSendParameters
+type RTPParameters struct {
+	SSRC          uint32
+	SelectedCodec *webrtc.RTPCodec
+	Codecs        []*webrtc.RTPCodec
+}