diff --git a/examples/transform/README.md b/examples/transform/README.md new file mode 100644 index 0000000..6d7bd34 --- /dev/null +++ b/examples/transform/README.md @@ -0,0 +1,29 @@ +## Instructions + +### Download transform example + +``` +go get github.com/pion/mediadevices/examples/transform +``` + +### Open example page + +[jsfiddle.net](https://jsfiddle.net/z7ms3u5r/) you should see two text-areas and a 'Start Session' button + +### Run transform with your browsers SessionDescription as stdin + +In the jsfiddle the top textarea is your browser, copy that and: + +#### Linux + +Run `echo $BROWSER_SDP | transform` + +### Input transform's SessionDescription into your browser + +Copy the text that `transform` just emitted and copy into second text area + +### Hit 'Start Session' in jsfiddle, enjoy your video! + +A video should start playing in your browser above the input boxes, and will continue playing until you close the application. + +Congrats, you have used pion-WebRTC! Now start building something cool diff --git a/examples/transform/main.go b/examples/transform/main.go new file mode 100644 index 0000000..fdfb79f --- /dev/null +++ b/examples/transform/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "fmt" + "image" + + "github.com/pion/mediadevices" + "github.com/pion/mediadevices/examples/internal/signal" + _ "github.com/pion/mediadevices/pkg/codec/openh264" // This is required to register h264 video encoder + _ "github.com/pion/mediadevices/pkg/codec/opus" // This is required to register opus audio encoder + _ "github.com/pion/mediadevices/pkg/codec/vpx" + "github.com/pion/mediadevices/pkg/frame" + "github.com/pion/mediadevices/pkg/io/video" + "github.com/pion/webrtc/v2" +) + +const ( + videoCodecName = webrtc.VP8 +) + +func removeBlue(r video.Reader) video.Reader { + return video.ReaderFunc(func() (img image.Image, err error) { + img, err = r.Read() + if err != nil { + return + } + + yuvImg, ok := img.(*image.YCbCr) + if !ok { + return img, nil + } + + yuvImg.Cb = make([]uint8, len(yuvImg.Cb)) + return + }) +} + +func main() { + config := webrtc.Configuration{ + ICEServers: []webrtc.ICEServer{ + { + URLs: []string{"stun:stun.l.google.com:19302"}, + }, + }, + } + + // Wait for the offer to be pasted + offer := webrtc.SessionDescription{} + signal.Decode(signal.MustReadStdin(), &offer) + + // Create a new RTCPeerConnection + mediaEngine := webrtc.MediaEngine{} + if err := mediaEngine.PopulateFromSDP(offer); err != nil { + panic(err) + } + api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine)) + peerConnection, err := api.NewPeerConnection(config) + if err != nil { + panic(err) + } + + // Set the handler for ICE connection state + // This will notify you when the peer has connected/disconnected + peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + fmt.Printf("Connection State has changed %s \n", connectionState.String()) + }) + + md := mediadevices.NewMediaDevices(peerConnection) + + s, err := md.GetUserMedia(mediadevices.MediaStreamConstraints{ + Audio: func(c *mediadevices.MediaTrackConstraints) { + c.CodecName = webrtc.Opus + c.Enabled = true + c.BitRate = 32000 // 32kbps + }, + Video: func(c *mediadevices.MediaTrackConstraints) { + c.CodecName = videoCodecName + c.FrameFormat = frame.FormatI420 // most of the encoder accepts I420 + c.Enabled = true + c.Width = 640 + c.Height = 480 + c.BitRate = 100000 // 100kbps + c.VideoTransform = removeBlue + }, + }) + if err != nil { + panic(err) + } + + for _, tracker := range s.GetTracks() { + t := tracker.Track() + tracker.OnEnded(func(err error) { + fmt.Printf("Track (ID: %s, Label: %s) ended with error: %v\n", + t.ID(), t.Label(), err) + }) + _, err = peerConnection.AddTrack(t) + if err != nil { + panic(err) + } + } + + // Tweak transceiver direction to work with Firefox + for _, t := range peerConnection.GetTransceivers() { + t.Direction = webrtc.RTPTransceiverDirectionSendonly + } + + // Set the remote SessionDescription + err = peerConnection.SetRemoteDescription(offer) + if err != nil { + panic(err) + } + + // Create an answer + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + + // Sets the LocalDescription, and starts our UDP listeners + err = peerConnection.SetLocalDescription(answer) + if err != nil { + panic(err) + } + + // Output the answer in base64 so we can paste it in browser + fmt.Println(signal.Encode(answer)) + select {} +} diff --git a/mediadevices.go b/mediadevices.go index 5aaaecd..b5ea8e6 100644 --- a/mediadevices.go +++ b/mediadevices.go @@ -135,8 +135,10 @@ func selectBestDriver(filter driver.FilterFn, constraints MediaTrackConstraints) // Reset Codec because bestProp only contains either audio.Prop or video.Prop bestProp.Codec = constraints.Codec bestConstraint := MediaTrackConstraints{ - Media: bestProp, - Enabled: true, + Media: bestProp, + Enabled: true, + AudioTransform: constraints.AudioTransform, + VideoTransform: constraints.VideoTransform, } return bestDriver, bestConstraint, nil } diff --git a/mediastreamconstraints.go b/mediastreamconstraints.go index 548e92a..a91be4b 100644 --- a/mediastreamconstraints.go +++ b/mediastreamconstraints.go @@ -1,6 +1,8 @@ package mediadevices import ( + "github.com/pion/mediadevices/pkg/io/audio" + "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" ) @@ -13,6 +15,12 @@ type MediaStreamConstraints struct { type MediaTrackConstraints struct { prop.Media Enabled bool + // VideoTransform will be used to transform the video that's coming from the driver. + // So, basically it'll look like following: driver -> VideoTransform -> codec + VideoTransform video.TransformFunc + // AudioTransform will be used to transform the audio that's coming from the driver. + // So, basically it'll look like following: driver -> AudioTransform -> code + AudioTransform audio.TransformFunc } type MediaOption func(*MediaTrackConstraints) diff --git a/pkg/io/audio/audio.go b/pkg/io/audio/audio.go index 49eca98..5f27fcd 100644 --- a/pkg/io/audio/audio.go +++ b/pkg/io/audio/audio.go @@ -9,3 +9,22 @@ type ReaderFunc func(samples [][2]float32) (n int, err error) func (rf ReaderFunc) Read(samples [][2]float32) (n int, err error) { return rf(samples) } + +// TransformFunc produces a new Reader that will produces a transformed audio +type TransformFunc func(r Reader) Reader + +// Merge merges transforms and produces a new TransformFunc that will execute +// transforms in order +func Merge(transforms ...TransformFunc) TransformFunc { + return func(r Reader) Reader { + for _, transform := range transforms { + if transform == nil { + continue + } + + r = transform(r) + } + + return r + } +} diff --git a/pkg/io/video/video.go b/pkg/io/video/video.go index d51dace..5148087 100644 --- a/pkg/io/video/video.go +++ b/pkg/io/video/video.go @@ -1,6 +1,8 @@ package video -import "image" +import ( + "image" +) type Reader interface { Read() (img image.Image, err error) @@ -11,3 +13,22 @@ type ReaderFunc func() (img image.Image, err error) func (rf ReaderFunc) Read() (img image.Image, err error) { return rf() } + +// TransformFunc produces a new Reader that will produces a transformed video +type TransformFunc func(r Reader) Reader + +// Merge merges transforms and produces a new TransformFunc that will execute +// transforms in order +func Merge(transforms ...TransformFunc) TransformFunc { + return func(r Reader) Reader { + for _, transform := range transforms { + if transform == nil { + continue + } + + r = transform(r) + } + + return r + } +} diff --git a/track.go b/track.go index c9adb5e..3b5e1e4 100644 --- a/track.go +++ b/track.go @@ -99,6 +99,10 @@ func newVideoTrack(codecs []*webrtc.RTPCodec, d driver.Driver, constraints Media return nil, err } + if constraints.VideoTransform != nil { + r = constraints.VideoTransform(r) + } + encoder, err := codec.BuildVideoEncoder(r, constraints.Media) if err != nil { return nil, err @@ -170,6 +174,10 @@ func newAudioTrack(codecs []*webrtc.RTPCodec, d driver.Driver, constraints Media return nil, err } + if constraints.AudioTransform != nil { + reader = constraints.AudioTransform(reader) + } + encoder, err := codec.BuildAudioEncoder(reader, constraints.Media) if err != nil { return nil, err