Fix crash in x264 encoder close

Generated by Renovate Bot

.gitattributes

@@ -1 +0,0 @@
-*.gif filter=lfs diff=lfs merge=lfs -text

README.md

@@ -15,8 +15,6 @@
 
 `mediadevices` provides access to media input devices like cameras, microphones, and screen capture. It can also be used to encode your video/audio stream to various codec selections. `mediadevices` abstracts away the complexities of interacting with things like hardware and codecs allowing you to focus on building appilcations, interacting only with an amazingly simple, easy, and elegant API!
 
-![](img/demo.gif)
-
 ## Install
 
 `go get -u github.com/pion/mediadevices`
@@ -79,6 +77,7 @@ func main() {
 * [Face Detection](/examples/facedetection) - Use a machine learning algorithm to detect faces in a camera stream
 * [RTP Stream](examples/rtp) - Capture camera stream, encode it in H264/VP8/VP9, and send it to a RTP server
 * [HTTP Broadcast](/examples/http) - Broadcast camera stream through HTTP with MJPEG
+* [Archive](/examples/archive) - Archive H264 encoded video stream from a camera
 
 ## Available Media Inputs
 

examples/archive/README.md

@@ -0,0 +1,38 @@
+## Instructions
+
+### Install required codecs
+
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
+
+Installation steps:
+
+* [x264](https://github.com/pion/mediadevices#x264)
+
+### Download archive examplee
+
+```
+git clone https://github.com/pion/mediadevices.git
+```
+
+### Run archive example
+
+Run `cd mediadevices/examples/archive && go build && ./archive recorded.h264`
+
+To stop recording, press `Ctrl+c` or send a SIGINT signal.
+
+### Playback recorded video
+
+Install GStreamer and run:
+```
+gst-launch-1.0 playbin uri=file://${PWD}/recorded.h264
+```
+
+Or run VLC media plyer:
+```
+vlc recorded.h264
+```
+
+A video should start playing in your GStreamer or VLC window.
+
+Congrats, you have used pion-MediaDevices! Now start building something cool
+

examples/archive/main.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/pion/mediadevices"
+	"github.com/pion/mediadevices/pkg/codec/x264"      // This is required to use H264 video encoder
+	_ "github.com/pion/mediadevices/pkg/driver/camera" // This is required to register camera adapter
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("usage: %s <path/to/file.h264>\n", os.Args[0])
+		return
+	}
+	dest := os.Args[1]
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT)
+
+	x264Params, err := x264.NewParams()
+	must(err)
+	x264Params.Preset = x264.PresetMedium
+	x264Params.BitRate = 1_000_000 // 1mbps
+
+	codecSelector := mediadevices.NewCodecSelector(
+		mediadevices.WithVideoEncoders(&x264Params),
+	)
+
+	mediaStream, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
+		Video: func(c *mediadevices.MediaTrackConstraints) {
+			c.FrameFormat = prop.FrameFormat(frame.FormatYUY2)
+			c.Width = prop.Int(640)
+			c.Height = prop.Int(480)
+		},
+		Codec: codecSelector,
+	})
+	must(err)
+
+	videoTrack := mediaStream.GetVideoTracks()[0].(*mediadevices.VideoTrack)
+	defer videoTrack.Close()
+
+	videoTrack.Transform(video.TransformFunc(func(r video.Reader) video.Reader {
+		return video.ReaderFunc(func() (img image.Image, release func(), err error) {
+			// we send io.EOF signal to the encoder reader to stop reading. Therefore, io.Copy
+			// will finish its execution and the program will finish
+			select {
+			case <-sigs:
+				return nil, func() {}, io.EOF
+			default:
+			}
+
+			return r.Read()
+		})
+	}))
+
+	reader, err := videoTrack.NewEncodedReader(x264Params.RTPCodec().Name)
+	must(err)
+	defer reader.Close()
+
+	out, err := os.Create(dest)
+	must(err)
+
+	fmt.Println("Recording... Press Ctrl+c to stop")
+	_, err = io.Copy(out, reader)
+	must(err)
+	fmt.Println("Your video has been recorded to", dest)
+}

examples/rtp/README.md

@@ -2,17 +2,16 @@
 
 ### Install required codecs
 
-In this example, we'll be using x264 and opus as our video and audio codecs. Therefore, we need to make sure that these codecs are installed within our system. 
+In this example, we'll be using x264 as our video codec. Therefore, we need to make sure that these codecs are installed within our system. 
 
 Installation steps:
 
 * [x264](https://github.com/pion/mediadevices#x264)
-* [opus](https://github.com/pion/mediadevices#opus)
 
-### Download rtpexample
+### Download rtp example
 
 ```
-go get github.com/pion/mediadevices/examples/rtp
+git clone https://github.com/pion/mediadevices.git
 ```
 
 ### Listen RTP
@@ -30,7 +29,7 @@ vlc ./h264.sdp
 
 ### Run rtp
 
-Run `rtp localhost:5000`
+Run `cd mediadevices/examples/archive && go build && ./rtp localhost:5000`
 
 A video should start playing in your GStreamer or VLC window.
 It's not WebRTC, but pure RTP.

go.mod

@@ -4,10 +4,10 @@ go 1.13
 
 require (
 	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
-	github.com/gen2brain/malgo v0.10.19
+	github.com/gen2brain/malgo v0.10.24
 	github.com/lherman-cs/opus v0.0.2
 	github.com/pion/logging v0.2.2
-	github.com/pion/rtp v1.6.0
+	github.com/pion/rtp v1.6.1
 	github.com/pion/webrtc/v2 v2.2.26
 	github.com/satori/go.uuid v1.2.0
 	golang.org/x/image v0.0.0-20200927104501-e162460cd6b5

go.sum

@@ -7,8 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gen2brain/malgo v0.10.19 h1:IUVF6WdVV7Txt47Kx2ajz0rWQ0MU0zO+tbcKmhva7l8=
-github.com/gen2brain/malgo v0.10.19/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
+github.com/gen2brain/malgo v0.10.24 h1:q9TFP4lRYpK8UbH3XSa/SNnMwMLUZraRyZt2u+qKYxg=
+github.com/gen2brain/malgo v0.10.24/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -53,6 +53,8 @@ github.com/pion/rtcp v1.2.3 h1:2wrhKnqgSz91Q5nzYTO07mQXztYPtxL8a0XOss4rJqA=
 github.com/pion/rtcp v1.2.3/go.mod h1:zGhIv0RPRF0Z1Wiij22pUt5W/c9fevqSzT4jje/oK7I=
 github.com/pion/rtp v1.6.0 h1:4Ssnl/T5W2LzxHj9ssYpGVEQh3YYhQFNVmSWO88MMwk=
 github.com/pion/rtp v1.6.0/go.mod h1:QgfogHsMBVE/RFNno467U/KBqfUywEH+HK+0rtnwsdI=
+github.com/pion/rtp v1.6.1 h1:2Y2elcVBrahYnHKN2X7rMHX/r1R4TEBMP1LaVu/wNhk=
+github.com/pion/rtp v1.6.1/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.7.10 h1:o3p3/hZB5Cx12RMGyWmItevJtZ6o2cpuxaw6GOS4x+8=
 github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sdp/v2 v2.4.0 h1:luUtaETR5x2KNNpvEMv/r4Y+/kzImzbz4Lm1z8eQNQI=

ioreader.go

@@ -0,0 +1,14 @@
+package mediadevices
+
+type encodedReadCloserImpl struct {
+	readFn  func([]byte) (int, error)
+	closeFn func() error
+}
+
+func (r *encodedReadCloserImpl) Read(b []byte) (int, error) {
+	return r.readFn(b)
+}
+
+func (r *encodedReadCloserImpl) Close() error {
+	return r.closeFn()
+}

mediastream_test.go

@@ -1,6 +1,7 @@
 package mediadevices
 
 import (
+	"io"
 	"testing"
 
 	"github.com/pion/webrtc/v2"
@@ -37,6 +38,10 @@ func (track *mockMediaStreamTrack) NewRTPReader(codecName string, mtu int) (RTPR
 	return nil, nil
 }
 
+func (track *mockMediaStreamTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	return nil, nil
+}
+
 func TestMediaStreamFilters(t *testing.T) {
 	audioTracks := []Track{
 		&mockMediaStreamTrack{AudioInput},

pkg/codec/vpx/vpx.go

@@ -237,10 +237,19 @@ func (e *encoder) Read() ([]byte, func(), error) {
 		e.raw.d_w, e.raw.d_h = C.uint(width), C.uint(height)
 	}
 
+	duration := t - e.tLastFrame
+	// VPX doesn't allow 0 duration. If 0 is given, vpx_codec_encode will fail with VPX_CODEC_INVALID_PARAM.
+	// 0 duration is possible because mediadevices first gets the frame meta data by reading from the source,
+	// and consequently the codec will read the first frame from the buffer. This means the first frame won't
+	// have a pause to the second frame, which means if the delay is <1 ms (vpx duration resolution), duration
+	// is going to be 0.
+	if duration == 0 {
+		duration = 1
+	}
 	var flags int
 	if ec := C.encode_wrapper(
 		e.codec, e.raw,
-		C.long(t-e.tStart), C.ulong(t-e.tLastFrame), C.long(flags), C.ulong(e.deadline),
+		C.long(t-e.tStart), C.ulong(duration), C.long(flags), C.ulong(e.deadline),
 		(*C.uchar)(&yuvImg.Y[0]), (*C.uchar)(&yuvImg.Cb[0]), (*C.uchar)(&yuvImg.Cr[0]),
 	); ec != C.VPX_CODEC_OK {
 		return nil, func() {}, fmt.Errorf("vpx_codec_encode failed (%d)", ec)

pkg/codec/x264/bridge.h

@@ -16,7 +16,7 @@ typedef struct Slice {
 
 typedef struct Encoder {
   x264_t *h;
-  x264_picture_t pic_in, pic_out;
+  x264_picture_t pic_in;
   x264_param_t param;
 } Encoder;
 
@@ -52,15 +52,21 @@ Encoder *enc_new(x264_param_t param, char *preset, int *rc) {
     goto fail;
   }
 
-  if (x264_picture_alloc(&e->pic_in, param.i_csp, param.i_width, param.i_height) < 0) {
+  x264_picture_t pic_in;
+  if (x264_picture_alloc(&pic_in, param.i_csp, param.i_width, param.i_height) < 0) {
     *rc = ERR_ALLOC_PICTURE;
     goto fail;
   }
 
+  // FIXME: we use x264_picture_alloc to set the metadata only, we don't need the allocated memory
+  //        to store the frame. Since we free the frame memory here, we don't need to call
+  //        x264_picture_clean later.
+  e->pic_in = pic_in;
+  x264_picture_clean(&pic_in);
+
   e->h = x264_encoder_open(&e->param);
   if (!e->h) {
     *rc = ERR_OPEN_ENGINE;
-    x264_picture_clean(&e->pic_in);
     goto fail;
   }
 
@@ -75,24 +81,24 @@ Slice enc_encode(Encoder *e, uint8_t *y, uint8_t *cb, uint8_t *cr, int *rc) {
   x264_nal_t *nal;
   int i_nal;
 
+  x264_picture_t pic_out;
   e->pic_in.img.plane[0] = y;
   e->pic_in.img.plane[1] = cb;
   e->pic_in.img.plane[2] = cr;
 
-  int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &e->pic_out);
+  int frame_size = x264_encoder_encode(e->h, &nal, &i_nal, &e->pic_in, &pic_out);
   Slice s = {.data_len = frame_size};
   if (frame_size <= 0) {
     *rc = ERR_ENCODE;
     return s;
   }
 
-  e->pic_in.i_pts++;
+  // e->pic_in.i_pts++;
   s.data = nal->p_payload;
   return s;
 }
 
 void enc_close(Encoder *e, int *rc) {
   x264_encoder_close(e->h);
-  x264_picture_clean(&e->pic_in);
   free(e);
 }

track.go

@@ -3,6 +3,7 @@ package mediadevices
 import (
 	"errors"
 	"image"
+	"io"
 	"math/rand"
 	"sync"
 
@@ -57,6 +58,8 @@ type Track interface {
 	// NewRTPReader creates a new reader from the source. The reader will encode the source, and packetize
 	// the encoded data in RTP format with given mtu size.
 	NewRTPReader(codecName string, mtu int) (RTPReadCloser, error)
+	// NewEncodedReader creates a new Go standard io.ReadCloser that reads the encoded data in codecName format
+	NewEncodedReader(codecName string) (io.ReadCloser, error)
 }
 
 type baseTrack struct {
@@ -182,6 +185,31 @@ func (track *baseTrack) unbind(pc *webrtc.PeerConnection) error {
 	return nil
 }
 
+func (track *baseTrack) newEncodedReader(reader codec.ReadCloser) (io.ReadCloser, error) {
+	var encoded []byte
+	release := func() {}
+	return &encodedReadCloserImpl{
+		readFn: func(b []byte) (int, error) {
+			var err error
+
+			if len(encoded) == 0 {
+				release()
+				encoded, release, err = reader.Read()
+				if err != nil {
+					reader.Close()
+					track.onError(err)
+					return 0, err
+				}
+			}
+
+			n := copy(b, encoded)
+			encoded = encoded[n:]
+			return n, nil
+		},
+		closeFn: reader.Close,
+	}, nil
+}
+
 func newTrackFromDriver(d driver.Driver, constraints MediaTrackConstraints, selector *CodecSelector) (Track, error) {
 	if err := d.Open(); err != nil {
 		return nil, err
@@ -298,6 +326,21 @@ func (track *VideoTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser,
 	}, nil
 }
 
+func (track *VideoTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentVideoProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, _, err := track.selector.selectVideoCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	return track.newEncodedReader(encodedReader)
+}
+
 // AudioTrack is a specific track type that contains audio source which allows multiple readers to access, and
 // manipulate.
 type AudioTrack struct {
@@ -380,7 +423,7 @@ func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser,
 		return nil, err
 	}
 
-	sample := newVideoSampler(selectedCodec.ClockRate)
+	sample := newAudioSampler(selectedCodec.ClockRate, inputProp.Latency)
 
 	// FIXME: not sure the best way to get unique ssrc. We probably should have a global keeper that can generate a random ID and does book keeping?
 	packetizer := rtp.NewPacketizer(mtu, selectedCodec.PayloadType, rand.Uint32(), selectedCodec.Payloader, rtp.NewRandomSequencer(), selectedCodec.ClockRate)
@@ -402,3 +445,18 @@ func (track *AudioTrack) NewRTPReader(codecName string, mtu int) (RTPReadCloser,
 		closeFn: encodedReader.Close,
 	}, nil
 }
+
+func (track *AudioTrack) NewEncodedReader(codecName string) (io.ReadCloser, error) {
+	reader := track.NewReader(false)
+	inputProp, err := detectCurrentAudioProp(track.Broadcaster)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedReader, _, err := track.selector.selectAudioCodecByNames(reader, inputProp, codecName)
+	if err != nil {
+		return nil, err
+	}
+
+	return track.newEncodedReader(encodedReader)
+}