update VP8/VP9 examples to decode/encode frames (#704)

README.md

@@ -67,17 +67,17 @@ Features:
 * [client-play-format-g722](examples/client-play-format-g722/main.go)
 * [client-play-format-h264](examples/client-play-format-h264/main.go)
 * [client-play-format-h264-to-jpeg](examples/client-play-format-h264-to-jpeg/main.go)
-* [client-play-format-h264-save-to-disk](examples/client-play-format-h264-save-to-disk/main.go)
-* [client-play-format-h264-mpeg4audio-save-to-disk](examples/client-play-format-h264-mpeg4audio-save-to-disk/main.go)
+* [client-play-format-h264-to-disk](examples/client-play-format-h264-to-disk/main.go)
+* [client-play-format-h264-mpeg4audio-to-disk](examples/client-play-format-h264-mpeg4audio-to-disk/main.go)
 * [client-play-format-h265](examples/client-play-format-h265/main.go)
 * [client-play-format-h265-to-jpeg](examples/client-play-format-h265-to-jpeg/main.go)
-* [client-play-format-h265-save-to-disk](examples/client-play-format-h265-save-to-disk/main.go)
+* [client-play-format-h265-to-disk](examples/client-play-format-h265-to-disk/main.go)
 * [client-play-format-lpcm](examples/client-play-format-lpcm/main.go)
 * [client-play-format-mjpeg](examples/client-play-format-mjpeg/main.go)
 * [client-play-format-mpeg4audio](examples/client-play-format-mpeg4audio/main.go)
-* [client-play-format-mpeg4audio-save-to-disk](examples/client-play-format-mpeg4audio-save-to-disk/main.go)
+* [client-play-format-mpeg4audio-to-disk](examples/client-play-format-mpeg4audio-to-disk/main.go)
 * [client-play-format-opus](examples/client-play-format-opus/main.go)
-* [client-play-format-opus-save-to-disk](examples/client-play-format-opus-save-to-disk/main.go)
+* [client-play-format-opus-to-disk](examples/client-play-format-opus-to-disk/main.go)
 * [client-play-format-vp8](examples/client-play-format-vp8/main.go)
 * [client-play-format-vp9](examples/client-play-format-vp9/main.go)
 * [client-record-options](examples/client-record-options/main.go)
@@ -97,7 +97,7 @@ Features:
 * [server](examples/server/main.go)
 * [server-tls](examples/server-tls/main.go)
 * [server-auth](examples/server-auth/main.go)
-* [server-h264-save-to-disk](examples/server-h264-save-to-disk/main.go)
+* [server-h264-to-disk](examples/server-h264-to-disk/main.go)
 * [proxy](examples/proxy/main.go)
 
 ## API Documentation

examples/client-play-format-mjpeg/main.go

@@ -16,7 +16,7 @@ import (
 // 1. connect to a RTSP server
 // 2. check if there's a M-JPEG format
 // 3. get JPEG images of that format
-// 4. decode JPEG images into raw images
+// 4. decode JPEG images into RGBA frames
 
 func main() {
 	c := gortsplib.Client{}
@@ -77,7 +77,7 @@ func main() {
 			return
 		}
 
-		// convert JPEG images into raw images
+		// convert JPEG images into RGBA frames
 		image, err := jpeg.Decode(bytes.NewReader(enc))
 		if err != nil {
 			panic(err)

examples/client-play-format-vp8/main.go

@@ -1,3 +1,5 @@
+//go:build cgo
+
 package main
 
 import (
@@ -12,8 +14,11 @@ import (
 
 // This example shows how to
 // 1. connect to a RTSP server
-// 2. check if there's a VP8 format
-// 3. get access units of that format
+// 2. check if there's an VP8 format
+// 3. decode the VP8 stream into RGBA frames
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
 
 func main() {
 	c := gortsplib.Client{}
@@ -44,12 +49,20 @@ func main() {
 		panic("media not found")
 	}
 
-	// create decoder
+	// setup RTP -> VP8 decoder
 	rtpDec, err := forma.CreateDecoder()
 	if err != nil {
 		panic(err)
 	}
 
+	// setup VP8 -> RGBA decoder
+	vp8Dec := &vp8Decoder{}
+	err = vp8Dec.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer vp8Dec.close()
+
 	// setup a single media
 	_, err = c.Setup(desc.BaseURL, medi, 0, 0)
 	if err != nil {
@@ -65,8 +78,8 @@ func main() {
 			return
 		}
 
-		// extract VP8 frames from RTP packets
-		vf, err := rtpDec.Decode(pkt)
+		// extract access units from RTP packets
+		au, err := rtpDec.Decode(pkt)
 		if err != nil {
 			if err != rtpvp8.ErrNonStartingPacketAndNoPrevious && err != rtpvp8.ErrMorePacketsNeeded {
 				log.Printf("ERR: %v", err)
@@ -74,7 +87,18 @@ func main() {
 			return
 		}
 
-		log.Printf("received frame with PTS %v size %d\n", pts, len(vf))
+		// convert VP8 access units into RGBA frames
+		img, err := vp8Dec.decode(au)
+		if err != nil {
+			panic(err)
+		}
+
+		// wait for a frame
+		if img == nil {
+			return
+		}
+
+		log.Printf("decoded frame with PTS %v and size %v", pts, img.Bounds().Max)
 	})
 
 	// start playing

examples/client-play-format-vp8/vp8_decoder.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"runtime"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libavutil/imgutils.h>
+// #include <libswscale/swscale.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// vp8Decoder is a wrapper around FFmpeg's VP8 decoder.
+type vp8Decoder struct {
+	codecCtx     *C.AVCodecContext
+	yuv420Frame  *C.AVFrame
+	rgbaFrame    *C.AVFrame
+	rgbaFramePtr []uint8
+	swsCtx       *C.struct_SwsContext
+}
+
+// initialize initializes a vp8Decoder.
+func (d *vp8Decoder) initialize() error {
+	codec := C.avcodec_find_decoder(C.AV_CODEC_ID_VP8)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_decoder() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.yuv420Frame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *vp8Decoder) close() {
+	if d.swsCtx != nil {
+		C.sws_freeContext(d.swsCtx)
+	}
+
+	if d.rgbaFrame != nil {
+		C.av_frame_free(&d.rgbaFrame)
+	}
+
+	C.av_frame_free(&d.yuv420Frame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// decode decodes a RGBA image from VP8.
+func (d *vp8Decoder) decode(au []byte) (*image.RGBA, error) {
+	// send access unit to decoder
+	var pkt C.AVPacket
+	ptr := &au[0]
+	var p runtime.Pinner
+	p.Pin(ptr)
+	pkt.data = (*C.uint8_t)(ptr)
+	pkt.size = (C.int)(len(au))
+	res := C.avcodec_send_packet(d.codecCtx, &pkt)
+	p.Unpin()
+	if res < 0 {
+		return nil, nil
+	}
+
+	// receive frame if available
+	res = C.avcodec_receive_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// if frame size has changed, allocate needed objects
+	if d.rgbaFrame == nil || d.rgbaFrame.width != d.yuv420Frame.width || d.rgbaFrame.height != d.yuv420Frame.height {
+		if d.swsCtx != nil {
+			C.sws_freeContext(d.swsCtx)
+		}
+
+		if d.rgbaFrame != nil {
+			C.av_frame_free(&d.rgbaFrame)
+		}
+
+		d.rgbaFrame = C.av_frame_alloc()
+		if d.rgbaFrame == nil {
+			return nil, fmt.Errorf("av_frame_alloc() failed")
+		}
+
+		d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+		d.rgbaFrame.width = d.yuv420Frame.width
+		d.rgbaFrame.height = d.yuv420Frame.height
+		d.rgbaFrame.color_range = C.AVCOL_RANGE_JPEG
+
+		res = C.av_frame_get_buffer(d.rgbaFrame, 1)
+		if res < 0 {
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
+		}
+
+		d.swsCtx = C.sws_getContext(d.yuv420Frame.width, d.yuv420Frame.height, int32(d.yuv420Frame.format),
+			d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format), C.SWS_BILINEAR, nil, nil, nil)
+		if d.swsCtx == nil {
+			return nil, fmt.Errorf("sws_getContext() failed")
+		}
+
+		rgbaFrameSize := C.av_image_get_buffer_size((int32)(d.rgbaFrame.format), d.rgbaFrame.width, d.rgbaFrame.height, 1)
+		d.rgbaFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.rgbaFrame.data[0]))[:rgbaFrameSize:rgbaFrameSize]
+	}
+
+	// convert color space from YUV420 to RGBA
+	res = C.sws_scale(d.swsCtx, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame),
+		0, d.yuv420Frame.height, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame))
+	if res < 0 {
+		return nil, fmt.Errorf("sws_scale() failed")
+	}
+
+	// embed frame into an image.RGBA
+	return &image.RGBA{
+		Pix:    d.rgbaFramePtr,
+		Stride: 4 * (int)(d.rgbaFrame.width),
+		Rect: image.Rectangle{
+			Max: image.Point{(int)(d.rgbaFrame.width), (int)(d.rgbaFrame.height)},
+		},
+	}, nil
+}

examples/client-play-format-vp9/main.go

@@ -1,3 +1,5 @@
+//go:build cgo
+
 package main
 
 import (
@@ -12,8 +14,11 @@ import (
 
 // This example shows how to
 // 1. connect to a RTSP server
-// 2. check if there's a VP9 format
-// 3. get access units of that format
+// 2. check if there's an VP9 format
+// 3. decode the VP9 stream into RGBA frames
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
 
 func main() {
 	c := gortsplib.Client{}
@@ -44,12 +49,20 @@ func main() {
 		panic("media not found")
 	}
 
-	// create decoder
+	// setup RTP -> VP9 decoder
 	rtpDec, err := forma.CreateDecoder()
 	if err != nil {
 		panic(err)
 	}
 
+	// setup VP9 -> RGBA decoder
+	vp9Dec := &vp9Decoder{}
+	err = vp9Dec.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer vp9Dec.close()
+
 	// setup a single media
 	_, err = c.Setup(desc.BaseURL, medi, 0, 0)
 	if err != nil {
@@ -65,8 +78,8 @@ func main() {
 			return
 		}
 
-		// extract VP9 frames from RTP packets
-		vf, err := rtpDec.Decode(pkt)
+		// extract access units from RTP packets
+		au, err := rtpDec.Decode(pkt)
 		if err != nil {
 			if err != rtpvp9.ErrNonStartingPacketAndNoPrevious && err != rtpvp9.ErrMorePacketsNeeded {
 				log.Printf("ERR: %v", err)
@@ -74,7 +87,18 @@ func main() {
 			return
 		}
 
-		log.Printf("received frame with PTS %v and size %d\n", pts, len(vf))
+		// convert VP9 access units into RGBA frames
+		img, err := vp9Dec.decode(au)
+		if err != nil {
+			panic(err)
+		}
+
+		// wait for a frame
+		if img == nil {
+			return
+		}
+
+		log.Printf("decoded frame with PTS %v and size %v", pts, img.Bounds().Max)
 	})
 
 	// start playing

examples/client-play-format-vp9/vp9_decoder.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"runtime"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libavutil/imgutils.h>
+// #include <libswscale/swscale.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// vp9Decoder is a wrapper around FFmpeg's VP9 decoder.
+type vp9Decoder struct {
+	codecCtx     *C.AVCodecContext
+	yuv420Frame  *C.AVFrame
+	rgbaFrame    *C.AVFrame
+	rgbaFramePtr []uint8
+	swsCtx       *C.struct_SwsContext
+}
+
+// initialize initializes a vp9Decoder.
+func (d *vp9Decoder) initialize() error {
+	codec := C.avcodec_find_decoder(C.AV_CODEC_ID_VP9)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_decoder() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.yuv420Frame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *vp9Decoder) close() {
+	if d.swsCtx != nil {
+		C.sws_freeContext(d.swsCtx)
+	}
+
+	if d.rgbaFrame != nil {
+		C.av_frame_free(&d.rgbaFrame)
+	}
+
+	C.av_frame_free(&d.yuv420Frame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// decode decodes a RGBA image from VP9.
+func (d *vp9Decoder) decode(au []byte) (*image.RGBA, error) {
+	// send access unit to decoder
+	var pkt C.AVPacket
+	ptr := &au[0]
+	var p runtime.Pinner
+	p.Pin(ptr)
+	pkt.data = (*C.uint8_t)(ptr)
+	pkt.size = (C.int)(len(au))
+	res := C.avcodec_send_packet(d.codecCtx, &pkt)
+	p.Unpin()
+	if res < 0 {
+		return nil, nil
+	}
+
+	// receive frame if available
+	res = C.avcodec_receive_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// if frame size has changed, allocate needed objects
+	if d.rgbaFrame == nil || d.rgbaFrame.width != d.yuv420Frame.width || d.rgbaFrame.height != d.yuv420Frame.height {
+		if d.swsCtx != nil {
+			C.sws_freeContext(d.swsCtx)
+		}
+
+		if d.rgbaFrame != nil {
+			C.av_frame_free(&d.rgbaFrame)
+		}
+
+		d.rgbaFrame = C.av_frame_alloc()
+		if d.rgbaFrame == nil {
+			return nil, fmt.Errorf("av_frame_alloc() failed")
+		}
+
+		d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+		d.rgbaFrame.width = d.yuv420Frame.width
+		d.rgbaFrame.height = d.yuv420Frame.height
+		d.rgbaFrame.color_range = C.AVCOL_RANGE_JPEG
+
+		res = C.av_frame_get_buffer(d.rgbaFrame, 1)
+		if res < 0 {
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
+		}
+
+		d.swsCtx = C.sws_getContext(d.yuv420Frame.width, d.yuv420Frame.height, int32(d.yuv420Frame.format),
+			d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format), C.SWS_BILINEAR, nil, nil, nil)
+		if d.swsCtx == nil {
+			return nil, fmt.Errorf("sws_getContext() failed")
+		}
+
+		rgbaFrameSize := C.av_image_get_buffer_size((int32)(d.rgbaFrame.format), d.rgbaFrame.width, d.rgbaFrame.height, 1)
+		d.rgbaFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.rgbaFrame.data[0]))[:rgbaFrameSize:rgbaFrameSize]
+	}
+
+	// convert color space from YUV420 to RGBA
+	res = C.sws_scale(d.swsCtx, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame),
+		0, d.yuv420Frame.height, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame))
+	if res < 0 {
+		return nil, fmt.Errorf("sws_scale() failed")
+	}
+
+	// embed frame into an image.RGBA
+	return &image.RGBA{
+		Pix:    d.rgbaFramePtr,
+		Stride: 4 * (int)(d.rgbaFrame.width),
+		Rect: image.Rectangle{
+			Max: image.Point{(int)(d.rgbaFrame.width), (int)(d.rgbaFrame.height)},
+		},
+	}, nil
+}

examples/client-record-format-av1/av1_encoder.go

@@ -67,8 +67,8 @@ func (d *av1Encoder) initialize() error {
 	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
 
 	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
-	d.codecCtx.width = (C.int)(d.Height)
-	d.codecCtx.height = (C.int)(d.Width)
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
 	d.codecCtx.time_base.num = 1
 	d.codecCtx.time_base.den = (C.int)(d.FPS)
 	d.codecCtx.gop_size = 10

examples/client-record-format-h264/h264_encoder.go

@@ -60,8 +60,8 @@ func (d *h264Encoder) initialize() error {
 	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
 
 	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
-	d.codecCtx.width = (C.int)(d.Height)
-	d.codecCtx.height = (C.int)(d.Width)
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
 	d.codecCtx.time_base.num = 1
 	d.codecCtx.time_base.den = (C.int)(d.FPS)
 	d.codecCtx.gop_size = 10

examples/client-record-format-h265/h265_encoder.go

@@ -60,8 +60,8 @@ func (d *h265Encoder) initialize() error {
 	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
 
 	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
-	d.codecCtx.width = (C.int)(d.Height)
-	d.codecCtx.height = (C.int)(d.Width)
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
 	d.codecCtx.time_base.num = 1
 	d.codecCtx.time_base.den = (C.int)(d.FPS)
 	d.codecCtx.gop_size = 10

examples/client-record-format-vp8/main.go

@@ -1,77 +1,152 @@
+//go:build cgo
+
 package main
 
 import (
+	"crypto/rand"
+	"image"
+	"image/color"
 	"log"
-	"net"
+	"time"
 
 	"github.com/bluenviron/gortsplib/v4"
 	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
-	"github.com/pion/rtp"
 )
 
 // This example shows how to
-// 1. generate a VP8 stream and RTP packets with GStreamer
-// 2. connect to a RTSP server, announce a VP8 format
-// 3. route the packets from GStreamer to the server
+// 1. connect to a RTSP server, announce a VP8 format
+// 2. generate dummy RGBA images
+// 3. encode images with VP8
+// 4. generate RTP packets from VP8
+// 5. write RTP packets to the server
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
+
+func multiplyAndDivide(v, m, d int64) int64 {
+	secs := v / d
+	dec := v % d
+	return (secs*m + dec*m/d)
+}
+
+func randUint32() (uint32, error) {
+	var b [4]byte
+	_, err := rand.Read(b[:])
+	if err != nil {
+		return 0, err
+	}
+	return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]), nil
+}
+
+func createDummyImage(i int) *image.RGBA {
+	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
+
+	var cl color.RGBA
+	switch i {
+	case 0:
+		cl = color.RGBA{255, 0, 0, 0}
+	case 1:
+		cl = color.RGBA{0, 255, 0, 0}
+	case 2:
+		cl = color.RGBA{0, 0, 255, 0}
+	}
+
+	for y := 0; y < img.Rect.Dy(); y++ {
+		for x := 0; x < img.Rect.Dx(); x++ {
+			img.SetRGBA(x, y, cl)
+		}
+	}
+
+	return img
+}
 
 func main() {
-	// open a listener to receive RTP/VP8 packets
-	pc, err := net.ListenPacket("udp", "localhost:9000")
-	if err != nil {
-		panic(err)
+	// create a stream description that contains a VP8 format
+	forma := &format.VP8{
+		PayloadTyp: 96,
 	}
-	defer pc.Close()
-
-	log.Println("Waiting for a RTP/VP8 stream on UDP port 9000 - you can send one with GStreamer:\n" +
-		"gst-launch-1.0 videotestsrc ! video/x-raw,width=1920,height=1080" +
-		" ! vp8enc cpu-used=8 deadline=1" +
-		" ! rtpvp8pay ! udpsink host=127.0.0.1 port=9000")
-
-	// wait for first packet
-	buf := make([]byte, 2048)
-	n, _, err := pc.ReadFrom(buf)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("stream connected")
-
-	// create a description that contains a VP8 format
 	desc := &description.Session{
 		Medias: []*description.Media{{
-			Type: description.MediaTypeVideo,
-			Formats: []format.Format{&format.VP8{
-				PayloadTyp: 96,
-			}},
+			Type:    description.MediaTypeVideo,
+			Formats: []format.Format{forma},
 		}},
 	}
 
-	// connect to the server and start recording
+	// connect to the server, announce the format and start recording
 	c := gortsplib.Client{}
-	err = c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
+	err := c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
 	if err != nil {
 		panic(err)
 	}
 	defer c.Close()
 
-	var pkt rtp.Packet
-	for {
-		// parse RTP packet
-		err = pkt.Unmarshal(buf[:n])
+	// setup RGBA -> VP8 encoder
+	vp8enc := &vp8Encoder{
+		Width:  640,
+		Height: 480,
+		FPS:    5,
+	}
+	err = vp8enc.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer vp8enc.close()
+
+	// setup VP8 -> RTP encoder
+	rtpEnc, err := forma.CreateEncoder()
+	if err != nil {
+		panic(err)
+	}
+
+	start := time.Now()
+
+	randomStart, err := randUint32()
+	if err != nil {
+		panic(err)
+	}
+
+	// setup a ticker to sleep between frames
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
+	i := 0
+
+	for range ticker.C {
+		// create a dummy image
+		img := createDummyImage(i)
+		i = (i + 1) % 3
+
+		// get current timestamp
+		pts := multiplyAndDivide(int64(time.Since(start)), int64(forma.ClockRate()), int64(time.Second))
+
+		// encode the image with VP8
+		au, pts, err := vp8enc.encode(img, pts)
 		if err != nil {
 			panic(err)
 		}
 
-		// route RTP packet to the server
-		err = c.WritePacketRTP(desc.Medias[0], &pkt)
+		// wait for a VP8 access unit
+		if au == nil {
+			continue
+		}
+
+		// generate RTP packets from the VP8 access unit
+		pkts, err := rtpEnc.Encode(au)
 		if err != nil {
 			panic(err)
 		}
 
-		// read another RTP packet from source
-		n, _, err = pc.ReadFrom(buf)
-		if err != nil {
-			panic(err)
+		log.Printf("writing RTP packets with PTS=%d, au=%d, pkts=%d", pts, len(au), len(pkts))
+
+		// write RTP packets to the server
+		for _, pkt := range pkts {
+			pkt.Timestamp = uint32(int64(randomStart) + pts)
+
+			err = c.WritePacketRTP(desc.Medias[0], pkt)
+			if err != nil {
+				panic(err)
+			}
 		}
 	}
 }

examples/client-record-format-vp8/vp8_encoder.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libswscale/swscale.h>
+// #include <libavutil/opt.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// vp8Encoder is a wrapper around FFmpeg's VP8 encoder.
+type vp8Encoder struct {
+	Width  int
+	Height int
+	FPS    int
+
+	codecCtx    *C.AVCodecContext
+	rgbaFrame   *C.AVFrame
+	yuv420Frame *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	pkt         *C.AVPacket
+}
+
+// initialize initializes a vp8Encoder.
+func (d *vp8Encoder) initialize() error {
+	codec := C.avcodec_find_encoder(C.AV_CODEC_ID_VP8)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_encoder_by_name() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
+	d.codecCtx.time_base.num = 1
+	d.codecCtx.time_base.den = (C.int)(d.FPS)
+	d.codecCtx.gop_size = 10
+	d.codecCtx.max_b_frames = 0
+	d.codecCtx.bit_rate = 600000
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.rgbaFrame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+	d.rgbaFrame.width = d.codecCtx.width
+	d.rgbaFrame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.rgbaFrame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.yuv420Frame.format = C.AV_PIX_FMT_YUV420P
+	d.yuv420Frame.width = d.codecCtx.width
+	d.yuv420Frame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.yuv420Frame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.swsCtx = C.sws_getContext(d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format),
+		d.yuv420Frame.width, d.yuv420Frame.height, (int32)(d.yuv420Frame.format), C.SWS_BILINEAR, nil, nil, nil)
+	if d.swsCtx == nil {
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("sws_getContext() failed")
+	}
+
+	d.pkt = C.av_packet_alloc()
+	if d.pkt == nil {
+		C.av_packet_free(&d.pkt)
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_packet_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *vp8Encoder) close() {
+	C.av_packet_free(&d.pkt)
+	C.sws_freeContext(d.swsCtx)
+	C.av_frame_free(&d.yuv420Frame)
+	C.av_frame_free(&d.rgbaFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// encode encodes a RGBA image into VP8.
+func (d *vp8Encoder) encode(img *image.RGBA, pts int64) ([]byte, int64, error) {
+	// pass image pointer to frame
+	d.rgbaFrame.data[0] = (*C.uint8_t)(&img.Pix[0])
+
+	// convert color space from RGBA to YUV420
+	res := C.sws_scale(d.swsCtx, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame),
+		0, d.rgbaFrame.height, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame))
+	if res < 0 {
+		return nil, 0, fmt.Errorf("sws_scale() failed")
+	}
+
+	// send frame to the encoder
+	d.yuv420Frame.pts = (C.int64_t)(pts)
+	res = C.avcodec_send_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_send_frame() failed")
+	}
+
+	// wait for result
+	res = C.avcodec_receive_packet(d.codecCtx, d.pkt)
+	if res == -C.EAGAIN {
+		return nil, 0, nil
+	}
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_receive_packet() failed")
+	}
+
+	// perform a deep copy of the data before unreferencing the packet
+	data := C.GoBytes(unsafe.Pointer(d.pkt.data), d.pkt.size)
+	pts = (int64)(d.pkt.pts)
+	C.av_packet_unref(d.pkt)
+
+	return data, pts, nil
+}

examples/client-record-format-vp9/main.go

@@ -1,77 +1,152 @@
+//go:build cgo
+
 package main
 
 import (
+	"crypto/rand"
+	"image"
+	"image/color"
 	"log"
-	"net"
+	"time"
 
 	"github.com/bluenviron/gortsplib/v4"
 	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
-	"github.com/pion/rtp"
 )
 
 // This example shows how to
-// 1. generate a VP9 stream and RTP packets with GStreamer
-// 2. connect to a RTSP server, announce a VP9 format
-// 3. route the packets from GStreamer to the server
+// 1. connect to a RTSP server, announce a VP9 format
+// 2. generate dummy RGBA images
+// 3. encode images with VP9
+// 4. generate RTP packets from VP9
+// 5. write RTP packets to the server
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
+
+func multiplyAndDivide(v, m, d int64) int64 {
+	secs := v / d
+	dec := v % d
+	return (secs*m + dec*m/d)
+}
+
+func randUint32() (uint32, error) {
+	var b [4]byte
+	_, err := rand.Read(b[:])
+	if err != nil {
+		return 0, err
+	}
+	return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]), nil
+}
+
+func createDummyImage(i int) *image.RGBA {
+	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
+
+	var cl color.RGBA
+	switch i {
+	case 0:
+		cl = color.RGBA{255, 0, 0, 0}
+	case 1:
+		cl = color.RGBA{0, 255, 0, 0}
+	case 2:
+		cl = color.RGBA{0, 0, 255, 0}
+	}
+
+	for y := 0; y < img.Rect.Dy(); y++ {
+		for x := 0; x < img.Rect.Dx(); x++ {
+			img.SetRGBA(x, y, cl)
+		}
+	}
+
+	return img
+}
 
 func main() {
-	// open a listener to receive RTP/VP9 packets
-	pc, err := net.ListenPacket("udp", "localhost:9000")
-	if err != nil {
-		panic(err)
+	// create a stream description that contains a VP9 format
+	forma := &format.VP9{
+		PayloadTyp: 96,
 	}
-	defer pc.Close()
-
-	log.Println("Waiting for a RTP/VP9 stream on UDP port 9000 - you can send one with GStreamer:\n" +
-		"gst-launch-1.0 videotestsrc ! video/x-raw,width=1920,height=1080" +
-		" ! vp9enc cpu-used=8 deadline=1" +
-		" ! rtpvp9pay ! udpsink host=127.0.0.1 port=9000")
-
-	// wait for first packet
-	buf := make([]byte, 2048)
-	n, _, err := pc.ReadFrom(buf)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("stream connected")
-
-	// create a description that contains a VP9 format
 	desc := &description.Session{
 		Medias: []*description.Media{{
-			Type: description.MediaTypeVideo,
-			Formats: []format.Format{&format.VP9{
-				PayloadTyp: 96,
-			}},
+			Type:    description.MediaTypeVideo,
+			Formats: []format.Format{forma},
 		}},
 	}
 
-	// connect to the server and start recording
+	// connect to the server, announce the format and start recording
 	c := gortsplib.Client{}
-	err = c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
+	err := c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
 	if err != nil {
 		panic(err)
 	}
 	defer c.Close()
 
-	var pkt rtp.Packet
-	for {
-		// parse RTP packet
-		err = pkt.Unmarshal(buf[:n])
+	// setup RGBA -> VP9 encoder
+	vp9enc := &vp9Encoder{
+		Width:  640,
+		Height: 480,
+		FPS:    5,
+	}
+	err = vp9enc.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer vp9enc.close()
+
+	// setup VP9 -> RTP encoder
+	rtpEnc, err := forma.CreateEncoder()
+	if err != nil {
+		panic(err)
+	}
+
+	start := time.Now()
+
+	randomStart, err := randUint32()
+	if err != nil {
+		panic(err)
+	}
+
+	// setup a ticker to sleep between frames
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
+	i := 0
+
+	for range ticker.C {
+		// create a dummy image
+		img := createDummyImage(i)
+		i = (i + 1) % 3
+
+		// get current timestamp
+		pts := multiplyAndDivide(int64(time.Since(start)), int64(forma.ClockRate()), int64(time.Second))
+
+		// encode the image with VP9
+		au, pts, err := vp9enc.encode(img, pts)
 		if err != nil {
 			panic(err)
 		}
 
-		// route RTP packet to the server
-		err = c.WritePacketRTP(desc.Medias[0], &pkt)
+		// wait for a VP9 access unit
+		if au == nil {
+			continue
+		}
+
+		// generate RTP packets from the VP9 access unit
+		pkts, err := rtpEnc.Encode(au)
 		if err != nil {
 			panic(err)
 		}
 
-		// read another RTP packet from source
-		n, _, err = pc.ReadFrom(buf)
-		if err != nil {
-			panic(err)
+		log.Printf("writing RTP packets with PTS=%d, au=%d, pkts=%d", pts, len(au), len(pkts))
+
+		// write RTP packets to the server
+		for _, pkt := range pkts {
+			pkt.Timestamp = uint32(int64(randomStart) + pts)
+
+			err = c.WritePacketRTP(desc.Medias[0], pkt)
+			if err != nil {
+				panic(err)
+			}
 		}
 	}
 }

examples/client-record-format-vp9/vp9_encoder.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libswscale/swscale.h>
+// #include <libavutil/opt.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// vp9Encoder is a wrapper around FFmpeg's VP9 encoder.
+type vp9Encoder struct {
+	Width  int
+	Height int
+	FPS    int
+
+	codecCtx    *C.AVCodecContext
+	rgbaFrame   *C.AVFrame
+	yuv420Frame *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	pkt         *C.AVPacket
+}
+
+// initialize initializes a vp9Encoder.
+func (d *vp9Encoder) initialize() error {
+	codec := C.avcodec_find_encoder(C.AV_CODEC_ID_VP9)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_encoder_by_name() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
+	d.codecCtx.time_base.num = 1
+	d.codecCtx.time_base.den = (C.int)(d.FPS)
+	d.codecCtx.gop_size = 10
+	d.codecCtx.max_b_frames = 0
+	d.codecCtx.bit_rate = 600000
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.rgbaFrame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+	d.rgbaFrame.width = d.codecCtx.width
+	d.rgbaFrame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.rgbaFrame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.yuv420Frame.format = C.AV_PIX_FMT_YUV420P
+	d.yuv420Frame.width = d.codecCtx.width
+	d.yuv420Frame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.yuv420Frame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.swsCtx = C.sws_getContext(d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format),
+		d.yuv420Frame.width, d.yuv420Frame.height, (int32)(d.yuv420Frame.format), C.SWS_BILINEAR, nil, nil, nil)
+	if d.swsCtx == nil {
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("sws_getContext() failed")
+	}
+
+	d.pkt = C.av_packet_alloc()
+	if d.pkt == nil {
+		C.av_packet_free(&d.pkt)
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_packet_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *vp9Encoder) close() {
+	C.av_packet_free(&d.pkt)
+	C.sws_freeContext(d.swsCtx)
+	C.av_frame_free(&d.yuv420Frame)
+	C.av_frame_free(&d.rgbaFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// encode encodes a RGBA image into VP9.
+func (d *vp9Encoder) encode(img *image.RGBA, pts int64) ([]byte, int64, error) {
+	// pass image pointer to frame
+	d.rgbaFrame.data[0] = (*C.uint8_t)(&img.Pix[0])
+
+	// convert color space from RGBA to YUV420
+	res := C.sws_scale(d.swsCtx, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame),
+		0, d.rgbaFrame.height, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame))
+	if res < 0 {
+		return nil, 0, fmt.Errorf("sws_scale() failed")
+	}
+
+	// send frame to the encoder
+	d.yuv420Frame.pts = (C.int64_t)(pts)
+	res = C.avcodec_send_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_send_frame() failed")
+	}
+
+	// wait for result
+	res = C.avcodec_receive_packet(d.codecCtx, d.pkt)
+	if res == -C.EAGAIN {
+		return nil, 0, nil
+	}
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_receive_packet() failed")
+	}
+
+	// perform a deep copy of the data before unreferencing the packet
+	data := C.GoBytes(unsafe.Pointer(d.pkt.data), d.pkt.size)
+	pts = (int64)(d.pkt.pts)
+	C.av_packet_unref(d.pkt)
+
+	return data, pts, nil
+}

examples/client-record-options/h264_encoder.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+
+	"github.com/bluenviron/mediacommon/v2/pkg/codecs/h264"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libswscale/swscale.h>
+// #include <libavutil/opt.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// h264Encoder is a wrapper around FFmpeg's H264 encoder.
+type h264Encoder struct {
+	Width  int
+	Height int
+	FPS    int
+
+	codecCtx    *C.AVCodecContext
+	rgbaFrame   *C.AVFrame
+	yuv420Frame *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	pkt         *C.AVPacket
+}
+
+// initialize initializes a h264Encoder.
+func (d *h264Encoder) initialize() error {
+	codec := C.avcodec_find_encoder(C.AV_CODEC_ID_H264)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_encoder() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	key := C.CString("tune")
+	defer C.free(unsafe.Pointer(key))
+	val := C.CString("zerolatency")
+	defer C.free(unsafe.Pointer(val))
+	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
+
+	key = C.CString("preset")
+	defer C.free(unsafe.Pointer(key))
+	val = C.CString("ultrafast")
+	defer C.free(unsafe.Pointer(val))
+	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
+
+	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
+	d.codecCtx.time_base.num = 1
+	d.codecCtx.time_base.den = (C.int)(d.FPS)
+	d.codecCtx.gop_size = 10
+	d.codecCtx.max_b_frames = 0
+	d.codecCtx.bit_rate = 600000
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.rgbaFrame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+	d.rgbaFrame.width = d.codecCtx.width
+	d.rgbaFrame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.rgbaFrame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.yuv420Frame.format = C.AV_PIX_FMT_YUV420P
+	d.yuv420Frame.width = d.codecCtx.width
+	d.yuv420Frame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.yuv420Frame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.swsCtx = C.sws_getContext(d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format),
+		d.yuv420Frame.width, d.yuv420Frame.height, (int32)(d.yuv420Frame.format), C.SWS_BILINEAR, nil, nil, nil)
+	if d.swsCtx == nil {
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("sws_getContext() failed")
+	}
+
+	d.pkt = C.av_packet_alloc()
+	if d.pkt == nil {
+		C.av_packet_free(&d.pkt)
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_packet_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *h264Encoder) close() {
+	C.av_packet_free(&d.pkt)
+	C.sws_freeContext(d.swsCtx)
+	C.av_frame_free(&d.yuv420Frame)
+	C.av_frame_free(&d.rgbaFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// encode encodes a RGBA image into H264.
+func (d *h264Encoder) encode(img *image.RGBA, pts int64) ([][]byte, int64, error) {
+	// pass image pointer to frame
+	d.rgbaFrame.data[0] = (*C.uint8_t)(&img.Pix[0])
+
+	// convert color space from RGBA to YUV420
+	res := C.sws_scale(d.swsCtx, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame),
+		0, d.rgbaFrame.height, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame))
+	if res < 0 {
+		return nil, 0, fmt.Errorf("sws_scale() failed")
+	}
+
+	// send frame to the encoder
+	d.yuv420Frame.pts = (C.int64_t)(pts)
+	res = C.avcodec_send_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_send_frame() failed")
+	}
+
+	// wait for result
+	res = C.avcodec_receive_packet(d.codecCtx, d.pkt)
+	if res == -C.EAGAIN {
+		return nil, 0, nil
+	}
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_receive_packet() failed")
+	}
+
+	// perform a deep copy of the data before unreferencing the packet
+	data := C.GoBytes(unsafe.Pointer(d.pkt.data), d.pkt.size)
+	pts = (int64)(d.pkt.pts)
+	C.av_packet_unref(d.pkt)
+
+	// decompress
+	var au h264.AnnexB
+	err := au.Unmarshal(data)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return au, pts, nil
+}

examples/client-record-options/main.go

@@ -1,50 +1,76 @@
+//go:build cgo
+
 package main
 
 import (
+	"crypto/rand"
+	"image"
+	"image/color"
 	"log"
-	"net"
 	"time"
 
 	"github.com/bluenviron/gortsplib/v4"
 	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
-	"github.com/pion/rtp"
 )
 
 // This example shows how to
 // 1. set additional client options
-// 2. read H264 frames from a file and generate RTP packets with GStreamer
-// 3. connect to a RTSP server, announce an H264 format
-// 4. write the frames to the server
+// 2. connect to a RTSP server, announce an H264 format
+// 3. generate dummy RGBA images
+// 4. encode images with H264
+// 5. generate RTP packets from H264
+// 6. write RTP packets to the server
 
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
+
+func multiplyAndDivide(v, m, d int64) int64 {
+	secs := v / d
+	dec := v % d
+	return (secs*m + dec*m/d)
+}
+
+func randUint32() (uint32, error) {
+	var b [4]byte
+	_, err := rand.Read(b[:])
+	if err != nil {
+		return 0, err
+	}
+	return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]), nil
+}
+
+func createDummyImage(i int) *image.RGBA {
+	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
+
+	var cl color.RGBA
+	switch i {
+	case 0:
+		cl = color.RGBA{255, 0, 0, 0}
+	case 1:
+		cl = color.RGBA{0, 255, 0, 0}
+	case 2:
+		cl = color.RGBA{0, 0, 255, 0}
+	}
+
+	for y := 0; y < img.Rect.Dy(); y++ {
+		for x := 0; x < img.Rect.Dx(); x++ {
+			img.SetRGBA(x, y, cl)
+		}
+	}
+
+	return img
+}
 func main() {
-	// open a listener to receive RTP/H264 frames
-	pc, err := net.ListenPacket("udp", "localhost:9000")
-	if err != nil {
-		panic(err)
-	}
-	defer pc.Close()
-
-	log.Println("Waiting for a RTP/H264 stream on port 9000 - you can send one with GStreamer:\n" +
-		"gst-launch-1.0 filesrc location=video.mp4 ! qtdemux ! video/x-h264" +
-		" ! h264parse config-interval=1 ! rtph264pay ! udpsink host=127.0.0.1 port=9000")
-
-	// wait for first packet
-	buf := make([]byte, 2048)
-	n, _, err := pc.ReadFrom(buf)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("stream connected")
-
 	// create a stream description that contains a H264 format
+	forma := &format.H264{
+		PayloadTyp:        96,
+		PacketizationMode: 1,
+	}
 	desc := &description.Session{
 		Medias: []*description.Media{{
-			Type: description.MediaTypeVideo,
-			Formats: []format.Format{&format.H264{
-				PayloadTyp:        96,
-				PacketizationMode: 1,
-			}},
+			Type:    description.MediaTypeVideo,
+			Formats: []format.Format{forma},
 		}},
 	}
 
@@ -58,31 +84,79 @@ func main() {
 		WriteTimeout: 10 * time.Second,
 	}
 
-	// connect to the server and start recording
-	err = c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
+	// connect to the server, announce the format and start recording
+	err := c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
 	if err != nil {
 		panic(err)
 	}
 	defer c.Close()
 
-	var pkt rtp.Packet
-	for {
-		// parse RTP packet
-		err = pkt.Unmarshal(buf[:n])
+	// setup RGBA -> H264 encoder
+	h264enc := &h264Encoder{
+		Width:  640,
+		Height: 480,
+		FPS:    5,
+	}
+	err = h264enc.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer h264enc.close()
+
+	// setup H264 -> RTP encoder
+	rtpEnc, err := forma.CreateEncoder()
+	if err != nil {
+		panic(err)
+	}
+
+	start := time.Now()
+
+	randomStart, err := randUint32()
+	if err != nil {
+		panic(err)
+	}
+
+	// setup a ticker to sleep between frames
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
+	i := 0
+
+	for range ticker.C {
+		// create a dummy image
+		img := createDummyImage(i)
+		i = (i + 1) % 3
+
+		// get current timestamp
+		pts := multiplyAndDivide(int64(time.Since(start)), int64(forma.ClockRate()), int64(time.Second))
+
+		// encode the image with H264
+		au, pts, err := h264enc.encode(img, pts)
 		if err != nil {
 			panic(err)
 		}
 
-		// route RTP packet to the server
-		err = c.WritePacketRTP(desc.Medias[0], &pkt)
+		// wait for a H264 access unit
+		if au == nil {
+			continue
+		}
+
+		// generate RTP packets from the H264 access unit
+		pkts, err := rtpEnc.Encode(au)
 		if err != nil {
 			panic(err)
 		}
 
-		// read another RTP packet from source
-		n, _, err = pc.ReadFrom(buf)
-		if err != nil {
-			panic(err)
+		log.Printf("writing RTP packets with PTS=%d, au=%d, pkts=%d", pts, len(au), len(pkts))
+
+		// write RTP packets to the server
+		for _, pkt := range pkts {
+			pkt.Timestamp = uint32(int64(randomStart) + pts)
+
+			err = c.WritePacketRTP(desc.Medias[0], pkt)
+			if err != nil {
+				panic(err)
+			}
 		}
 	}
 }

examples/client-record-pause/h264_encoder.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+
+	"github.com/bluenviron/mediacommon/v2/pkg/codecs/h264"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libswscale/swscale.h>
+// #include <libavutil/opt.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// h264Encoder is a wrapper around FFmpeg's H264 encoder.
+type h264Encoder struct {
+	Width  int
+	Height int
+	FPS    int
+
+	codecCtx    *C.AVCodecContext
+	rgbaFrame   *C.AVFrame
+	yuv420Frame *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	pkt         *C.AVPacket
+}
+
+// initialize initializes a h264Encoder.
+func (d *h264Encoder) initialize() error {
+	codec := C.avcodec_find_encoder(C.AV_CODEC_ID_H264)
+	if codec == nil {
+		return fmt.Errorf("avcodec_find_encoder() failed")
+	}
+
+	d.codecCtx = C.avcodec_alloc_context3(codec)
+	if d.codecCtx == nil {
+		return fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	key := C.CString("tune")
+	defer C.free(unsafe.Pointer(key))
+	val := C.CString("zerolatency")
+	defer C.free(unsafe.Pointer(val))
+	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
+
+	key = C.CString("preset")
+	defer C.free(unsafe.Pointer(key))
+	val = C.CString("ultrafast")
+	defer C.free(unsafe.Pointer(val))
+	C.av_opt_set(d.codecCtx.priv_data, key, val, 0)
+
+	d.codecCtx.pix_fmt = C.AV_PIX_FMT_YUV420P
+	d.codecCtx.width = (C.int)(d.Width)
+	d.codecCtx.height = (C.int)(d.Height)
+	d.codecCtx.time_base.num = 1
+	d.codecCtx.time_base.den = (C.int)(d.FPS)
+	d.codecCtx.gop_size = 10
+	d.codecCtx.max_b_frames = 0
+	d.codecCtx.bit_rate = 600000
+
+	res := C.avcodec_open2(d.codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("avcodec_open2() failed")
+	}
+
+	d.rgbaFrame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.rgbaFrame.format = C.AV_PIX_FMT_RGBA
+	d.rgbaFrame.width = d.codecCtx.width
+	d.rgbaFrame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.rgbaFrame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.yuv420Frame = C.av_frame_alloc()
+	if d.rgbaFrame == nil {
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	d.yuv420Frame.format = C.AV_PIX_FMT_YUV420P
+	d.yuv420Frame.width = d.codecCtx.width
+	d.yuv420Frame.height = d.codecCtx.height
+
+	res = C.av_frame_get_buffer(d.yuv420Frame, 0)
+	if res < 0 {
+		return fmt.Errorf("av_frame_get_buffer() failed")
+	}
+
+	d.swsCtx = C.sws_getContext(d.rgbaFrame.width, d.rgbaFrame.height, (int32)(d.rgbaFrame.format),
+		d.yuv420Frame.width, d.yuv420Frame.height, (int32)(d.yuv420Frame.format), C.SWS_BILINEAR, nil, nil, nil)
+	if d.swsCtx == nil {
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("sws_getContext() failed")
+	}
+
+	d.pkt = C.av_packet_alloc()
+	if d.pkt == nil {
+		C.av_packet_free(&d.pkt)
+		C.av_frame_free(&d.yuv420Frame)
+		C.av_frame_free(&d.rgbaFrame)
+		C.avcodec_close(d.codecCtx)
+		return fmt.Errorf("av_packet_alloc() failed")
+	}
+
+	return nil
+}
+
+// close closes the decoder.
+func (d *h264Encoder) close() {
+	C.av_packet_free(&d.pkt)
+	C.sws_freeContext(d.swsCtx)
+	C.av_frame_free(&d.yuv420Frame)
+	C.av_frame_free(&d.rgbaFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+// encode encodes a RGBA image into H264.
+func (d *h264Encoder) encode(img *image.RGBA, pts int64) ([][]byte, int64, error) {
+	// pass image pointer to frame
+	d.rgbaFrame.data[0] = (*C.uint8_t)(&img.Pix[0])
+
+	// convert color space from RGBA to YUV420
+	res := C.sws_scale(d.swsCtx, frameData(d.rgbaFrame), frameLineSize(d.rgbaFrame),
+		0, d.rgbaFrame.height, frameData(d.yuv420Frame), frameLineSize(d.yuv420Frame))
+	if res < 0 {
+		return nil, 0, fmt.Errorf("sws_scale() failed")
+	}
+
+	// send frame to the encoder
+	d.yuv420Frame.pts = (C.int64_t)(pts)
+	res = C.avcodec_send_frame(d.codecCtx, d.yuv420Frame)
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_send_frame() failed")
+	}
+
+	// wait for result
+	res = C.avcodec_receive_packet(d.codecCtx, d.pkt)
+	if res == -C.EAGAIN {
+		return nil, 0, nil
+	}
+	if res < 0 {
+		return nil, 0, fmt.Errorf("avcodec_receive_packet() failed")
+	}
+
+	// perform a deep copy of the data before unreferencing the packet
+	data := C.GoBytes(unsafe.Pointer(d.pkt.data), d.pkt.size)
+	pts = (int64)(d.pkt.pts)
+	C.av_packet_unref(d.pkt)
+
+	// decompress
+	var au h264.AnnexB
+	err := au.Unmarshal(data)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return au, pts, nil
+}

examples/client-record-pause/main.go

@@ -1,86 +1,166 @@
+//go:build cgo
+
 package main
 
 import (
+	"crypto/rand"
+	"image"
+	"image/color"
 	"log"
-	"net"
 	"time"
 
 	"github.com/bluenviron/gortsplib/v4"
 	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
-	"github.com/pion/rtp"
 )
 
 // This example shows how to
-// 1. read H264 frames from a file and generate RTP packets with GStreamer
-// 2. connect to a RTSP server, announce an H264 format
-// 3. write the frames to the server for 5 seconds
-// 4. pause for 5 seconds
-// 5. repeat
+// 1. connect to a RTSP server, announce an H264 format
+// 2. generate dummy RGBA images
+// 3. encode images with H264
+// 4. generate RTP packets from H264
+// 5. write RTP packets to the server for 5 seconds
+// 6. pause for 5 seconds
+// 7. repeat
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
+
+func multiplyAndDivide(v, m, d int64) int64 {
+	secs := v / d
+	dec := v % d
+	return (secs*m + dec*m/d)
+}
+
+func randUint32() (uint32, error) {
+	var b [4]byte
+	_, err := rand.Read(b[:])
+	if err != nil {
+		return 0, err
+	}
+	return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]), nil
+}
+
+func createDummyImage(i int) *image.RGBA {
+	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
+
+	var cl color.RGBA
+	switch i {
+	case 0:
+		cl = color.RGBA{255, 0, 0, 0}
+	case 1:
+		cl = color.RGBA{0, 255, 0, 0}
+	case 2:
+		cl = color.RGBA{0, 0, 255, 0}
+	}
+
+	for y := 0; y < img.Rect.Dy(); y++ {
+		for x := 0; x < img.Rect.Dx(); x++ {
+			img.SetRGBA(x, y, cl)
+		}
+	}
+
+	return img
+}
 
 func main() {
-	// open a listener to receive RTP/H264 frames
-	pc, err := net.ListenPacket("udp", "localhost:9000")
-	if err != nil {
-		panic(err)
-	}
-	defer pc.Close()
-
-	log.Println("Waiting for a RTP/H264 stream on port 9000 - you can send one with GStreamer:\n" +
-		"gst-launch-1.0 filesrc location=video.mp4 ! qtdemux ! video/x-h264" +
-		" ! h264parse config-interval=1 ! rtph264pay ! udpsink host=127.0.0.1 port=9000")
-
-	// wait for first packet
-	buf := make([]byte, 2048)
-	n, _, err := pc.ReadFrom(buf)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("stream connected")
-
 	// create a stream description that contains a H264 format
+	forma := &format.H264{
+		PayloadTyp:        96,
+		PacketizationMode: 1,
+	}
 	desc := &description.Session{
 		Medias: []*description.Media{{
-			Type: description.MediaTypeVideo,
-			Formats: []format.Format{&format.H264{
-				PayloadTyp:        96,
-				PacketizationMode: 1,
-			}},
+			Type:    description.MediaTypeVideo,
+			Formats: []format.Format{forma},
 		}},
 	}
 
 	// connect to the server and start recording
 	c := gortsplib.Client{}
-	err = c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
+	err := c.StartRecording("rtsp://myuser:mypass@localhost:8554/mystream", desc)
 	if err != nil {
 		panic(err)
 	}
 	defer c.Close()
 
-	for {
-		go func() {
-			var pkt rtp.Packet
-			for {
-				// parse RTP packet
-				err = pkt.Unmarshal(buf[:n])
-				if err != nil {
-					panic(err)
-				}
+	// setup RGBA -> H264 encoder
+	h264enc := &h264Encoder{
+		Width:  640,
+		Height: 480,
+		FPS:    5,
+	}
+	err = h264enc.initialize()
+	if err != nil {
+		panic(err)
+	}
+	defer h264enc.close()
 
-				// route RTP packet to the server
-				c.WritePacketRTP(desc.Medias[0], &pkt)
+	// setup H264 -> RTP encoder
+	rtpEnc, err := forma.CreateEncoder()
+	if err != nil {
+		panic(err)
+	}
 
-				// read another RTP packet from source
-				n, _, err = pc.ReadFrom(buf)
+	start := time.Now()
+
+	randomStart, err := randUint32()
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		// setup a ticker to sleep between frames
+		ticker := time.NewTicker(200 * time.Millisecond)
+		defer ticker.Stop()
+
+		i := 0
+
+		for range ticker.C {
+			// create a dummy image
+			img := createDummyImage(i)
+			i = (i + 1) % 3
+
+			// get current timestamp
+			pts := multiplyAndDivide(int64(time.Since(start)), int64(forma.ClockRate()), int64(time.Second))
+
+			// encode the image with H264
+			au, pts, err := h264enc.encode(img, pts)
+			if err != nil {
+				panic(err)
+			}
+
+			// wait for a H264 access unit
+			if au == nil {
+				continue
+			}
+
+			// generate RTP packets from the H264 access unit
+			pkts, err := rtpEnc.Encode(au)
+			if err != nil {
+				panic(err)
+			}
+
+			log.Printf("writing RTP packets with PTS=%d, au=%d, pkts=%d", pts, len(au), len(pkts))
+
+			// write RTP packets to the server
+			for _, pkt := range pkts {
+				pkt.Timestamp = uint32(int64(randomStart) + pts)
+
+				err = c.WritePacketRTP(desc.Medias[0], pkt)
 				if err != nil {
 					panic(err)
 				}
 			}
-		}()
+		}
+	}()
 
+	for {
 		// wait
 		time.Sleep(5 * time.Second)
 
+		log.Println("pausing")
+
 		// pause
 		_, err := c.Pause()
 		if err != nil {
@@ -90,6 +170,8 @@ func main() {
 		// wait
 		time.Sleep(5 * time.Second)
 
+		log.Println("recording")
+
 		// record again
 		_, err = c.Record()
 		if err != nil {