improve H264/H265 examples (#355) (#375) (#398)

README.md

@@ -64,6 +64,8 @@ Features:
 * [client-read-format-h264-convert-to-jpeg](examples/client-read-format-h264-convert-to-jpeg/main.go)
 * [client-read-format-h264-save-to-disk](examples/client-read-format-h264-save-to-disk/main.go)
 * [client-read-format-h265](examples/client-read-format-h265/main.go)
+* [client-read-format-h265-convert-to-jpeg](examples/client-read-format-h265-convert-to-jpeg/main.go)
+* [client-read-format-h265-save-to-disk](examples/client-read-format-h265-save-to-disk/main.go)
 * [client-read-format-lpcm](examples/client-read-format-lpcm/main.go)
 * [client-read-format-mjpeg](examples/client-read-format-mjpeg/main.go)
 * [client-read-format-mpeg4audio](examples/client-read-format-mpeg4audio/main.go)

examples/client-read-format-h264-convert-to-jpeg/h264_decoder.go

@@ -109,24 +109,24 @@ func (d *h264Decoder) decode(nalu []byte) (image.Image, error) {
 		d.dstFrame.color_range = C.AVCOL_RANGE_JPEG
 		res = C.av_frame_get_buffer(d.dstFrame, 1)
 		if res < 0 {
-			return nil, fmt.Errorf("av_frame_get_buffer() err")
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
 		}
 
 		d.swsCtx = C.sws_getContext(d.srcFrame.width, d.srcFrame.height, C.AV_PIX_FMT_YUV420P,
 			d.dstFrame.width, d.dstFrame.height, (int32)(d.dstFrame.format), C.SWS_BILINEAR, nil, nil, nil)
 		if d.swsCtx == nil {
-			return nil, fmt.Errorf("sws_getContext() err")
+			return nil, fmt.Errorf("sws_getContext() failed")
 		}
 
 		dstFrameSize := C.av_image_get_buffer_size((int32)(d.dstFrame.format), d.dstFrame.width, d.dstFrame.height, 1)
 		d.dstFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.dstFrame.data[0]))[:dstFrameSize:dstFrameSize]
 	}
 
-	// convert color space from YUV420 to RGB
+	// convert color space from YUV420 to RGBA
 	res = C.sws_scale(d.swsCtx, frameData(d.srcFrame), frameLineSize(d.srcFrame),
 		0, d.srcFrame.height, frameData(d.dstFrame), frameLineSize(d.dstFrame))
 	if res < 0 {
-		return nil, fmt.Errorf("sws_scale() err")
+		return nil, fmt.Errorf("sws_scale() failed")
 	}
 
 	// embed frame into an image.Image

examples/client-read-format-h264-convert-to-jpeg/main.go

@@ -12,16 +12,17 @@ import (
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
 	"github.com/bluenviron/gortsplib/v4/pkg/format/rtph264"
 	"github.com/bluenviron/gortsplib/v4/pkg/url"
+	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/pion/rtp"
 )
 
 // This example shows how to
 // 1. connect to a RTSP server
-// 2. check if there's a H264 media
+// 2. check if there's a H264 media stream
-// 3. decode H264 into RGBA frames
+// 3. decode the H264 media stream into RGBA frames
-// 4. encode the frames into JPEG images and save them on disk
+// 4. convert frames to JPEG images and save them on disk
 
-// This example requires the ffmpeg libraries, that can be installed in this way:
+// This example requires the FFmpeg libraries, that can be installed with this command:
 // apt install -y libavformat-dev libswscale-dev gcc pkg-config
 
 func saveToFile(img image.Image) error {
@@ -77,18 +78,18 @@ func main() {
 	}
 
 	// setup H264 -> raw frames decoder
-	h264RawDec, err := newH264Decoder()
+	frameDec, err := newH264Decoder()
 	if err != nil {
 		panic(err)
 	}
-	defer h264RawDec.close()
+	defer frameDec.close()
 
 	// if SPS and PPS are present into the SDP, send them to the decoder
 	if forma.SPS != nil {
-		h264RawDec.decode(forma.SPS)
+		frameDec.decode(forma.SPS)
 	}
 	if forma.PPS != nil {
-		h264RawDec.decode(forma.PPS)
+		frameDec.decode(forma.PPS)
 	}
 
 	// setup a single media
@@ -97,6 +98,7 @@ func main() {
 		panic(err)
 	}
 
+	iframeReceived := false
 	saveCount := 0
 
 	// called when a RTP packet arrives
@@ -110,9 +112,18 @@ func main() {
 			return
 		}
 
+		// wait for an I-frame
+		if !iframeReceived {
+			if !h264.IDRPresent(au) {
+				log.Printf("waiting for an I-frame")
+				return
+			}
+			iframeReceived = true
+		}
+
 		for _, nalu := range au {
 			// convert NALUs into RGBA frames
-			img, err := h264RawDec.decode(nalu)
+			img, err := frameDec.decode(nalu)
 			if err != nil {
 				panic(err)
 			}

examples/client-read-format-h264-save-to-disk/main.go

@@ -67,6 +67,7 @@ func main() {
 		// decode timestamp
 		pts, ok := c.PacketPTS(medi, pkt)
 		if !ok {
+			log.Printf("waiting for timestamp")
 			return
 		}
 

examples/client-read-format-h264-save-to-disk/mpegts_muxer.go

@@ -58,7 +58,7 @@ func (e *mpegtsMuxer) close() {
 // encode encodes a H264 access unit into MPEG-TS.
 func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 	// prepend an AUD. This is required by some players
-	filteredNALUs := [][]byte{
+	filteredAU := [][]byte{
 		{byte(h264.NALUTypeAccessUnitDelimiter), 240},
 	}
 
@@ -69,11 +69,11 @@ func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 		typ := h264.NALUType(nalu[0] & 0x1F)
 		switch typ {
 		case h264.NALUTypeSPS:
-			e.sps = append([]byte(nil), nalu...)
+			e.sps = nalu
 			continue
 
 		case h264.NALUTypePPS:
-			e.pps = append([]byte(nil), nalu...)
+			e.pps = nalu
 			continue
 
 		case h264.NALUTypeAccessUnitDelimiter:
@@ -86,16 +86,16 @@ func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 			nonIDRPresent = true
 		}
 
-		filteredNALUs = append(filteredNALUs, nalu)
+		filteredAU = append(filteredAU, nalu)
 	}
 
-	au = filteredNALUs
+	au = filteredAU
 
 	if len(au) <= 1 || (!nonIDRPresent && !idrPresent) {
 		return nil
 	}
 
-	// add SPS and PPS before every group that contains an IDR
+	// add SPS and PPS before access unit that contains an IDR
 	if idrPresent {
 		au = append([][]byte{e.sps, e.pps}, au...)
 	}

examples/client-read-format-h264/h264_decoder.go

@@ -109,24 +109,24 @@ func (d *h264Decoder) decode(nalu []byte) (image.Image, error) {
 		d.dstFrame.color_range = C.AVCOL_RANGE_JPEG
 		res = C.av_frame_get_buffer(d.dstFrame, 1)
 		if res < 0 {
-			return nil, fmt.Errorf("av_frame_get_buffer() err")
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
 		}
 
 		d.swsCtx = C.sws_getContext(d.srcFrame.width, d.srcFrame.height, C.AV_PIX_FMT_YUV420P,
 			d.dstFrame.width, d.dstFrame.height, (int32)(d.dstFrame.format), C.SWS_BILINEAR, nil, nil, nil)
 		if d.swsCtx == nil {
-			return nil, fmt.Errorf("sws_getContext() err")
+			return nil, fmt.Errorf("sws_getContext() failed")
 		}
 
 		dstFrameSize := C.av_image_get_buffer_size((int32)(d.dstFrame.format), d.dstFrame.width, d.dstFrame.height, 1)
 		d.dstFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.dstFrame.data[0]))[:dstFrameSize:dstFrameSize]
 	}
 
-	// convert color space from YUV420 to RGB
+	// convert color space from YUV420 to RGBA
 	res = C.sws_scale(d.swsCtx, frameData(d.srcFrame), frameLineSize(d.srcFrame),
 		0, d.srcFrame.height, frameData(d.dstFrame), frameLineSize(d.dstFrame))
 	if res < 0 {
-		return nil, fmt.Errorf("sws_scale() err")
+		return nil, fmt.Errorf("sws_scale() failed")
 	}
 
 	// embed frame into an image.Image

examples/client-read-format-h264/main.go

@@ -12,10 +12,10 @@ import (
 
 // This example shows how to
 // 1. connect to a RTSP server
-// 2. check if there's an H264 media
+// 2. check if there's an H264 media stream
-// 3. decode H264 into RGBA frames
+// 3. decode the H264 media stream into RGBA frames
 
-// This example requires the ffmpeg libraries, that can be installed in this way:
+// This example requires the FFmpeg libraries, that can be installed with this command:
 // apt install -y libavformat-dev libswscale-dev gcc pkg-config
 
 func main() {
@@ -54,18 +54,18 @@ func main() {
 	}
 
 	// setup H264 -> raw frames decoder
-	h264RawDec, err := newH264Decoder()
+	frameDec, err := newH264Decoder()
 	if err != nil {
 		panic(err)
 	}
-	defer h264RawDec.close()
+	defer frameDec.close()
 
 	// if SPS and PPS are present into the SDP, send them to the decoder
 	if forma.SPS != nil {
-		h264RawDec.decode(forma.SPS)
+		frameDec.decode(forma.SPS)
 	}
 	if forma.PPS != nil {
-		h264RawDec.decode(forma.PPS)
+		frameDec.decode(forma.PPS)
 	}
 
 	// setup a single media
@@ -79,6 +79,7 @@ func main() {
 		// decode timestamp
 		pts, ok := c.PacketPTS(medi, pkt)
 		if !ok {
+			log.Printf("waiting for timestamp")
 			return
 		}
 
@@ -93,7 +94,7 @@ func main() {
 
 		for _, nalu := range au {
 			// convert NALUs into RGBA frames
-			img, err := h264RawDec.decode(nalu)
+			img, err := frameDec.decode(nalu)
 			if err != nil {
 				panic(err)
 			}
@@ -103,7 +104,7 @@ func main() {
 				continue
 			}
 
-			log.Printf("decoded frame with PTS %v and size %v and pts %v", pts, img.Bounds().Max)
+			log.Printf("decoded frame with PTS %v and size %v", pts, img.Bounds().Max)
 		}
 	})
 

examples/client-read-format-h265-convert-to-jpeg/h265_decoder.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libavutil/imgutils.h>
+// #include <libswscale/swscale.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// h265Decoder is a wrapper around FFmpeg's H265 decoder.
+type h265Decoder struct {
+	codecCtx    *C.AVCodecContext
+	srcFrame    *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	dstFrame    *C.AVFrame
+	dstFramePtr []uint8
+}
+
+// newH265Decoder allocates a new h265Decoder.
+func newH265Decoder() (*h265Decoder, error) {
+	codec := C.avcodec_find_decoder(C.AV_CODEC_ID_H265)
+	if codec == nil {
+		return nil, fmt.Errorf("avcodec_find_decoder() failed")
+	}
+
+	codecCtx := C.avcodec_alloc_context3(codec)
+	if codecCtx == nil {
+		return nil, fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	res := C.avcodec_open2(codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(codecCtx)
+		return nil, fmt.Errorf("avcodec_open2() failed")
+	}
+
+	srcFrame := C.av_frame_alloc()
+	if srcFrame == nil {
+		C.avcodec_close(codecCtx)
+		return nil, fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	return &h265Decoder{
+		codecCtx: codecCtx,
+		srcFrame: srcFrame,
+	}, nil
+}
+
+// close closes the decoder.
+func (d *h265Decoder) close() {
+	if d.dstFrame != nil {
+		C.av_frame_free(&d.dstFrame)
+	}
+
+	if d.swsCtx != nil {
+		C.sws_freeContext(d.swsCtx)
+	}
+
+	C.av_frame_free(&d.srcFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+func (d *h265Decoder) decode(nalu []byte) (image.Image, error) {
+	nalu = append([]uint8{0x00, 0x00, 0x00, 0x01}, []uint8(nalu)...)
+
+	// send NALU to decoder
+	var avPacket C.AVPacket
+	avPacket.data = (*C.uint8_t)(C.CBytes(nalu))
+	defer C.free(unsafe.Pointer(avPacket.data))
+	avPacket.size = C.int(len(nalu))
+	res := C.avcodec_send_packet(d.codecCtx, &avPacket)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// receive frame if available
+	res = C.avcodec_receive_frame(d.codecCtx, d.srcFrame)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// if frame size has changed, allocate needed objects
+	if d.dstFrame == nil || d.dstFrame.width != d.srcFrame.width || d.dstFrame.height != d.srcFrame.height {
+		if d.dstFrame != nil {
+			C.av_frame_free(&d.dstFrame)
+		}
+
+		if d.swsCtx != nil {
+			C.sws_freeContext(d.swsCtx)
+		}
+
+		d.dstFrame = C.av_frame_alloc()
+		d.dstFrame.format = C.AV_PIX_FMT_RGBA
+		d.dstFrame.width = d.srcFrame.width
+		d.dstFrame.height = d.srcFrame.height
+		d.dstFrame.color_range = C.AVCOL_RANGE_JPEG
+		res = C.av_frame_get_buffer(d.dstFrame, 1)
+		if res < 0 {
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
+		}
+
+		d.swsCtx = C.sws_getContext(d.srcFrame.width, d.srcFrame.height, C.AV_PIX_FMT_YUV420P,
+			d.dstFrame.width, d.dstFrame.height, (int32)(d.dstFrame.format), C.SWS_BILINEAR, nil, nil, nil)
+		if d.swsCtx == nil {
+			return nil, fmt.Errorf("sws_getContext() failed")
+		}
+
+		dstFrameSize := C.av_image_get_buffer_size((int32)(d.dstFrame.format), d.dstFrame.width, d.dstFrame.height, 1)
+		d.dstFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.dstFrame.data[0]))[:dstFrameSize:dstFrameSize]
+	}
+
+	// convert color space from YUV420 to RGBA
+	res = C.sws_scale(d.swsCtx, frameData(d.srcFrame), frameLineSize(d.srcFrame),
+		0, d.srcFrame.height, frameData(d.dstFrame), frameLineSize(d.dstFrame))
+	if res < 0 {
+		return nil, fmt.Errorf("sws_scale() failed")
+	}
+
+	// embed frame into an image.Image
+	return &image.RGBA{
+		Pix:    d.dstFramePtr,
+		Stride: 4 * (int)(d.dstFrame.width),
+		Rect: image.Rectangle{
+			Max: image.Point{(int)(d.dstFrame.width), (int)(d.dstFrame.height)},
+		},
+	}, nil
+}

examples/client-read-format-h265-convert-to-jpeg/main.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"image"
+	"image/jpeg"
+	"log"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/bluenviron/gortsplib/v4"
+	"github.com/bluenviron/gortsplib/v4/pkg/format"
+	"github.com/bluenviron/gortsplib/v4/pkg/format/rtph265"
+	"github.com/bluenviron/gortsplib/v4/pkg/url"
+	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
+	"github.com/pion/rtp"
+)
+
+// This example shows how to
+// 1. connect to a RTSP server
+// 2. check if there's a H265 media stream
+// 3. decode the H265 media stream into RGBA frames
+// 4. convert frames to JPEG images and save them on disk
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
+
+func saveToFile(img image.Image) error {
+	// create file
+	fname := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) + ".jpg"
+	f, err := os.Create(fname)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	log.Println("saving", fname)
+
+	// convert to jpeg
+	return jpeg.Encode(f, img, &jpeg.Options{
+		Quality: 60,
+	})
+}
+
+func main() {
+	c := gortsplib.Client{}
+
+	// parse URL
+	u, err := url.Parse("rtsp://localhost:8554/mystream")
+	if err != nil {
+		panic(err)
+	}
+
+	// connect to the server
+	err = c.Start(u.Scheme, u.Host)
+	if err != nil {
+		panic(err)
+	}
+	defer c.Close()
+
+	// find published medias
+	desc, _, err := c.Describe(u)
+	if err != nil {
+		panic(err)
+	}
+
+	// find the H265 media and format
+	var forma *format.H265
+	medi := desc.FindFormat(&forma)
+	if medi == nil {
+		panic("media not found")
+	}
+
+	// setup RTP/H265 -> H265 decoder
+	rtpDec, err := forma.CreateDecoder()
+	if err != nil {
+		panic(err)
+	}
+
+	// setup H265 -> raw frames decoder
+	frameDec, err := newH265Decoder()
+	if err != nil {
+		panic(err)
+	}
+	defer frameDec.close()
+
+	// if VPS, SPS and PPS are present into the SDP, send them to the decoder
+	if forma.VPS != nil {
+		frameDec.decode(forma.VPS)
+	}
+	if forma.SPS != nil {
+		frameDec.decode(forma.SPS)
+	}
+	if forma.PPS != nil {
+		frameDec.decode(forma.PPS)
+	}
+
+	// setup a single media
+	_, err = c.Setup(desc.BaseURL, medi, 0, 0)
+	if err != nil {
+		panic(err)
+	}
+
+	iframeReceived := false
+	saveCount := 0
+
+	// called when a RTP packet arrives
+	c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
+		// extract access units from RTP packets
+		au, err := rtpDec.Decode(pkt)
+		if err != nil {
+			if err != rtph265.ErrNonStartingPacketAndNoPrevious && err != rtph265.ErrMorePacketsNeeded {
+				log.Printf("ERR: %v", err)
+			}
+			return
+		}
+
+		// wait for an I-frame
+		if !iframeReceived {
+			if !h265.IsRandomAccess(au) {
+				log.Printf("waiting for an I-frame")
+				return
+			}
+			iframeReceived = true
+		}
+
+		for _, nalu := range au {
+			// convert NALUs into RGBA frames
+			img, err := frameDec.decode(nalu)
+			if err != nil {
+				panic(err)
+			}
+
+			// wait for a frame
+			if img == nil {
+				continue
+			}
+
+			// convert frame to JPEG and save to file
+			err = saveToFile(img)
+			if err != nil {
+				panic(err)
+			}
+
+			saveCount++
+			if saveCount == 5 {
+				log.Printf("saved 5 images, exiting")
+				os.Exit(1)
+			}
+		}
+	})
+
+	// start playing
+	_, err = c.Play(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// wait until a fatal error
+	panic(c.Wait())
+}

examples/client-read-format-h265-save-to-disk/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"log"
+
+	"github.com/bluenviron/gortsplib/v4"
+	"github.com/bluenviron/gortsplib/v4/pkg/format"
+	"github.com/bluenviron/gortsplib/v4/pkg/format/rtph265"
+	"github.com/bluenviron/gortsplib/v4/pkg/url"
+	"github.com/pion/rtp"
+)
+
+// This example shows how to
+// 1. connect to a RTSP server
+// 2. check if there's a H265 media
+// 3. save the content of the media into a file in MPEG-TS format
+
+func main() {
+	c := gortsplib.Client{}
+
+	// parse URL
+	u, err := url.Parse("rtsp://localhost:8554/mystream")
+	if err != nil {
+		panic(err)
+	}
+
+	// connect to the server
+	err = c.Start(u.Scheme, u.Host)
+	if err != nil {
+		panic(err)
+	}
+	defer c.Close()
+
+	// find published medias
+	desc, _, err := c.Describe(u)
+	if err != nil {
+		panic(err)
+	}
+
+	// find the H265 media and format
+	var forma *format.H265
+	medi := desc.FindFormat(&forma)
+	if medi == nil {
+		panic("media not found")
+	}
+
+	// setup RTP/H265 -> H265 decoder
+	rtpDec, err := forma.CreateDecoder()
+	if err != nil {
+		panic(err)
+	}
+
+	// setup H265 -> MPEG-TS muxer
+	mpegtsMuxer, err := newMPEGTSMuxer(forma.VPS, forma.SPS, forma.PPS)
+	if err != nil {
+		panic(err)
+	}
+
+	// setup a single media
+	_, err = c.Setup(desc.BaseURL, medi, 0, 0)
+	if err != nil {
+		panic(err)
+	}
+
+	// called when a RTP packet arrives
+	c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
+		// decode timestamp
+		pts, ok := c.PacketPTS(medi, pkt)
+		if !ok {
+			log.Printf("waiting for timestamp")
+			return
+		}
+
+		// extract access unit from RTP packets
+		au, err := rtpDec.Decode(pkt)
+		if err != nil {
+			if err != rtph265.ErrNonStartingPacketAndNoPrevious && err != rtph265.ErrMorePacketsNeeded {
+				log.Printf("ERR: %v", err)
+			}
+			return
+		}
+
+		// encode the access unit into MPEG-TS
+		err = mpegtsMuxer.encode(au, pts)
+		if err != nil {
+			log.Printf("ERR: %v", err)
+			return
+		}
+
+		log.Printf("saved TS packet")
+	})
+
+	// start playing
+	_, err = c.Play(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// wait until a fatal error
+	panic(c.Wait())
+}

examples/client-read-format-h265-save-to-disk/mpegts_muxer.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"time"
+
+	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
+	"github.com/bluenviron/mediacommon/pkg/formats/mpegts"
+)
+
+func durationGoToMPEGTS(v time.Duration) int64 {
+	return int64(v.Seconds() * 90000)
+}
+
+// mpegtsMuxer allows to save a H265 stream into a MPEG-TS file.
+type mpegtsMuxer struct {
+	vps []byte
+	sps []byte
+	pps []byte
+
+	f            *os.File
+	b            *bufio.Writer
+	w            *mpegts.Writer
+	track        *mpegts.Track
+	dtsExtractor *h265.DTSExtractor
+}
+
+// newMPEGTSMuxer allocates a mpegtsMuxer.
+func newMPEGTSMuxer(vps []byte, sps []byte, pps []byte) (*mpegtsMuxer, error) {
+	f, err := os.Create("mystream.ts")
+	if err != nil {
+		return nil, err
+	}
+	b := bufio.NewWriter(f)
+
+	track := &mpegts.Track{
+		Codec: &mpegts.CodecH265{},
+	}
+
+	w := mpegts.NewWriter(b, []*mpegts.Track{track})
+
+	return &mpegtsMuxer{
+		vps:   vps,
+		sps:   sps,
+		pps:   pps,
+		f:     f,
+		b:     b,
+		w:     w,
+		track: track,
+	}, nil
+}
+
+// close closes all the mpegtsMuxer resources.
+func (e *mpegtsMuxer) close() {
+	e.b.Flush()
+	e.f.Close()
+}
+
+// encode encodes a H265 access unit into MPEG-TS.
+func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
+	// prepend an AUD. This is required by some players
+	filteredAU := [][]byte{
+		{byte(h265.NALUType_AUD_NUT) << 1, 1, 0x50},
+	}
+
+	isRandomAccess := false
+
+	for _, nalu := range au {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+		switch typ {
+		case h265.NALUType_VPS_NUT:
+			e.vps = nalu
+			continue
+
+		case h265.NALUType_SPS_NUT:
+			e.sps = nalu
+			continue
+
+		case h265.NALUType_PPS_NUT:
+			e.pps = nalu
+			continue
+
+		case h265.NALUType_AUD_NUT:
+			continue
+
+		case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+			isRandomAccess = true
+		}
+
+		filteredAU = append(filteredAU, nalu)
+	}
+
+	au = filteredAU
+
+	if len(au) <= 1 {
+		return nil
+	}
+
+	// add VPS, SPS and PPS before random access access unit
+	if isRandomAccess {
+		au = append([][]byte{e.vps, e.sps, e.pps}, au...)
+	}
+
+	var dts time.Duration
+
+	if e.dtsExtractor == nil {
+		// skip samples silently until we find one with a IDR
+		if !isRandomAccess {
+			return nil
+		}
+		e.dtsExtractor = h265.NewDTSExtractor()
+	}
+
+	var err error
+	dts, err = e.dtsExtractor.Extract(au, pts)
+	if err != nil {
+		return err
+	}
+
+	// encode into MPEG-TS
+	return e.w.WriteH26x(e.track, durationGoToMPEGTS(pts), durationGoToMPEGTS(dts), isRandomAccess, au)
+}

examples/client-read-format-h265/h265_decoder.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"unsafe"
+)
+
+// #cgo pkg-config: libavcodec libavutil libswscale
+// #include <libavcodec/avcodec.h>
+// #include <libavutil/imgutils.h>
+// #include <libswscale/swscale.h>
+import "C"
+
+func frameData(frame *C.AVFrame) **C.uint8_t {
+	return (**C.uint8_t)(unsafe.Pointer(&frame.data[0]))
+}
+
+func frameLineSize(frame *C.AVFrame) *C.int {
+	return (*C.int)(unsafe.Pointer(&frame.linesize[0]))
+}
+
+// h265Decoder is a wrapper around FFmpeg's H265 decoder.
+type h265Decoder struct {
+	codecCtx    *C.AVCodecContext
+	srcFrame    *C.AVFrame
+	swsCtx      *C.struct_SwsContext
+	dstFrame    *C.AVFrame
+	dstFramePtr []uint8
+}
+
+// newH265Decoder allocates a new h265Decoder.
+func newH265Decoder() (*h265Decoder, error) {
+	codec := C.avcodec_find_decoder(C.AV_CODEC_ID_H265)
+	if codec == nil {
+		return nil, fmt.Errorf("avcodec_find_decoder() failed")
+	}
+
+	codecCtx := C.avcodec_alloc_context3(codec)
+	if codecCtx == nil {
+		return nil, fmt.Errorf("avcodec_alloc_context3() failed")
+	}
+
+	res := C.avcodec_open2(codecCtx, codec, nil)
+	if res < 0 {
+		C.avcodec_close(codecCtx)
+		return nil, fmt.Errorf("avcodec_open2() failed")
+	}
+
+	srcFrame := C.av_frame_alloc()
+	if srcFrame == nil {
+		C.avcodec_close(codecCtx)
+		return nil, fmt.Errorf("av_frame_alloc() failed")
+	}
+
+	return &h265Decoder{
+		codecCtx: codecCtx,
+		srcFrame: srcFrame,
+	}, nil
+}
+
+// close closes the decoder.
+func (d *h265Decoder) close() {
+	if d.dstFrame != nil {
+		C.av_frame_free(&d.dstFrame)
+	}
+
+	if d.swsCtx != nil {
+		C.sws_freeContext(d.swsCtx)
+	}
+
+	C.av_frame_free(&d.srcFrame)
+	C.avcodec_close(d.codecCtx)
+}
+
+func (d *h265Decoder) decode(nalu []byte) (image.Image, error) {
+	nalu = append([]uint8{0x00, 0x00, 0x00, 0x01}, []uint8(nalu)...)
+
+	// send NALU to decoder
+	var avPacket C.AVPacket
+	avPacket.data = (*C.uint8_t)(C.CBytes(nalu))
+	defer C.free(unsafe.Pointer(avPacket.data))
+	avPacket.size = C.int(len(nalu))
+	res := C.avcodec_send_packet(d.codecCtx, &avPacket)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// receive frame if available
+	res = C.avcodec_receive_frame(d.codecCtx, d.srcFrame)
+	if res < 0 {
+		return nil, nil
+	}
+
+	// if frame size has changed, allocate needed objects
+	if d.dstFrame == nil || d.dstFrame.width != d.srcFrame.width || d.dstFrame.height != d.srcFrame.height {
+		if d.dstFrame != nil {
+			C.av_frame_free(&d.dstFrame)
+		}
+
+		if d.swsCtx != nil {
+			C.sws_freeContext(d.swsCtx)
+		}
+
+		d.dstFrame = C.av_frame_alloc()
+		d.dstFrame.format = C.AV_PIX_FMT_RGBA
+		d.dstFrame.width = d.srcFrame.width
+		d.dstFrame.height = d.srcFrame.height
+		d.dstFrame.color_range = C.AVCOL_RANGE_JPEG
+		res = C.av_frame_get_buffer(d.dstFrame, 1)
+		if res < 0 {
+			return nil, fmt.Errorf("av_frame_get_buffer() failed")
+		}
+
+		d.swsCtx = C.sws_getContext(d.srcFrame.width, d.srcFrame.height, C.AV_PIX_FMT_YUV420P,
+			d.dstFrame.width, d.dstFrame.height, (int32)(d.dstFrame.format), C.SWS_BILINEAR, nil, nil, nil)
+		if d.swsCtx == nil {
+			return nil, fmt.Errorf("sws_getContext() failed")
+		}
+
+		dstFrameSize := C.av_image_get_buffer_size((int32)(d.dstFrame.format), d.dstFrame.width, d.dstFrame.height, 1)
+		d.dstFramePtr = (*[1 << 30]uint8)(unsafe.Pointer(d.dstFrame.data[0]))[:dstFrameSize:dstFrameSize]
+	}
+
+	// convert color space from YUV420 to RGBA
+	res = C.sws_scale(d.swsCtx, frameData(d.srcFrame), frameLineSize(d.srcFrame),
+		0, d.srcFrame.height, frameData(d.dstFrame), frameLineSize(d.dstFrame))
+	if res < 0 {
+		return nil, fmt.Errorf("sws_scale() failed")
+	}
+
+	// embed frame into an image.Image
+	return &image.RGBA{
+		Pix:    d.dstFramePtr,
+		Stride: 4 * (int)(d.dstFrame.width),
+		Rect: image.Rectangle{
+			Max: image.Point{(int)(d.dstFrame.width), (int)(d.dstFrame.height)},
+		},
+	}, nil
+}

examples/client-read-format-h265/main.go

@@ -12,8 +12,11 @@ import (
 
 // This example shows how to
 // 1. connect to a RTSP server
-// 2. check if there's an H265 media
+// 2. check if there's an H265 media stream
-// 3. get access units of that media
+// 3. decode the H264 media stream into RGBA frames
+
+// This example requires the FFmpeg libraries, that can be installed with this command:
+// apt install -y libavformat-dev libswscale-dev gcc pkg-config
 
 func main() {
 	c := gortsplib.Client{}
@@ -50,6 +53,24 @@ func main() {
 		panic(err)
 	}
 
+	// setup H265 -> raw frames decoder
+	frameDec, err := newH265Decoder()
+	if err != nil {
+		panic(err)
+	}
+	defer frameDec.close()
+
+	// if VPS, SPS and PPS are present into the SDP, send them to the decoder
+	if forma.VPS != nil {
+		frameDec.decode(forma.VPS)
+	}
+	if forma.SPS != nil {
+		frameDec.decode(forma.SPS)
+	}
+	if forma.PPS != nil {
+		frameDec.decode(forma.PPS)
+	}
+
 	// setup a single media
 	_, err = c.Setup(desc.BaseURL, medi, 0, 0)
 	if err != nil {
@@ -61,6 +82,7 @@ func main() {
 		// decode timestamp
 		pts, ok := c.PacketPTS(medi, pkt)
 		if !ok {
+			log.Printf("waiting for timestamp")
 			return
 		}
 
@@ -74,7 +96,18 @@ func main() {
 		}
 
 		for _, nalu := range au {
-			log.Printf("received NALU with PTS %v and size %d\n", pts, len(nalu))
+			// convert NALUs into RGBA frames
+			img, err := frameDec.decode(nalu)
+			if err != nil {
+				panic(err)
+			}
+
+			// wait for a frame
+			if img == nil {
+				continue
+			}
+
+			log.Printf("decoded frame with PTS %v and size %v", pts, img.Bounds().Max)
 		}
 	})
 

examples/server-h264-save-to-disk/mpegts_muxer.go

@@ -58,7 +58,7 @@ func (e *mpegtsMuxer) close() {
 // encode encodes a H264 access unit into MPEG-TS.
 func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 	// prepend an AUD. This is required by some players
-	filteredNALUs := [][]byte{
+	filteredAU := [][]byte{
 		{byte(h264.NALUTypeAccessUnitDelimiter), 240},
 	}
 
@@ -69,11 +69,11 @@ func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 		typ := h264.NALUType(nalu[0] & 0x1F)
 		switch typ {
 		case h264.NALUTypeSPS:
-			e.sps = append([]byte(nil), nalu...)
+			e.sps = nalu
 			continue
 
 		case h264.NALUTypePPS:
-			e.pps = append([]byte(nil), nalu...)
+			e.pps = nalu
 			continue
 
 		case h264.NALUTypeAccessUnitDelimiter:
@@ -86,16 +86,16 @@ func (e *mpegtsMuxer) encode(au [][]byte, pts time.Duration) error {
 			nonIDRPresent = true
 		}
 
-		filteredNALUs = append(filteredNALUs, nalu)
+		filteredAU = append(filteredAU, nalu)
 	}
 
-	au = filteredNALUs
+	au = filteredAU
 
 	if len(au) <= 1 || (!nonIDRPresent && !idrPresent) {
 		return nil
 	}
 
-	// add SPS and PPS before every group that contains an IDR
+	// add SPS and PPS before every access unit that contains an IDR
 	if idrPresent {
 		au = append([][]byte{e.sps, e.pps}, au...)
 	}

go.mod

@@ -3,7 +3,7 @@ module github.com/bluenviron/gortsplib/v4
 go 1.19
 
 require (
-	github.com/bluenviron/mediacommon v1.1.0
+	github.com/bluenviron/mediacommon v1.1.1
 	github.com/google/uuid v1.3.1
 	github.com/pion/rtcp v1.2.10
 	github.com/pion/rtp v1.8.1

go.sum

@@ -2,8 +2,8 @@ github.com/asticode/go-astikit v0.30.0 h1:DkBkRQRIxYcknlaU7W7ksNfn4gMFsB0tqMJflx
 github.com/asticode/go-astikit v0.30.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0=
 github.com/asticode/go-astits v1.13.0 h1:XOgkaadfZODnyZRR5Y0/DWkA9vrkLLPLeeOvDwfKZ1c=
 github.com/asticode/go-astits v1.13.0/go.mod h1:QSHmknZ51pf6KJdHKZHJTLlMegIrhega3LPWz3ND/iI=
-github.com/bluenviron/mediacommon v1.1.0 h1:04PQsjITYRerO81Fwc9B1wQqXnvZeLhX74MccIATeRA=
+github.com/bluenviron/mediacommon v1.1.1 h1:cGFy5hnLBeiTrpPyEOf23En+E86r3s99tudlQHY9eEA=
-github.com/bluenviron/mediacommon v1.1.0/go.mod h1:/vlOVSebDwzdRtQONOKLua0fOSJg1tUDHpP+h9a0uqM=
+github.com/bluenviron/mediacommon v1.1.1/go.mod h1:/vlOVSebDwzdRtQONOKLua0fOSJg1tUDHpP+h9a0uqM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=