From 3e3988a67f00f872606675dc1a73863efe534655 Mon Sep 17 00:00:00 2001 From: seydx Date: Sat, 25 Jan 2025 16:11:39 +0100 Subject: [PATCH 1/7] minor improvements --- pkg/ring/client.go | 5 ++--- pkg/ring/snapshot.go | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/ring/client.go b/pkg/ring/client.go index 7014213d..4c473276 100644 --- a/pkg/ring/client.go +++ b/pkg/ring/client.go @@ -514,7 +514,6 @@ func (c *Client) Stop() error { if c.prod != nil { _ = c.prod.Stop() - c.prod = nil } if c.ws != nil { @@ -537,6 +536,6 @@ func (c *Client) MarshalJSON() ([]byte, error) { if webrtcProd, ok := c.prod.(*webrtc.Conn); ok { return webrtcProd.MarshalJSON() } - - return nil, errors.New("ring: can't marshal") + + return json.Marshal(c.prod) } diff --git a/pkg/ring/snapshot.go b/pkg/ring/snapshot.go index 84da0fd3..f64e4f79 100644 --- a/pkg/ring/snapshot.go +++ b/pkg/ring/snapshot.go @@ -20,6 +20,7 @@ func NewSnapshotProducer(client *RingRestClient, camera *CameraData) *SnapshotPr ID: core.NewID(), FormatName: "ring/snapshot", Protocol: "https", + RemoteAddr: "app-snaps.ring.com", Medias: []*core.Media{ { Kind: core.KindVideo, @@ -43,7 +44,7 @@ func (p *SnapshotProducer) Start() error { // Fetch snapshot response, err := p.client.Request("GET", fmt.Sprintf("https://app-snaps.ring.com/snapshots/next/%d", int(p.camera.ID)), nil) if err != nil { - return fmt.Errorf("failed to get snapshot: %w", err) + return err } pkt := &rtp.Packet{ @@ -51,10 +52,7 @@ func (p *SnapshotProducer) Start() error { Payload: response, } - // Send to all receivers - for _, receiver := range p.Receivers { - receiver.WriteRTP(pkt) - } + p.Receivers[0].WriteRTP(pkt) return nil } From 36547a7343ff80c5167cc739ebce6dc67a766e43 Mon Sep 17 00:00:00 2001 From: Alex X Date: Sun, 26 Jan 2025 16:09:50 +0300 Subject: [PATCH 2/7] Add support H264, H265, NV12 for V4L2 source #1546 --- pkg/h264/annexb/annexb_test.go | 12 ++++++++++++ pkg/v4l2/device/device.go | 26 +++++++++++++++++--------- pkg/v4l2/device/formats.go | 26 ++++++++++++++++++++++++-- pkg/v4l2/producer.go | 33 +++++++++++++++++++++++++++------ 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/pkg/h264/annexb/annexb_test.go b/pkg/h264/annexb/annexb_test.go index 7220f570..cbc382fe 100644 --- a/pkg/h264/annexb/annexb_test.go +++ b/pkg/h264/annexb/annexb_test.go @@ -83,3 +83,15 @@ func TestDahua(t *testing.T) { n := naluTypes(b) require.Equal(t, []byte{0x40, 0x42, 0x44, 0x26}, n) } + +func TestUSB(t *testing.T) { + s := "00 00 00 01 67 4D 00 1F 8D 8D 40 28 02 DD 37 01 01 01 40 00 01 C2 00 00 57 E4 01 00 00 00 01 68 EE 3C 80 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 65 88 80 00" + b := EncodeToAVCC(decode(s)) + n := naluTypes(b) + require.Equal(t, []byte{0x67, 0x68, 0x65}, n) + + s = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 41 9A 00 4C" + b = EncodeToAVCC(decode(s)) + n = naluTypes(b) + require.Equal(t, []byte{0x41}, n) +} diff --git a/pkg/v4l2/device/device.go b/pkg/v4l2/device/device.go index 7f16fd23..c77d60f5 100644 --- a/pkg/v4l2/device/device.go +++ b/pkg/v4l2/device/device.go @@ -11,8 +11,9 @@ import ( ) type Device struct { - fd int - bufs [][]byte + fd int + bufs [][]byte + pixFmt uint32 } func Open(path string) (*Device, error) { @@ -119,6 +120,8 @@ func (d *Device) ListFrameRates(pixFmt, width, height uint32) ([]uint32, error) } func (d *Device) SetFormat(width, height, pixFmt uint32) error { + d.pixFmt = pixFmt + f := v4l2_format{ typ: V4L2_BUF_TYPE_VIDEO_CAPTURE, pix: v4l2_pix_format{ @@ -196,7 +199,7 @@ func (d *Device) StreamOff() (err error) { return ioctl(d.fd, VIDIOC_REQBUFS, unsafe.Pointer(&rb)) } -func (d *Device) Capture(planarYUV bool) ([]byte, error) { +func (d *Device) Capture() ([]byte, error) { dec := v4l2_buffer{ typ: V4L2_BUF_TYPE_VIDEO_CAPTURE, memory: V4L2_MEMORY_MMAP, @@ -205,11 +208,16 @@ func (d *Device) Capture(planarYUV bool) ([]byte, error) { return nil, err } - buf := make([]byte, dec.bytesused) - if planarYUV { - YUYV2YUV(buf, d.bufs[dec.index][:dec.bytesused]) - } else { - copy(buf, d.bufs[dec.index][:dec.bytesused]) + src := d.bufs[dec.index][:dec.bytesused] + dst := make([]byte, dec.bytesused) + + switch d.pixFmt { + case V4L2_PIX_FMT_YUYV: + YUYVtoYUV(dst, src) + case V4L2_PIX_FMT_NV12: + NV12toYUV(dst, src) + default: + copy(dst, d.bufs[dec.index][:dec.bytesused]) } enc := v4l2_buffer{ @@ -221,7 +229,7 @@ func (d *Device) Capture(planarYUV bool) ([]byte, error) { return nil, err } - return buf, nil + return dst, nil } func (d *Device) Close() error { diff --git a/pkg/v4l2/device/formats.go b/pkg/v4l2/device/formats.go index fb54bbd1..a0b41082 100644 --- a/pkg/v4l2/device/formats.go +++ b/pkg/v4l2/device/formats.go @@ -2,7 +2,10 @@ package device const ( V4L2_PIX_FMT_YUYV = 'Y' | 'U'<<8 | 'Y'<<16 | 'V'<<24 + V4L2_PIX_FMT_NV12 = 'N' | 'V'<<8 | '1'<<16 | '2'<<24 V4L2_PIX_FMT_MJPEG = 'M' | 'J'<<8 | 'P'<<16 | 'G'<<24 + V4L2_PIX_FMT_H264 = 'H' | '2'<<8 | '6'<<16 | '4'<<24 + V4L2_PIX_FMT_HEVC = 'H' | 'E'<<8 | 'V'<<16 | 'C'<<24 ) type Format struct { @@ -13,11 +16,13 @@ type Format struct { var Formats = []Format{ {V4L2_PIX_FMT_YUYV, "YUV 4:2:2", "yuyv422"}, + {V4L2_PIX_FMT_NV12, "Y/UV 4:2:0", "nv12"}, {V4L2_PIX_FMT_MJPEG, "Motion-JPEG", "mjpeg"}, + {V4L2_PIX_FMT_H264, "H.264", "h264"}, + {V4L2_PIX_FMT_HEVC, "HEVC", "hevc"}, } -// YUYV2YUV convert packed YUV to planar YUV -func YUYV2YUV(dst, src []byte) { +func YUYVtoYUV(dst, src []byte) { n := len(src) i0 := 0 iy := 0 @@ -38,3 +43,20 @@ func YUYV2YUV(dst, src []byte) { iv++ } } + +func NV12toYUV(dst, src []byte) { + n := len(src) + k := n / 6 + i0 := k * 4 + iu := i0 + iv := i0 + k + copy(dst, src[:i0]) // copy Y + for i0 < n { + dst[iu] = src[i0] + i0++ + iu++ + dst[iv] = src[i0] + i0++ + iv++ + } +} diff --git a/pkg/v4l2/producer.go b/pkg/v4l2/producer.go index 87199762..663d0a9e 100644 --- a/pkg/v4l2/producer.go +++ b/pkg/v4l2/producer.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/AlexxIT/go2rtc/pkg/core" + "github.com/AlexxIT/go2rtc/pkg/h264/annexb" "github.com/AlexxIT/go2rtc/pkg/v4l2/device" "github.com/pion/rtp" ) @@ -46,17 +47,29 @@ func Open(rawURL string) (*Producer, error) { } switch query.Get("input_format") { - case "mjpeg": - codec.Name = core.CodecJPEG - pixFmt = device.V4L2_PIX_FMT_MJPEG case "yuyv422": if codec.FmtpLine == "" { return nil, errors.New("v4l2: invalid video_size") } - codec.Name = core.CodecRAW codec.FmtpLine += ";colorspace=422" pixFmt = device.V4L2_PIX_FMT_YUYV + case "nv12": + if codec.FmtpLine == "" { + return nil, errors.New("v4l2: invalid video_size") + } + codec.Name = core.CodecRAW + codec.FmtpLine += ";colorspace=420mpeg2" // maybe 420jpeg + pixFmt = device.V4L2_PIX_FMT_NV12 + case "mjpeg": + codec.Name = core.CodecJPEG + pixFmt = device.V4L2_PIX_FMT_MJPEG + case "h264": + codec.Name = core.CodecH264 + pixFmt = device.V4L2_PIX_FMT_H264 + case "hevc": + codec.Name = core.CodecH265 + pixFmt = device.V4L2_PIX_FMT_HEVC default: return nil, errors.New("v4l2: invalid input_format") } @@ -93,10 +106,14 @@ func (c *Producer) Start() error { return err } - planarYUV := c.Medias[0].Codecs[0].Name == core.CodecRAW + var bitstream bool + switch c.Medias[0].Codecs[0].Name { + case core.CodecH264, core.CodecH265: + bitstream = true + } for { - buf, err := c.dev.Capture(planarYUV) + buf, err := c.dev.Capture() if err != nil { return err } @@ -107,6 +124,10 @@ func (c *Producer) Start() error { continue } + if bitstream { + buf = annexb.EncodeToAVCC(buf) + } + pkt := &rtp.Packet{ Header: rtp.Header{Timestamp: core.Now90000()}, Payload: buf, From 9b392a22e1a19992fe9baa8ae4c8977d3a7fb78d Mon Sep 17 00:00:00 2001 From: Alex X Date: Sun, 2 Feb 2025 11:01:44 +0300 Subject: [PATCH 3/7] Ignore unknown NAL unit types for RTP/H264 #1570 --- pkg/h264/rtp.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/h264/rtp.go b/pkg/h264/rtp.go index b4a9dafb..d093254f 100644 --- a/pkg/h264/rtp.go +++ b/pkg/h264/rtp.go @@ -22,7 +22,7 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { buf := make([]byte, 0, 512*1024) // 512K return func(packet *rtp.Packet) { - //log.Printf("[RTP] codec: %s, nalu: %2d, size: %6d, ts: %10d, pt: %2d, ssrc: %d, seq: %d, %v", track.Codec.Name, packet.Payload[0]&0x1F, len(packet.Payload), packet.Timestamp, packet.PayloadType, packet.SSRC, packet.SequenceNumber, packet.Marker) + //log.Printf("[RTP] codec: %s, nalu: %2d, size: %6d, ts: %10d, pt: %2d, ssrc: %d, seq: %d, %v", codec.Name, packet.Payload[0]&0x1F, len(packet.Payload), packet.Timestamp, packet.PayloadType, packet.SSRC, packet.SequenceNumber, packet.Marker) payload, err := depack.Unmarshal(packet.Payload) if len(payload) == 0 || err != nil { @@ -68,6 +68,9 @@ func RTPDepay(codec *core.Codec, handler core.HandlerFunc) core.HandlerFunc { payload = payload[i:] continue + case NALUTypePFrame, NALUTypeSPS, NALUTypePPS: // pass + default: + return // skip any unknown NAL unit type } break } From b14aa4f0dcf9cb2a577195b4abb529004adef40d Mon Sep 17 00:00:00 2001 From: Alex X Date: Sun, 2 Feb 2025 14:46:37 +0300 Subject: [PATCH 4/7] Improve delay for MSE player --- www/video-rtc.js | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/www/video-rtc.js b/www/video-rtc.js index 52fb5dda..fb872b45 100644 --- a/www/video-rtc.js +++ b/www/video-rtc.js @@ -439,24 +439,30 @@ export class VideoRTC extends HTMLElement { const sb = ms.addSourceBuffer(msg.value); sb.mode = 'segments'; // segments or sequence sb.addEventListener('updateend', () => { - if (sb.updating) return; - - try { - if (bufLen > 0) { + if (!sb.updating && bufLen > 0) { + try { const data = buf.slice(0, bufLen); - bufLen = 0; sb.appendBuffer(data); - } else if (sb.buffered && sb.buffered.length) { - const end = sb.buffered.end(sb.buffered.length - 1) - 15; - const start = sb.buffered.start(0); - if (end > start) { - sb.remove(start, end); - ms.setLiveSeekableRange(end, end + 15); - } - // console.debug("VideoRTC.buffered", start, end); + bufLen = 0; + } catch (e) { + // console.debug(e); } - } catch (e) { - // console.debug(e); + } + + if (!sb.updating && sb.buffered && sb.buffered.length) { + const end = sb.buffered.end(sb.buffered.length - 1); + const start = end - 5; + const start0 = sb.buffered.start(0); + if (start > start0) { + sb.remove(start0, start); + ms.setLiveSeekableRange(start, end); + } + if (this.video.currentTime < start) { + this.video.currentTime = start; + } + const gap = end - this.video.currentTime; + this.video.playbackRate = gap > 0.1 ? gap : 0.1; + // console.debug('VideoRTC.buffered', gap, this.video.playbackRate, this.video.readyState); } }); @@ -468,7 +474,7 @@ export class VideoRTC extends HTMLElement { const b = new Uint8Array(data); buf.set(b, bufLen); bufLen += b.byteLength; - // console.debug("VideoRTC.buffer", b.byteLength, bufLen); + // console.debug('VideoRTC.buffer', b.byteLength, bufLen); } else { try { sb.appendBuffer(data); From b139b8fdd6f69f716d5ac8421501d0c8b9ab6031 Mon Sep 17 00:00:00 2001 From: Alex X Date: Sun, 2 Feb 2025 15:41:30 +0300 Subject: [PATCH 5/7] Add readme for V4L2 module --- internal/v4l2/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 internal/v4l2/README.md diff --git a/internal/v4l2/README.md b/internal/v4l2/README.md new file mode 100644 index 00000000..1c5dd390 --- /dev/null +++ b/internal/v4l2/README.md @@ -0,0 +1,39 @@ +# V4L2 + +What you should to know about [V4L2](https://en.wikipedia.org/wiki/Video4Linux): + +- V4L2 (Video for Linux API version 2) works only in Linux +- supports USB cameras and other similar devices +- one device can only be connected to one software simultaneously +- cameras support a fixed list of formats, resolutions and frame rates +- basic cameras supports only RAW (non-compressed) pixel formats +- regular cameras supports MJPEG format (series of JPEG frames) +- advances cameras support H264 format (MSE/MP4, WebRTC compatible) +- using MJPEG and H264 formats (if the camera supports them) won't cost you the CPU usage +- transcoding RAW format to MJPEG or H264 - will cost you a significant CPU usage +- H265 (HEVC) format is also supported (if the camera supports it) + +Tests show that the basic Keenetic router with MIPS processor can broadcast three MJPEG cameras in the following resolutions: 1600х1200 + 640х480 + 640х480. The USB bus bandwidth is no more enough for larger resolutions. CPU consumption is no more than 5%. + +Supported formats for your camera can be found here: **Go2rtc > WebUI > Add > V4L2**. + +## RAW format + +Example: + +```yaml +streams: + camera1: v4l2:device?video=/dev/video0&input_format=yuyv422&video_size=1280x720&framerate=10 +``` + +Go2rtc supports built-in transcoding of RAW to MJPEG format. This does not need to be additionally configured. + +``` +ffplay http://localhost:1984/api/stream.mjpeg?src=camera1 +``` + +**Important.** You don't have to transcode the RAW format to transmit it over the network. You can stream it in `y4m` format, which is perfectly supported by ffmpeg. It won't cost you a CPU usage. But will require high network bandwidth. + +``` +ffplay http://localhost:1984/api/stream.y4m?src=camera1 +``` From da809bb9d74a846ac66715aef91a9badc1421f93 Mon Sep 17 00:00:00 2001 From: Julian Date: Sun, 2 Feb 2025 22:15:19 +0000 Subject: [PATCH 6/7] Update build.yml Fix Build binaries This request has been automatically failed because it uses a deprecated version of `actions/upload-artifact: v3`. Learn more: https://github.blog/changelog/2024-04-16-deprecation-notice-v3-of-the-artifact-actions/ --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1bb01ca7..0bc21d11 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,14 +102,14 @@ jobs: env: { GOOS: freebsd, GOARCH: amd64 } run: go build -ldflags "-s -w" -trimpath - name: Upload go2rtc_freebsd_amd64 - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: { name: go2rtc_freebsd_amd64, path: go2rtc } - name: Build go2rtc_freebsd_arm64 env: { GOOS: freebsd, GOARCH: arm64 } run: go build -ldflags "-s -w" -trimpath - name: Upload go2rtc_freebsd_arm64 - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: { name: go2rtc_freebsd_arm64, path: go2rtc } docker-master: From e935885cd346bb4e57e9c94ef37001d908a0a1bb Mon Sep 17 00:00:00 2001 From: Alex X Date: Fri, 7 Feb 2025 10:12:32 +0300 Subject: [PATCH 7/7] Update general H265 support for WebRTC #1439 --- pkg/webrtc/consumer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/webrtc/consumer.go b/pkg/webrtc/consumer.go index fb90442c..e9d7b2e5 100644 --- a/pkg/webrtc/consumer.go +++ b/pkg/webrtc/consumer.go @@ -56,11 +56,11 @@ func (c *Conn) AddTrack(media *core.Media, codec *core.Codec, track *core.Receiv } case core.CodecH265: - // SafariPay because it is the only browser in the world - // that supports WebRTC + H265 - sender.Handler = h265.SafariPay(1200, sender.Handler) + sender.Handler = h265.RTPPay(1200, sender.Handler) if track.Codec.IsRTP() { sender.Handler = h265.RTPDepay(track.Codec, sender.Handler) + } else { + sender.Handler = h265.RepairAVCC(track.Codec, sender.Handler) } case core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecPCML: