diff --git a/README.md b/README.md index 5f7ece62..f052fcf0 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ Ultimate camera streaming application with support RTSP, WebRTC, HomeKit, FFmpeg - zero-dependency and zero-config [small app](#go2rtc-binary) for all OS (Windows, macOS, Linux, ARM) - zero-delay for many supported protocols (lowest possible streaming latency) -- streaming from [RTSP](#source-rtsp), [RTMP](#source-rtmp), [MJPEG](#source-ffmpeg), [HLS](#source-ffmpeg), [USB Cameras](#source-ffmpeg-device), [files](#source-ffmpeg) and [other sources](#module-streams) -- streaming to [RTSP](#module-rtsp), [WebRTC](#module-webrtc) or [MSE](#module-mp4) +- streaming from [RTSP](#source-rtsp), [RTMP](#source-rtmp), [MJPEG](#source-ffmpeg), [HLS/HTTP](#source-ffmpeg), [USB Cameras](#source-ffmpeg-device) and [other sources](#module-streams) +- streaming to [RTSP](#module-rtsp), [WebRTC](#module-webrtc), [MSE/MP4](#module-mp4) or [MJPEG](#module-mjpeg) - first project in the World with support streaming from [HomeKit Cameras](#source-homekit) - on the fly transcoding for unsupported codecs via [FFmpeg](#source-ffmpeg) - multi-source 2-way [codecs negotiation](#codecs-negotiation) @@ -327,13 +327,9 @@ api: ### Module: RTSP -You can get any stream as RTSP-stream with codecs filter: +You can get any stream as RTSP-stream: `rtsp://192.168.1.123:8554/{stream_name}` -``` -rtsp://192.168.1.123/{stream_name}?video={codec}&audio={codec1}&audio={codec2} -``` - -- you can omit the codecs, so one first video and one first audio will be selected +- you can omit the codec filters, so one first video and one first audio will be selected - you can set `?video=copy` or just `?video`, so only one first video without audio will be selected - you can set multiple video or audio, so all of them will be selected @@ -497,6 +493,19 @@ Provides several features: 2. Camera snapshots in MP4 format (single frame), can be sent to [Telegram](https://www.telegram.org/) 3. Progressive MP4 stream - bad format for streaming because of high latency, doesn't work in Safari +### Module: MJPEG + +**Important.** For stream as MJPEG format, your source MUST contain the MJPEG codec. If your camera outputs H264/H265 - you SHOULD use transcoding. With this example, your stream will have both H264 and MJPEG codecs: + +```yaml +streams: + camera1: + - rtsp://rtsp:12345678@192.168.1.123/av_stream/ch0 + - ffmpeg:rtsp://rtsp:12345678@192.168.1.123/av_stream/ch0#video=mjpeg +``` + +Example link to MJPEG: `http://192.168.1.123:1984/api/stream.mjpeg?src=camera1` + ### Module: Log You can set different log levels for different modules. diff --git a/cmd/ffmpeg/README.md b/cmd/ffmpeg/README.md index 6c573035..4bb5af64 100644 --- a/cmd/ffmpeg/README.md +++ b/cmd/ffmpeg/README.md @@ -39,3 +39,5 @@ - https://html5test.com/ - https://trac.ffmpeg.org/wiki/Capture/Webcam - https://trac.ffmpeg.org/wiki/DirectShow +- https://stackoverflow.com/questions/53207692/libav-mjpeg-encoding-and-huffman-table +- https://github.com/tuupola/esp_video/blob/master/README.md diff --git a/cmd/ffmpeg/ffmpeg.go b/cmd/ffmpeg/ffmpeg.go index 80260c8f..7017766d 100644 --- a/cmd/ffmpeg/ffmpeg.go +++ b/cmd/ffmpeg/ffmpeg.go @@ -37,6 +37,7 @@ func Init() { "h264/ultra": "-codec:v libx264 -g 30 -preset ultrafast -tune zerolatency", "h264/high": "-codec:v libx264 -g 30 -preset superfast -tune zerolatency", "h265": "-codec:v libx265 -g 30 -preset ultrafast -tune zerolatency", + "mjpeg": "-codec:v mjpeg -force_duplicated_matrix 1 -huffman 0 -pix_fmt yuvj420p", "opus": "-codec:a libopus -ar 48000 -ac 2", "pcmu": "-codec:a pcm_mulaw -ar 8000 -ac 1", "pcmu/16000": "-codec:a pcm_mulaw -ar 16000 -ac 1", diff --git a/cmd/mjpeg/mjpeg.go b/cmd/mjpeg/mjpeg.go new file mode 100644 index 00000000..6e8e8ceb --- /dev/null +++ b/cmd/mjpeg/mjpeg.go @@ -0,0 +1,54 @@ +package mjpeg + +import ( + "github.com/AlexxIT/go2rtc/cmd/api" + "github.com/AlexxIT/go2rtc/cmd/streams" + "github.com/AlexxIT/go2rtc/pkg/mjpeg" + "github.com/rs/zerolog/log" + "net/http" + "strconv" +) + +func Init() { + api.HandleFunc("api/stream.mjpeg", handler) +} + +const header = "--frame\r\nContent-Type: image/jpeg\r\nContent-Length: " + +func handler(w http.ResponseWriter, r *http.Request) { + src := r.URL.Query().Get("src") + stream := streams.GetOrNew(src) + if stream == nil { + return + } + + exit := make(chan struct{}) + + cons := &mjpeg.Consumer{} + cons.Listen(func(msg interface{}) { + switch msg := msg.(type) { + case []byte: + data := []byte(header + strconv.Itoa(len(msg))) + data = append(data, 0x0D, 0x0A, 0x0D, 0x0A) + data = append(data, msg...) + data = append(data, 0x0D, 0x0A) + + if _, err := w.Write(data); err != nil { + exit <- struct{}{} + } + } + }) + + if err := stream.AddConsumer(cons); err != nil { + log.Error().Err(err).Msg("[api.mjpeg] add consumer") + return + } + + w.Header().Set("Content-Type", `multipart/x-mixed-replace; boundary=frame`) + + <-exit + + stream.RemoveConsumer(cons) + + //log.Trace().Msg("[api.mjpeg] close") +} diff --git a/main.go b/main.go index 8123c026..0ca26111 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "github.com/AlexxIT/go2rtc/cmd/hass" "github.com/AlexxIT/go2rtc/cmd/homekit" "github.com/AlexxIT/go2rtc/cmd/ivideon" + "github.com/AlexxIT/go2rtc/cmd/mjpeg" "github.com/AlexxIT/go2rtc/cmd/mp4" "github.com/AlexxIT/go2rtc/cmd/ngrok" "github.com/AlexxIT/go2rtc/cmd/rtmp" @@ -38,6 +39,7 @@ func main() { webrtc.Init() mp4.Init() + mjpeg.Init() srtp.Init() homekit.Init() diff --git a/pkg/mjpeg/README.md b/pkg/mjpeg/README.md new file mode 100644 index 00000000..f23c88c2 --- /dev/null +++ b/pkg/mjpeg/README.md @@ -0,0 +1,4 @@ +## Useful links + +- https://www.rfc-editor.org/rfc/rfc2435 +- https://github.com/GStreamer/gst-plugins-good/blob/master/gst/rtp/gstrtpjpegdepay.c diff --git a/pkg/mjpeg/consumer.go b/pkg/mjpeg/consumer.go new file mode 100644 index 00000000..c2c3b9bf --- /dev/null +++ b/pkg/mjpeg/consumer.go @@ -0,0 +1,87 @@ +package mjpeg + +import ( + "github.com/AlexxIT/go2rtc/pkg/streamer" + "github.com/pion/rtp" +) + +type Consumer struct { + streamer.Element + + UserAgent string + RemoteAddr string + + codecs []*streamer.Codec + start bool + + send int +} + +func (c *Consumer) GetMedias() []*streamer.Media { + return []*streamer.Media{{ + Kind: streamer.KindVideo, + Direction: streamer.DirectionRecvonly, + Codecs: []*streamer.Codec{{Name: streamer.CodecJPEG}}, + }} +} + +func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *streamer.Track { + var header, payload []byte + + push := func(packet *rtp.Packet) error { + //fmt.Printf( + // "[RTP] codec: %s, size: %6d, ts: %10d, pt: %2d, ssrc: %d, seq: %d, mark: %v\n", + // track.Codec.Name, len(packet.Payload), packet.Timestamp, + // packet.PayloadType, packet.SSRC, packet.SequenceNumber, packet.Marker, + //) + + // https://www.rfc-editor.org/rfc/rfc2435#section-3.1 + b := packet.Payload + + // 3.1. JPEG header + t := b[4] + + // 3.1.7. Restart Marker header + if 64 <= t && t <= 127 { + b = b[12:] // skip it + } else { + b = b[8:] + } + + if header == nil { + var lqt, cqt []byte + + // 3.1.8. Quantization Table header + q := packet.Payload[5] + if q >= 128 { + lqt = b[4:68] + cqt = b[68:132] + b = b[132:] + } else { + lqt, cqt = MakeTables(q) + } + + w := uint16(packet.Payload[6]) << 3 + h := uint16(packet.Payload[7]) << 3 + //fmt.Printf("t: %d, q: %d, w: %d, h: %d\n", t, q, w, h) + header = MakeHeaders(t, w, h, lqt, cqt) + } + + // 3.1.9. JPEG Payload + payload = append(payload, b...) + + if packet.Marker { + b = append(header, payload...) + if end := b[len(b)-2:]; end[0] != 0xFF && end[1] != 0xD9 { + b = append(b, 0xFF, 0xD9) + } + c.Fire(b) + + header = nil + payload = nil + } + + return nil + } + return track.Bind(push) +} diff --git a/pkg/mjpeg/rfc2435.go b/pkg/mjpeg/rfc2435.go new file mode 100644 index 00000000..5e3278a6 --- /dev/null +++ b/pkg/mjpeg/rfc2435.go @@ -0,0 +1,182 @@ +package mjpeg + +// RFC 2435. Appendix A + +var jpeg_luma_quantizer = []byte{ + 16, 11, 10, 16, 24, 40, 51, 61, + 12, 12, 14, 19, 26, 58, 60, 55, + 14, 13, 16, 24, 40, 57, 69, 56, + 14, 17, 22, 29, 51, 87, 80, 62, + 18, 22, 37, 56, 68, 109, 103, 77, + 24, 35, 55, 64, 81, 104, 113, 92, + 49, 64, 78, 87, 103, 121, 120, 101, + 72, 92, 95, 98, 112, 100, 103, 99, +} +var jpeg_chroma_quantizer = []byte{ + 17, 18, 24, 47, 99, 99, 99, 99, + 18, 21, 26, 66, 99, 99, 99, 99, + 24, 26, 56, 99, 99, 99, 99, 99, + 47, 66, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, +} + +func MakeTables(q byte) (lqt, cqt []byte) { + var factor int + + switch { + case q < 1: + factor = 1 + case q > 99: + factor = 99 + default: + factor = int(q) + } + + if q < 50 { + factor = 5000 / factor + } else if q > 99 { + factor = 200 - factor*2 + } + + lqt = make([]byte, 64) + cqt = make([]byte, 64) + + for i := 0; i < 64; i++ { + lq := (int(jpeg_luma_quantizer[i])*factor + 50) / 100 + cq := (int(jpeg_chroma_quantizer[i])*factor + 50) / 100 + + /* Limit the quantizers to 1 <= q <= 255 */ + switch { + case lq < 1: + lqt[i] = 1 + case lq > 255: + lqt[i] = 255 + default: + lqt[i] = byte(lq) + } + + switch { + case cq < 1: + cqt[i] = 1 + case cq > 255: + cqt[i] = 255 + default: + cqt[i] = byte(cq) + } + } + + return +} + +// RFC 2435. Appendix B + +var lum_dc_codelens = []byte{ + 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, +} +var lum_dc_symbols = []byte{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, +} +var lum_ac_codelens = []byte{ + 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d, +} +var lum_ac_symbols = []byte{ + 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, + 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, + 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, + 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, + 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, + 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, + 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, + 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, + 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, + 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, +} +var chm_dc_codelens = []byte{ + 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, +} +var chm_dc_symbols = []byte{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, +} +var chm_ac_codelens = []byte{ + 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77, +} +var chm_ac_symbols = []byte{ + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, + 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, + 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, + 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, + 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, + 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, + 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, + 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, + 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, + 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, +} + +func MakeHeaders(t byte, w, h uint16, lqt, cqt []byte) []byte { + // Appendix A from https://www.rfc-editor.org/rfc/rfc2435 + p := []byte{0xFF, 0xD8} + + p = MakeQuantHeader(p, lqt, 0) + p = MakeQuantHeader(p, cqt, 1) + + if t == 0 { + t = 0x21 + } else { + t = 0x22 + } + + p = append(p, + 0xFF, 0xC0, 0, 17, 8, + byte(h>>8), byte(h&0xFF), + byte(w>>8), byte(w&0xFF), + 3, 0, t, 0, 1, 0x11, 1, 2, 0x11, 1, + ) + + p = MakeHuffmanHeader(p, lum_dc_codelens, lum_dc_symbols, 0, 0) + p = MakeHuffmanHeader(p, lum_ac_codelens, lum_ac_symbols, 0, 1) + p = MakeHuffmanHeader(p, chm_dc_codelens, chm_dc_symbols, 1, 0) + p = MakeHuffmanHeader(p, chm_ac_codelens, chm_ac_symbols, 1, 1) + + return append(p, 0xFF, 0xDA, 0, 12, 3, 0, 0, 1, 0x11, 2, 0x11, 0, 63, 0) +} + +func MakeQuantHeader(p []byte, qt []byte, tableNo byte) []byte { + p = append(p, 0xFF, 0xDB, 0, 67, tableNo) + return append(p, qt...) +} + +func MakeHuffmanHeader(p, codelens, symbols []byte, tableNo, tableClass byte) []byte { + p = append(p, + 0xFF, 0xC4, 0, + byte(3+len(codelens)+len(symbols)), + (tableClass<<4)|tableNo, + ) + p = append(p, codelens...) + return append(p, symbols...) +} diff --git a/pkg/streamer/media.go b/pkg/streamer/media.go index 70a7a020..c14e3945 100644 --- a/pkg/streamer/media.go +++ b/pkg/streamer/media.go @@ -25,6 +25,7 @@ const ( CodecVP8 = "VP8" CodecVP9 = "VP9" CodecAV1 = "AV1" + CodecJPEG = "JPEG" // payloadType: 26 CodecPCMU = "PCMU" // payloadType: 0 CodecPCMA = "PCMA" // payloadType: 8 @@ -36,7 +37,7 @@ const ( func GetKind(name string) string { switch name { - case CodecH264, CodecH265, CodecVP8, CodecVP9, CodecAV1: + case CodecH264, CodecH265, CodecVP8, CodecVP9, CodecAV1, CodecJPEG: return KindVideo case CodecPCMU, CodecPCMA, CodecAAC, CodecOpus, CodecG722, CodecMPA: return KindAudio @@ -129,12 +130,14 @@ type Codec struct { func NewCodec(name string) *Codec { name = strings.ToUpper(name) switch name { - case CodecH264, CodecH265, CodecVP8, CodecVP9, CodecAV1: + case CodecH264, CodecH265, CodecVP8, CodecVP9, CodecAV1, CodecJPEG: return &Codec{Name: name, ClockRate: 90000} case CodecPCMU, CodecPCMA: return &Codec{Name: name, ClockRate: 8000} case CodecOpus: return &Codec{Name: name, ClockRate: 48000, Channels: 2} + case "MJPEG": + return &Codec{Name: CodecJPEG, ClockRate: 90000} } panic(fmt.Sprintf("unsupported codec: %s", name)) @@ -257,6 +260,7 @@ func UnmarshalCodec(md *sdp.MediaDescription, payloadType string) *Codec { } if c.Name == "" { + // https://en.wikipedia.org/wiki/RTP_payload_formats switch payloadType { case "0": c.Name = CodecPCMU @@ -267,6 +271,9 @@ func UnmarshalCodec(md *sdp.MediaDescription, payloadType string) *Codec { case "14": c.Name = CodecMPA c.ClockRate = 44100 + case "26": + c.Name = CodecJPEG + c.ClockRate = 90000 default: c.Name = payloadType }