support decoding M-JPEG streams with any quantization (#566) (#583)

pkg/format/rtpmjpeg/decoder.go

@@ -91,6 +91,65 @@ var chmAcSymbols = []byte{ //nolint:dupl
 	0xf9, 0xfa,
 }
 
+var lumaQuantizers = []int{
+	0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
+	0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
+	0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
+	0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
+	0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
+	0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
+	0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
+	0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
+}
+
+var chromaQuantizers = []int{
+	0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
+	0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+}
+
+func makeQuantizationTables(q uint8) [][]byte {
+	var scale int
+	if q < 50 {
+		scale = 5000 / int(q)
+	} else {
+		scale = 200 - 2*int(q)
+	}
+
+	tables := make([][]byte, 2)
+
+	tables[0] = make([]byte, 64)
+
+	for i := 0; i < 64; i++ {
+		v := (lumaQuantizers[i]*scale + 50) / 100
+		if v > 255 {
+			v = 255
+		} else if v == 0 {
+			v = 1
+		}
+		tables[0][i] = byte(v)
+	}
+
+	tables[1] = make([]byte, 64)
+
+	for i := 0; i < 64; i++ {
+		v := (chromaQuantizers[i]*scale + 50) / 100
+		if v > 255 {
+			v = 255
+		} else if v == 0 {
+			v = 1
+		}
+		tables[1][i] = byte(v)
+	}
+
+	return tables
+}
+
 func joinFragments(fragments [][]byte, size int) []byte {
 	ret := make([]byte, size)
 	n := 0
@@ -107,7 +166,7 @@ type Decoder struct {
 	fragmentsSize       int
 	fragments           [][]byte
 	firstJpegHeader     *headerJPEG
-	firstQTHeader       *headerQuantizationTable
+	quantizationTables  [][]byte
 }
 
 // Init initializes the decoder.
@@ -127,11 +186,11 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {
 	byts = byts[n:]
 
 	if jh.Width > maxDimension {
-		return nil, fmt.Errorf("Width of %d is not supported", jh.Width)
+		return nil, fmt.Errorf("width of %d is not supported", jh.Width)
 	}
 
 	if jh.Height > maxDimension {
-		return nil, fmt.Errorf("Height of %d is not supported", jh.Height)
+		return nil, fmt.Errorf("height of %d is not supported", jh.Height)
 	}
 
 	if jh.FragmentOffset == 0 {
@@ -140,14 +199,15 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {
 		d.firstPacketReceived = true
 
 		if jh.Quantization >= 128 {
-			d.firstQTHeader = &headerQuantizationTable{}
+			var hqt headerQuantizationTable
-			n, err := d.firstQTHeader.unmarshal(byts)
+			n, err := hqt.unmarshal(byts)
 			if err != nil {
 				return nil, err
 			}
+			d.quantizationTables = hqt.Tables
 			byts = byts[n:]
 		} else {
-			d.firstQTHeader = nil
+			d.quantizationTables = makeQuantizationTables(jh.Quantization)
 		}
 
 		d.fragments = append(d.fragments, byts)
@@ -186,10 +246,10 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {
 
 	var dqt jpeg.DefineQuantizationTable
 	id := uint8(0)
-	for i := 0; i < len(d.firstQTHeader.Tables); i += 64 {
+	for _, table := range d.quantizationTables {
 		dqt.Tables = append(dqt.Tables, jpeg.QuantizationTable{
 			ID:   id,
-			Data: d.firstQTHeader.Tables[i : i+64],
+			Data: table,
 		})
 		id++
 	}

pkg/format/rtpmjpeg/decoder_test.go

@@ -28,6 +28,108 @@ func TestDecode(t *testing.T) {
 	}
 }
 
+func TestDecodeFixedQuantizationTable(t *testing.T) {
+	d := &Decoder{}
+	err := d.Init()
+	require.NoError(t, err)
+
+	image, err := d.Decode(&rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			Marker:         true,
+			PayloadType:    26,
+			SequenceNumber: 17645,
+			SSRC:           0x9dbb7812,
+		},
+		Payload: []byte{
+			// JPEG header
+			0x00, 0x00, 0x00, 0x00, 0x01, 63, 0xf0, 0x87,
+			// JPEG data
+			1, 2,
+		},
+	})
+	require.NoError(t, err)
+
+	require.Equal(t, []byte{
+		0xff, 0xd8, 0xff, 0xdb, 0x00, 0x84, 0x00, 0x0c,
+		0x08, 0x09, 0x0a, 0x09, 0x07, 0x0c, 0x0a, 0x0a,
+		0x0a, 0x0d, 0x0d, 0x0c, 0x0e, 0x12, 0x1e, 0x13,
+		0x12, 0x10, 0x10, 0x12, 0x24, 0x1a, 0x1b, 0x15,
+		0x1e, 0x2b, 0x26, 0x2d, 0x2c, 0x2a, 0x26, 0x29,
+		0x29, 0x2f, 0x35, 0x44, 0x3a, 0x2f, 0x32, 0x40,
+		0x33, 0x29, 0x29, 0x3b, 0x51, 0x3c, 0x40, 0x46,
+		0x49, 0x4c, 0x4d, 0x4c, 0x2e, 0x39, 0x54, 0x5a,
+		0x53, 0x4a, 0x59, 0x44, 0x4b, 0x4c, 0x49, 0x01,
+		0x0d, 0x0d, 0x0d, 0x12, 0x10, 0x12, 0x23, 0x13,
+		0x13, 0x23, 0x49, 0x31, 0x29, 0x31, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
+		0xff, 0xc0, 0x00, 0x11, 0x08, 0x04, 0x38, 0x07,
+		0x80, 0x03, 0x00, 0x22, 0x00, 0x01, 0x11, 0x01,
+		0x02, 0x11, 0x01, 0xff, 0xc4, 0x00, 0x1f, 0x00,
+		0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
+		0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5,
+		0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04,
+		0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01,
+		0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05,
+		0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
+		0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1,
+		0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1,
+		0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a,
+		0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27,
+		0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38,
+		0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
+		0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+		0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
+		0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+		0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
+		0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+		0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6,
+		0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5,
+		0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4,
+		0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3,
+		0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1,
+		0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
+		0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+		0xf8, 0xf9, 0xfa, 0xff, 0xc4, 0x00, 0x1f, 0x01,
+		0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+		0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5,
+		0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03,
+		0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02,
+		0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05,
+		0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61,
+		0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42,
+		0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52,
+		0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24,
+		0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a,
+		0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37,
+		0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47,
+		0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57,
+		0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67,
+		0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77,
+		0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86,
+		0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95,
+		0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4,
+		0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3,
+		0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2,
+		0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca,
+		0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9,
+		0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8,
+		0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+		0xf8, 0xf9, 0xfa, 0xff, 0xda, 0x00, 0x0c, 0x03,
+		0x00, 0x00, 0x01, 0x11, 0x02, 0x11, 0x00, 0x3f,
+		0x00, 0x01, 0x02, 0xff, 0xd9,
+	}, image)
+}
+
 func FuzzDecoder(f *testing.F) {
 	f.Fuzz(func(_ *testing.T, a []byte, am bool, b []byte, bm bool) {
 		d := &Decoder{}

pkg/format/rtpmjpeg/encoder.go

@@ -229,6 +229,7 @@ outer:
 
 			qth := headerQuantizationTable{}
 
+			// gather and sort tables IDs
 			ids := make([]uint8, len(quantizationTables))
 			i := 0
 			for id := range quantizationTables {
@@ -238,8 +239,10 @@ outer:
 			sort.Slice(ids, func(i, j int) bool {
 				return ids[i] < ids[j]
 			})
+
+			// add tables sorted by ID
 			for _, id := range ids {
-				qth.Tables = append(qth.Tables, quantizationTables[id]...)
+				qth.Tables = append(qth.Tables, quantizationTables[id])
 			}
 
 			buf = qth.marshal(buf)

pkg/format/rtpmjpeg/header_jpeg.go

@@ -23,12 +23,13 @@ func (h *headerJPEG) unmarshal(byts []byte) (int, error) {
 
 	h.Type = byts[4]
 	if h.Type != 1 {
-		return 0, fmt.Errorf("Type %d is not supported", h.Type)
+		return 0, fmt.Errorf("type %d is not supported", h.Type)
 	}
 
 	h.Quantization = byts[5]
-	if h.Quantization != 255 {
+	if h.Quantization == 0 ||
-		return 0, fmt.Errorf("Q %d is not supported", h.Quantization)
+		(h.Quantization > 99 && h.Quantization < 127) {
+		return 0, fmt.Errorf("quantization %d is invalid", h.Quantization)
 	}
 
 	h.Width = int(byts[6]) * 8

pkg/format/rtpmjpeg/header_quantization_table.go

@@ -7,7 +7,7 @@ import (
 type headerQuantizationTable struct {
 	MBZ       uint8
 	Precision uint8
-	Tables    []byte
+	Tables    [][]byte
 }
 
 func (h *headerQuantizationTable) unmarshal(byts []byte) (int, error) {
@@ -18,29 +18,42 @@ func (h *headerQuantizationTable) unmarshal(byts []byte) (int, error) {
 	h.MBZ = byts[0]
 	h.Precision = byts[1]
 	if h.Precision != 0 {
-		return 0, fmt.Errorf("Precision %d is not supported", h.Precision)
+		return 0, fmt.Errorf("precision %d is not supported", h.Precision)
 	}
 
 	length := int(byts[2])<<8 | int(byts[3])
 	switch length {
 	case 64, 128:
 	default:
-		return 0, fmt.Errorf("Quantization table length %d is not supported", length)
+		return 0, fmt.Errorf("table length %d is not supported", length)
 	}
 
 	if (len(byts) - 4) < length {
 		return 0, fmt.Errorf("buffer is too short")
 	}
 
-	h.Tables = byts[4 : 4+length]
+	tableCount := length / 64
+	h.Tables = make([][]byte, tableCount)
+	n := 0
+
+	for i := 0; i < tableCount; i++ {
+		h.Tables[i] = byts[4+n : 4+64+n]
+		n += 64
+	}
+
 	return 4 + length, nil
 }
 
 func (h headerQuantizationTable) marshal(byts []byte) []byte {
 	byts = append(byts, h.MBZ)
 	byts = append(byts, h.Precision)
-	l := len(h.Tables)
+
+	l := len(h.Tables) * 64
 	byts = append(byts, []byte{byte(l >> 8), byte(l)}...)
-	byts = append(byts, h.Tables...)
+
+	for i := 0; i < len(h.Tables); i++ {
+		byts = append(byts, h.Tables[i]...)
+	}
+
 	return byts
 }

pkg/format/rtpmjpeg/header_quantization_table_test.go

@@ -28,7 +28,7 @@ var casesQuantizationTable = []struct {
 		headerQuantizationTable{
 			MBZ:       1,
 			Precision: 0,
-			Tables:    bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 64/4),
+			Tables:    [][]byte{bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 64/4)},
 		},
 	},
 }