package/mediadevices
1
0
Fork 0
mirror of https://github.com/pion/mediadevices.git synced 2025-09-26 20:41:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix scaling functions (#385)

Browse Source 
This commit corrects two small errors that causes the improper
results reported in #312.

- A copy paste typo (most likely) which sets the Dy component
  of the image rectangle to Dx.
- The tmp array in fastBoxSampling was not properly
  reset because the last argument in memset should be
  the number of bytes to fill in and not the number of elements.
This commit is contained in:
f-fl0
2022-03-18 15:04:36 +09:00
committed by GitHub
parent 79f9fc31f6
commit 548cdac668
4 changed files with 387 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/io/video/scale.go
View File

@@ -109,7 +109,7 @@ func Scale(width, height int, scaler Scaler) TransformFunc {
yDy := rect.Dy()
cRect := fixedRect(rect, i1.SubsampleRatio)
cDx := cRect.Dx()
cDy := cRect.Dx()
cDy := cRect.Dy()
yLen := yDx * yDy
cLen := cDx * cDy
if len(imgDst.Y) < yLen {

372
pkg/io/video/scale_test.go
View File

@@ -36,6 +36,32 @@ func TestScale(t *testing.T) {
Rect: image.Rect(0, 0, 2, 2),
},
},
"RGBASameSize": {
src: &image.RGBA{
Pix: []uint8{
// R G B A | R G B A | R G B A | R G B A
0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
},
Stride: 16,
Rect: image.Rect(0, 0, 4, 4),
},
width: 4,
height: 4,
expected: &image.RGBA{
Pix: []uint8{
// R G B A | R G B A | R G B A | R G B A
0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
0x80, 0x00, 0x00, 0xFF, 0x80, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF,
0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
0x00, 0x40, 0x00, 0xFF, 0x00, 0x40, 0x00, 0xFF, 0x00, 0x00, 0x60, 0xFF, 0x00, 0x00, 0x60, 0xFF,
},
Stride: 16,
Rect: image.Rect(0, 0, 4, 4),
},
},
"I444": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio444,
@@ -91,6 +117,70 @@ func TestScale(t *testing.T) {
Rect: image.Rect(0, 0, 3, 3),
},
},
"I444SameSize": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio444,
Y: []uint8{
0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
},
YStride: 6,
CStride: 6,
Rect: image.Rect(0, 0, 6, 6),
},
width: 6,
height: 6,
expected: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio444,
Y: []uint8{
0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
0x80, 0x80, 0x80, 0x80, 0xC0, 0xC0,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
0x80, 0x80, 0x40, 0x40, 0x80, 0x80,
},
YStride: 6,
CStride: 6,
Rect: image.Rect(0, 0, 6, 6),
},
},
"I422": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio422,
@@ -155,6 +245,82 @@ func TestScale(t *testing.T) {
Rect: image.Rect(0, 0, 4, 4),
},
},
"I422SameSize": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio422,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0xE0, 0xE0,
0x80, 0x80, 0xE0, 0xE0,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0xF0, 0xF0, 0x40, 0x40,
0xF0, 0xF0, 0x40, 0x40,
},
YStride: 8,
CStride: 4,
Rect: image.Rect(0, 0, 8, 8),
},
width: 8,
height: 8,
expected: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio422,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0xE0, 0xE0,
0x80, 0x80, 0xE0, 0xE0,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80,
0xF0, 0xF0, 0x40, 0x40,
0xF0, 0xF0, 0x40, 0x40,
},
YStride: 8,
CStride: 4,
Rect: image.Rect(0, 0, 8, 8),
},
},
"I420": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
@@ -207,6 +373,118 @@ func TestScale(t *testing.T) {
Rect: image.Rect(0, 0, 4, 4),
},
},
"I420SameSize": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80,
0x80, 0x80, 0xE0, 0xE0,
0x80, 0x80, 0xE0, 0xE0,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80,
0xF0, 0xF0, 0x40, 0x40,
0xF0, 0xF0, 0x40, 0x40,
},
YStride: 8,
CStride: 4,
Rect: image.Rect(0, 0, 8, 8),
},
width: 8,
height: 8,
expected: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80,
0x20, 0x20, 0x80, 0x80,
0x80, 0x80, 0xE0, 0xE0,
0x80, 0x80, 0xE0, 0xE0,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80,
0xE0, 0xE0, 0x80, 0x80,
0xF0, 0xF0, 0x40, 0x40,
0xF0, 0xF0, 0x40, 0x40,
},
YStride: 8,
CStride: 4,
Rect: image.Rect(0, 0, 8, 8),
},
},
"I420NonSquareImage": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
},
YStride: 12,
CStride: 6,
Rect: image.Rect(0, 0, 12, 8),
},
width: 6,
height: 4,
expected: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0x10, 0x00, 0x00, 0xF0, 0x10,
0x00, 0x00, 0x40, 0x00, 0x40, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x80, 0x30, 0x00, 0x30, 0x00,
},
Cb: []uint8{
0x20, 0x80, 0x50,
0x80, 0xE0, 0x30,
},
Cr: []uint8{
0xE0, 0x80, 0xB0,
0xF0, 0x40, 0xC0,
},
YStride: 6,
CStride: 3,
Rect: image.Rect(0, 0, 6, 4),
},
},
}
for name, algo := range scalerTestAlgos {
algo := algo
@@ -247,6 +525,100 @@ func TestScale(t *testing.T) {
}
}
func TestScaleFastBoxSampling(t *testing.T) {
cases := map[string]struct {
src image.Image
width, height int
expected image.Image
}{
"I420NonSquareImage": {
src: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
},
Cb: []uint8{
0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
},
Cr: []uint8{
0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
0xE0, 0xE0, 0x80, 0x80, 0xB0, 0xB0,
0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
0xF0, 0xF0, 0x40, 0x40, 0xC0, 0xC0,
},
YStride: 12,
CStride: 6,
Rect: image.Rect(0, 0, 12, 8),
},
width: 6,
height: 4,
expected: &image.YCbCr{
SubsampleRatio: image.YCbCrSubsampleRatio420,
Y: []uint8{
0xF0, 0x80, 0x08, 0x00, 0x78, 0x80,
0x08, 0x00, 0x20, 0x20, 0x20, 0x20,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x40, 0x58, 0x18, 0x18, 0x18,
},
Cb: []uint8{
0x20, 0x50, 0x68,
0x68, 0xB0, 0x88,
},
Cr: []uint8{
0xE0, 0xB0, 0x98,
0xD0, 0x98, 0x80,
},
YStride: 6,
CStride: 3,
Rect: image.Rect(0, 0, 6, 4),
},
},
}
for name, c := range cases {
c := c
t.Run(name, func(t *testing.T) {
trans := Scale(c.width, c.height, ScalerFastBoxSampling)
r := trans(ReaderFunc(func() (image.Image, func(), error) {
return c.src, func() {}, nil
}))
for i := 0; i < 4; i++ {
out, _, err := r.Read()
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if !reflect.DeepEqual(c.expected, out) {
t.Errorf("Expected output image:\n%v\ngot:\n%v\nrepeat: %d", c.expected, out, i)
}
// Destroy output contents
switch v := out.(type) {
case *image.RGBA:
v.Stride = 10
v.Pix = v.Pix[:1]
v.Rect.Max.X = 1
case *image.YCbCr:
v.YStride = 10
v.CStride = 100
v.Y = v.Y[:1]
v.Cb = v.Cb[:2]
v.Cr = v.Cr[:1]
v.Rect.Max.X = 1
}
}
})
}
}
func BenchmarkScale(b *testing.B) {
for name, algo := range scalerBenchAlgos {
algo := algo

2
pkg/io/video/scaler_cgo.c
View File

@@ -29,7 +29,7 @@ void fastBoxSampling(
const int sw, const int sh, const int sstride,
uint32_t* tmp)
{
memset(tmp, 0, dw * dh * ch);
memset(tmp, 0, dw * dh * ch * sizeof(tmp[0]));
for (int sy = 0; sy < sh; sy++)
{

18
pkg/io/video/scaleycrcb.go
View File

@@ -30,10 +30,18 @@ func (p *rgbLikeYCbCr) At(x, y int) color.Color {
}
func (p *rgbLikeYCbCr) Set(x, y int, c color.Color) {
rgb := c.(*color.RGBA64)
p.y.SetGray(x, y, color.Gray{uint8(rgb.R / 0x100)})
if (image.Point{x, y}.In(p.cb.Rect)) {
p.cb.SetGray(x, y, color.Gray{uint8(rgb.G / 0x100)})
p.cr.SetGray(x, y, color.Gray{uint8(rgb.B / 0x100)})
switch v := c.(type) {
case color.RGBA:
p.y.SetGray(x, y, color.Gray{v.R})
if (image.Point{x, y}.In(p.cb.Rect)) {
p.cb.SetGray(x, y, color.Gray{v.G})
p.cr.SetGray(x, y, color.Gray{v.B})
}
case *color.RGBA64:
p.y.SetGray(x, y, color.Gray{uint8(v.R / 0x100)})
if (image.Point{x, y}.In(p.cb.Rect)) {
p.cb.SetGray(x, y, color.Gray{uint8(v.G / 0x100)})
p.cr.SetGray(x, y, color.Gray{uint8(v.B / 0x100)})
}
}
}