package/mediadevices
1
0
Fork 0
mirror of https://github.com/pion/mediadevices.git synced 2025-09-26 20:41:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixes avfoundation panic when using NV12 frame (#302)

Browse Source 
* Fixes frame.FormatI420 in avfoundation

* Fixes avfoundation: frame size

which causes panic when using NV12

* avfoundation: uses CVPixelBufferGetDataSize() as frame buffer size

frame: fix frame size checking for YUY2, UYVY

* add camera_darwin_test
This commit is contained in:
Markus
2021-02-22 04:13:04 +08:00
committed by GitHub
parent a1087f7f4e
commit b5b0653697
6 changed files with 106 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
pkg/avfoundation/AVFoundationBind/AVFoundationBind.m
View File

@@ -1,17 +1,17 @@
// MIT License
//
//
// Copyright (c) 2019-2020 Pion
//
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -76,29 +76,23 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
!CMSampleBufferDataIsReady(sampleBuffer)) {
return;
}
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (imageBuffer == NULL) {
return;
}
imageBuffer = CVBufferRetain(imageBuffer);
CVReturn ret =
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
if (ret != kCVReturnSuccess) {
return;
}
size_t heightY = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
size_t bytesPerRowY = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
size_t heightUV = CVPixelBufferGetHeightOfPlane(imageBuffer, 1);
size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 1);
int len = (int)((heightY * bytesPerRowY) + (2 * heightUV * bytesPerRowUV));
void *buf = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
_mCallback(_mPUserData, buf, len);
size_t dataSize = CVPixelBufferGetDataSize(imageBuffer);
_mCallback(_mPUserData, buf, (int)dataSize);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
CVBufferRelease(imageBuffer);
}
@@ -136,6 +130,9 @@ STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
// Useful mapping reference from ffmpeg:
// https://github.com/FFmpeg/FFmpeg/blob/c810a9502cebe32e1dd08ee3d0d17053dde44aa9/libavdevice/avfoundation.m#L53-L80
switch (format) {
case AVBindFrameFormatI420:
*pFourCC = kCVPixelFormatType_420YpCbCr8Planar;
break;
case AVBindFrameFormatNV21:
*pFourCC = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
break;
@@ -158,6 +155,9 @@ STATUS frameFormatToFourCC(AVBindFrameFormat format, FourCharCode *pFourCC) {
STATUS frameFormatFromFourCC(FourCharCode fourCC, AVBindFrameFormat *pFormat) {
STATUS retStatus = STATUS_OK;
switch (fourCC) {
case kCVPixelFormatType_420YpCbCr8Planar:
*pFormat = AVBindFrameFormatI420;
break;
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
*pFormat = AVBindFrameFormatNV21;
break;
@@ -184,7 +184,7 @@ STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *p
NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
CHK(mediaType == AVBindMediaTypeVideo || mediaType == AVBindMediaTypeAudio, STATUS_UNSUPPORTED_MEDIA_TYPE);
CHK(ppDevices != NULL && pLen != NULL, STATUS_NULL_ARG);
PAVBindDevice pDevice;
AVMediaType _mediaType = mediaType == AVBindMediaTypeVideo ? AVMediaTypeVideo : AVMediaTypeAudio;
NSArray *refAllTypes = @[
@@ -196,22 +196,22 @@ STATUS AVBindDevices(AVBindMediaType mediaType, PAVBindDevice *ppDevices, int *p
discoverySessionWithDeviceTypes: refAllTypes
mediaType: _mediaType
position: AVCaptureDevicePositionUnspecified];
int i = 0;
for (AVCaptureDevice *refDevice in refSession.devices) {
if (i >= MAX_DEVICES) {
break;
}
pDevice = devices + i;
strncpy(pDevice->uid, refDevice.uniqueID.UTF8String, MAX_DEVICE_UID_CHARS);
pDevice->uid[MAX_DEVICE_UID_CHARS] = '\0';
i++;
}
*ppDevices = devices;
*pLen = i;
cleanup:
[refPool drain];
return retStatus;
@@ -231,7 +231,7 @@ STATUS AVBindSessionInit(AVBindDevice device, PAVBindSession *ppSessionResult) {
pSession->device = device;
pSession->refCaptureSession = NULL;
*ppSessionResult = pSession;
cleanup:
return retStatus;
}
@@ -258,15 +258,15 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
STATUS retStatus = STATUS_OK;
NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
CHK(pSession != NULL && dataCallback != NULL, STATUS_NULL_ARG);
AVCaptureDeviceInput *refInput;
NSError *refErr = NULL;
NSString *refUID = [NSString stringWithUTF8String: pSession->device.uid];
AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refUID];
refInput = [[AVCaptureDeviceInput alloc] initWithDevice: refDevice error: &refErr];
CHK(refErr == NULL, STATUS_DEVICE_INIT_FAILED);
AVCaptureSession *refCaptureSession = [[AVCaptureSession alloc] init];
refCaptureSession.sessionPreset = AVCaptureSessionPresetMedium;
[refCaptureSession addInput: refInput];
@@ -275,7 +275,7 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
VideoDataDelegate *pDelegate = [[VideoDataDelegate alloc]
init: dataCallback
withUserData: pUserData];
AVCaptureVideoDataOutput *pOutput = [[AVCaptureVideoDataOutput alloc] init];
FourCharCode fourCC;
CHK_STATUS(frameFormatToFourCC(property.frameFormat, &fourCC));
@@ -293,10 +293,10 @@ STATUS AVBindSessionOpen(PAVBindSession pSession,
} else {
// TODO: implement audio pipeline
}
pSession->refCaptureSession = [refCaptureSession retain];
[refCaptureSession startRunning];
cleanup:
[refPool drain];
return retStatus;
@@ -307,11 +307,11 @@ STATUS AVBindSessionClose(PAVBindSession pSession) {
STATUS retStatus = STATUS_OK;
CHK(pSession != NULL, STATUS_NULL_ARG);
CHK(pSession->refCaptureSession != NULL, STATUS_OK);
[pSession->refCaptureSession stopRunning];
[pSession->refCaptureSession release];
pSession->refCaptureSession = NULL;
cleanup:
return retStatus;
}
@@ -330,7 +330,7 @@ STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *pp
STATUS retStatus = STATUS_OK;
NSAutoreleasePool *refPool = [[NSAutoreleasePool alloc] init];
CHK(pSession != NULL && ppProperties != NULL && pLen != NULL, STATUS_NULL_ARG);
NSString *refDeviceUID = [NSString stringWithUTF8String: pSession->device.uid];
AVCaptureDevice *refDevice = [AVCaptureDevice deviceWithUniqueID: refDeviceUID];
FourCharCode fourCC;
@@ -346,14 +346,14 @@ STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *pp
NSLog(@"[WARNING] skipping the rest of properties due to MAX_PROPERTIES");
break;
}
if ([refFormat.mediaType isEqual:AVMediaTypeVideo]) {
fourCC = CMFormatDescriptionGetMediaSubType(refFormat.formatDescription);
if (frameFormatFromFourCC(fourCC, &pProperty->frameFormat) != STATUS_OK) {
NSLog(@"[WARNING] skipping %@ %dx%d since it's not supported", FourCCString(fourCC), videoDimensions.width, videoDimensions.height);
continue;
}
videoFormat = (CMVideoFormatDescriptionRef) refFormat.formatDescription;
videoDimensions = CMVideoFormatDescriptionGetDimensions(videoFormat);
pProperty->height = videoDimensions.height;
@@ -361,16 +361,16 @@ STATUS AVBindSessionProperties(PAVBindSession pSession, PAVBindMediaProperty *pp
} else {
// TODO: Get audio properties
}
pProperty++;
len++;
}
*ppProperties = pSession->properties;
*pLen = len;
cleanup:
[refPool drain];
return retStatus;
}

4
pkg/avfoundation/avfoundation_callback_darwin.go
View File

@@ -1,6 +1,5 @@
package avfoundation
// extern void onData(void*, void*, int);
import "C"
import (
"sync"
@@ -18,11 +17,10 @@ type handleID int
//export onData
func onData(userData unsafe.Pointer, buf unsafe.Pointer, length C.int) {
data := C.GoBytes(buf, length)
handleNum := (*C.int)(userData)
cb, ok := lookup(handleID(*handleNum))
if ok {
data := C.GoBytes(buf, length)
cb(data)
}
}

2
pkg/driver/camera/camera_bench_test.go
View File

@@ -56,7 +56,7 @@ func BenchmarkRead(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := r.Read()
_, _, err := r.Read()
if err != nil {
b.Fatalf("Failed to read: %v", err)
}

63
pkg/driver/camera/camera_darwin_test.go Normal file
View File

@@ -0,0 +1,63 @@
// +build darwin
// $ go test -v . -tags darwin -run="^TestCameraFrameFormatSupport$"
package camera
import (
"testing"
"github.com/pion/mediadevices/pkg/avfoundation"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/prop"
)
func TestCameraFrameFormatSupport(t *testing.T) {
devices, err := avfoundation.Devices(avfoundation.Video)
if err != nil {
t.Fatal(err)
}
if len(devices) > 0 {
c := newCamera(devices[0])
if err := c.Open(); err != nil {
t.Fatal(err)
}
defer c.Close()
supportedFormats := make(map[frame.Format]struct{})
for _, p := range c.Properties() {
supportedFormats[p.FrameFormat] = struct{}{}
}
for _, format := range []frame.Format{
frame.FormatI420,
frame.FormatNV12,
frame.FormatNV21,
frame.FormatYUY2,
frame.FormatUYVY,
} {
if _, ok := supportedFormats[format]; !ok {
t.Logf("[%v] UNSUPPORTED", format)
continue
}
r, err := c.VideoRecord(prop.Media{
Video: prop.Video{
Width: 640,
Height: 480,
FrameFormat: format,
}})
if err != nil {
t.Logf("[%v] Failed to capture image: %v", format, err)
continue
}
for i := 0; i < 10; i++ {
_, _, err := r.Read()
if err != nil {
t.Logf("[%v] Failed to read: %v", format, err)
continue
}
}
t.Logf("[%v] OK", format)
}
}
}

4
pkg/frame/yuv_cgo.go
View File

@@ -17,7 +17,7 @@ func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
ci := yi / 2
fi := yi + 2*ci
if len(frame) != fi {
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
@@ -49,7 +49,7 @@ func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
ci := yi / 2
fi := yi + 2*ci
if len(frame) != fi {
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}

4
pkg/frame/yuv_nocgo.go
View File

@@ -12,7 +12,7 @@ func decodeYUY2(frame []byte, width, height int) (image.Image, func(), error) {
ci := yi / 2
fi := yi + 2*ci
if len(frame) != fi {
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}
@@ -47,7 +47,7 @@ func decodeUYVY(frame []byte, width, height int) (image.Image, func(), error) {
ci := yi / 2
fi := yi + 2*ci
if len(frame) != fi {
if len(frame) < fi {
return nil, func() {}, fmt.Errorf("frame length (%d) less than expected (%d)", len(frame), fi)
}