From bd53d48f9a9ac9e3bd8453d2c01ec480a773d5af Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Thu, 17 Nov 2022 10:59:47 +0800 Subject: [PATCH] [Bug Fix] fix CopyShutterBitmap and async camera pause (#617) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView * [Android] Update Android SDK usage docs * [Android] Add facedet Android app example * [cmake] fix FastDeploy.cmake.in errors for Android * [Doc] update SetProcLibCpuNumThreads API doc * [Bug Fix] fix CopyShutterBitmap and async camera pause --- .../detection/DetectionMainActivity.java | 15 +- .../examples/facedet/FaceDetMainActivity.java | 37 ++-- .../res/layout/segmentation_activity_main.xml | 12 +- .../res/layout/segmentation_camera_page.xml | 155 ++++++++++++++++- .../res/layout/segmentation_result_page.xml | 158 +++++++++++++++++- .../layout/segmentation_result_page_item.xml | 26 ++- .../src/main/res/xml/segmentation_setting.xml | 39 ++++- 7 files changed, 413 insertions(+), 29 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index e890b42f0..e3da008bb 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -38,6 +38,7 @@ import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; @@ -72,6 +73,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; @@ -148,6 +150,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe resultPageView.setVisibility(View.GONE); cameraPageView.setVisibility(View.VISIBLE); TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; svPreview.onResume(); break; } @@ -189,7 +192,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe } private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { - if (ARGB8888ImageBitmap == null) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { return; } if (!ARGB8888ImageBitmap.isRecycled()) { @@ -197,7 +200,8 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } - SystemClock.sleep(TIME_SLEEP_INTERVAL); // 50ms + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; } } @@ -253,10 +257,10 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe boolean modified = false; long tc = System.currentTimeMillis(); - DetectionResult result = predictor.predict( - ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); + DetectionResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); - frameCounter++; + + Visualize.visDetection(ARGB8888ImageBitmap, result, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); if (!savedImagePath.isEmpty()) { @@ -265,6 +269,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe } } + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index 56cc8e815..c7c270faa 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -37,6 +37,7 @@ import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.FaceDetectionResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.facedet.SCRFD; import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; @@ -65,12 +66,13 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private ImageView backInResult; private SeekBar confidenceSeekbar; private TextView seekbarText; - private float resultConfThreshold = 1.0f; + private float resultNum = 1.0f; private ResultListView detectResultView; private Bitmap shutterBitmap; private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; @@ -147,6 +149,7 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene resultPageView.setVisibility(View.GONE); cameraPageView.setVisibility(View.VISIBLE); TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; svPreview.onResume(); break; } @@ -169,8 +172,8 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene svPreview.onPause(); cameraPageView.setVisibility(View.GONE); resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); if (shutterBitmap != null && !shutterBitmap.isRecycled()) { resultImage.setImageBitmap(shutterBitmap); } else { @@ -188,7 +191,7 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene } private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { - if (ARGB8888ImageBitmap == null) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { return; } if (!ARGB8888ImageBitmap.isRecycled()) { @@ -197,6 +200,7 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; } } @@ -207,8 +211,8 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene if (resultCode == Activity.RESULT_OK) { cameraPageView.setVisibility(View.GONE); resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); @@ -228,6 +232,7 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + // Camera is still working but detecting loop is on pause. svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @@ -253,10 +258,11 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene long tc = System.currentTimeMillis(); FaceDetectionResult result = predictor.predict( - ARGB8888ImageBitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); + ARGB8888ImageBitmap, FaceDetSettingsActivity.scoreThreshold, 0.4f); timeElapsed += (System.currentTimeMillis() - tc); - frameCounter++; + + Visualize.visFaceDetection(ARGB8888ImageBitmap, result); modified = result.initialized(); if (!savedImagePath.isEmpty()) { @@ -265,6 +271,7 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene } } + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @@ -346,9 +353,9 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { float resultConfidence = seekBar.getProgress() / 100f; BigDecimal bd = new BigDecimal(resultConfidence); - resultConfThreshold = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); } @Override @@ -364,19 +371,19 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms if (!picBitmap.isRecycled()) { - predictor.predict(picBitmap, true, resultConfThreshold, 0.4f); + predictor.predict(picBitmap, true, resultNum, 0.4f); resultImage.setImageBitmap(picBitmap); picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } - resultConfThreshold = 1.0f; + resultNum = 1.0f; } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms if (!shutterBitmap.isRecycled()) { - predictor.predict(shutterBitmap, true, resultConfThreshold, 0.4f); + predictor.predict(shutterBitmap, true, resultNum, 0.4f); resultImage.setImageBitmap(shutterBitmap); shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } - resultConfThreshold = 1.0f; + resultNum = 1.0f; } } }); diff --git a/java/android/app/src/main/res/layout/segmentation_activity_main.xml b/java/android/app/src/main/res/layout/segmentation_activity_main.xml index ae843ab71..2c5ab44f0 100644 --- a/java/android/app/src/main/res/layout/segmentation_activity_main.xml +++ b/java/android/app/src/main/res/layout/segmentation_activity_main.xml @@ -1,6 +1,14 @@ - - \ No newline at end of file + + + + diff --git a/java/android/app/src/main/res/layout/segmentation_camera_page.xml b/java/android/app/src/main/res/layout/segmentation_camera_page.xml index ae843ab71..baaedf594 100644 --- a/java/android/app/src/main/res/layout/segmentation_camera_page.xml +++ b/java/android/app/src/main/res/layout/segmentation_camera_page.xml @@ -1,6 +1,159 @@ + android:layout_height="match_parent" + android:keepScreenOn="true" + tools:context=".segmentation.SegmentationMainActivity"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page.xml b/java/android/app/src/main/res/layout/segmentation_result_page.xml index ae843ab71..958a85940 100644 --- a/java/android/app/src/main/res/layout/segmentation_result_page.xml +++ b/java/android/app/src/main/res/layout/segmentation_result_page.xml @@ -1,6 +1,160 @@ - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page_item.xml b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml index ae843ab71..6a2b09ebf 100644 --- a/java/android/app/src/main/res/layout/segmentation_result_page_item.xml +++ b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml @@ -1,6 +1,26 @@ - + android:layout_height="wrap_content" + android:background="@drawable/result_page_border_section_bk"> - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/xml/segmentation_setting.xml b/java/android/app/src/main/res/xml/segmentation_setting.xml index 624ed13ae..91082dd8f 100644 --- a/java/android/app/src/main/res/xml/segmentation_setting.xml +++ b/java/android/app/src/main/res/xml/segmentation_setting.xml @@ -1,4 +1,41 @@ - + + + + + + \ No newline at end of file