[Android] Run button shutter in sub Ui Thread (#598)

* [Backend] fix lite backend save model error

* [Backend] fixed typos

* [FlyCV] optimize the integration of FlyCV

* [cmake] close some tests options

* [cmake] close some test option

* [FlyCV] remove un-need warnings

* [FlyCV] remove un-need GetMat method

* [FlyCV] optimize FlyCV codes

* [cmake] remove un-need cmake function in examples/CMakelists

* [cmake] support gflags for Android

* [Android] Run button shutter in sub Ui Thread

* [Android] Update CameraSurfaceView

java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java

@@ -47,6 +47,8 @@ import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.microedition.khronos.opengles.GL10;
+
 public class DetectionMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener {
     private static final String TAG = DetectionMainActivity.class.getSimpleName();
 
@@ -75,10 +77,12 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
     public static final int TYPE_UNKNOWN = -1;
     public static final int BTN_SHUTTER = 0;
     public static final int ALBUM_SELECT = 1;
-    private static int TYPE = TYPE_UNKNOWN;
+    public static final int REALTIME_DETECT = 2;
+    private static int TYPE = REALTIME_DETECT;
 
     private static final int REQUEST_PERMISSION_CODE_STORAGE = 101;
     private static final int INTENT_CODE_PICK_IMAGE = 100;
+    private static final int TIME_SLEEP_INTERVAL = 50; // ms
 
     String savedImagePath = "result.jpg";
     int lastFrameIndex = 0;
@@ -118,12 +122,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
                 break;
             case R.id.btn_shutter:
                 TYPE = BTN_SHUTTER;
-                svPreview.onPause();
+                runOnShutterUiThread();
-                cameraPageView.setVisibility(View.GONE);
-                resultPageView.setVisibility(View.VISIBLE);
-                seekbarText.setText(resultNum + "");
-                confidenceSeekbar.setProgress((int) (resultNum * 100));
-                resultImage.setImageBitmap(shutterBitmap);
                 break;
             case R.id.btn_settings:
                 startActivity(new Intent(DetectionMainActivity.this, DetectionSettingsActivity.class));
@@ -149,9 +148,49 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
             case R.id.back_in_result:
                 resultPageView.setVisibility(View.GONE);
                 cameraPageView.setVisibility(View.VISIBLE);
+                TYPE = REALTIME_DETECT;
                 svPreview.onResume();
                 break;
+        }
+    }
 
+    private void runOnShutterUiThread() {
+        runOnUiThread(new Runnable() {
+            @SuppressLint("SetTextI18n")
+            public void run() {
+                try {
+                    Thread.sleep(TIME_SLEEP_INTERVAL);
+
+                    svPreview.onPause();
+                    cameraPageView.setVisibility(View.GONE);
+                    resultPageView.setVisibility(View.VISIBLE);
+                    seekbarText.setText(resultNum + "");
+                    confidenceSeekbar.setProgress((int) (resultNum * 100));
+                    if (shutterBitmap != null && !shutterBitmap.isRecycled()) {
+                        resultImage.setImageBitmap(shutterBitmap);
+                    } else {
+                        new AlertDialog.Builder(DetectionMainActivity.this)
+                                .setTitle("Empty Result!")
+                                .setMessage("Current picture is empty, please shutting it again!")
+                                .setCancelable(true)
+                                .show();
+                    }
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+    }
+
+    private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) {
+        if (ARGB8888ImageBitmap == null) {
+            return;
+        }
+        if (!ARGB8888ImageBitmap.isRecycled()) {
+            synchronized (this) {
+                shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true);
+                originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true);
+            }
         }
     }
 
@@ -194,18 +233,17 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
 
     @Override
     public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) {
+        synchronized (this) {
+            if (TYPE == BTN_SHUTTER) {
+                copyBitmapFromCamera(ARGB8888ImageBitmap);
+                return false;
+            }
+        }
+
         String savedImagePath = "";
         synchronized (this) {
             savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg";
         }
-        if (TYPE == BTN_SHUTTER) {
-            shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true);
-            originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true);
-        } else {
-            // Only reference in predict loops.
-            shutterBitmap = ARGB8888ImageBitmap;
-            originShutterBitmap = ARGB8888ImageBitmap;
-        }
 
         boolean modified = false;
         DetectionResult result = predictor.predict(
@@ -258,7 +296,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
     }
 
     public void initView() {
-        TYPE = BTN_SHUTTER;
+        TYPE = REALTIME_DETECT;
         svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview);
         svPreview.setOnTextureChangedListener(this);
         tvStatus = (TextView) findViewById(R.id.tv_status);
@@ -313,16 +351,20 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe
                     @Override
                     public void run() {
                         if (TYPE == ALBUM_SELECT) {
-                            SystemClock.sleep(500);
+                            SystemClock.sleep(TIME_SLEEP_INTERVAL * 10);
-                            predictor.predict(picBitmap, savedImagePath, resultNum);
+                            if (!picBitmap.isRecycled()) {
-                            resultImage.setImageBitmap(picBitmap);
+                                predictor.predict(picBitmap, true, resultNum);
-                            picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true);
+                                resultImage.setImageBitmap(picBitmap);
+                                picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true);
+                            }
                             resultNum = 1.0f;
                         } else {
-                            SystemClock.sleep(500);
+                            SystemClock.sleep(TIME_SLEEP_INTERVAL * 10);
-                            predictor.predict(shutterBitmap, savedImagePath, resultNum);
+                            if (!shutterBitmap.isRecycled()) {
-                            resultImage.setImageBitmap(shutterBitmap);
+                                predictor.predict(shutterBitmap, true, resultNum);
-                            shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true);
+                                resultImage.setImageBitmap(shutterBitmap);
+                                shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true);
+                            }
                             resultNum = 1.0f;
                         }
                     }

java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java

@@ -46,6 +46,9 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer,
     protected int textureWidth = 0;
     protected int textureHeight = 0;
 
+    protected Bitmap ARGB8888ImageBitmap;
+    protected boolean bitmapReleaseMode = true;
+
     // In order to manipulate the camera preview data and render the modified one
     // to the screen, three textures are created and the data flow is shown as following:
     // previewdata->camTextureId->fboTexureId->drawTexureId->framebuffer
@@ -198,9 +201,12 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer,
             // Read pixels of FBO to a bitmap
             ByteBuffer pixelBuffer = ByteBuffer.allocate(textureWidth * textureHeight * 4);
             GLES20.glReadPixels(0, 0, textureWidth, textureHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer);
-            Bitmap ARGB8888ImageBitmap = Bitmap.createBitmap(textureWidth, textureHeight, Bitmap.Config.ARGB_8888);
+
+            ARGB8888ImageBitmap = Bitmap.createBitmap(textureWidth, textureHeight, Bitmap.Config.ARGB_8888);
             ARGB8888ImageBitmap.copyPixelsFromBuffer(pixelBuffer);
+
             boolean modified = onTextureChangedListener.onTextureChanged(ARGB8888ImageBitmap);
+
             if (modified) {
                 targetTexureId = drawTexureId[0];
                 // Update a bitmap to the GL texture if modified
@@ -209,7 +215,9 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer,
                 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, targetTexureId);
                 GLUtils.texImage2D(GL10.GL_TEXTURE_2D, 0, ARGB8888ImageBitmap, 0);
             }
-            ARGB8888ImageBitmap.recycle();
+            if (bitmapReleaseMode) {
+                ARGB8888ImageBitmap.recycle();
+            }
         }
 
         // fboTexureId/drawTexureId->Screen
@@ -229,6 +237,16 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer,
         GLES20.glFlush();
     }
 
+    public void setBitmapReleaseMode(boolean mode) {
+        synchronized (this) {
+            bitmapReleaseMode = mode;
+        }
+    }
+
+    public Bitmap getBitmap() {
+        return ARGB8888ImageBitmap; // may null or recycled.
+    }
+
     private float[] transformTextureCoordinates(float[] coords, float[] matrix) {
         float[] result = new float[coords.length];
         float[] vt = new float[4];

java/android/app/src/main/res/xml/ocr_settings.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <PreferenceScreen xmlns:android="http://schemas.android.com/apk/res/android">
     <ListPreference
-        android:defaultValue="@string/MODEL_DIR_DEFAULT"
+        android:defaultValue="@string/OCR_MODEL_DIR_DEFAULT"
         android:key="@string/CHOOSE_PRE_INSTALLED_MODEL_KEY"
         android:negativeButtonText="@null"
         android:positiveButtonText="@null"