[Android] Add CxxBuffer to native and Java PaddleSegModel (#677)

[Android] Add CxxBuffer to native PaddleSegModel

java/android/README.md

@@ -197,6 +197,14 @@ public SegmentationResult predict(Bitmap ARGB8888Bitmap)；
 // 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上
 public SegmentationResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float weight);
 public SegmentationResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float weight); // 只渲染 不保存图片
+// 修改result，而非返回result，关注性能的用户可以将以下接口与SegmentationResult的CxxBuffer一起使用
+public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result)；
+public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, String savedImagePath, float weight);
+public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, boolean rendering, float weight);
+```
+- 设置竖屏或横屏模式: 对于 PP-HumanSeg系列模型，必须要调用该方法设置竖屏模式为true.
+```java  
+public void setVerticalScreenFlag(boolean flag);
 ```
 - 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。
 ```java
@@ -311,6 +319,10 @@ public class SegmentationResult {
   public float[] mScoreMap; // 预测到的得分 map 每个像素位置对应一个score HxW
   public long[] mShape; // label map实际的shape (H,W)
   public boolean mContainScoreMap = false; // 是否包含 score map
+  // 用户可以选择直接使用CxxBuffer，而非通过JNI拷贝到Java层，
+  // 该方式可以一定程度上提升性能
+  public void setCxxBufferFlag(boolean flag); // 设置是否为CxxBuffer模式
+  public boolean releaseCxxBuffer(); // 手动释放CxxBuffer!!!
   public boolean initialized(); // 检测结果是否有效
 }  
 ```

java/android/app/build.gradle

@@ -1,5 +1,3 @@
-import java.security.MessageDigest
-
 apply plugin: 'com.android.application'
 
 android {
@@ -90,8 +88,6 @@ task downloadAndExtractModels(type: DefaultTask) {
             mkdir "${cachePath}"
         }
         FD_MODEL.eachWithIndex { model, index ->
-            MessageDigest messageDigest = MessageDigest.getInstance('MD5')
-            messageDigest.update(model.src.bytes)
             String[] modelPaths = model.src.split("/")
             String modelName = modelPaths[modelPaths.length - 1]
             // Download the target model if not exists

java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java

@@ -19,6 +19,7 @@ import android.preference.PreferenceManager;
 import android.support.annotation.NonNull;
 import android.support.v4.app.ActivityCompat;
 import android.support.v4.content.ContextCompat;
+import android.util.Log;
 import android.view.View;
 import android.view.ViewGroup;
 import android.view.Window;
@@ -251,12 +252,18 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
         boolean modified = false;
 
         long tc = System.currentTimeMillis();
-        SegmentationResult result = predictor.predict(ARGB8888ImageBitmap);
+
+        SegmentationResult result = new SegmentationResult();
+        result.setCxxBufferFlag(true);
+
+        predictor.predict(ARGB8888ImageBitmap, result);
         timeElapsed += (System.currentTimeMillis() - tc);
 
         Visualize.visSegmentation(ARGB8888ImageBitmap, result);
         modified = result.initialized();
 
+        result.releaseCxxBuffer();
+
         frameCounter++;
         if (frameCounter >= 30) {
             final int fps = (int) (1000 / (timeElapsed / 30));
@@ -304,7 +311,8 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
         // should mean 'width' if the camera display orientation is 90 | 270 degree
         // (Hold the phone upright to record video)
         // (2) Smaller resolution is more suitable for Lite Portrait HumanSeg.
-        // So, we set this preview size (480,480) here.
+        // So, we set this preview size (480,480) here. Reference:
+        // https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.6/contrib/PP-HumanSeg/README_cn.md
         CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 480;
         CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480;
         svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview);
@@ -360,6 +368,7 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
             if (Boolean.parseBoolean(SegmentationSettingsActivity.enableLiteFp16)) {
                 option.enableLiteFp16();
             }
+            predictor.setVerticalScreenFlag(true);
             predictor.init(modelFile, paramsFile, configFile, option);
         }
     }

java/android/fastdeploy/build.gradle

@@ -1,5 +1,3 @@
-import java.security.MessageDigest
-
 apply plugin: 'com.android.library'
 
 
@@ -56,26 +54,23 @@ task downloadAndExtractLibs(type: DefaultTask) {
         println "Downloading and extracting fastdeploy android c++ lib ..."
     }
     doLast {
-        // Prepare cache folder for archives
         String cachePath = "cache"
         if (!file("${cachePath}").exists()) {
             mkdir "${cachePath}"
         }
         FD_CXX_LIB.eachWithIndex { lib, index ->
-            MessageDigest messageDigest = MessageDigest.getInstance('MD5')
+            String[] libPaths = lib.src.split("/")
-            messageDigest.update(lib.src.bytes)
+            String libName = libPaths[libPaths.length - 1]
-            String cacheName = new BigInteger(1, messageDigest.digest()).toString(32)
+            libName = libName.split("\\.")[0]
-            // Download the target archive if not exists
+            boolean copyFiles = !file("${lib.dest}/${libName}").exists()
-            boolean copyFiles = !file("${lib.dest}").exists()
+            if (!file("${cachePath}/${libName}.tgz").exists()) {
-            if (!file("${cachePath}/${cacheName}.tgz").exists()) {
+                println "Downloading ${lib.src} -> ${cachePath}/${libName}.tgz"
-                ant.get(src: lib.src, dest: file("${cachePath}/${cacheName}.tgz"))
+                ant.get(src: lib.src, dest: file("${cachePath}/${libName}.tgz"))
                 copyFiles = true
-                // force to copy files from the latest archive files
             }
-            // Extract the target archive if its dest path does not exists
             if (copyFiles) {
                 copy {
-                    from tarTree("${cachePath}/${cacheName}.tgz")
+                    from tarTree("${cachePath}/${libName}.tgz")
                     into "${lib.dest}"
                 }
             }

java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc

@@ -283,6 +283,10 @@ bool AllocateJavaSegmentationResultFromCxx(
       j_seg_result_clazz, "mContainScoreMap", "Z");
   const jfieldID j_seg_score_map_id = env->GetFieldID(
       j_seg_result_clazz, "mScoreMap", "[F");
+  const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
+      j_seg_result_clazz, "mEnableCxxBuffer", "Z");
+  const jfieldID  j_cxx_buffer_id = env->GetFieldID(
+      j_seg_result_clazz, "mCxxBuffer", "J");
   const jfieldID j_seg_initialized_id = env->GetFieldID(
       j_seg_result_clazz, "mInitialized", "Z");
 
@@ -290,6 +294,18 @@ bool AllocateJavaSegmentationResultFromCxx(
     return false;
   }
 
+  // If 'mEnableCxxBuffer' set as true, then, we only setup the cxx result
+  // pointer to the value of 'mCxxBuffer' field. Some users may want
+  // to use this method to boost the performance of segmentation.
+  jboolean j_enable_cxx_buffer =
+      env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id);
+  if (j_enable_cxx_buffer == JNI_TRUE) {
+    jlong j_cxx_buffer = reinterpret_cast<jlong>(c_result_ptr);
+    env->SetLongField(j_seg_result_obj, j_cxx_buffer_id, j_cxx_buffer);
+    env->SetBooleanField(j_seg_result_obj, j_seg_initialized_id, JNI_TRUE);
+    return true;
+  }
+
   // mLabelMap int[] shape (n):        [I
   const auto &label_map_uint8 = c_result_ptr->label_map;
   jbyteArray j_seg_label_map_byte_arr = env->NewByteArray(len);
@@ -832,6 +848,10 @@ bool AllocateSegmentationResultFromJava(
       j_seg_result_clazz_cc, "mContainScoreMap", "Z");
   const jfieldID j_seg_score_map_id_cc = env->GetFieldID(
       j_seg_result_clazz_cc, "mScoreMap", "[F");
+  const jfieldID j_enable_cxx_buffer_id_cc = env->GetFieldID(
+      j_seg_result_clazz_cc, "mEnableCxxBuffer", "Z");
+  const jfieldID  j_cxx_buffer_id_cc = env->GetFieldID(
+      j_seg_result_clazz_cc, "mCxxBuffer", "J");
   const jfieldID j_seg_initialized_id_cc = env->GetFieldID(
       j_seg_result_clazz_cc, "mInitialized", "Z");
 
@@ -839,6 +859,38 @@ bool AllocateSegmentationResultFromJava(
     return false;
   }
 
+  // If 'mEnableCxxBuffer' set as true, then, we only Allocate from
+  // cxx context to cxx result. Some users may want to use this
+  // method to boost the performance of segmentation.
+  jboolean j_enable_cxx_buffer =
+      env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id_cc);
+
+  if (j_enable_cxx_buffer == JNI_TRUE) {
+    jlong j_cxx_buffer = env->GetLongField(j_seg_result_obj, j_cxx_buffer_id_cc);
+    if (j_cxx_buffer == 0) {
+      return false;
+    }
+    // Allocate from cxx context to cxx result
+    auto c_cxx_buffer = reinterpret_cast<vision::SegmentationResult *>(j_cxx_buffer);
+    // TODO: May use 'swap' to exchange the administrative privileges ?
+    // c_result_ptr->shape.swap(c_cxx_buffer->shape);
+    // c_result_ptr->label_map.swap(c_cxx_buffer->label_map);
+    // c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map;
+    // if (c_cxx_buffer->contain_score_map) {
+    //   c_result_ptr->score_map.swap(c_cxx_buffer->score_map);
+    // }
+    c_result_ptr->shape.assign(
+        c_cxx_buffer->shape.begin(), c_cxx_buffer->shape.end());
+    c_result_ptr->label_map.assign(
+        c_cxx_buffer->label_map.begin(), c_cxx_buffer->label_map.end());
+    c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map;
+    if (c_cxx_buffer->contain_score_map) {
+      c_result_ptr->score_map.assign(
+          c_cxx_buffer->score_map.begin(), c_cxx_buffer->score_map.end());
+    }
+    return true;
+  }
+
   // mInitialized boolean:         Z
   jboolean j_seg_initialized =
       env->GetBooleanField(j_seg_result_obj, j_seg_initialized_id_cc);
@@ -1030,3 +1082,43 @@ bool AllocateCxxResultFromJava(
 
 }  // namespace jni
 }  // namespace fastdeploy
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT jboolean JNICALL
+Java_com_baidu_paddle_fastdeploy_vision_SegmentationResult_releaseCxxBufferNative(
+    JNIEnv *env, jobject thiz) {
+  const jclass j_seg_result_clazz = env->GetObjectClass(thiz);
+  const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
+      j_seg_result_clazz, "mEnableCxxBuffer", "Z");
+  const jfieldID  j_cxx_buffer_id = env->GetFieldID(
+      j_seg_result_clazz, "mCxxBuffer", "J");
+  const jfieldID j_seg_initialized_id = env->GetFieldID(
+      j_seg_result_clazz, "mInitialized", "Z");
+
+  jboolean j_enable_cxx_buffer =
+      env->GetBooleanField(thiz, j_enable_cxx_buffer_id);
+  if (j_enable_cxx_buffer == JNI_FALSE) {
+    return JNI_FALSE;
+  }
+  jlong j_cxx_buffer = env->GetLongField(thiz, j_cxx_buffer_id);
+  if (j_cxx_buffer == 0) {
+    return JNI_FALSE;
+  }
+  auto c_result_ptr = reinterpret_cast<
+      fastdeploy::vision::SegmentationResult *>(j_cxx_buffer);
+  delete c_result_ptr;
+  LOGD("[End] Release SegmentationResult in native !");
+
+  env->SetBooleanField(thiz, j_seg_initialized_id, JNI_FALSE);
+  env->DeleteLocalRef(j_seg_result_clazz);
+
+  return JNI_TRUE;
+}
+
+#ifdef __cplusplus
+}
+#endif

java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc

@@ -41,6 +41,17 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_bindNative(
 #ifdef ENABLE_RUNTIME_PERF
   c_model_ptr->EnableRecordTimeOfRuntime();
 #endif
+
+  // Setup is_vertical_screen param
+  const jclass j_ppseg_clazz = env->GetObjectClass(thiz);
+  const jfieldID j_is_vertical_screen_id = env->GetFieldID(
+      j_ppseg_clazz, "mIsVerticalScreen", "Z");
+  jboolean j_is_vertical_screen = env->GetBooleanField(
+      thiz, j_is_vertical_screen_id);
+  bool c_is_vertical_screen = static_cast<jboolean>(j_is_vertical_screen);
+  c_model_ptr->is_vertical_screen = c_is_vertical_screen;
+  env->DeleteLocalRef(j_ppseg_clazz);
+
   vision::EnableFlyCV();
   return reinterpret_cast<jlong>(c_model_ptr);
 }
@@ -70,6 +81,48 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativ
                                    vision::ResultType::SEGMENTATION);
 }
 
+JNIEXPORT jboolean JNICALL
+Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativeV2(
+    JNIEnv *env, jobject thiz, jlong cxx_context, jobject argb8888_bitmap,
+    jobject result, jboolean save_image, jstring save_path, jboolean rendering,
+    jfloat weight) {
+  if (cxx_context == 0) {
+    return JNI_FALSE;
+  }
+  cv::Mat c_bgr;
+  if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) {
+    return JNI_FALSE;
+  }
+  auto c_model_ptr = reinterpret_cast<segmentation::PaddleSegModel *>(cxx_context);
+  const jclass j_seg_result_clazz = env->GetObjectClass(result);
+  const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
+      j_seg_result_clazz, "mEnableCxxBuffer", "Z");
+  jboolean j_enable_cxx_buffer =
+      env->GetBooleanField(result, j_enable_cxx_buffer_id);
+
+  auto c_result_ptr = new vision::SegmentationResult();
+  auto t = fni::GetCurrentTime();
+  c_model_ptr->Predict(&c_bgr, c_result_ptr);
+  PERF_TIME_OF_RUNTIME(c_model_ptr, t)
+
+  if (rendering) {
+    fni::RenderingSegmentation(env, c_bgr, *c_result_ptr, argb8888_bitmap,
+                               save_image, weight, save_path);
+  }
+  if (!fni::AllocateJavaResultFromCxx(
+      env, result, reinterpret_cast<void *>(c_result_ptr),
+      vision::ResultType::SEGMENTATION)) {
+    delete c_result_ptr;
+    return JNI_FALSE;
+  }
+  // Users need to release cxx result buffer manually
+  // if mEnableCxxBuffer is set as true.
+  if (j_enable_cxx_buffer == JNI_FALSE) {
+    delete c_result_ptr;
+  }
+  return JNI_TRUE;
+}
+
 JNIEXPORT jboolean JNICALL
 Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNative(
     JNIEnv *env, jobject thiz, jlong cxx_context) {
@@ -87,3 +140,4 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNativ
 #ifdef __cplusplus
 }
 #endif
+

java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java

@@ -2,6 +2,8 @@ package com.baidu.paddle.fastdeploy.vision;
 
 import android.support.annotation.NonNull;
 
+import com.baidu.paddle.fastdeploy.FastDeployInitializer;
+
 public class SegmentationResult {
     // Init from native
     public byte[] mLabelMap;
@@ -9,6 +11,11 @@ public class SegmentationResult {
     public long[] mShape;
     public boolean mContainScoreMap = false;
     public boolean mInitialized = false;
+    // Cxx result context, some users may want to use
+    // result pointer from native directly to boost
+    // the performance of segmentation.
+    public long mCxxBuffer = 0;
+    public boolean mEnableCxxBuffer = false;
 
     public SegmentationResult() {
         mInitialized = false;
@@ -18,6 +25,17 @@ public class SegmentationResult {
         return mInitialized;
     }
 
+    public void setCxxBufferFlag(boolean flag) {
+        mEnableCxxBuffer = flag;
+    }
+
+    public boolean releaseCxxBuffer() {
+        if (mCxxBuffer == 0 || !mEnableCxxBuffer) {
+            return false;
+        }
+        return releaseCxxBufferNative();
+    }
+
     public void setLabelMap(@NonNull byte[] labelMapBuffer) {
         if (labelMapBuffer.length > 0) {
             mLabelMap = labelMapBuffer.clone();
@@ -39,4 +57,11 @@ public class SegmentationResult {
     public void setContainScoreMap(boolean containScoreMap) {
         mContainScoreMap = containScoreMap;
     }
+
+    private native boolean releaseCxxBufferNative();
+
+    // Initializes at the beginning.
+    static {
+        FastDeployInitializer.init();
+    }
 }

java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java

@@ -7,6 +7,7 @@ import com.baidu.paddle.fastdeploy.RuntimeOption;
 import com.baidu.paddle.fastdeploy.vision.SegmentationResult;
 
 public class PaddleSegModel {
+    public boolean mIsVerticalScreen = false;
     protected long mCxxContext = 0; // Context from native.
     protected boolean mInitialized = false;
 
@@ -21,6 +22,12 @@ public class PaddleSegModel {
         init_(modelFile, paramsFile, configFile, new RuntimeOption());
     }
 
+    // Is vertical screen or not, for PP-HumanSeg on vertical screen,
+    // this flag must be 'true'.
+    public void setVerticalScreenFlag(boolean flag) {
+        mIsVerticalScreen = flag;
+    }
+
     // Constructor with custom runtime option
     public PaddleSegModel(String modelFile,
                           String paramsFile,
@@ -87,13 +94,48 @@ public class PaddleSegModel {
         // Only support ARGB8888 bitmap in native now.
         SegmentationResult result = predictNative(
                 mCxxContext, ARGB8888Bitmap, true,
-                savedImagePath,true, weight);
+                savedImagePath, true, weight);
         if (result == null) {
             return new SegmentationResult();
         }
         return result;
     }
 
+    // Reset the values of input SegmentationResult directly instead of
+    // return a SegmentationResult from native.
+    public boolean predict(Bitmap ARGB8888Bitmap,
+                           SegmentationResult result) {
+        if (mCxxContext == 0) {
+            return false;
+        }
+        return predictNativeV2(mCxxContext, ARGB8888Bitmap, result,
+                false, "", false, 0.5f);
+    }
+
+    public boolean predict(Bitmap ARGB8888Bitmap,
+                           SegmentationResult result,
+                           boolean rendering,
+                           float weight) {
+        if (mCxxContext == 0) {
+            return false;
+        }
+        return predictNativeV2(mCxxContext, ARGB8888Bitmap, result,
+                false, "", rendering, weight);
+    }
+
+    public boolean predict(Bitmap ARGB8888Bitmap,
+                           SegmentationResult result,
+                           String savedImagePath,
+                           float weight) {
+        if (mCxxContext == 0) {
+            return false;
+        }
+        return predictNativeV2(
+                mCxxContext, ARGB8888Bitmap, result, true,
+                savedImagePath, true, weight);
+    }
+
+
     private boolean init_(String modelFile,
                           String paramsFile,
                           String configFile,
@@ -139,6 +181,15 @@ public class PaddleSegModel {
                                                     boolean rendering,
                                                     float weight);
 
+    // Get cxx result pointer from native
+    private native boolean predictNativeV2(long CxxContext,
+                                           Bitmap ARGB8888Bitmap,
+                                           SegmentationResult result,
+                                           boolean saveImage,
+                                           String savePath,
+                                           boolean rendering,
+                                           float weight);
+
     // Release buffers allocated in native context.
     private native boolean releaseNative(long CxxContext);