From ff5fb248ff08f0beddc888977aff02746c61dfbe Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Wed, 23 Nov 2022 16:10:11 +0800 Subject: [PATCH] [Android] Add CxxBuffer to native and Java PaddleSegModel (#677) [Android] Add CxxBuffer to native PaddleSegModel --- java/android/README.md | 12 +++ java/android/app/build.gradle | 4 - .../SegmentationMainActivity.java | 13 ++- java/android/fastdeploy/build.gradle | 21 ++--- .../cpp/fastdeploy_jni/vision/results_jni.cc | 92 +++++++++++++++++++ .../segmentation/paddleseg_model_jni.cc | 54 +++++++++++ .../fastdeploy/vision/SegmentationResult.java | 25 +++++ .../vision/segmentation/PaddleSegModel.java | 53 ++++++++++- 8 files changed, 254 insertions(+), 20 deletions(-) diff --git a/java/android/README.md b/java/android/README.md index a032ae24b..4647d916b 100644 --- a/java/android/README.md +++ b/java/android/README.md @@ -197,6 +197,14 @@ public SegmentationResult predict(Bitmap ARGB8888Bitmap)； // 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 public SegmentationResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float weight); public SegmentationResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float weight); // 只渲染 不保存图片 +// 修改result，而非返回result，关注性能的用户可以将以下接口与SegmentationResult的CxxBuffer一起使用 +public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result)； +public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, String savedImagePath, float weight); +public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, boolean rendering, float weight); +``` +- 设置竖屏或横屏模式: 对于 PP-HumanSeg系列模型，必须要调用该方法设置竖屏模式为true. +```java +public void setVerticalScreenFlag(boolean flag); ``` - 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 ```java @@ -311,6 +319,10 @@ public class SegmentationResult { public float[] mScoreMap; // 预测到的得分 map 每个像素位置对应一个score HxW public long[] mShape; // label map实际的shape (H,W) public boolean mContainScoreMap = false; // 是否包含 score map + // 用户可以选择直接使用CxxBuffer，而非通过JNI拷贝到Java层， + // 该方式可以一定程度上提升性能 + public void setCxxBufferFlag(boolean flag); // 设置是否为CxxBuffer模式 + public boolean releaseCxxBuffer(); // 手动释放CxxBuffer!!! public boolean initialized(); // 检测结果是否有效 } ``` diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle index b66459244..a92d0a13c 100644 --- a/java/android/app/build.gradle +++ b/java/android/app/build.gradle @@ -1,5 +1,3 @@ -import java.security.MessageDigest - apply plugin: 'com.android.application' android { @@ -90,8 +88,6 @@ task downloadAndExtractModels(type: DefaultTask) { mkdir "${cachePath}" } FD_MODEL.eachWithIndex { model, index -> - MessageDigest messageDigest = MessageDigest.getInstance('MD5') - messageDigest.update(model.src.bytes) String[] modelPaths = model.src.split("/") String modelName = modelPaths[modelPaths.length - 1] // Download the target model if not exists diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 9e0df55b5..08ecda611 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -19,6 +19,7 @@ import android.preference.PreferenceManager; import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; +import android.util.Log; import android.view.View; import android.view.ViewGroup; import android.view.Window; @@ -251,12 +252,18 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi boolean modified = false; long tc = System.currentTimeMillis(); - SegmentationResult result = predictor.predict(ARGB8888ImageBitmap); + + SegmentationResult result = new SegmentationResult(); + result.setCxxBufferFlag(true); + + predictor.predict(ARGB8888ImageBitmap, result); timeElapsed += (System.currentTimeMillis() - tc); Visualize.visSegmentation(ARGB8888ImageBitmap, result); modified = result.initialized(); + result.releaseCxxBuffer(); + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -304,7 +311,8 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi // should mean 'width' if the camera display orientation is 90 | 270 degree // (Hold the phone upright to record video) // (2) Smaller resolution is more suitable for Lite Portrait HumanSeg. - // So, we set this preview size (480,480) here. + // So, we set this preview size (480,480) here. Reference: + // https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.6/contrib/PP-HumanSeg/README_cn.md CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 480; CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); @@ -360,6 +368,7 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi if (Boolean.parseBoolean(SegmentationSettingsActivity.enableLiteFp16)) { option.enableLiteFp16(); } + predictor.setVerticalScreenFlag(true); predictor.init(modelFile, paramsFile, configFile, option); } } diff --git a/java/android/fastdeploy/build.gradle b/java/android/fastdeploy/build.gradle index a7bc3f67e..d73e1e36f 100644 --- a/java/android/fastdeploy/build.gradle +++ b/java/android/fastdeploy/build.gradle @@ -1,5 +1,3 @@ -import java.security.MessageDigest - apply plugin: 'com.android.library' @@ -56,26 +54,23 @@ task downloadAndExtractLibs(type: DefaultTask) { println "Downloading and extracting fastdeploy android c++ lib ..." } doLast { - // Prepare cache folder for archives String cachePath = "cache" if (!file("${cachePath}").exists()) { mkdir "${cachePath}" } FD_CXX_LIB.eachWithIndex { lib, index -> - MessageDigest messageDigest = MessageDigest.getInstance('MD5') - messageDigest.update(lib.src.bytes) - String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) - // Download the target archive if not exists - boolean copyFiles = !file("${lib.dest}").exists() - if (!file("${cachePath}/${cacheName}.tgz").exists()) { - ant.get(src: lib.src, dest: file("${cachePath}/${cacheName}.tgz")) + String[] libPaths = lib.src.split("/") + String libName = libPaths[libPaths.length - 1] + libName = libName.split("\\.")[0] + boolean copyFiles = !file("${lib.dest}/${libName}").exists() + if (!file("${cachePath}/${libName}.tgz").exists()) { + println "Downloading ${lib.src} -> ${cachePath}/${libName}.tgz" + ant.get(src: lib.src, dest: file("${cachePath}/${libName}.tgz")) copyFiles = true - // force to copy files from the latest archive files } - // Extract the target archive if its dest path does not exists if (copyFiles) { copy { - from tarTree("${cachePath}/${cacheName}.tgz") + from tarTree("${cachePath}/${libName}.tgz") into "${lib.dest}" } } diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc index 9ae6054aa..8f6199465 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc @@ -283,6 +283,10 @@ bool AllocateJavaSegmentationResultFromCxx( j_seg_result_clazz, "mContainScoreMap", "Z"); const jfieldID j_seg_score_map_id = env->GetFieldID( j_seg_result_clazz, "mScoreMap", "[F"); + const jfieldID j_enable_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mEnableCxxBuffer", "Z"); + const jfieldID j_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mCxxBuffer", "J"); const jfieldID j_seg_initialized_id = env->GetFieldID( j_seg_result_clazz, "mInitialized", "Z"); @@ -290,6 +294,18 @@ bool AllocateJavaSegmentationResultFromCxx( return false; } + // If 'mEnableCxxBuffer' set as true, then, we only setup the cxx result + // pointer to the value of 'mCxxBuffer' field. Some users may want + // to use this method to boost the performance of segmentation. + jboolean j_enable_cxx_buffer = + env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id); + if (j_enable_cxx_buffer == JNI_TRUE) { + jlong j_cxx_buffer = reinterpret_cast(c_result_ptr); + env->SetLongField(j_seg_result_obj, j_cxx_buffer_id, j_cxx_buffer); + env->SetBooleanField(j_seg_result_obj, j_seg_initialized_id, JNI_TRUE); + return true; + } + // mLabelMap int[] shape (n): [I const auto &label_map_uint8 = c_result_ptr->label_map; jbyteArray j_seg_label_map_byte_arr = env->NewByteArray(len); @@ -832,6 +848,10 @@ bool AllocateSegmentationResultFromJava( j_seg_result_clazz_cc, "mContainScoreMap", "Z"); const jfieldID j_seg_score_map_id_cc = env->GetFieldID( j_seg_result_clazz_cc, "mScoreMap", "[F"); + const jfieldID j_enable_cxx_buffer_id_cc = env->GetFieldID( + j_seg_result_clazz_cc, "mEnableCxxBuffer", "Z"); + const jfieldID j_cxx_buffer_id_cc = env->GetFieldID( + j_seg_result_clazz_cc, "mCxxBuffer", "J"); const jfieldID j_seg_initialized_id_cc = env->GetFieldID( j_seg_result_clazz_cc, "mInitialized", "Z"); @@ -839,6 +859,38 @@ bool AllocateSegmentationResultFromJava( return false; } + // If 'mEnableCxxBuffer' set as true, then, we only Allocate from + // cxx context to cxx result. Some users may want to use this + // method to boost the performance of segmentation. + jboolean j_enable_cxx_buffer = + env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id_cc); + + if (j_enable_cxx_buffer == JNI_TRUE) { + jlong j_cxx_buffer = env->GetLongField(j_seg_result_obj, j_cxx_buffer_id_cc); + if (j_cxx_buffer == 0) { + return false; + } + // Allocate from cxx context to cxx result + auto c_cxx_buffer = reinterpret_cast(j_cxx_buffer); + // TODO: May use 'swap' to exchange the administrative privileges ? + // c_result_ptr->shape.swap(c_cxx_buffer->shape); + // c_result_ptr->label_map.swap(c_cxx_buffer->label_map); + // c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map; + // if (c_cxx_buffer->contain_score_map) { + // c_result_ptr->score_map.swap(c_cxx_buffer->score_map); + // } + c_result_ptr->shape.assign( + c_cxx_buffer->shape.begin(), c_cxx_buffer->shape.end()); + c_result_ptr->label_map.assign( + c_cxx_buffer->label_map.begin(), c_cxx_buffer->label_map.end()); + c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map; + if (c_cxx_buffer->contain_score_map) { + c_result_ptr->score_map.assign( + c_cxx_buffer->score_map.begin(), c_cxx_buffer->score_map.end()); + } + return true; + } + // mInitialized boolean: Z jboolean j_seg_initialized = env->GetBooleanField(j_seg_result_obj, j_seg_initialized_id_cc); @@ -1030,3 +1082,43 @@ bool AllocateCxxResultFromJava( } // namespace jni } // namespace fastdeploy + + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_SegmentationResult_releaseCxxBufferNative( + JNIEnv *env, jobject thiz) { + const jclass j_seg_result_clazz = env->GetObjectClass(thiz); + const jfieldID j_enable_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mEnableCxxBuffer", "Z"); + const jfieldID j_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mCxxBuffer", "J"); + const jfieldID j_seg_initialized_id = env->GetFieldID( + j_seg_result_clazz, "mInitialized", "Z"); + + jboolean j_enable_cxx_buffer = + env->GetBooleanField(thiz, j_enable_cxx_buffer_id); + if (j_enable_cxx_buffer == JNI_FALSE) { + return JNI_FALSE; + } + jlong j_cxx_buffer = env->GetLongField(thiz, j_cxx_buffer_id); + if (j_cxx_buffer == 0) { + return JNI_FALSE; + } + auto c_result_ptr = reinterpret_cast< + fastdeploy::vision::SegmentationResult *>(j_cxx_buffer); + delete c_result_ptr; + LOGD("[End] Release SegmentationResult in native !"); + + env->SetBooleanField(thiz, j_seg_initialized_id, JNI_FALSE); + env->DeleteLocalRef(j_seg_result_clazz); + + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc index 0bdad7a4d..44686076a 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc @@ -41,6 +41,17 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_bindNative( #ifdef ENABLE_RUNTIME_PERF c_model_ptr->EnableRecordTimeOfRuntime(); #endif + + // Setup is_vertical_screen param + const jclass j_ppseg_clazz = env->GetObjectClass(thiz); + const jfieldID j_is_vertical_screen_id = env->GetFieldID( + j_ppseg_clazz, "mIsVerticalScreen", "Z"); + jboolean j_is_vertical_screen = env->GetBooleanField( + thiz, j_is_vertical_screen_id); + bool c_is_vertical_screen = static_cast(j_is_vertical_screen); + c_model_ptr->is_vertical_screen = c_is_vertical_screen; + env->DeleteLocalRef(j_ppseg_clazz); + vision::EnableFlyCV(); return reinterpret_cast(c_model_ptr); } @@ -70,6 +81,48 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativ vision::ResultType::SEGMENTATION); } +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativeV2( + JNIEnv *env, jobject thiz, jlong cxx_context, jobject argb8888_bitmap, + jobject result, jboolean save_image, jstring save_path, jboolean rendering, + jfloat weight) { + if (cxx_context == 0) { + return JNI_FALSE; + } + cv::Mat c_bgr; + if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return JNI_FALSE; + } + auto c_model_ptr = reinterpret_cast(cxx_context); + const jclass j_seg_result_clazz = env->GetObjectClass(result); + const jfieldID j_enable_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mEnableCxxBuffer", "Z"); + jboolean j_enable_cxx_buffer = + env->GetBooleanField(result, j_enable_cxx_buffer_id); + + auto c_result_ptr = new vision::SegmentationResult(); + auto t = fni::GetCurrentTime(); + c_model_ptr->Predict(&c_bgr, c_result_ptr); + PERF_TIME_OF_RUNTIME(c_model_ptr, t) + + if (rendering) { + fni::RenderingSegmentation(env, c_bgr, *c_result_ptr, argb8888_bitmap, + save_image, weight, save_path); + } + if (!fni::AllocateJavaResultFromCxx( + env, result, reinterpret_cast(c_result_ptr), + vision::ResultType::SEGMENTATION)) { + delete c_result_ptr; + return JNI_FALSE; + } + // Users need to release cxx result buffer manually + // if mEnableCxxBuffer is set as true. + if (j_enable_cxx_buffer == JNI_FALSE) { + delete c_result_ptr; + } + return JNI_TRUE; +} + JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNative( JNIEnv *env, jobject thiz, jlong cxx_context) { @@ -87,3 +140,4 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNativ #ifdef __cplusplus } #endif + diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java index 974abeb66..217e0eaa5 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java @@ -2,6 +2,8 @@ package com.baidu.paddle.fastdeploy.vision; import android.support.annotation.NonNull; +import com.baidu.paddle.fastdeploy.FastDeployInitializer; + public class SegmentationResult { // Init from native public byte[] mLabelMap; @@ -9,6 +11,11 @@ public class SegmentationResult { public long[] mShape; public boolean mContainScoreMap = false; public boolean mInitialized = false; + // Cxx result context, some users may want to use + // result pointer from native directly to boost + // the performance of segmentation. + public long mCxxBuffer = 0; + public boolean mEnableCxxBuffer = false; public SegmentationResult() { mInitialized = false; @@ -18,6 +25,17 @@ public class SegmentationResult { return mInitialized; } + public void setCxxBufferFlag(boolean flag) { + mEnableCxxBuffer = flag; + } + + public boolean releaseCxxBuffer() { + if (mCxxBuffer == 0 || !mEnableCxxBuffer) { + return false; + } + return releaseCxxBufferNative(); + } + public void setLabelMap(@NonNull byte[] labelMapBuffer) { if (labelMapBuffer.length > 0) { mLabelMap = labelMapBuffer.clone(); @@ -39,4 +57,11 @@ public class SegmentationResult { public void setContainScoreMap(boolean containScoreMap) { mContainScoreMap = containScoreMap; } + + private native boolean releaseCxxBufferNative(); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } } diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java index d2bd9f7ae..8ca586cb3 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java @@ -7,6 +7,7 @@ import com.baidu.paddle.fastdeploy.RuntimeOption; import com.baidu.paddle.fastdeploy.vision.SegmentationResult; public class PaddleSegModel { + public boolean mIsVerticalScreen = false; protected long mCxxContext = 0; // Context from native. protected boolean mInitialized = false; @@ -21,6 +22,12 @@ public class PaddleSegModel { init_(modelFile, paramsFile, configFile, new RuntimeOption()); } + // Is vertical screen or not, for PP-HumanSeg on vertical screen, + // this flag must be 'true'. + public void setVerticalScreenFlag(boolean flag) { + mIsVerticalScreen = flag; + } + // Constructor with custom runtime option public PaddleSegModel(String modelFile, String paramsFile, @@ -87,13 +94,48 @@ public class PaddleSegModel { // Only support ARGB8888 bitmap in native now. SegmentationResult result = predictNative( mCxxContext, ARGB8888Bitmap, true, - savedImagePath,true, weight); + savedImagePath, true, weight); if (result == null) { return new SegmentationResult(); } return result; } + // Reset the values of input SegmentationResult directly instead of + // return a SegmentationResult from native. + public boolean predict(Bitmap ARGB8888Bitmap, + SegmentationResult result) { + if (mCxxContext == 0) { + return false; + } + return predictNativeV2(mCxxContext, ARGB8888Bitmap, result, + false, "", false, 0.5f); + } + + public boolean predict(Bitmap ARGB8888Bitmap, + SegmentationResult result, + boolean rendering, + float weight) { + if (mCxxContext == 0) { + return false; + } + return predictNativeV2(mCxxContext, ARGB8888Bitmap, result, + false, "", rendering, weight); + } + + public boolean predict(Bitmap ARGB8888Bitmap, + SegmentationResult result, + String savedImagePath, + float weight) { + if (mCxxContext == 0) { + return false; + } + return predictNativeV2( + mCxxContext, ARGB8888Bitmap, result, true, + savedImagePath, true, weight); + } + + private boolean init_(String modelFile, String paramsFile, String configFile, @@ -139,6 +181,15 @@ public class PaddleSegModel { boolean rendering, float weight); + // Get cxx result pointer from native + private native boolean predictNativeV2(long CxxContext, + Bitmap ARGB8888Bitmap, + SegmentationResult result, + boolean saveImage, + String savePath, + boolean rendering, + float weight); + // Release buffers allocated in native context. private native boolean releaseNative(long CxxContext);