apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 16:48:03 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Android] Add CxxBuffer to native and Java PaddleSegModel (#677)

Browse Source 
[Android] Add CxxBuffer to native PaddleSegModel
This commit is contained in:
DefTruth
2022-11-23 16:10:11 +08:00
committed by GitHub
parent b5b2732366
commit ff5fb248ff
8 changed files with 254 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
java/android/README.md
View File

@@ -197,6 +197,14 @@ public SegmentationResult predict(Bitmap ARGB8888Bitmap)；
// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上
public SegmentationResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float weight);
public SegmentationResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float weight); // 只渲染 不保存图片
// 修改result，而非返回result，关注性能的用户可以将以下接口与SegmentationResult的CxxBuffer一起使用
public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result)；
public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, String savedImagePath, float weight);
public boolean predict(Bitmap ARGB8888Bitmap, SegmentationResult result, boolean rendering, float weight);
```
- 设置竖屏或横屏模式: 对于 PP-HumanSeg系列模型，必须要调用该方法设置竖屏模式为true.
```java
public void setVerticalScreenFlag(boolean flag);
```
- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。
```java
@@ -311,6 +319,10 @@ public class SegmentationResult {
public float[] mScoreMap; // 预测到的得分 map 每个像素位置对应一个score HxW
public long[] mShape; // label map实际的shape (H,W)
public boolean mContainScoreMap = false; // 是否包含 score map
// 用户可以选择直接使用CxxBuffer，而非通过JNI拷贝到Java层，
// 该方式可以一定程度上提升性能
public void setCxxBufferFlag(boolean flag); // 设置是否为CxxBuffer模式
public boolean releaseCxxBuffer(); // 手动释放CxxBuffer!!!
public boolean initialized(); // 检测结果是否有效
}
```

4
java/android/app/build.gradle
View File

@@ -1,5 +1,3 @@
import java.security.MessageDigest
apply plugin: 'com.android.application'
android {
@@ -90,8 +88,6 @@ task downloadAndExtractModels(type: DefaultTask) {
mkdir "${cachePath}"
}
FD_MODEL.eachWithIndex { model, index ->
MessageDigest messageDigest = MessageDigest.getInstance('MD5')
messageDigest.update(model.src.bytes)
String[] modelPaths = model.src.split("/")
String modelName = modelPaths[modelPaths.length - 1]
// Download the target model if not exists

13
java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java
View File

@@ -19,6 +19,7 @@ import android.preference.PreferenceManager;
import android.support.annotation.NonNull;
import android.support.v4.app.ActivityCompat;
import android.support.v4.content.ContextCompat;
import android.util.Log;
import android.view.View;
import android.view.ViewGroup;
import android.view.Window;
@@ -251,12 +252,18 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
boolean modified = false;
long tc = System.currentTimeMillis();
SegmentationResult result = predictor.predict(ARGB8888ImageBitmap);
SegmentationResult result = new SegmentationResult();
result.setCxxBufferFlag(true);
predictor.predict(ARGB8888ImageBitmap, result);
timeElapsed += (System.currentTimeMillis() - tc);
Visualize.visSegmentation(ARGB8888ImageBitmap, result);
modified = result.initialized();
result.releaseCxxBuffer();
frameCounter++;
if (frameCounter >= 30) {
final int fps = (int) (1000 / (timeElapsed / 30));
@@ -304,7 +311,8 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
// should mean 'width' if the camera display orientation is 90 | 270 degree
// (Hold the phone upright to record video)
// (2) Smaller resolution is more suitable for Lite Portrait HumanSeg.
// So, we set this preview size (480,480) here.
// So, we set this preview size (480,480) here. Reference:
// https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.6/contrib/PP-HumanSeg/README_cn.md
CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 480;
CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480;
svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview);
@@ -360,6 +368,7 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi
if (Boolean.parseBoolean(SegmentationSettingsActivity.enableLiteFp16)) {
option.enableLiteFp16();
}
predictor.setVerticalScreenFlag(true);
predictor.init(modelFile, paramsFile, configFile, option);
}
}

21
java/android/fastdeploy/build.gradle
View File

@@ -1,5 +1,3 @@
import java.security.MessageDigest
apply plugin: 'com.android.library'
@@ -56,26 +54,23 @@ task downloadAndExtractLibs(type: DefaultTask) {
println "Downloading and extracting fastdeploy android c++ lib ..."
}
doLast {
// Prepare cache folder for archives
String cachePath = "cache"
if (!file("${cachePath}").exists()) {
mkdir "${cachePath}"
}
FD_CXX_LIB.eachWithIndex { lib, index ->
MessageDigest messageDigest = MessageDigest.getInstance('MD5')
messageDigest.update(lib.src.bytes)
String cacheName = new BigInteger(1, messageDigest.digest()).toString(32)
// Download the target archive if not exists
boolean copyFiles = !file("${lib.dest}").exists()
if (!file("${cachePath}/${cacheName}.tgz").exists()) {
ant.get(src: lib.src, dest: file("${cachePath}/${cacheName}.tgz"))
String[] libPaths = lib.src.split("/")
String libName = libPaths[libPaths.length - 1]
libName = libName.split("\\.")[0]
boolean copyFiles = !file("${lib.dest}/${libName}").exists()
if (!file("${cachePath}/${libName}.tgz").exists()) {
println "Downloading ${lib.src} -> ${cachePath}/${libName}.tgz"
ant.get(src: lib.src, dest: file("${cachePath}/${libName}.tgz"))
copyFiles = true
// force to copy files from the latest archive files
}
// Extract the target archive if its dest path does not exists
if (copyFiles) {
copy {
from tarTree("${cachePath}/${cacheName}.tgz")
from tarTree("${cachePath}/${libName}.tgz")
into "${lib.dest}"
}
}

92
java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc
View File

@@ -283,6 +283,10 @@ bool AllocateJavaSegmentationResultFromCxx(
j_seg_result_clazz, "mContainScoreMap", "Z");
const jfieldID j_seg_score_map_id = env->GetFieldID(
j_seg_result_clazz, "mScoreMap", "[F");
const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
j_seg_result_clazz, "mEnableCxxBuffer", "Z");
const jfieldID j_cxx_buffer_id = env->GetFieldID(
j_seg_result_clazz, "mCxxBuffer", "J");
const jfieldID j_seg_initialized_id = env->GetFieldID(
j_seg_result_clazz, "mInitialized", "Z");
@@ -290,6 +294,18 @@ bool AllocateJavaSegmentationResultFromCxx(
return false;
}
// If 'mEnableCxxBuffer' set as true, then, we only setup the cxx result
// pointer to the value of 'mCxxBuffer' field. Some users may want
// to use this method to boost the performance of segmentation.
jboolean j_enable_cxx_buffer =
env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id);
if (j_enable_cxx_buffer == JNI_TRUE) {
jlong j_cxx_buffer = reinterpret_cast<jlong>(c_result_ptr);
env->SetLongField(j_seg_result_obj, j_cxx_buffer_id, j_cxx_buffer);
env->SetBooleanField(j_seg_result_obj, j_seg_initialized_id, JNI_TRUE);
return true;
}
// mLabelMap int[] shape (n): [I
const auto &label_map_uint8 = c_result_ptr->label_map;
jbyteArray j_seg_label_map_byte_arr = env->NewByteArray(len);
@@ -832,6 +848,10 @@ bool AllocateSegmentationResultFromJava(
j_seg_result_clazz_cc, "mContainScoreMap", "Z");
const jfieldID j_seg_score_map_id_cc = env->GetFieldID(
j_seg_result_clazz_cc, "mScoreMap", "[F");
const jfieldID j_enable_cxx_buffer_id_cc = env->GetFieldID(
j_seg_result_clazz_cc, "mEnableCxxBuffer", "Z");
const jfieldID j_cxx_buffer_id_cc = env->GetFieldID(
j_seg_result_clazz_cc, "mCxxBuffer", "J");
const jfieldID j_seg_initialized_id_cc = env->GetFieldID(
j_seg_result_clazz_cc, "mInitialized", "Z");
@@ -839,6 +859,38 @@ bool AllocateSegmentationResultFromJava(
return false;
}
// If 'mEnableCxxBuffer' set as true, then, we only Allocate from
// cxx context to cxx result. Some users may want to use this
// method to boost the performance of segmentation.
jboolean j_enable_cxx_buffer =
env->GetBooleanField(j_seg_result_obj, j_enable_cxx_buffer_id_cc);
if (j_enable_cxx_buffer == JNI_TRUE) {
jlong j_cxx_buffer = env->GetLongField(j_seg_result_obj, j_cxx_buffer_id_cc);
if (j_cxx_buffer == 0) {
return false;
}
// Allocate from cxx context to cxx result
auto c_cxx_buffer = reinterpret_cast<vision::SegmentationResult *>(j_cxx_buffer);
// TODO: May use 'swap' to exchange the administrative privileges ?
// c_result_ptr->shape.swap(c_cxx_buffer->shape);
// c_result_ptr->label_map.swap(c_cxx_buffer->label_map);
// c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map;
// if (c_cxx_buffer->contain_score_map) {
// c_result_ptr->score_map.swap(c_cxx_buffer->score_map);
// }
c_result_ptr->shape.assign(
c_cxx_buffer->shape.begin(), c_cxx_buffer->shape.end());
c_result_ptr->label_map.assign(
c_cxx_buffer->label_map.begin(), c_cxx_buffer->label_map.end());
c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map;
if (c_cxx_buffer->contain_score_map) {
c_result_ptr->score_map.assign(
c_cxx_buffer->score_map.begin(), c_cxx_buffer->score_map.end());
}
return true;
}
// mInitialized boolean: Z
jboolean j_seg_initialized =
env->GetBooleanField(j_seg_result_obj, j_seg_initialized_id_cc);
@@ -1030,3 +1082,43 @@ bool AllocateCxxResultFromJava(
} // namespace jni
} // namespace fastdeploy
#ifdef __cplusplus
extern "C" {
#endif
JNIEXPORT jboolean JNICALL
Java_com_baidu_paddle_fastdeploy_vision_SegmentationResult_releaseCxxBufferNative(
JNIEnv *env, jobject thiz) {
const jclass j_seg_result_clazz = env->GetObjectClass(thiz);
const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
j_seg_result_clazz, "mEnableCxxBuffer", "Z");
const jfieldID j_cxx_buffer_id = env->GetFieldID(
j_seg_result_clazz, "mCxxBuffer", "J");
const jfieldID j_seg_initialized_id = env->GetFieldID(
j_seg_result_clazz, "mInitialized", "Z");
jboolean j_enable_cxx_buffer =
env->GetBooleanField(thiz, j_enable_cxx_buffer_id);
if (j_enable_cxx_buffer == JNI_FALSE) {
return JNI_FALSE;
}
jlong j_cxx_buffer = env->GetLongField(thiz, j_cxx_buffer_id);
if (j_cxx_buffer == 0) {
return JNI_FALSE;
}
auto c_result_ptr = reinterpret_cast<
fastdeploy::vision::SegmentationResult *>(j_cxx_buffer);
delete c_result_ptr;
LOGD("[End] Release SegmentationResult in native !");
env->SetBooleanField(thiz, j_seg_initialized_id, JNI_FALSE);
env->DeleteLocalRef(j_seg_result_clazz);
return JNI_TRUE;
}
#ifdef __cplusplus
}
#endif

54
java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc
View File

@@ -41,6 +41,17 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_bindNative(
#ifdef ENABLE_RUNTIME_PERF
c_model_ptr->EnableRecordTimeOfRuntime();
#endif
// Setup is_vertical_screen param
const jclass j_ppseg_clazz = env->GetObjectClass(thiz);
const jfieldID j_is_vertical_screen_id = env->GetFieldID(
j_ppseg_clazz, "mIsVerticalScreen", "Z");
jboolean j_is_vertical_screen = env->GetBooleanField(
thiz, j_is_vertical_screen_id);
bool c_is_vertical_screen = static_cast<jboolean>(j_is_vertical_screen);
c_model_ptr->is_vertical_screen = c_is_vertical_screen;
env->DeleteLocalRef(j_ppseg_clazz);
vision::EnableFlyCV();
return reinterpret_cast<jlong>(c_model_ptr);
}
@@ -70,6 +81,48 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativ
vision::ResultType::SEGMENTATION);
}
JNIEXPORT jboolean JNICALL
Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativeV2(
JNIEnv *env, jobject thiz, jlong cxx_context, jobject argb8888_bitmap,
jobject result, jboolean save_image, jstring save_path, jboolean rendering,
jfloat weight) {
if (cxx_context == 0) {
return JNI_FALSE;
}
cv::Mat c_bgr;
if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) {
return JNI_FALSE;
}
auto c_model_ptr = reinterpret_cast<segmentation::PaddleSegModel *>(cxx_context);
const jclass j_seg_result_clazz = env->GetObjectClass(result);
const jfieldID j_enable_cxx_buffer_id = env->GetFieldID(
j_seg_result_clazz, "mEnableCxxBuffer", "Z");
jboolean j_enable_cxx_buffer =
env->GetBooleanField(result, j_enable_cxx_buffer_id);
auto c_result_ptr = new vision::SegmentationResult();
auto t = fni::GetCurrentTime();
c_model_ptr->Predict(&c_bgr, c_result_ptr);
PERF_TIME_OF_RUNTIME(c_model_ptr, t)
if (rendering) {
fni::RenderingSegmentation(env, c_bgr, *c_result_ptr, argb8888_bitmap,
save_image, weight, save_path);
}
if (!fni::AllocateJavaResultFromCxx(
env, result, reinterpret_cast<void *>(c_result_ptr),
vision::ResultType::SEGMENTATION)) {
delete c_result_ptr;
return JNI_FALSE;
}
// Users need to release cxx result buffer manually
// if mEnableCxxBuffer is set as true.
if (j_enable_cxx_buffer == JNI_FALSE) {
delete c_result_ptr;
}
return JNI_TRUE;
}
JNIEXPORT jboolean JNICALL
Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNative(
JNIEnv *env, jobject thiz, jlong cxx_context) {
@@ -87,3 +140,4 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_releaseNativ
#ifdef __cplusplus
}
#endif

25
java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java
View File

@@ -2,6 +2,8 @@ package com.baidu.paddle.fastdeploy.vision;
import android.support.annotation.NonNull;
import com.baidu.paddle.fastdeploy.FastDeployInitializer;
public class SegmentationResult {
// Init from native
public byte[] mLabelMap;
@@ -9,6 +11,11 @@ public class SegmentationResult {
public long[] mShape;
public boolean mContainScoreMap = false;
public boolean mInitialized = false;
// Cxx result context, some users may want to use
// result pointer from native directly to boost
// the performance of segmentation.
public long mCxxBuffer = 0;
public boolean mEnableCxxBuffer = false;
public SegmentationResult() {
mInitialized = false;
@@ -18,6 +25,17 @@ public class SegmentationResult {
return mInitialized;
}
public void setCxxBufferFlag(boolean flag) {
mEnableCxxBuffer = flag;
}
public boolean releaseCxxBuffer() {
if (mCxxBuffer == 0 || !mEnableCxxBuffer) {
return false;
}
return releaseCxxBufferNative();
}
public void setLabelMap(@NonNull byte[] labelMapBuffer) {
if (labelMapBuffer.length > 0) {
mLabelMap = labelMapBuffer.clone();
@@ -39,4 +57,11 @@ public class SegmentationResult {
public void setContainScoreMap(boolean containScoreMap) {
mContainScoreMap = containScoreMap;
}
private native boolean releaseCxxBufferNative();
// Initializes at the beginning.
static {
FastDeployInitializer.init();
}
}

53
java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java
View File

@@ -7,6 +7,7 @@ import com.baidu.paddle.fastdeploy.RuntimeOption;
import com.baidu.paddle.fastdeploy.vision.SegmentationResult;
public class PaddleSegModel {
public boolean mIsVerticalScreen = false;
protected long mCxxContext = 0; // Context from native.
protected boolean mInitialized = false;
@@ -21,6 +22,12 @@ public class PaddleSegModel {
init_(modelFile, paramsFile, configFile, new RuntimeOption());
}
// Is vertical screen or not, for PP-HumanSeg on vertical screen,
// this flag must be 'true'.
public void setVerticalScreenFlag(boolean flag) {
mIsVerticalScreen = flag;
}
// Constructor with custom runtime option
public PaddleSegModel(String modelFile,
String paramsFile,
@@ -87,13 +94,48 @@ public class PaddleSegModel {
// Only support ARGB8888 bitmap in native now.
SegmentationResult result = predictNative(
mCxxContext, ARGB8888Bitmap, true,
savedImagePath,true, weight);
savedImagePath, true, weight);
if (result == null) {
return new SegmentationResult();
}
return result;
}
// Reset the values of input SegmentationResult directly instead of
// return a SegmentationResult from native.
public boolean predict(Bitmap ARGB8888Bitmap,
SegmentationResult result) {
if (mCxxContext == 0) {
return false;
}
return predictNativeV2(mCxxContext, ARGB8888Bitmap, result,
false, "", false, 0.5f);
}
public boolean predict(Bitmap ARGB8888Bitmap,
SegmentationResult result,
boolean rendering,
float weight) {
if (mCxxContext == 0) {
return false;
}
return predictNativeV2(mCxxContext, ARGB8888Bitmap, result,
false, "", rendering, weight);
}
public boolean predict(Bitmap ARGB8888Bitmap,
SegmentationResult result,
String savedImagePath,
float weight) {
if (mCxxContext == 0) {
return false;
}
return predictNativeV2(
mCxxContext, ARGB8888Bitmap, result, true,
savedImagePath, true, weight);
}
private boolean init_(String modelFile,
String paramsFile,
String configFile,
@@ -139,6 +181,15 @@ public class PaddleSegModel {
boolean rendering,
float weight);
// Get cxx result pointer from native
private native boolean predictNativeV2(long CxxContext,
Bitmap ARGB8888Bitmap,
SegmentationResult result,
boolean saveImage,
String savePath,
boolean rendering,
float weight);
// Release buffers allocated in native context.
private native boolean releaseNative(long CxxContext);