mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-04 16:22:57 +08:00
[Model] Add tinypose single && pipeline model (#177)
* Add tinypose model * Add PPTinypose python API * Fix picodet preprocess bug && Add Tinypose examples * Update tinypose example code * Update ppseg preprocess if condition * Update ppseg backend support type * Update permute.h * Update README.md * Update code with comments * Move files dir * Delete premute.cc * Add single model pptinypose * Delete pptinypose old code in ppdet * Code format * Add ppdet + pptinypose pipeline model * Fix bug for posedetpipeline * Change Frontend to ModelFormat * Change Frontend to ModelFormat in __init__.py * Add python posedetpipeline/ * Update pptinypose example dir name * Update README.md * Update README.md * Update README.md * Update README.md * Create keypointdetection_result.md * Create README.md * Create README.md * Create README.md * Update README.md * Update README.md * Create README.md * Fix det_keypoint_unite_infer.py bug * Create README.md * Update PP-Tinypose by comment * Update by comment * Add pipeline directory * Add pptinypose dir * Update pptinypose to align accuracy * Addd warpAffine processor * Update GetCpuMat to GetOpenCVMat * Add comment for pptinypose && pipline * Update docs/main_page.md * Add README.md for pptinypose * Add README for det_keypoint_unite * Remove ENABLE_PIPELINE option * Remove ENABLE_PIPELINE option * Change pptinypose default backend * PP-TinyPose Pipeline support multi PP-Detection models * Update pp-tinypose comment * Update by comments * Add single test example Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
7
CMakeLists.txt
Executable file → Normal file
7
CMakeLists.txt
Executable file → Normal file
@@ -184,10 +184,11 @@ file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastde
|
||||
file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc)
|
||||
file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc)
|
||||
file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc)
|
||||
file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*.cc)
|
||||
file(GLOB_RECURSE DEPLOY_VISION_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cu)
|
||||
file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc)
|
||||
file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc)
|
||||
list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS})
|
||||
list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS})
|
||||
|
||||
set(DEPEND_LIBS "")
|
||||
|
||||
@@ -389,6 +390,7 @@ if(ENABLE_VISION)
|
||||
list(APPEND DEPLOY_VISION_SRCS ${DEPLOY_VISION_CUDA_SRCS})
|
||||
endif()
|
||||
list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_VISION_SRCS})
|
||||
list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PIPELINE_SRCS})
|
||||
include_directories(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include)
|
||||
include(${PROJECT_SOURCE_DIR}/cmake/opencv.cmake)
|
||||
|
||||
@@ -586,7 +588,8 @@ if(BUILD_FASTDEPLOY_PYTHON)
|
||||
|
||||
if(NOT ENABLE_VISION)
|
||||
file(GLOB_RECURSE VISION_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*_pybind.cc)
|
||||
list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${VISION_PYBIND_SRCS})
|
||||
file(GLOB_RECURSE PIPELINE_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*_pybind.cc)
|
||||
list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${VISION_PYBIND_SRCS} ${PIPELINE_PYBIND_SRCS})
|
||||
endif()
|
||||
|
||||
if (NOT ENABLE_TEXT)
|
||||
|
@@ -6,8 +6,9 @@ FastDeploy根据视觉模型的任务类型,定义了不同的结构体(`fastd
|
||||
| :----- | :--- | :---- | :------- |
|
||||
| ClassifyResult | [C++/Python文档](./classification_result.md) | 图像分类返回结果 | ResNet50、MobileNetV3等 |
|
||||
| SegmentationResult | [C++/Python文档](./segmentation_result.md) | 图像分割返回结果 | PP-HumanSeg、PP-LiteSeg等 |
|
||||
| DetectionResult | [C++/Python文档](./detection_result.md) | 目标检测返回结果 | PPYOLOE、YOLOv7系列模型等 |
|
||||
| DetectionResult | [C++/Python文档](./detection_result.md) | 目标检测返回结果 | PP-YOLOE、YOLOv7系列模型等 |
|
||||
| FaceDetectionResult | [C++/Python文档](./face_detection_result.md) | 目标检测返回结果 | SCRFD、RetinaFace系列模型等 |
|
||||
| KeyPointDetectionResult | [C++/Python文档](./keypointdetection_result.md) | 关键点检测返回结果 | PP-Tinypose系列模型等 |
|
||||
| FaceRecognitionResult | [C++/Python文档](./face_recognition_result.md) | 目标检测返回结果 | ArcFace、CosFace系列模型等 |
|
||||
| MattingResult | [C++/Python文档](./matting_result.md) | 目标检测返回结果 | MODNet系列模型等 |
|
||||
| OCRResult | [C++/Python文档](./ocr_result.md) | 文本框检测,分类和文本识别返回结果 | OCR系列模型等 |
|
||||
|
45
docs/api/vision_results/keypointdetection_result.md
Normal file
45
docs/api/vision_results/keypointdetection_result.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# KeyPointDetectionResult 目标检测结果
|
||||
|
||||
KeyPointDetectionResult 代码定义在`fastdeploy/vision/common/result.h`中,用于表明图像中目标行为的各个关键点坐标和置信度。
|
||||
|
||||
## C++ 定义
|
||||
|
||||
`fastdeploy::vision::KeyPointDetectionResult`
|
||||
|
||||
```c++
|
||||
struct KeyPointDetectionResult {
|
||||
std::vector<std::array<float, 2>> keypoints;
|
||||
std::vector<float> scores;
|
||||
int num_joints = -1;
|
||||
void Clear();
|
||||
std::string Str();
|
||||
};
|
||||
```
|
||||
|
||||
- **keypoints**: 成员变量,表示识别到的目标行为的关键点坐标。`keypoints.size()= N * J * 2`,
|
||||
- `N`:图片中的目标数量
|
||||
- `J`:num_joints(一个目标的关键点数量)
|
||||
- `3`:坐标信息[x, y]
|
||||
- **scores**: 成员变量,表示识别到的目标行为的关键点坐标的置信度。`scores.size()= N * J`
|
||||
- `N`:图片中的目标数量
|
||||
- `J`:num_joints(一个目标的关键点数量)
|
||||
- **num_joints**: 成员变量,一个目标的关键点数量
|
||||
|
||||
- **num_joints**: 成员变量,一个目标的关键点数量
|
||||
- **Clear()**: 成员函数,用于清除结构体中存储的结果
|
||||
- **Str()**: 成员函数,将结构体中的信息以字符串形式输出(用于Debug)
|
||||
|
||||
## Python 定义
|
||||
|
||||
`fastdeploy.vision.KeyPointDetectionResult`
|
||||
|
||||
- **keypoints**(list of list(float)): 成员变量,表示识别到的目标行为的关键点坐标。
|
||||
`keypoints.size()= N * J * 2`
|
||||
`N`:图片中的目标数量
|
||||
`J`:num_joints(关键点数量)
|
||||
`3`:坐标信息[x, y, conf]
|
||||
- **scores**(list of float): 成员变量,表示识别到的目标行为的关键点坐标的置信度。
|
||||
`scores.size()= N * J`
|
||||
`N`:图片中的目标数量
|
||||
`J`:num_joints(一个目标的关键点数量)
|
||||
- **num_joints**(int): 成员变量,一个目标的关键点数量
|
1
docs/api_docs/cpp/main_page.md
Executable file → Normal file
1
docs/api_docs/cpp/main_page.md
Executable file → Normal file
@@ -26,5 +26,6 @@ Currently, FastDeploy supported backends listed as below,
|
||||
| Task | Model | API | Example |
|
||||
| :---- | :---- | :---- | :----- |
|
||||
| object detection | PaddleDetection/PPYOLOE | [fastdeploy::vision::detection::PPYOLOE](./classfastdeploy_1_1vision_1_1detection_1_1PPYOLOE.html) | [C++](./)/[Python](./) |
|
||||
| keypoint detection | PaddleDetection/PPTinyPose | [fastdeploy::vision::keypointdetection::PPTinyPose](./classfastdeploy_1_1vision_1_1keypointdetection_1_1PPTinyPose.html) | [C++](./)/[Python](./) |
|
||||
| image classification | PaddleClassification serials | [fastdeploy::vision::classification::PaddleClasModel](./classfastdeploy_1_1vision_1_1classification_1_1PaddleClasModel.html) | [C++](./)/[Python](./) |
|
||||
| semantic segmentation | PaddleSegmentation serials | [fastdeploy::vision::classification::PaddleSegModel](./classfastdeploy_1_1vision_1_1segmentation_1_1PaddleSegModel.html) | [C++](./)/[Python](./) |
|
||||
|
@@ -1,3 +1,9 @@
|
||||
# Keypoint Detection API
|
||||
|
||||
comming soon...
|
||||
## fastdeploy.vision.keypointdetection.PPTinyPose
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: fastdeploy.vision.keypointdetection.PPTinyPose
|
||||
:members:
|
||||
:inherited-members:
|
||||
```
|
||||
|
@@ -40,6 +40,19 @@ API:`fastdeploy.vision.FaceDetectionResult` , 该结果返回:
|
||||
- **landmarks**(list of list(float)): 成员变量,表示单张图片检测出来的所有人脸的关键点.
|
||||
- **landmarks_per_face**(int): 成员变量,表示每个人脸框中的关键点的数量.
|
||||
|
||||
## KeyPointDetectionResult
|
||||
KeyPointDetectionResult 代码定义在`fastdeploy/vision/common/result.h`中,用于表明图像中目标行为的各个关键点坐标和置信度。
|
||||
|
||||
API:`fastdeploy.vision.KeyPointDetectionResult` , 该结果返回:
|
||||
- **keypoints**(list of list(float)): 成员变量,表示识别到的目标行为的关键点坐标。`keypoints.size()= N * J * 2`,
|
||||
- `N`:图片中的目标数量
|
||||
- `J`:num_joints(一个目标的关键点数量)
|
||||
- `3`:坐标信息[x, y]
|
||||
- **scores**(list of float): 成员变量,表示识别到的目标行为的关键点坐标的置信度。`scores.size()= N * J`
|
||||
- `N`:图片中的目标数量
|
||||
- `J`:num_joints(一个目标的关键点数量)
|
||||
- **num_joints**(int): 成员变量,表示一个目标的关键点数量
|
||||
|
||||
|
||||
## FaceRecognitionResult
|
||||
FaceRecognitionResult 代码定义在`fastdeploy/vision/common/result.h`中,用于表明人脸识别模型对图像特征的embedding.
|
||||
|
@@ -40,6 +40,19 @@ API: `fastdeploy.vision.FaceDetectionResult`, The FaceDetectionResult will retur
|
||||
- **landmarks**(list of list(float)): Member variables that represent the key points of all faces detected by a single image.
|
||||
- **landmarks_per_face**(int):Member variable indicating the number of key points in each face frame.
|
||||
|
||||
## KeyPointDetectionResult
|
||||
The KeyPointDetectionResult code is defined in `fastdeploy/vision/common/result.h` and is used to indicate the coordinates and confidence of each keypoint of the target behavior in the image.
|
||||
|
||||
API:`fastdeploy.vision.KeyPointDetectionResult`, The KeyPointDetectionResult will return:
|
||||
- **keypoints**(list of list(float)): Member variable, representing the key point coordinates of the identified target behavior. `keypoints.size()= N * J * 2`,
|
||||
- `N`: number of objects in the picture
|
||||
- `J`: num_joints(number of keypoints for a target)
|
||||
- `3`: 坐标信息[x, y]
|
||||
- **scores**(list of float): Member variable, representing the confidence of the keypoint coordinates of the recognized target behavior. `scores.size()= N * J`
|
||||
- `N`: number of objects in the picture
|
||||
- `J`: num_joints(number of keypoints for a target)
|
||||
- **num_joints**(int): Member variable, representing the number of keypoints for a target
|
||||
|
||||
## FaceRecognitionResult
|
||||
The FaceRecognitionResult code is defined in `fastdeploy/vision/common/result.h` and is used to indicate the embedding of the image features by the face recognition model.
|
||||
|
||||
|
@@ -8,6 +8,7 @@
|
||||
| Segmentation | 语义分割,输入图像,给出图像中每个像素的分类及置信度 | [SegmentationResult](../../docs/api/vision_results/segmentation_result.md) |
|
||||
| Classification | 图像分类,输入图像,给出图像的分类结果和置信度 | [ClassifyResult](../../docs/api/vision_results/classification_result.md) |
|
||||
| FaceDetection | 人脸检测,输入图像,检测图像中人脸位置,并返回检测框坐标及人脸关键点 | [FaceDetectionResult](../../docs/api/vision_results/face_detection_result.md) |
|
||||
| KeypointDetection | 关键点检测,输入图像,返回图像中人物行为的各个关键点坐标和置信度 | [KeyPointDetectionResult](../../docs/api/vision_results/keypointdetection_result.md) |
|
||||
| FaceRecognition | 人脸识别,输入图像,返回可用于相似度计算的人脸特征的embedding | [FaceRecognitionResult](../../docs/api/vision_results/face_recognition_result.md) |
|
||||
| Matting | 抠图,输入图像,返回图片的前景每个像素点的Alpha值 | [MattingResult](../../docs/api/vision_results/matting_result.md) |
|
||||
| OCR | 文本框检测,分类,文本框内容识别,输入图像,返回文本框坐标,文本框的方向类别以及框内的文本内容 | [OCRResult](../../docs/api/vision_results/ocr_result.md) |
|
||||
|
17
examples/vision/keypointdetection/README.md
Normal file
17
examples/vision/keypointdetection/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# 关键点检测模型
|
||||
|
||||
FastDeploy目前支持两种关键点检测任务方式的部署
|
||||
|
||||
| 任务 | 说明 | 模型格式 | 示例 | 版本 |
|
||||
| :---| :--- | :--- | :------- | :--- |
|
||||
| 单人关键点检测 | 部署PP-TinyPose系列模型,输入图像仅包含单人 | Paddle | 参考[tinypose目录](./tiny_pose/) | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
||||
| 单人/多人关键点检测 | 部署PicoDet + PP-TinyPose的模型串联任务,输入图像先通过检测模型,得到独立的人像子图后,再经过PP-TinyPose模型检测关键点 | Paddle | 参考[det_keypoint_unite目录](./det_keypoint_unite/) |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
||||
|
||||
# 预训练模型准备
|
||||
本文档提供了如下预训练模型,开发者可直接下载使用
|
||||
| 模型 | 说明 | 模型格式 | 版本 |
|
||||
| :--- | :--- | :------- | :--- |
|
||||
| [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 单人关键点检测模型 | Paddle | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
||||
| [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 单人关键点检测模型 | Paddle | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
||||
| [PicoDet-S-Lcnet-Pedestrian-192x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_192x192_infer.tgz) + [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 单人关键点检测串联配置 | Paddle |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
||||
| [PicoDet-S-Lcnet-Pedestrian-320x320](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz) + [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 多人关键点检测串联配置 | Paddle |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) |
|
@@ -0,0 +1,38 @@
|
||||
# PP-PicoDet + PP-TinyPose 联合部署(Pipeline)
|
||||
|
||||
## 模型版本说明
|
||||
|
||||
- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5)
|
||||
|
||||
目前FastDeploy支持如下模型的部署
|
||||
|
||||
- [PP-PicoDet + PP-TinyPose系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md)
|
||||
|
||||
## 准备PP-TinyPose部署模型
|
||||
|
||||
PP-TinyPose以及PP-PicoDet模型导出,请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md)
|
||||
|
||||
**注意**:导出的推理模型包含`model.pdmodel`、`model.pdiparams`和`infer_cfg.yml`三个文件,FastDeploy会从yaml文件中获取模型在推理时需要的预处理信息。
|
||||
|
||||
## 下载预训练模型
|
||||
|
||||
为了方便开发者的测试,下面提供了PP-PicoDet + PP-TinyPose(Pipeline)导出的部分模型,开发者可直接下载使用。
|
||||
|
||||
| 应用场景 | 模型 | 参数文件大小 | AP(业务数据集) | AP(COCO Val 单人/多人) | 单人/多人推理耗时 (FP32) | 单人/多人推理耗时(FP16) |
|
||||
|:-------------------------------|:--------------------------------- |:----- |:----- | :----- | :----- | :----- |
|
||||
| 单人模型配置 |[PicoDet-S-Lcnet-Pedestrian-192x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_192x192_infer.tgz) + [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 4.6MB + 5.3MB | 86.2% | 52.8% | 12.90ms | 9.61ms |
|
||||
| 多人模型配置 |[PicoDet-S-Lcnet-Pedestrian-320x320](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz) + [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 4.6M + 5.3MB | 85.7% | 49.9% | 47.63ms | 34.62ms |
|
||||
|
||||
**说明**
|
||||
- 关键点检测模型的精度指标是基于对应行人检测模型检测得到的检测框。
|
||||
- 精度测试中去除了flip操作,且检测置信度阈值要求0.5。
|
||||
- 速度测试环境为qualcomm snapdragon 865,采用arm8下4线程推理。
|
||||
- Pipeline速度包含模型的预处理、推理及后处理部分。
|
||||
- 精度测试中,为了公平比较,多人数据去除了6人以上(不含6人)的图像。
|
||||
|
||||
更多信息请参考:[PP-TinyPose 官方文档](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md)
|
||||
|
||||
## 详细部署文档
|
||||
|
||||
- [Python部署](python)
|
||||
- [C++部署](cpp)
|
@@ -0,0 +1,14 @@
|
||||
PROJECT(infer_demo C CXX)
|
||||
CMAKE_MINIMUM_REQUIRED (VERSION 3.12)
|
||||
|
||||
# 指定下载解压后的fastdeploy库路径
|
||||
option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
|
||||
|
||||
include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
|
||||
|
||||
# 添加FastDeploy依赖头文件
|
||||
include_directories(${FASTDEPLOY_INCS})
|
||||
|
||||
add_executable(infer_demo ${PROJECT_SOURCE_DIR}/det_keypoint_unite_infer.cc)
|
||||
# 添加FastDeploy库依赖
|
||||
target_link_libraries(infer_demo ${FASTDEPLOY_LIBS})
|
@@ -0,0 +1,84 @@
|
||||
# PP-PicoDet + PP-TinyPose (Pipeline) C++部署示例
|
||||
|
||||
本目录下提供`det_keypoint_unite_infer.cc`快速完成多人模型配置 PP-PicoDet + PP-TinyPose 在CPU/GPU,以及GPU上通过TensorRT加速部署的`单图多人关键点检测`示例。执行如下脚本即可完成
|
||||
>> **注意**: PP-TinyPose单模型独立部署,请参考[PP-TinyPose 单模型](../../tiny_pose/cpp/README.md)
|
||||
|
||||
在部署前,需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. 根据开发环境,下载预编译部署库和samples代码,参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
|
||||
以Linux上推理为例,在本目录执行如下命令即可完成编译测试
|
||||
|
||||
```bash
|
||||
wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.3.0.tgz
|
||||
tar xvf fastdeploy-linux-x64-gpu-0.3.0.tgz
|
||||
cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/keypointdetection/tiny_pose/cpp/
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.3.0
|
||||
make -j
|
||||
|
||||
# 下载PP-TinyPose和PP-PicoDet模型文件和测试图片
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz
|
||||
tar -xvf PP_TinyPose_256x192_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz
|
||||
tar -xvf PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/000000018491.jpg
|
||||
|
||||
# CPU推理
|
||||
./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 0
|
||||
# GPU推理
|
||||
./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 1
|
||||
# GPU上TensorRT推理
|
||||
./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 2
|
||||
```
|
||||
|
||||
运行完成可视化结果如下图所示
|
||||
<div align="center">
|
||||
<img src="https://user-images.githubusercontent.com/16222477/196393343-eeb6b68f-0bc6-4927-871f-5ac610da7293.jpeg", width=359px, height=423px />
|
||||
</div>
|
||||
|
||||
以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考:
|
||||
- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md)
|
||||
|
||||
## PP-TinyPose C++接口
|
||||
|
||||
### PP-TinyPose类
|
||||
|
||||
```c++
|
||||
fastdeploy::pipeline::PPTinyPose(
|
||||
fastdeploy::vision::detection::PPYOLOE* det_model,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model)
|
||||
```
|
||||
|
||||
PPTinyPose Pipeline模型加载和初始化。
|
||||
|
||||
**参数**
|
||||
|
||||
> * **model_det_modelfile**(fastdeploy::vision::detection): 初始化后的检测模型,参考[PP-TinyPose](../../tiny_pose/README.md)
|
||||
> * **pptinypose_model**(fastdeploy::vision::keypointdetection): 初始化后的检测模型[Detection](../../../detection/paddledetection/README.md),暂时只提供PaddleDetection系列
|
||||
|
||||
#### Predict函数
|
||||
|
||||
> ```c++
|
||||
> PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result)
|
||||
> ```
|
||||
>
|
||||
> 模型预测接口,输入图像直接输出关键点检测结果。
|
||||
>
|
||||
> **参数**
|
||||
>
|
||||
> > * **im**: 输入图像,注意需为HWC,BGR格式
|
||||
> > * **result**: 关键点检测结果,包括关键点的坐标以及关键点对应的概率值, KeyPointDetectionResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
|
||||
### 类成员属性
|
||||
#### 后处理参数
|
||||
> > * **detection_model_score_threshold**(bool):
|
||||
输入PP-TinyPose模型前,Detectin模型过滤检测框的分数阈值
|
||||
|
||||
- [模型介绍](../../)
|
||||
- [Python部署](../python)
|
||||
- [视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md)
|
@@ -0,0 +1,196 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision.h"
|
||||
#include "fastdeploy/pipeline.h"
|
||||
|
||||
#ifdef WIN32
|
||||
const char sep = '\\';
|
||||
#else
|
||||
const char sep = '/';
|
||||
#endif
|
||||
|
||||
void CpuInfer(const std::string& det_model_dir,
|
||||
const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto det_model_file = det_model_dir + sep + "model.pdmodel";
|
||||
auto det_params_file = det_model_dir + sep + "model.pdiparams";
|
||||
auto det_config_file = det_model_dir + sep + "infer_cfg.yml";
|
||||
auto det_model = fastdeploy::vision::detection::PicoDet(
|
||||
det_model_file, det_params_file, det_config_file);
|
||||
if (!det_model.Initialized()) {
|
||||
std::cerr << "Detection Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
|
||||
auto pipeline =fastdeploy::pipeline::PPTinyPose(&det_model, &tinypose_model);
|
||||
pipeline.detection_model_score_threshold = 0.5;
|
||||
if (!pipeline.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测框结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.2);
|
||||
cv::imwrite("vis_result.jpg", vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void GpuInfer(const std::string& det_model_dir,
|
||||
const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto option = fastdeploy::RuntimeOption();
|
||||
option.UseGpu();
|
||||
auto det_model_file = det_model_dir + sep + "model.pdmodel";
|
||||
auto det_params_file = det_model_dir + sep + "model.pdiparams";
|
||||
auto det_config_file = det_model_dir + sep + "infer_cfg.yml";
|
||||
auto det_model = fastdeploy::vision::detection::PicoDet(
|
||||
det_model_file, det_params_file, det_config_file, option);
|
||||
if (!det_model.Initialized()) {
|
||||
std::cerr << "Detection Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file, option);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
|
||||
auto pipeline =
|
||||
fastdeploy::pipeline::PPTinyPose(
|
||||
&det_model, &tinypose_model);
|
||||
pipeline.detection_model_score_threshold = 0.5;
|
||||
if (!pipeline.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测框结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.2);
|
||||
cv::imwrite("vis_result.jpg", vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void TrtInfer(const std::string& det_model_dir,
|
||||
const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto det_model_file = det_model_dir + sep + "model.pdmodel";
|
||||
auto det_params_file = det_model_dir + sep + "model.pdiparams";
|
||||
auto det_config_file = det_model_dir + sep + "infer_cfg.yml";
|
||||
|
||||
auto det_option = fastdeploy::RuntimeOption();
|
||||
det_option.UseGpu();
|
||||
det_option.UseTrtBackend();
|
||||
det_option.SetTrtInputShape("image", {1, 3, 320, 320});
|
||||
det_option.SetTrtInputShape("scale_factor", {1, 2});
|
||||
auto det_model = fastdeploy::vision::detection::PicoDet(
|
||||
det_model_file, det_params_file, det_config_file, det_option);
|
||||
if (!det_model.Initialized()) {
|
||||
std::cerr << "Detection Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_option = fastdeploy::RuntimeOption();
|
||||
tinypose_option.UseGpu();
|
||||
tinypose_option.UseTrtBackend();
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file,
|
||||
tinypose_option);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
|
||||
auto pipeline =
|
||||
fastdeploy::pipeline::PPTinyPose(
|
||||
&det_model, &tinypose_model);
|
||||
pipeline.detection_model_score_threshold = 0.5;
|
||||
if (!pipeline.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测关键点结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.2);
|
||||
cv::imwrite("vis_result.jpg", vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 5) {
|
||||
std::cout << "Usage: infer_demo path/to/detection_model_dir "
|
||||
"path/to/pptinypose_model_dir path/to/image run_option, "
|
||||
"e.g ./infer_model ./picodet_model_dir ./pptinypose_model_dir "
|
||||
"./test.jpeg 0"
|
||||
<< std::endl;
|
||||
std::cout << "The data type of run_option is int, 0: run with cpu; 1: run "
|
||||
"with gpu; 2: run with gpu and use tensorrt backend."
|
||||
<< std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (std::atoi(argv[4]) == 0) {
|
||||
CpuInfer(argv[1], argv[2], argv[3]);
|
||||
} else if (std::atoi(argv[4]) == 1) {
|
||||
GpuInfer(argv[1], argv[2], argv[3]);
|
||||
} else if (std::atoi(argv[4]) == 2) {
|
||||
TrtInfer(argv[1], argv[2], argv[3]);
|
||||
}
|
||||
return 0;
|
||||
}
|
@@ -0,0 +1,74 @@
|
||||
# PP-PicoDet + PP-TinyPose (Pipeline) Python部署示例
|
||||
|
||||
在部署前,需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. 根据开发环境,下载预编译部署库和samples代码,参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
本目录下提供`det_keypoint_unite_infer.py`快速完成多人模型配置 PP-PicoDet + PP-TinyPose 在CPU/GPU,以及GPU上通过TensorRT加速部署的`单图多人关键点检测`示例。执行如下脚本即可完成
|
||||
>> **注意**: PP-TinyPose单模型独立部署,请参考[PP-TinyPose 单模型](../../tiny_pose//python/README.md)
|
||||
|
||||
```bash
|
||||
#下载部署示例代码
|
||||
git clone https://github.com/PaddlePaddle/FastDeploy.git
|
||||
cd FastDeploy/examples/vision/keypointdetection/det_keypoint_unite/python
|
||||
|
||||
# 下载PP-TinyPose模型文件和测试图片
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz
|
||||
tar -xvf PP_TinyPose_256x192_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz
|
||||
tar -xvf PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/000000018491.jpg
|
||||
# CPU推理
|
||||
python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device cpu
|
||||
# GPU推理
|
||||
python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device gpu
|
||||
# GPU上使用TensorRT推理 (注意:TensorRT推理第一次运行,有序列化模型的操作,有一定耗时,需要耐心等待)
|
||||
python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device gpu --use_trt True
|
||||
```
|
||||
|
||||
运行完成可视化结果如下图所示
|
||||
<div align="center">
|
||||
<img src="https://user-images.githubusercontent.com/16222477/196393343-eeb6b68f-0bc6-4927-871f-5ac610da7293.jpeg", width=640px, height=427px />
|
||||
</div>
|
||||
|
||||
## PPTinyPosePipeline Python接口
|
||||
|
||||
```python
|
||||
fd.pipeline.PPTinyPose(det_model=None, pptinypose_model=None)
|
||||
```
|
||||
|
||||
PPTinyPosePipeline模型加载和初始化,其中det_model是使用`fd.vision.detection.PicoDet`[参考Detection文档](../../../detection/paddledetection/python/)初始化的检测模型,pptinypose_model是使用`fd.vision.keypointdetection.PPTinyPose`[参考PP-TinyPose文档](../../tiny_pose/python/)初始化的检测模型
|
||||
|
||||
**参数**
|
||||
|
||||
> * **det_model**(str): 初始化后的检测模型
|
||||
> * **pptinypose_model**(str): 初始化后的PP-TinyPose模型
|
||||
|
||||
### predict函数
|
||||
|
||||
> ```python
|
||||
> PPTinyPosePipeline.predict(input_image)
|
||||
> ```
|
||||
>
|
||||
> 模型预测结口,输入图像直接输出检测结果。
|
||||
>
|
||||
> **参数**
|
||||
>
|
||||
> > * **input_image**(np.ndarray): 输入数据,注意需为HWC,BGR格式
|
||||
|
||||
> **返回**
|
||||
>
|
||||
> > 返回`fastdeploy.vision.KeyPointDetectionResult`结构体,结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
|
||||
### 类成员属性
|
||||
#### 后处理参数
|
||||
> > * **detection_model_score_threshold**(bool):
|
||||
输入PP-TinyPose模型前,Detectin模型过滤检测框的分数阈值
|
||||
|
||||
## 其它文档
|
||||
|
||||
- [Pipeline 模型介绍](..)
|
||||
- [Pipeline C++部署](../cpp)
|
||||
- [模型预测结果说明](../../../../../docs/api/vision_results/)
|
||||
- [如何切换模型推理后端引擎](../../../../../docs/runtime/how_to_change_backend.md)
|
@@ -0,0 +1,91 @@
|
||||
import fastdeploy as fd
|
||||
import cv2
|
||||
import os
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
import argparse
|
||||
import ast
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--tinypose_model_dir",
|
||||
required=True,
|
||||
help="path of paddletinypose model directory")
|
||||
parser.add_argument(
|
||||
"--det_model_dir", help="path of paddledetection model directory")
|
||||
parser.add_argument(
|
||||
"--image", required=True, help="path of test image file.")
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default='cpu',
|
||||
help="type of inference device, support 'cpu' or 'gpu'.")
|
||||
parser.add_argument(
|
||||
"--use_trt",
|
||||
type=ast.literal_eval,
|
||||
default=False,
|
||||
help="wether to use tensorrt.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_picodet_option(args):
|
||||
option = fd.RuntimeOption()
|
||||
|
||||
if args.device.lower() == "gpu":
|
||||
option.use_gpu()
|
||||
|
||||
if args.use_trt:
|
||||
option.use_trt_backend()
|
||||
option.set_trt_input_shape("image", [1, 3, 320, 320])
|
||||
option.set_trt_input_shape("scale_factor", [1, 2])
|
||||
return option
|
||||
|
||||
|
||||
def build_tinypose_option(args):
|
||||
option = fd.RuntimeOption()
|
||||
|
||||
if args.device.lower() == "gpu":
|
||||
option.use_gpu()
|
||||
|
||||
if args.use_trt:
|
||||
option.use_trt_backend()
|
||||
option.set_trt_input_shape("image", [1, 3, 256, 192])
|
||||
return option
|
||||
|
||||
|
||||
args = parse_arguments()
|
||||
picodet_model_file = os.path.join(args.det_model_dir, "model.pdmodel")
|
||||
picodet_params_file = os.path.join(args.det_model_dir, "model.pdiparams")
|
||||
picodet_config_file = os.path.join(args.det_model_dir, "infer_cfg.yml")
|
||||
|
||||
# 配置runtime,加载模型
|
||||
runtime_option = build_picodet_option(args)
|
||||
det_model = fd.vision.detection.PicoDet(
|
||||
picodet_model_file,
|
||||
picodet_params_file,
|
||||
picodet_config_file,
|
||||
runtime_option=runtime_option)
|
||||
|
||||
tinypose_model_file = os.path.join(args.tinypose_model_dir, "model.pdmodel")
|
||||
tinypose_params_file = os.path.join(args.tinypose_model_dir, "model.pdiparams")
|
||||
tinypose_config_file = os.path.join(args.tinypose_model_dir, "infer_cfg.yml")
|
||||
# 配置runtime,加载模型
|
||||
runtime_option = build_tinypose_option(args)
|
||||
tinypose_model = fd.vision.keypointdetection.PPTinyPose(
|
||||
tinypose_model_file,
|
||||
tinypose_params_file,
|
||||
tinypose_config_file,
|
||||
runtime_option=runtime_option)
|
||||
|
||||
# 预测图片检测结果
|
||||
im = cv2.imread(args.image)
|
||||
pipeline = fd.pipeline.PPTinyPose(det_model, tinypose_model)
|
||||
pipeline.detection_model_score_threshold = 0.5
|
||||
pipeline_result = pipeline.predict(im)
|
||||
print("Paddle TinyPose Result:\n", pipeline_result)
|
||||
|
||||
# 预测结果可视化
|
||||
vis_im = fd.vision.vis_keypoint_detection(
|
||||
im, pipeline_result, conf_threshold=0.2)
|
||||
cv2.imwrite("visualized_result.jpg", vis_im)
|
||||
print("TinyPose visualized result save in ./visualized_result.jpg")
|
37
examples/vision/keypointdetection/tiny_pose/README.md
Normal file
37
examples/vision/keypointdetection/tiny_pose/README.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# PP-TinyPose 模型部署
|
||||
|
||||
## 模型版本说明
|
||||
|
||||
- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5)
|
||||
|
||||
目前FastDeploy支持如下模型的部署
|
||||
|
||||
- [PP-TinyPose系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md)
|
||||
|
||||
## 准备PP-TinyPose部署模型
|
||||
|
||||
PP-TinyPose模型导出,请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md)
|
||||
|
||||
**注意**:PP-TinyPose导出的模型包含`model.pdmodel`、`model.pdiparams`和`infer_cfg.yml`三个文件,FastDeploy会从yaml文件中获取模型在推理时需要的预处理信息。
|
||||
|
||||
|
||||
## 下载预训练模型
|
||||
|
||||
为了方便开发者的测试,下面提供了PP-TinyPose导出的部分模型,开发者可直接下载使用。
|
||||
|
||||
| 模型 | 参数文件大小 |输入Shape | AP(业务数据集) | AP(COCO Val) | FLOPS | 单人推理耗时 (FP32) | 单人推理耗时(FP16) |
|
||||
|:---------------------------------------------------------------- |:----- |:----- | :----- | :----- | :----- | :----- | :----- |
|
||||
| [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 5.3MB | 128x96 | 84.3% | 58.4% | 81.56 M | 4.57ms | 3.27ms |
|
||||
| [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 5.3M | 256x96 | 91.0% | 68.3% | 326.24M | 14.07ms | 8.33ms |
|
||||
|
||||
**说明**
|
||||
- 关键点检测模型使用`COCO train2017`和`AI Challenger trainset`作为训练集。使用`COCO person keypoints val2017`作为测试集。
|
||||
- 关键点检测模型的精度指标所依赖的检测框为ground truth标注得到。
|
||||
- 推理速度测试环境为 Qualcomm Snapdragon 865,采用arm8下4线程推理得到。
|
||||
|
||||
更多信息请参考:[PP-TinyPose 官方文档](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md)
|
||||
|
||||
## 详细部署文档
|
||||
|
||||
- [Python部署](python)
|
||||
- [C++部署](cpp)
|
@@ -0,0 +1,13 @@
|
||||
PROJECT(infer_demo C CXX)
|
||||
CMAKE_MINIMUM_REQUIRED (VERSION 3.12)
|
||||
|
||||
# 指定下载解压后的fastdeploy库路径
|
||||
option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
|
||||
|
||||
include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
|
||||
|
||||
# 添加FastDeploy依赖头文件
|
||||
include_directories(${FASTDEPLOY_INCS})
|
||||
|
||||
add_executable(infer_tinypose_demo ${PROJECT_SOURCE_DIR}/pptinypose_infer.cc)
|
||||
target_link_libraries(infer_tinypose_demo ${FASTDEPLOY_LIBS})
|
88
examples/vision/keypointdetection/tiny_pose/cpp/README.md
Normal file
88
examples/vision/keypointdetection/tiny_pose/cpp/README.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# PP-TinyPose C++部署示例
|
||||
|
||||
本目录下提供`pptinypose_infer.cc`快速完成PP-TinyPose在CPU/GPU,以及GPU上通过TensorRT加速部署的`单图单人关键点检测`示例
|
||||
>> **注意**: PP-Tinypose单模型目前只支持单图单人关键点检测,因此输入的图片应只包含一个人或者进行过裁剪的图像。多人关键点检测请参考[PP-TinyPose Pipeline](../../det_keypoint_unite/cpp/README.md)
|
||||
|
||||
在部署前,需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. 根据开发环境,下载预编译部署库和samples代码,参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
|
||||
以Linux上推理为例,在本目录执行如下命令即可完成编译测试
|
||||
|
||||
```bash
|
||||
wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.3.0.tgz
|
||||
tar xvf fastdeploy-linux-x64-gpu-0.3.0.tgz
|
||||
cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/keypointdetection/tiny_pose/cpp/
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.3.0
|
||||
make -j
|
||||
|
||||
# 下载PP-TinyPose模型文件和测试图片
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz
|
||||
tar -xvf PP_TinyPose_256x192_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/hrnet_demo.jpg
|
||||
|
||||
|
||||
# CPU推理
|
||||
./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 0
|
||||
# GPU推理
|
||||
./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 1
|
||||
# GPU上TensorRT推理
|
||||
./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 2
|
||||
```
|
||||
|
||||
运行完成可视化结果如下图所示
|
||||
<div align="center">
|
||||
<img src="https://user-images.githubusercontent.com/16222477/196386764-dd51ad56-c410-4c54-9580-643f282f5a83.jpeg", width=359px, height=423px />
|
||||
</div>
|
||||
|
||||
以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考:
|
||||
- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md)
|
||||
|
||||
## PP-TinyPose C++接口
|
||||
|
||||
### PP-TinyPose类
|
||||
|
||||
```c++
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
const string& model_file,
|
||||
const string& params_file = "",
|
||||
const string& config_file,
|
||||
const RuntimeOption& runtime_option = RuntimeOption(),
|
||||
const ModelFormat& model_format = ModelFormat::PADDLE)
|
||||
```
|
||||
|
||||
PPTinyPose模型加载和初始化,其中model_file为导出的Paddle模型格式。
|
||||
|
||||
**参数**
|
||||
|
||||
> * **model_file**(str): 模型文件路径
|
||||
> * **params_file**(str): 参数文件路径
|
||||
> * **config_file**(str): 推理部署配置文件
|
||||
> * **runtime_option**(RuntimeOption): 后端推理配置,默认为None,即采用默认配置
|
||||
> * **model_format**(ModelFormat): 模型格式,默认为Paddle格式
|
||||
|
||||
#### Predict函数
|
||||
|
||||
> ```c++
|
||||
> PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result)
|
||||
> ```
|
||||
>
|
||||
> 模型预测接口,输入图像直接输出关键点检测结果。
|
||||
>
|
||||
> **参数**
|
||||
>
|
||||
> > * **im**: 输入图像,注意需为HWC,BGR格式
|
||||
> > * **result**: 关键点检测结果,包括关键点的坐标以及关键点对应的概率值, KeyPointDetectionResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
|
||||
### 类成员属性
|
||||
#### 后处理参数
|
||||
> > * **use_dark**(bool): 是否使用DARK进行后处理[参考论文](https://arxiv.org/abs/1910.06278)
|
||||
|
||||
- [模型介绍](../../)
|
||||
- [Python部署](../python)
|
||||
- [视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md)
|
@@ -0,0 +1,143 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision.h"
|
||||
|
||||
#ifdef WIN32
|
||||
const char sep = '\\';
|
||||
#else
|
||||
const char sep = '/';
|
||||
#endif
|
||||
|
||||
void CpuInfer(const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
if (!tinypose_model.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测框结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto tinypose_vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.5);
|
||||
cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void GpuInfer(const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto option = fastdeploy::RuntimeOption();
|
||||
option.UseGpu();
|
||||
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file, option);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
if (!tinypose_model.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测框结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto tinypose_vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.5);
|
||||
cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void TrtInfer(const std::string& tinypose_model_dir,
|
||||
const std::string& image_file) {
|
||||
auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel";
|
||||
auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams";
|
||||
auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml";
|
||||
auto tinypose_option = fastdeploy::RuntimeOption();
|
||||
tinypose_option.UseGpu();
|
||||
tinypose_option.UseTrtBackend();
|
||||
auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose(
|
||||
tinypose_model_file, tinypose_params_file, tinypose_config_file,
|
||||
tinypose_option);
|
||||
if (!tinypose_model.Initialized()) {
|
||||
std::cerr << "TinyPose Model Failed to initialize." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
auto im = cv::imread(image_file);
|
||||
fastdeploy::vision::KeyPointDetectionResult res;
|
||||
if (!tinypose_model.Predict(&im, &res)) {
|
||||
std::cerr << "TinyPose Prediction Failed." << std::endl;
|
||||
return;
|
||||
} else {
|
||||
std::cout << "TinyPose Prediction Done!" << std::endl;
|
||||
}
|
||||
// 输出预测框结果
|
||||
std::cout << res.Str() << std::endl;
|
||||
|
||||
// 可视化预测结果
|
||||
auto tinypose_vis_im =
|
||||
fastdeploy::vision::VisKeypointDetection(im, res, 0.5);
|
||||
cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im);
|
||||
std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 4) {
|
||||
std::cout << "Usage: infer_demo path/to/pptinypose_model_dir path/to/image "
|
||||
"run_option, "
|
||||
"e.g ./infer_model ./pptinypose_model_dir ./test.jpeg 0"
|
||||
<< std::endl;
|
||||
std::cout << "The data type of run_option is int, 0: run with cpu; 1: run "
|
||||
"with gpu; 2: run with gpu and use tensorrt backend."
|
||||
<< std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (std::atoi(argv[3]) == 0) {
|
||||
CpuInfer(argv[1], argv[2]);
|
||||
} else if (std::atoi(argv[3]) == 1) {
|
||||
GpuInfer(argv[1], argv[2]);
|
||||
} else if (std::atoi(argv[3]) == 2) {
|
||||
TrtInfer(argv[1], argv[2]);
|
||||
}
|
||||
return 0;
|
||||
}
|
79
examples/vision/keypointdetection/tiny_pose/python/README.md
Normal file
79
examples/vision/keypointdetection/tiny_pose/python/README.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# PP-TinyPose Python部署示例
|
||||
|
||||
在部署前,需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. 根据开发环境,下载预编译部署库和samples代码,参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
本目录下提供`pptinypose_infer.py`快速完成PP-TinyPose在CPU/GPU,以及GPU上通过TensorRT加速部署的`单图单人关键点检测`示例。执行如下脚本即可完成
|
||||
|
||||
>> **注意**: PP-Tinypose单模型目前只支持单图单人关键点检测,因此输入的图片应只包含一个人或者进行过裁剪的图像。多人关键点检测请参考[PP-TinyPose Pipeline](../../det_keypoint_unite/python/README.md)
|
||||
|
||||
```bash
|
||||
#下载部署示例代码
|
||||
git clone https://github.com/PaddlePaddle/FastDeploy.git
|
||||
cd FastDeploy/examples/vision/keypointdetection/tiny_pose/python
|
||||
|
||||
# 下载PP-TinyPose模型文件和测试图片
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz
|
||||
tar -xvf PP_TinyPose_256x192_infer.tgz
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/hrnet_demo.jpg
|
||||
|
||||
# CPU推理
|
||||
python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device cpu
|
||||
# GPU推理
|
||||
python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device gpu
|
||||
# GPU上使用TensorRT推理 (注意:TensorRT推理第一次运行,有序列化模型的操作,有一定耗时,需要耐心等待)
|
||||
python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device gpu --use_trt True
|
||||
```
|
||||
|
||||
运行完成可视化结果如下图所示
|
||||
<div align="center">
|
||||
<img src="https://user-images.githubusercontent.com/16222477/196386764-dd51ad56-c410-4c54-9580-643f282f5a83.jpeg", width=359px, height=423px />
|
||||
</div>
|
||||
|
||||
## PP-TinyPose Python接口
|
||||
|
||||
```python
|
||||
fd.vision.keypointdetection.PPTinyPose(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE)
|
||||
```
|
||||
|
||||
PP-TinyPose模型加载和初始化,其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件,具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md)
|
||||
|
||||
**参数**
|
||||
|
||||
> * **model_file**(str): 模型文件路径
|
||||
> * **params_file**(str): 参数文件路径
|
||||
> * **config_file**(str): 推理部署配置文件
|
||||
> * **runtime_option**(RuntimeOption): 后端推理配置,默认为None,即采用默认配置
|
||||
> * **model_format**(ModelFormat): 模型格式,默认为Paddle格式
|
||||
|
||||
### predict函数
|
||||
|
||||
> ```python
|
||||
> PPTinyPose.predict(input_image)
|
||||
> ```
|
||||
>
|
||||
> 模型预测结口,输入图像直接输出检测结果。
|
||||
>
|
||||
> **参数**
|
||||
>
|
||||
> > * **input_image**(np.ndarray): 输入数据,注意需为HWC,BGR格式
|
||||
|
||||
> **返回**
|
||||
>
|
||||
> > 返回`fastdeploy.vision.KeyPointDetectionResult`结构体,结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/)
|
||||
|
||||
### 类成员属性
|
||||
#### 后处理参数
|
||||
用户可按照自己的实际需求,修改下列后处理参数,从而影响最终的推理和部署效果
|
||||
|
||||
> > * **use_dark**(bool): 是否使用DARK进行后处理[参考论文](https://arxiv.org/abs/1910.06278)
|
||||
|
||||
|
||||
## 其它文档
|
||||
|
||||
- [PP-TinyPose 模型介绍](..)
|
||||
- [PP-TinyPose C++部署](../cpp)
|
||||
- [模型预测结果说明](../../../../../docs/api/vision_results/)
|
||||
- [如何切换模型推理后端引擎](../../../../../docs/runtime/how_to_change_backend.md)
|
@@ -0,0 +1,62 @@
|
||||
import fastdeploy as fd
|
||||
import cv2
|
||||
import os
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
import argparse
|
||||
import ast
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--tinypose_model_dir",
|
||||
required=True,
|
||||
help="path of paddletinypose model directory")
|
||||
parser.add_argument(
|
||||
"--image", required=True, help="path of test image file.")
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default='cpu',
|
||||
help="type of inference device, support 'cpu' or 'gpu'.")
|
||||
parser.add_argument(
|
||||
"--use_trt",
|
||||
type=ast.literal_eval,
|
||||
default=False,
|
||||
help="wether to use tensorrt.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_tinypose_option(args):
|
||||
option = fd.RuntimeOption()
|
||||
|
||||
if args.device.lower() == "gpu":
|
||||
option.use_gpu()
|
||||
|
||||
if args.use_trt:
|
||||
option.use_trt_backend()
|
||||
option.set_trt_input_shape("image", [1, 3, 256, 192])
|
||||
return option
|
||||
|
||||
|
||||
args = parse_arguments()
|
||||
|
||||
tinypose_model_file = os.path.join(args.tinypose_model_dir, "model.pdmodel")
|
||||
tinypose_params_file = os.path.join(args.tinypose_model_dir, "model.pdiparams")
|
||||
tinypose_config_file = os.path.join(args.tinypose_model_dir, "infer_cfg.yml")
|
||||
# 配置runtime,加载模型
|
||||
runtime_option = build_tinypose_option(args)
|
||||
tinypose_model = fd.vision.keypointdetection.PPTinyPose(
|
||||
tinypose_model_file,
|
||||
tinypose_params_file,
|
||||
tinypose_config_file,
|
||||
runtime_option=runtime_option)
|
||||
# 预测图片检测结果
|
||||
im = cv2.imread(args.image)
|
||||
tinypose_result = tinypose_model.predict(im)
|
||||
print("Paddle TinyPose Result:\n", tinypose_result)
|
||||
|
||||
# 预测结果可视化
|
||||
vis_im = fd.vision.vis_keypoint_detection(
|
||||
im, tinypose_result, conf_threshold=0.5)
|
||||
cv2.imwrite("visualized_result.jpg", vis_im)
|
||||
print("TinyPose visualized result save in ./visualized_result.jpg")
|
@@ -52,6 +52,10 @@ void ShareTensorFromFDTensor(paddle_infer::Tensor* tensor,
|
||||
tensor->CopyFromCpu(static_cast<const int64_t*>(fd_tensor.Data()));
|
||||
}
|
||||
return;
|
||||
} else if (fd_tensor.dtype == FDDataType::UINT8) {
|
||||
tensor->ShareExternalData(static_cast<const uint8_t*>(fd_tensor.Data()),
|
||||
shape, paddle_infer::PlaceType::kCPU);
|
||||
return;
|
||||
}
|
||||
FDASSERT(false, "Unexpected data type(%s) while infer with PaddleBackend.",
|
||||
Str(fd_tensor.dtype).c_str());
|
||||
|
21
fastdeploy/pipeline.h
Normal file
21
fastdeploy/pipeline.h
Normal file
@@ -0,0 +1,21 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "fastdeploy/core/config.h"
|
||||
#ifdef ENABLE_VISION
|
||||
#include "fastdeploy/pipeline/pptinypose/pipeline.h"
|
||||
#endif
|
||||
|
||||
#include "fastdeploy/vision/visualize/visualize.h"
|
24
fastdeploy/pipeline/pipeline_pybind.cc
Normal file
24
fastdeploy/pipeline/pipeline_pybind.cc
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/pybind/main.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
|
||||
void BindPPTinyPosePipeline(pybind11::module& m);
|
||||
|
||||
void BindPipeline(pybind11::module& m) {
|
||||
BindPPTinyPosePipeline(m);
|
||||
}
|
||||
} // namespace fastdeploy
|
70
fastdeploy/pipeline/pptinypose/pipeline.cc
Normal file
70
fastdeploy/pipeline/pptinypose/pipeline.cc
Normal file
@@ -0,0 +1,70 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/pipeline/pptinypose/pipeline.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace pipeline {
|
||||
PPTinyPose::PPTinyPose(
|
||||
fastdeploy::vision::detection::PPYOLOE* det_model,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model)
|
||||
: detector_(det_model), pptinypose_model_(pptinypose_model) {}
|
||||
|
||||
bool PPTinyPose::Detect(
|
||||
cv::Mat* img, fastdeploy::vision::DetectionResult* detection_res) {
|
||||
if (!detector_->Predict(img, detection_res)) {
|
||||
FDERROR << "There's a error while detectiong human box in image."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::KeypointDetect(
|
||||
cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result,
|
||||
fastdeploy::vision::DetectionResult& detection_result) {
|
||||
if (!pptinypose_model_->Predict(img, result, detection_result)) {
|
||||
FDERROR << "There's a error while detecting keypoint in image "
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::Predict(
|
||||
cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result) {
|
||||
result->Clear();
|
||||
fastdeploy::vision::DetectionResult detection_res;
|
||||
if (nullptr != detector_ && !Detect(img, &detection_res)) {
|
||||
FDERROR << "Failed to detect image." << std::endl;
|
||||
return false;
|
||||
}
|
||||
fastdeploy::vision::DetectionResult filter_detection_res;
|
||||
for (size_t i = 0; i < detection_res.boxes.size(); ++i) {
|
||||
if (detection_res.scores[i] > detection_model_score_threshold) {
|
||||
filter_detection_res.boxes.push_back(detection_res.boxes[i]);
|
||||
filter_detection_res.scores.push_back(detection_res.scores[i]);
|
||||
filter_detection_res.label_ids.push_back(detection_res.label_ids[i]);
|
||||
}
|
||||
}
|
||||
if (nullptr != pptinypose_model_ &&
|
||||
!KeypointDetect(img, result, filter_detection_res)) {
|
||||
FDERROR << "Failed to detect keypoint in image " << std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
} // namespace pipeline
|
||||
} // namespace fastdeploy
|
67
fastdeploy/pipeline/pptinypose/pipeline.h
Normal file
67
fastdeploy/pipeline/pptinypose/pipeline.h
Normal file
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fastdeploy/fastdeploy_model.h"
|
||||
#include "fastdeploy/vision/common/result.h"
|
||||
#include "fastdeploy/vision/detection/ppdet/model.h"
|
||||
#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
/** \brief All pipeline model APIs are defined inside this namespace
|
||||
*
|
||||
*/
|
||||
namespace pipeline {
|
||||
|
||||
/*! @brief PPTinyPose Pipeline object used when to load a detection model + pptinypose model
|
||||
*/
|
||||
class FASTDEPLOY_DECL PPTinyPose {
|
||||
public:
|
||||
/** \brief Set initialized detection model object and pptinypose model object
|
||||
*
|
||||
* \param[in] det_model Initialized detection model object
|
||||
* \param[in] pptinypose_model Initialized pptinypose model object
|
||||
*/
|
||||
PPTinyPose(
|
||||
fastdeploy::vision::detection::PPYOLOE* det_model,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model);
|
||||
|
||||
/** \brief Predict the keypoint detection result for an input image
|
||||
*
|
||||
* \param[in] img The input image data, comes from cv::imread()
|
||||
* \param[in] result The output keypoint detection result will be writen to this structure
|
||||
* \return true if the prediction successed, otherwise false
|
||||
*/
|
||||
virtual bool Predict(cv::Mat* img,
|
||||
fastdeploy::vision::KeyPointDetectionResult* result);
|
||||
|
||||
/* \brief The score threshold for detectin model to filter bbox before inputting pptinypose model
|
||||
*/
|
||||
float detection_model_score_threshold = 0;
|
||||
|
||||
protected:
|
||||
fastdeploy::vision::detection::PPYOLOE* detector_ = nullptr;
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model_ =
|
||||
nullptr;
|
||||
|
||||
virtual bool Detect(cv::Mat* img,
|
||||
fastdeploy::vision::DetectionResult* result);
|
||||
virtual bool KeypointDetect(
|
||||
cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result,
|
||||
fastdeploy::vision::DetectionResult& detection_result);
|
||||
};
|
||||
|
||||
} // namespace pipeline
|
||||
} // namespace fastdeploy
|
58
fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc
Normal file
58
fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc
Normal file
@@ -0,0 +1,58 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <pybind11/stl.h>
|
||||
#include "fastdeploy/pybind/main.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
void BindPPTinyPosePipeline(pybind11::module& m) {
|
||||
pybind11::class_<pipeline::PPTinyPose>(m, "PPTinyPose")
|
||||
|
||||
// explicitly pybind more kinds of detection models here
|
||||
.def(pybind11::init<fastdeploy::vision::detection::PPYOLOE*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::PicoDet*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::PPYOLO*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::PPYOLOv2*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::PaddleYOLOX*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::FasterRCNN*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::YOLOv3*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def(pybind11::init<fastdeploy::vision::detection::MaskRCNN*,
|
||||
fastdeploy::vision::keypointdetection::PPTinyPose*>())
|
||||
|
||||
.def("predict", [](pipeline::PPTinyPose& self,
|
||||
pybind11::array& data) {
|
||||
auto mat = PyArrayToCvMat(data);
|
||||
vision::KeyPointDetectionResult res;
|
||||
self.Predict(&mat, &res);
|
||||
return res;
|
||||
})
|
||||
|
||||
.def_readwrite("detection_model_score_threshold",
|
||||
&pipeline::PPTinyPose::detection_model_score_threshold);
|
||||
}
|
||||
|
||||
} // namespace fastdeploy
|
@@ -20,6 +20,7 @@ void BindRuntime(pybind11::module&);
|
||||
void BindFDModel(pybind11::module&);
|
||||
void BindVision(pybind11::module&);
|
||||
void BindText(pybind11::module&);
|
||||
void BindPipeline(pybind11::module&);
|
||||
|
||||
pybind11::dtype FDDataTypeToNumpyDataType(const FDDataType& fd_dtype) {
|
||||
pybind11::dtype dt;
|
||||
@@ -154,6 +155,9 @@ PYBIND11_MODULE(@PY_LIBRARY_NAME@, m) {
|
||||
auto vision_module =
|
||||
m.def_submodule("vision", "Vision module of FastDeploy.");
|
||||
BindVision(vision_module);
|
||||
auto pipeline_module =
|
||||
m.def_submodule("pipeline", "Pipeline module of FastDeploy.");
|
||||
BindPipeline(pipeline_module);
|
||||
#endif
|
||||
#ifdef ENABLE_TEXT
|
||||
auto text_module =
|
||||
|
@@ -24,6 +24,7 @@
|
||||
|
||||
#ifdef ENABLE_VISION
|
||||
#include "fastdeploy/vision.h"
|
||||
#include "fastdeploy/pipeline.h"
|
||||
#endif
|
||||
|
||||
#ifdef ENABLE_TEXT
|
||||
@@ -37,6 +38,7 @@ namespace fastdeploy {
|
||||
void BindBackend(pybind11::module&);
|
||||
void BindVision(pybind11::module&);
|
||||
void BindText(pybind11::module& m);
|
||||
void BindPipeline(pybind11::module& m);
|
||||
|
||||
pybind11::dtype FDDataTypeToNumpyDataType(const FDDataType& fd_dtype);
|
||||
|
||||
|
@@ -38,6 +38,7 @@
|
||||
#include "fastdeploy/vision/faceid/contrib/insightface_rec.h"
|
||||
#include "fastdeploy/vision/faceid/contrib/partial_fc.h"
|
||||
#include "fastdeploy/vision/faceid/contrib/vpl.h"
|
||||
#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h"
|
||||
#include "fastdeploy/vision/matting/contrib/modnet.h"
|
||||
#include "fastdeploy/vision/matting/ppmatting/ppmatting.h"
|
||||
#include "fastdeploy/vision/ocr/ppocr/classifier.h"
|
||||
|
@@ -35,7 +35,9 @@ class Processor {
|
||||
virtual bool ImplByOpenCV(Mat* mat) = 0;
|
||||
|
||||
virtual bool ImplByFalconCV(Mat* mat) {
|
||||
FDASSERT(false, "%s is not implemented with FalconCV, please use OpenCV instead.", Name().c_str());
|
||||
FDASSERT(false,
|
||||
"%s is not implemented with FalconCV, please use OpenCV instead.",
|
||||
Name().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@@ -27,6 +27,7 @@ class Crop : public Processor {
|
||||
width_ = width;
|
||||
height_ = height;
|
||||
}
|
||||
|
||||
bool ImplByOpenCV(Mat* mat);
|
||||
std::string Name() { return "Crop"; }
|
||||
|
||||
|
@@ -23,8 +23,8 @@
|
||||
#endif
|
||||
|
||||
namespace fcv {
|
||||
class Mat;
|
||||
}
|
||||
class Mat;
|
||||
} // namespace fcv
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
@@ -56,6 +56,11 @@ struct FASTDEPLOY_DECL Mat {
|
||||
return &cpu_mat;
|
||||
}
|
||||
|
||||
inline const cv::Mat* GetOpenCVMat() const {
|
||||
FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure.");
|
||||
return &cpu_mat;
|
||||
}
|
||||
|
||||
#ifdef ENABLE_FALCONCV
|
||||
void SetMat(const fcv::Mat& mat) {
|
||||
fcv_mat = mat;
|
||||
|
@@ -30,3 +30,4 @@
|
||||
#include "fastdeploy/vision/common/processors/resize_by_long.h"
|
||||
#include "fastdeploy/vision/common/processors/resize_by_short.h"
|
||||
#include "fastdeploy/vision/common/processors/stride_pad.h"
|
||||
#include "fastdeploy/vision/common/processors/warp_affine.h"
|
||||
|
51
fastdeploy/vision/common/processors/warp_affine.cc
Normal file
51
fastdeploy/vision/common/processors/warp_affine.cc
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision/common/processors/warp_affine.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
|
||||
bool WarpAffine::ImplByOpenCV(Mat* mat) {
|
||||
if (mat->layout != Layout::HWC) {
|
||||
FDERROR << "WarpAffine: The format of input is not HWC." << std::endl;
|
||||
return false;
|
||||
}
|
||||
cv::Mat* im = mat->GetOpenCVMat();
|
||||
if (width_ > 0 && height_ > 0) {
|
||||
cv::warpAffine(*im, *im, trans_matrix_, cv::Size(width_, height_), interp_, border_mode_, borderValue_);
|
||||
} else {
|
||||
FDERROR << "WarpAffine: the parameters must satisfy (width > 0 && height > 0) ."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
mat->SetWidth(im->cols);
|
||||
mat->SetHeight(im->rows);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WarpAffine::Run(Mat* mat,
|
||||
const cv::Mat& trans_matrix,
|
||||
int width, int height,
|
||||
int interp,
|
||||
int border_mode,
|
||||
const cv::Scalar& borderValue,
|
||||
ProcLib lib) {
|
||||
auto r = WarpAffine(trans_matrix, width, height, interp, border_mode, borderValue);
|
||||
return r(mat, lib);
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
66
fastdeploy/vision/common/processors/warp_affine.h
Normal file
66
fastdeploy/vision/common/processors/warp_affine.h
Normal file
@@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fastdeploy/vision/common/processors/base.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
|
||||
class WarpAffine : public Processor {
|
||||
public:
|
||||
WarpAffine(const cv::Mat& trans_matrix,
|
||||
int width, int height,
|
||||
int interp = 1,
|
||||
int border_mode = 0,
|
||||
const cv::Scalar& borderValue = cv::Scalar()) {
|
||||
trans_matrix_ = trans_matrix;
|
||||
width_ = width;
|
||||
height_ = height;
|
||||
interp_ = interp;
|
||||
border_mode_ = border_mode;
|
||||
borderValue_ = borderValue;
|
||||
}
|
||||
|
||||
bool ImplByOpenCV(Mat* mat);
|
||||
std::string Name() { return "WarpAffine"; }
|
||||
|
||||
bool SetTransformMatrix(const cv::Mat &trans_matrix) {
|
||||
trans_matrix_ = trans_matrix;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::tuple<int, int> GetWidthAndHeight() {
|
||||
return std::make_tuple(width_, height_);
|
||||
}
|
||||
|
||||
static bool Run(Mat* mat,
|
||||
const cv::Mat& trans_matrix,
|
||||
int width, int height,
|
||||
int interp = 1,
|
||||
int border_mode = 0,
|
||||
const cv::Scalar& borderValue = cv::Scalar(),
|
||||
ProcLib lib = ProcLib::OPENCV);
|
||||
|
||||
private:
|
||||
cv::Mat trans_matrix_;
|
||||
int width_;
|
||||
int height_;
|
||||
int interp_;
|
||||
int border_mode_;
|
||||
cv::Scalar borderValue_;
|
||||
};
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
@@ -117,6 +117,29 @@ std::string DetectionResult::Str() {
|
||||
return out;
|
||||
}
|
||||
|
||||
void KeyPointDetectionResult::Clear() {
|
||||
std::vector<std::array<float, 2>>().swap(keypoints);
|
||||
std::vector<float>().swap(scores);
|
||||
num_joints = -1;
|
||||
}
|
||||
|
||||
void KeyPointDetectionResult::Reserve(int size) { keypoints.reserve(size); }
|
||||
|
||||
void KeyPointDetectionResult::Resize(int size) { keypoints.resize(size); }
|
||||
|
||||
std::string KeyPointDetectionResult::Str() {
|
||||
std::string out;
|
||||
|
||||
out = "KeyPointDetectionResult: [x, y, conf]\n";
|
||||
for (size_t i = 0; i < keypoints.size(); ++i) {
|
||||
out = out + std::to_string(keypoints[i][0]) + "," +
|
||||
std::to_string(keypoints[i][1]) + ", " +
|
||||
std::to_string(scores[i]) + "\n";
|
||||
}
|
||||
out += "num_joints:" + std::to_string(num_joints) + "\n";
|
||||
return out;
|
||||
}
|
||||
|
||||
void OCRResult::Clear() {
|
||||
boxes.clear();
|
||||
text.clear();
|
||||
|
@@ -29,7 +29,8 @@ enum FASTDEPLOY_DECL ResultType {
|
||||
FACE_DETECTION,
|
||||
FACE_RECOGNITION,
|
||||
MATTING,
|
||||
MASK
|
||||
MASK,
|
||||
KEYPOINT_DETECTION
|
||||
};
|
||||
|
||||
struct FASTDEPLOY_DECL BaseResult {
|
||||
@@ -114,6 +115,29 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult {
|
||||
std::string Str();
|
||||
};
|
||||
|
||||
/*! @brief KeyPoint Detection result structure for all the keypoint detection models
|
||||
*/
|
||||
struct FASTDEPLOY_DECL KeyPointDetectionResult : public BaseResult {
|
||||
/** \brief All the coordinates of detected keypoints for an input image, the size of `keypoints` is num_detected_objects * num_joints, and the element of `keypoint` is a array of 2 float values, means [x, y]
|
||||
*/
|
||||
std::vector<std::array<float, 2>> keypoints;
|
||||
//// The confidence for all the detected points
|
||||
std::vector<float> scores;
|
||||
//// Number of joints for a detected object
|
||||
int num_joints = -1;
|
||||
|
||||
ResultType type = ResultType::KEYPOINT_DETECTION;
|
||||
/// Clear detection result
|
||||
void Clear();
|
||||
|
||||
void Reserve(int size);
|
||||
|
||||
void Resize(int size);
|
||||
|
||||
/// Debug function, convert the result to string to print
|
||||
std::string Str();
|
||||
};
|
||||
|
||||
struct FASTDEPLOY_DECL OCRResult : public BaseResult {
|
||||
std::vector<std::array<int, 8>> boxes;
|
||||
|
||||
|
@@ -89,6 +89,7 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() {
|
||||
|
||||
processors_.push_back(std::make_shared<BGR2RGB>());
|
||||
|
||||
bool has_permute = false;
|
||||
for (const auto& op : cfg["Preprocess"]) {
|
||||
std::string op_name = op["type"].as<std::string>();
|
||||
if (op_name == "NormalizeImage") {
|
||||
@@ -132,6 +133,7 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() {
|
||||
}
|
||||
} else if (op_name == "Permute") {
|
||||
// Do nothing, do permute as the last operation
|
||||
has_permute = true;
|
||||
continue;
|
||||
// processors_.push_back(std::make_shared<HWC2CHW>());
|
||||
} else if (op_name == "Pad") {
|
||||
@@ -150,7 +152,14 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (has_permute) {
|
||||
// permute = cast<float> + HWC2CHW
|
||||
processors_.push_back(std::make_shared<Cast>("float"));
|
||||
processors_.push_back(std::make_shared<HWC2CHW>());
|
||||
} else {
|
||||
processors_.push_back(std::make_shared<HWC2CHW>());
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
26
fastdeploy/vision/keypointdet/keypointdet_pybind.cc
Normal file
26
fastdeploy/vision/keypointdet/keypointdet_pybind.cc
Normal file
@@ -0,0 +1,26 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/pybind/main.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
|
||||
void BindPPTinyPose(pybind11::module& m);
|
||||
|
||||
void BindKeyPointDetection(pybind11::module& m) {
|
||||
auto keypointdetection_module = m.def_submodule(
|
||||
"keypointdetection", "Image object keypoint detection models.");
|
||||
BindPPTinyPose(keypointdetection_module);
|
||||
}
|
||||
} // namespace fastdeploy
|
262
fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc
Normal file
262
fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc
Normal file
@@ -0,0 +1,262 @@
|
||||
#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h"
|
||||
#include "fastdeploy/vision/utils/utils.h"
|
||||
#include "yaml-cpp/yaml.h"
|
||||
#ifdef ENABLE_PADDLE_FRONTEND
|
||||
#include "paddle2onnx/converter.h"
|
||||
#endif
|
||||
#include "fastdeploy/vision.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
namespace keypointdetection {
|
||||
|
||||
PPTinyPose::PPTinyPose(const std::string& model_file,
|
||||
const std::string& params_file,
|
||||
const std::string& config_file,
|
||||
const RuntimeOption& custom_option,
|
||||
const ModelFormat& model_format) {
|
||||
config_file_ = config_file;
|
||||
valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO};
|
||||
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
|
||||
runtime_option = custom_option;
|
||||
runtime_option.model_format = model_format;
|
||||
runtime_option.model_file = model_file;
|
||||
runtime_option.params_file = params_file;
|
||||
initialized = Initialize();
|
||||
}
|
||||
|
||||
bool PPTinyPose::Initialize() {
|
||||
if (!BuildPreprocessPipelineFromConfig()) {
|
||||
FDERROR << "Failed to build preprocess pipeline from configuration file."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
if (!InitRuntime()) {
|
||||
FDERROR << "Failed to initialize fastdeploy backend." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::BuildPreprocessPipelineFromConfig() {
|
||||
processors_.clear();
|
||||
YAML::Node cfg;
|
||||
try {
|
||||
cfg = YAML::LoadFile(config_file_);
|
||||
} catch (YAML::BadFile& e) {
|
||||
FDERROR << "Failed to load yaml file " << config_file_
|
||||
<< ", maybe you should check this file." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string arch = cfg["arch"].as<std::string>();
|
||||
if (arch != "HRNet" && arch != "HigherHRNet") {
|
||||
FDERROR << "Require the arch of model is HRNet or HigherHRNet, but arch "
|
||||
<< "defined in "
|
||||
<< "config file is " << arch << "." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
processors_.push_back(std::make_shared<BGR2RGB>());
|
||||
|
||||
for (const auto& op : cfg["Preprocess"]) {
|
||||
std::string op_name = op["type"].as<std::string>();
|
||||
if (op_name == "NormalizeImage") {
|
||||
auto mean = op["mean"].as<std::vector<float>>();
|
||||
auto std = op["std"].as<std::vector<float>>();
|
||||
bool is_scale = op["is_scale"].as<bool>();
|
||||
processors_.push_back(std::make_shared<Normalize>(mean, std, is_scale));
|
||||
} else if (op_name == "Permute") {
|
||||
// permute = cast<float> + HWC2CHW
|
||||
processors_.push_back(std::make_shared<Cast>("float"));
|
||||
processors_.push_back(std::make_shared<HWC2CHW>());
|
||||
} else if (op_name == "TopDownEvalAffine") {
|
||||
auto trainsize = op["trainsize"].as<std::vector<int>>();
|
||||
int height = trainsize[1];
|
||||
int width = trainsize[0];
|
||||
cv::Mat trans_matrix(2, 3, CV_64FC1);
|
||||
processors_.push_back(
|
||||
std::make_shared<WarpAffine>(trans_matrix, width, height, 1));
|
||||
} else {
|
||||
FDERROR << "Unexcepted preprocess operator: " << op_name << "."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::Preprocess(Mat* mat, std::vector<FDTensor>* outputs) {
|
||||
for (size_t i = 0; i < processors_.size(); ++i) {
|
||||
if (processors_[i]->Name().compare("WarpAffine") == 0) {
|
||||
auto processor = dynamic_cast<WarpAffine*>(processors_[i].get());
|
||||
float origin_width = static_cast<float>(mat->Width());
|
||||
float origin_height = static_cast<float>(mat->Height());
|
||||
std::vector<float> center = {origin_width / 2.0f, origin_height / 2.0f};
|
||||
std::vector<float> scale = {origin_width, origin_height};
|
||||
int resize_width = -1;
|
||||
int resize_height = -1;
|
||||
std::tie(resize_width, resize_height) = processor->GetWidthAndHeight();
|
||||
cv::Mat trans_matrix(2, 3, CV_64FC1);
|
||||
GetAffineTransform(center, scale, 0, {resize_width, resize_height}, &trans_matrix, 0);
|
||||
if (!(processor->SetTransformMatrix(trans_matrix))) {
|
||||
FDERROR << "Failed to set transform matrix of "
|
||||
<< processors_[i]->Name()
|
||||
<< " processor." << std::endl;
|
||||
}
|
||||
}
|
||||
if (!(*(processors_[i].get()))(mat)) {
|
||||
FDERROR << "Failed to process image data in " << processors_[i]->Name()
|
||||
<< "." << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
outputs->resize(1);
|
||||
(*outputs)[0].name = InputInfoOfRuntime(0).name;
|
||||
mat->ShareWithTensor(&((*outputs)[0]));
|
||||
|
||||
// reshape to [1, c, h, w]
|
||||
(*outputs)[0].ExpandDim(0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::Postprocess(std::vector<FDTensor>& infer_result,
|
||||
KeyPointDetectionResult* result,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float>& scale) {
|
||||
FDASSERT(infer_result[1].shape[0] == 1,
|
||||
"Only support batch = 1 in FastDeploy now.");
|
||||
result->Clear();
|
||||
|
||||
// Calculate output length
|
||||
int outdata_size =
|
||||
std::accumulate(infer_result[0].shape.begin(),
|
||||
infer_result[0].shape.end(), 1, std::multiplies<int>());
|
||||
int idxdata_size =
|
||||
std::accumulate(infer_result[1].shape.begin(),
|
||||
infer_result[1].shape.end(), 1, std::multiplies<int>());
|
||||
|
||||
if (outdata_size < 6) {
|
||||
FDWARNING << "PPTinyPose No object detected." << std::endl;
|
||||
}
|
||||
float* out_data = static_cast<float*>(infer_result[0].Data());
|
||||
void* idx_data = infer_result[1].Data();
|
||||
int idx_dtype = infer_result[1].dtype;
|
||||
std::vector<int> out_data_shape(infer_result[0].shape.begin(),
|
||||
infer_result[0].shape.end());
|
||||
std::vector<int> idx_data_shape(infer_result[1].shape.begin(),
|
||||
infer_result[1].shape.end());
|
||||
std::vector<float> preds(out_data_shape[1] * 3, 0);
|
||||
std::vector<float> heatmap(out_data, out_data + outdata_size);
|
||||
std::vector<int64_t> idxout(idxdata_size);
|
||||
if (idx_dtype == FDDataType::INT32) {
|
||||
std::copy(static_cast<int32_t*>(idx_data),
|
||||
static_cast<int32_t*>(idx_data) + idxdata_size, idxout.begin());
|
||||
} else if (idx_dtype == FDDataType::INT64) {
|
||||
std::copy(static_cast<int64_t*>(idx_data),
|
||||
static_cast<int64_t*>(idx_data) + idxdata_size, idxout.begin());
|
||||
} else {
|
||||
FDERROR << "Only support process inference result with INT32/INT64 data type, but now it's " << idx_dtype << "." << std::endl;
|
||||
}
|
||||
GetFinalPredictions(heatmap, out_data_shape, idxout, center, scale, &preds,
|
||||
this->use_dark);
|
||||
result->Reserve(outdata_size);
|
||||
result->num_joints = out_data_shape[1];
|
||||
result->keypoints.clear();
|
||||
for (int i = 0; i < out_data_shape[1]; i++) {
|
||||
result->keypoints.push_back({preds[i * 3 + 1], preds[i * 3 + 2]});
|
||||
result->scores.push_back(preds[i * 3]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result) {
|
||||
std::vector<float> center = {round(im->cols / 2.0f), round(im->rows / 2.0f)};
|
||||
std::vector<float> scale = {static_cast<float>(im->cols), static_cast<float>(im->rows)};
|
||||
Mat mat(*im);
|
||||
std::vector<FDTensor> processed_data;
|
||||
if (!Preprocess(&mat, &processed_data)) {
|
||||
FDERROR << "Failed to preprocess input data while using model:"
|
||||
<< ModelName() << "." << std::endl;
|
||||
return false;
|
||||
}
|
||||
std::vector<FDTensor> infer_result;
|
||||
if (!Infer(processed_data, &infer_result)) {
|
||||
FDERROR << "Failed to inference while using model:" << ModelName() << "."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
if (!Postprocess(infer_result, result, center, scale)) {
|
||||
FDERROR << "Failed to postprocess while using model:" << ModelName() << "."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result,
|
||||
const DetectionResult& detection_result) {
|
||||
std::vector<Mat> crop_imgs;
|
||||
std::vector<std::vector<float>> center_bs;
|
||||
std::vector<std::vector<float>> scale_bs;
|
||||
int crop_imgs_num = 0;
|
||||
int box_num = detection_result.boxes.size();
|
||||
for (int i = 0; i < box_num; i++) {
|
||||
auto box = detection_result.boxes[i];
|
||||
auto label_id = detection_result.label_ids[i];
|
||||
int channel = im->channels();
|
||||
cv::Mat cv_crop_img(0, 0, CV_32SC(channel));
|
||||
Mat crop_img(cv_crop_img);
|
||||
std::vector<float> rect(box.begin(), box.end());
|
||||
std::vector<float> center;
|
||||
std::vector<float> scale;
|
||||
if (label_id == 0) {
|
||||
Mat mat(*im);
|
||||
utils::CropImageByBox(mat, &crop_img, rect, ¢er, &scale);
|
||||
center_bs.emplace_back(center);
|
||||
scale_bs.emplace_back(scale);
|
||||
crop_imgs.emplace_back(crop_img);
|
||||
crop_imgs_num += 1;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < crop_imgs_num; i++) {
|
||||
std::vector<FDTensor> processed_data;
|
||||
if (!Preprocess(&crop_imgs[i], &processed_data)) {
|
||||
FDERROR << "Failed to preprocess input data while using model:"
|
||||
<< ModelName() << "." << std::endl;
|
||||
return false;
|
||||
}
|
||||
std::vector<FDTensor> infer_result;
|
||||
if (!Infer(processed_data, &infer_result)) {
|
||||
FDERROR << "Failed to inference while using model:" << ModelName() << "."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
KeyPointDetectionResult one_cropimg_result;
|
||||
if (!Postprocess(infer_result, &one_cropimg_result, center_bs[i],
|
||||
scale_bs[i])) {
|
||||
FDERROR << "Failed to postprocess while using model:" << ModelName()
|
||||
<< "." << std::endl;
|
||||
return false;
|
||||
}
|
||||
if (result->num_joints == -1) {
|
||||
result->num_joints = one_cropimg_result.num_joints;
|
||||
}
|
||||
std::copy(one_cropimg_result.keypoints.begin(),
|
||||
one_cropimg_result.keypoints.end(),
|
||||
std::back_inserter(result->keypoints));
|
||||
std::copy(one_cropimg_result.scores.begin(),
|
||||
one_cropimg_result.scores.end(),
|
||||
std::back_inserter(result->scores));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace keypointdetection
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
90
fastdeploy/vision/keypointdet/pptinypose/pptinypose.h
Normal file
90
fastdeploy/vision/keypointdet/pptinypose/pptinypose.h
Normal file
@@ -0,0 +1,90 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "fastdeploy/fastdeploy_model.h"
|
||||
#include "fastdeploy/vision/common/processors/transform.h"
|
||||
#include "fastdeploy/vision/common/result.h"
|
||||
|
||||
#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
/** \brief All keypoint detection model APIs are defined inside this namespace
|
||||
*
|
||||
*/
|
||||
namespace keypointdetection {
|
||||
|
||||
/*! @brief PPTinyPose model object used when to load a PPTinyPose model exported by PaddleDetection
|
||||
*/
|
||||
class FASTDEPLOY_DECL PPTinyPose : public FastDeployModel {
|
||||
public:
|
||||
/** \brief Set path of model file and configuration file, and the configuration of runtime
|
||||
*
|
||||
* \param[in] model_file Path of model file, e.g pptinypose/model.pdmodel
|
||||
* \param[in] params_file Path of parameter file, e.g pptinypose/model.pdiparams, if the model format is ONNX, this parameter will be ignored
|
||||
* \param[in] config_file Path of configuration file for deployment, e.g pptinypose/infer_cfg.yml
|
||||
* \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`
|
||||
* \param[in] model_format Model format of the loaded model, default is Paddle format
|
||||
*/
|
||||
PPTinyPose(const std::string& model_file, const std::string& params_file,
|
||||
const std::string& config_file,
|
||||
const RuntimeOption& custom_option = RuntimeOption(),
|
||||
const ModelFormat& model_format = ModelFormat::PADDLE);
|
||||
|
||||
/// Get model's name
|
||||
std::string ModelName() const { return "PaddleDetection/PPTinyPose"; }
|
||||
|
||||
/** \brief Predict the keypoint detection result for an input image
|
||||
*
|
||||
* \param[in] im The input image data, comes from cv::imread()
|
||||
* \param[in] result The output keypoint detection result will be writen to this structure
|
||||
* \return true if the keypoint prediction successed, otherwise false
|
||||
*/
|
||||
bool Predict(cv::Mat* im, KeyPointDetectionResult* result);
|
||||
|
||||
/** \brief Predict the keypoint detection result with given detection result for an input image
|
||||
*
|
||||
* \param[in] im The input image data, comes from cv::imread()
|
||||
* \param[in] result The output keypoint detection result will be writen to this structure
|
||||
* \param[in] detection_result The structure strores pedestrian detection result, which is used to crop image for multi-persons keypoint detection
|
||||
* \return true if the keypoint prediction successed, otherwise false
|
||||
*/
|
||||
bool Predict(cv::Mat* im, KeyPointDetectionResult* result,
|
||||
const DetectionResult& detection_result);
|
||||
|
||||
/** \brief Whether using Distribution-Aware Coordinate Representation for Human Pose Estimation(DARK for short) in postprocess, default is true
|
||||
*/
|
||||
bool use_dark = true;
|
||||
|
||||
protected:
|
||||
bool Initialize();
|
||||
/// Build the preprocess pipeline from the loaded model
|
||||
bool BuildPreprocessPipelineFromConfig();
|
||||
/// Preprocess an input image, and set the preprocessed results to `outputs`
|
||||
bool Preprocess(Mat* mat, std::vector<FDTensor>* outputs);
|
||||
|
||||
/// Postprocess the inferenced results, and set the final result to `result`
|
||||
bool Postprocess(std::vector<FDTensor>& infer_result,
|
||||
KeyPointDetectionResult* result,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float>& scale);
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<Processor>> processors_;
|
||||
std::string config_file_;
|
||||
};
|
||||
} // namespace keypointdetection
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
@@ -0,0 +1,42 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "fastdeploy/pybind/main.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
void BindPPTinyPose(pybind11::module& m) {
|
||||
pybind11::class_<vision::keypointdetection::PPTinyPose, FastDeployModel>(
|
||||
m, "PPTinyPose")
|
||||
.def(pybind11::init<std::string, std::string, std::string, RuntimeOption,
|
||||
ModelFormat>())
|
||||
.def("predict",
|
||||
[](vision::keypointdetection::PPTinyPose& self,
|
||||
pybind11::array& data) {
|
||||
auto mat = PyArrayToCvMat(data);
|
||||
vision::KeyPointDetectionResult res;
|
||||
self.Predict(&mat, &res);
|
||||
return res;
|
||||
})
|
||||
.def(
|
||||
"predict",
|
||||
[](vision::keypointdetection::PPTinyPose& self, pybind11::array& data,
|
||||
vision::DetectionResult& detection_result) {
|
||||
auto mat = PyArrayToCvMat(data);
|
||||
vision::KeyPointDetectionResult res;
|
||||
self.Predict(&mat, &res, detection_result);
|
||||
return res;
|
||||
})
|
||||
.def_readwrite("use_dark",
|
||||
&vision::keypointdetection::PPTinyPose::use_dark);
|
||||
}
|
||||
} // namespace fastdeploy
|
125
fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc
Normal file
125
fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc
Normal file
@@ -0,0 +1,125 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h"
|
||||
#define PI 3.1415926535
|
||||
#define HALF_CIRCLE_DEGREE 180
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
namespace keypointdetection {
|
||||
|
||||
cv::Point2f Get3dPoint(const cv::Point2f& a, const cv::Point2f& b) {
|
||||
cv::Point2f direct{a.x - b.x, a.y - b.y};
|
||||
return cv::Point2f(a.x - direct.y, a.y + direct.x);
|
||||
}
|
||||
|
||||
std::vector<float> GetDir(const float src_point_x, const float src_point_y,
|
||||
const float rot_rad) {
|
||||
float sn = sin(rot_rad);
|
||||
float cs = cos(rot_rad);
|
||||
std::vector<float> src_result{0.0, 0.0};
|
||||
src_result[0] = src_point_x * cs - src_point_y * sn;
|
||||
src_result[1] = src_point_x * sn + src_point_y * cs;
|
||||
return src_result;
|
||||
}
|
||||
|
||||
void AffineTransform(const float pt_x, const float pt_y, const cv::Mat& trans,
|
||||
std::vector<float>* preds, const int p) {
|
||||
double new1[3] = {pt_x, pt_y, 1.0};
|
||||
cv::Mat new_pt(3, 1, trans.type(), new1);
|
||||
cv::Mat w = trans * new_pt;
|
||||
(*preds)[p * 3 + 1] = static_cast<float>(w.at<double>(0, 0));
|
||||
(*preds)[p * 3 + 2] = static_cast<float>(w.at<double>(1, 0));
|
||||
}
|
||||
|
||||
void GetAffineTransform(const std::vector<float>& center,
|
||||
const std::vector<float>& scale, const float rot,
|
||||
const std::vector<int>& output_size, cv::Mat* trans,
|
||||
const int inv) {
|
||||
float src_w = scale[0];
|
||||
float dst_w = static_cast<float>(output_size[0]);
|
||||
float dst_h = static_cast<float>(output_size[1]);
|
||||
float rot_rad = rot * PI / HALF_CIRCLE_DEGREE;
|
||||
std::vector<float> src_dir = GetDir(-0.5 * src_w, 0, rot_rad);
|
||||
std::vector<float> dst_dir{-0.5f * dst_w, 0.0};
|
||||
cv::Point2f srcPoint2f[3], dstPoint2f[3];
|
||||
srcPoint2f[0] = cv::Point2f(center[0], center[1]);
|
||||
srcPoint2f[1] = cv::Point2f(center[0] + src_dir[0], center[1] + src_dir[1]);
|
||||
srcPoint2f[2] = Get3dPoint(srcPoint2f[0], srcPoint2f[1]);
|
||||
|
||||
dstPoint2f[0] = cv::Point2f(dst_w * 0.5, dst_h * 0.5);
|
||||
dstPoint2f[1] =
|
||||
cv::Point2f(dst_w * 0.5 + dst_dir[0], dst_h * 0.5 + dst_dir[1]);
|
||||
dstPoint2f[2] = Get3dPoint(dstPoint2f[0], dstPoint2f[1]);
|
||||
if (inv == 0) {
|
||||
(*trans) = cv::getAffineTransform(srcPoint2f, dstPoint2f);
|
||||
} else {
|
||||
(*trans) = cv::getAffineTransform(dstPoint2f, srcPoint2f);
|
||||
}
|
||||
}
|
||||
|
||||
void TransformPreds(std::vector<float>& coords,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float>& scale,
|
||||
const std::vector<int>& output_size,
|
||||
const std::vector<int>& dim,
|
||||
std::vector<float>* target_coords) {
|
||||
cv::Mat trans(2, 3, CV_64FC1);
|
||||
GetAffineTransform(center, scale, 0, output_size, &trans, 1);
|
||||
for (int p = 0; p < dim[1]; ++p) {
|
||||
AffineTransform(coords[p * 2], coords[p * 2 + 1], trans, target_coords, p);
|
||||
}
|
||||
}
|
||||
|
||||
void GetFinalPredictions(const std::vector<float>& heatmap,
|
||||
const std::vector<int>& dim,
|
||||
const std::vector<int64_t>& idxout,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float> scale,
|
||||
std::vector<float>* preds, const bool DARK) {
|
||||
std::vector<float> coords(dim[1] * 2);
|
||||
|
||||
int heatmap_height = dim[2];
|
||||
int heatmap_width = dim[3];
|
||||
for (int j = 0; j < dim[1]; ++j) {
|
||||
int index = j * dim[2] * dim[3];
|
||||
int idx = idxout[j];
|
||||
(*preds)[j * 3] = heatmap[index + idx];
|
||||
coords[j * 2] = idx % heatmap_width;
|
||||
coords[j * 2 + 1] = idx / heatmap_width;
|
||||
int px = int(coords[j * 2] + 0.5);
|
||||
int py = int(coords[j * 2 + 1] + 0.5);
|
||||
if (DARK && px > 1 && px < heatmap_width - 2) {
|
||||
utils::DarkParse(heatmap, dim, &coords, px, py, index, j);
|
||||
} else {
|
||||
if (px > 0 && px < heatmap_width - 1) {
|
||||
float diff_x = heatmap[index + py * dim[3] + px + 1] -
|
||||
heatmap[index + py * dim[3] + px - 1];
|
||||
coords[j * 2] += diff_x > 0 ? 1 : -1 * 0.25;
|
||||
}
|
||||
if (py > 0 && py < heatmap_height - 1) {
|
||||
float diff_y = heatmap[index + (py + 1) * dim[3] + px] -
|
||||
heatmap[index + (py - 1) * dim[3] + px];
|
||||
coords[j * 2 + 1] += diff_y > 0 ? 1 : -1 * 0.25;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::vector<int> img_size{heatmap_width, heatmap_height};
|
||||
TransformPreds(coords, center, scale, img_size, dim, preds);
|
||||
}
|
||||
|
||||
} // namespace detection
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
51
fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h
Normal file
51
fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "fastdeploy/vision/utils/utils.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
namespace keypointdetection {
|
||||
|
||||
cv::Point2f Get3dPoint(const cv::Point2f& a, const cv::Point2f& b);
|
||||
|
||||
std::vector<float> GetDir(const float src_point_x, const float src_point_y,
|
||||
const float rot_rad);
|
||||
|
||||
void GetAffineTransform(const std::vector<float>& center,
|
||||
const std::vector<float>& scale, const float rot,
|
||||
const std::vector<int>& output_size, cv::Mat* trans,
|
||||
const int inv);
|
||||
|
||||
void AffineTransform(const float pt_x, const float pt_y, const cv::Mat& trans,
|
||||
std::vector<float>* preds, const int p);
|
||||
|
||||
void TransformPreds(std::vector<float>& coords,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float>& scale,
|
||||
const std::vector<int>& output_size,
|
||||
const std::vector<int>& dim,
|
||||
std::vector<float>* target_coords);
|
||||
|
||||
void GetFinalPredictions(const std::vector<float>& heatmap,
|
||||
const std::vector<int>& dim,
|
||||
const std::vector<int64_t>& idxout,
|
||||
const std::vector<float>& center,
|
||||
const std::vector<float> scale,
|
||||
std::vector<float>* preds, const bool DARK);
|
||||
|
||||
} // namespace keypointdetection
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
@@ -143,9 +143,9 @@ bool PaddleSegModel::Preprocess(Mat* mat, FDTensor* output) {
|
||||
int resize_height = -1;
|
||||
std::tie(resize_width, resize_height) = processor->GetWidthAndHeight();
|
||||
if (is_vertical_screen && (resize_width > resize_height)) {
|
||||
if (processor->SetWidthAndHeight(resize_height, resize_width)) {
|
||||
FDERROR << "Failed to set Resize processor width and height "
|
||||
<< processors_[i]->Name() << "." << std::endl;
|
||||
if (!(processor->SetWidthAndHeight(resize_height, resize_width))) {
|
||||
FDERROR << "Failed to set width and height of "
|
||||
<< processors_[i]->Name() << " processor." << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
61
fastdeploy/vision/utils/crop_image.cc
Normal file
61
fastdeploy/vision/utils/crop_image.cc
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision/utils/utils.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
namespace utils {
|
||||
|
||||
bool CropImageByBox(const Mat& src_im, Mat* dst_im,
|
||||
const std::vector<float>& box, std::vector<float>* center,
|
||||
std::vector<float>* scale, const float expandratio) {
|
||||
const cv::Mat* img = src_im.GetOpenCVMat();
|
||||
cv::Mat* crop_img = dst_im->GetOpenCVMat();
|
||||
int xmin = static_cast<int>(box[0]);
|
||||
int ymin = static_cast<int>(box[1]);
|
||||
int xmax = static_cast<int>(box[2]);
|
||||
int ymax = static_cast<int>(box[3]);
|
||||
float centerx = (xmin + xmax) / 2.0f;
|
||||
float centery = (ymin + ymax) / 2.0f;
|
||||
float half_h = (ymax - ymin) * (1 + expandratio) / 2.0f;
|
||||
float half_w = (xmax - xmin) * (1 + expandratio) / 2.0f;
|
||||
// adjust h or w to keep image ratio, expand the shorter edge
|
||||
if (half_h * 3 > half_w * 4) {
|
||||
half_w = half_h * 0.75;
|
||||
}
|
||||
int crop_xmin =std::max(0, static_cast<int>(centerx - half_w));
|
||||
int crop_ymin =std::max(0, static_cast<int>(centery - half_h));
|
||||
int crop_xmax = std::min(img->cols - 1, static_cast<int>(centerx + half_w));
|
||||
int crop_ymax = std::min(img->rows - 1, static_cast<int>(centery + half_h));
|
||||
|
||||
crop_img->create(crop_ymax - crop_ymin, crop_xmax - crop_xmin, img->type());
|
||||
*crop_img =
|
||||
(*img)(cv::Range(crop_ymin, crop_ymax), cv::Range(crop_xmin, crop_xmax));
|
||||
center->clear();
|
||||
center->emplace_back((crop_xmin + crop_xmax) / 2.0f);
|
||||
center->emplace_back((crop_ymin + crop_ymax) / 2.0f);
|
||||
|
||||
scale->clear();
|
||||
scale->emplace_back((crop_xmax - crop_xmin));
|
||||
scale->emplace_back((crop_ymax - crop_ymin));
|
||||
|
||||
dst_im->SetWidth(crop_img->cols);
|
||||
dst_im->SetHeight(crop_img->rows);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace utils
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
81
fastdeploy/vision/utils/dark_parse.cc
Normal file
81
fastdeploy/vision/utils/dark_parse.cc
Normal file
@@ -0,0 +1,81 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fastdeploy/vision/utils/utils.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
namespace utils {
|
||||
|
||||
void DarkParse(const std::vector<float>& heatmap, const std::vector<int>& dim,
|
||||
std::vector<float>* coords, const int px, const int py,
|
||||
const int index, const int ch) {
|
||||
/*DARK postpocessing, Zhang et al. Distribution-Aware Coordinate
|
||||
Representation for Human Pose Estimation (CVPR 2020).
|
||||
1) offset = - hassian.inv() * derivative
|
||||
2) dx = (heatmap[x+1] - heatmap[x-1])/2.
|
||||
3) dxx = (dx[x+1] - dx[x-1])/2.
|
||||
4) derivative = Mat([dx, dy])
|
||||
5) hassian = Mat([[dxx, dxy], [dxy, dyy]])
|
||||
*/
|
||||
std::vector<float>::const_iterator first1 = heatmap.begin() + index;
|
||||
std::vector<float>::const_iterator last1 =
|
||||
heatmap.begin() + index + dim[2] * dim[3];
|
||||
std::vector<float> heatmap_ch(first1, last1);
|
||||
cv::Mat heatmap_mat = cv::Mat(heatmap_ch).reshape(0, dim[2]);
|
||||
heatmap_mat.convertTo(heatmap_mat, CV_32FC1);
|
||||
cv::GaussianBlur(heatmap_mat, heatmap_mat, cv::Size(3, 3), 0, 0);
|
||||
heatmap_mat = heatmap_mat.reshape(1, 1);
|
||||
heatmap_ch = std::vector<float>(heatmap_mat.reshape(1, 1));
|
||||
|
||||
float epsilon = 1e-10;
|
||||
// sample heatmap to get values in around target location
|
||||
float xy = log(fmax(heatmap_ch[py * dim[3] + px], epsilon));
|
||||
float xr = log(fmax(heatmap_ch[py * dim[3] + px + 1], epsilon));
|
||||
float xl = log(fmax(heatmap_ch[py * dim[3] + px - 1], epsilon));
|
||||
|
||||
float xr2 = log(fmax(heatmap_ch[py * dim[3] + px + 2], epsilon));
|
||||
float xl2 = log(fmax(heatmap_ch[py * dim[3] + px - 2], epsilon));
|
||||
float yu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px], epsilon));
|
||||
float yd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px], epsilon));
|
||||
float yu2 = log(fmax(heatmap_ch[(py + 2) * dim[3] + px], epsilon));
|
||||
float yd2 = log(fmax(heatmap_ch[(py - 2) * dim[3] + px], epsilon));
|
||||
float xryu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px + 1], epsilon));
|
||||
float xryd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px + 1], epsilon));
|
||||
float xlyu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px - 1], epsilon));
|
||||
float xlyd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px - 1], epsilon));
|
||||
|
||||
// compute dx/dy and dxx/dyy with sampled values
|
||||
float dx = 0.5 * (xr - xl);
|
||||
float dy = 0.5 * (yu - yd);
|
||||
float dxx = 0.25 * (xr2 - 2 * xy + xl2);
|
||||
float dxy = 0.25 * (xryu - xryd - xlyu + xlyd);
|
||||
float dyy = 0.25 * (yu2 - 2 * xy + yd2);
|
||||
|
||||
// finally get offset by derivative and hassian, which combined by dx/dy and
|
||||
// dxx/dyy
|
||||
if (dxx * dyy - dxy * dxy != 0) {
|
||||
float M[2][2] = {dxx, dxy, dxy, dyy};
|
||||
float D[2] = {dx, dy};
|
||||
cv::Mat hassian(2, 2, CV_32F, M);
|
||||
cv::Mat derivative(2, 1, CV_32F, D);
|
||||
cv::Mat offset = -hassian.inv() * derivative;
|
||||
(*coords)[ch * 2] += offset.at<float>(0, 0);
|
||||
(*coords)[ch * 2 + 1] += offset.at<float>(1, 0);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace utils
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
@@ -14,6 +14,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <opencv2/opencv.hpp>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include "fastdeploy/core/fd_tensor.h"
|
||||
@@ -24,6 +25,7 @@
|
||||
#include "fastdeploy/function/reduce.h"
|
||||
#include "fastdeploy/function/softmax.h"
|
||||
#include "fastdeploy/function/transpose.h"
|
||||
#include "fastdeploy/vision/common/processors/mat.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
@@ -73,6 +75,27 @@ FASTDEPLOY_DECL float CosineSimilarity(const std::vector<float>& a,
|
||||
const std::vector<float>& b,
|
||||
bool normalized = true);
|
||||
|
||||
bool CropImageByBox(const Mat& src_im, Mat* dst_im,
|
||||
const std::vector<float>& box, std::vector<float>* center,
|
||||
std::vector<float>* scale, const float expandratio = 0.3);
|
||||
|
||||
/**
|
||||
* Function: for keypoint detection model, fine positioning of keypoints in postprocess
|
||||
* Parameters:
|
||||
* heatmap: model inference results for keypoint detection models
|
||||
* dim: shape information of the inference result
|
||||
* coords: coordinates after refined positioning
|
||||
* px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions
|
||||
* py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions
|
||||
* index: index information of heatmap pixels
|
||||
* ch: channel
|
||||
* Paper reference: DARK postpocessing, Zhang et al. Distribution-Aware Coordinate
|
||||
* Representation for Human Pose Estimation (CVPR 2020).
|
||||
*/
|
||||
void DarkParse(const std::vector<float>& heatmap, const std::vector<int>& dim,
|
||||
std::vector<float>* coords, const int px, const int py,
|
||||
const int index, const int ch);
|
||||
|
||||
} // namespace utils
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
||||
|
@@ -23,6 +23,7 @@ void BindMatting(pybind11::module& m);
|
||||
void BindFaceDet(pybind11::module& m);
|
||||
void BindFaceId(pybind11::module& m);
|
||||
void BindOcr(pybind11::module& m);
|
||||
void BindKeyPointDetection(pybind11::module& m);
|
||||
#ifdef ENABLE_VISION_VISUALIZE
|
||||
void BindVisualize(pybind11::module& m);
|
||||
#endif
|
||||
@@ -95,6 +96,15 @@ void BindVision(pybind11::module& m) {
|
||||
.def("__repr__", &vision::MattingResult::Str)
|
||||
.def("__str__", &vision::MattingResult::Str);
|
||||
|
||||
pybind11::class_<vision::KeyPointDetectionResult>(m,
|
||||
"KeyPointDetectionResult")
|
||||
.def(pybind11::init())
|
||||
.def_readwrite("keypoints", &vision::KeyPointDetectionResult::keypoints)
|
||||
.def_readwrite("scores", &vision::KeyPointDetectionResult::scores)
|
||||
.def_readwrite("num_joints", &vision::KeyPointDetectionResult::num_joints)
|
||||
.def("__repr__", &vision::KeyPointDetectionResult::Str)
|
||||
.def("__str__", &vision::KeyPointDetectionResult::Str);
|
||||
|
||||
BindDetection(m);
|
||||
BindClassification(m);
|
||||
BindSegmentation(m);
|
||||
@@ -102,6 +112,7 @@ void BindVision(pybind11::module& m) {
|
||||
BindFaceId(m);
|
||||
BindMatting(m);
|
||||
BindOcr(m);
|
||||
BindKeyPointDetection(m);
|
||||
#ifdef ENABLE_VISION_VISUALIZE
|
||||
BindVisualize(m);
|
||||
#endif
|
||||
|
62
fastdeploy/vision/visualize/keypoint.cc
Normal file
62
fastdeploy/vision/visualize/keypoint.cc
Normal file
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifdef ENABLE_VISION_VISUALIZE
|
||||
|
||||
#include "fastdeploy/vision/visualize/visualize.h"
|
||||
|
||||
namespace fastdeploy {
|
||||
namespace vision {
|
||||
|
||||
cv::Mat VisKeypointDetection(const cv::Mat& im,
|
||||
const KeyPointDetectionResult& results,
|
||||
float conf_threshold) {
|
||||
const int edge[][2] = {{0, 1}, {0, 2}, {1, 3}, {2, 4}, {3, 5},
|
||||
{4, 6}, {5, 7}, {6, 8}, {7, 9}, {8, 10},
|
||||
{5, 11}, {6, 12}, {11, 13}, {12, 14}, {13, 15},
|
||||
{14, 16}, {11, 12}};
|
||||
auto colormap = GenerateColorMap();
|
||||
cv::Mat vis_img = im.clone();
|
||||
int detection_nums = results.keypoints.size() / 17;
|
||||
for (int i = 0; i < detection_nums; i++){
|
||||
int index = i * 17;
|
||||
bool is_over_threshold = true;
|
||||
for (int j = 0; j < results.num_joints; j++) {
|
||||
if (results.scores[index + j] < conf_threshold) {
|
||||
is_over_threshold = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (is_over_threshold) {
|
||||
for (int k = 0; k < results.num_joints; k++) {
|
||||
int x_coord = int(results.keypoints[index + k][0]);
|
||||
int y_coord = int(results.keypoints[index + k][1]);
|
||||
cv::circle(vis_img, cv::Point2d(x_coord, y_coord), 1,
|
||||
cv::Scalar(0, 0, 255), 2);
|
||||
int x_start = int(results.keypoints[index + edge[k][0]][0]);
|
||||
int y_start = int(results.keypoints[index + edge[k][0]][1]);
|
||||
int x_end = int(results.keypoints[index + edge[k][1]][0]);
|
||||
int y_end = int(results.keypoints[index + edge[k][1]][1]);
|
||||
cv::line(vis_img, cv::Point2d(x_start, y_start), cv::Point2d(x_end, y_end),
|
||||
colormap[k], 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return vis_img;
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
||||
#endif
|
@@ -84,6 +84,9 @@ FASTDEPLOY_DECL cv::Mat SwapBackground(const cv::Mat& im,
|
||||
const cv::Mat& background,
|
||||
const SegmentationResult& result,
|
||||
int background_label);
|
||||
FASTDEPLOY_DECL cv::Mat VisKeypointDetection(const cv::Mat& im,
|
||||
const KeyPointDetectionResult& results,
|
||||
float conf_threshold = 0.5f);
|
||||
|
||||
} // namespace vision
|
||||
} // namespace fastdeploy
|
||||
|
@@ -98,6 +98,17 @@ void BindVisualize(pybind11::module& m) {
|
||||
vision::Mat(vis_im).ShareWithTensor(&out);
|
||||
return TensorToPyArray(out);
|
||||
})
|
||||
.def_static(
|
||||
"vis_keypoint_detection",
|
||||
[](pybind11::array& im_data, vision::KeyPointDetectionResult& result,
|
||||
float conf_threshold) {
|
||||
auto im = PyArrayToCvMat(im_data);
|
||||
auto vis_im = vision::VisKeypointDetection(
|
||||
im, result, conf_threshold);
|
||||
FDTensor out;
|
||||
vision::Mat(vis_im).ShareWithTensor(&out);
|
||||
return TensorToPyArray(out);
|
||||
})
|
||||
.def_static(
|
||||
"vis_face_detection",
|
||||
[](pybind11::array& im_data, vision::FaceDetectionResult& result,
|
||||
|
@@ -25,5 +25,6 @@ from .runtime import Runtime, RuntimeOption
|
||||
from .model import FastDeployModel
|
||||
from . import c_lib_wrap as C
|
||||
from . import vision
|
||||
from . import pipeline
|
||||
from . import text
|
||||
from .download import download, download_and_decompress
|
||||
|
16
python/fastdeploy/pipeline/__init__.py
Normal file
16
python/fastdeploy/pipeline/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import absolute_import
|
||||
|
||||
from .pptinypose import PPTinyPose
|
55
python/fastdeploy/pipeline/pptinypose/__init__.py
Normal file
55
python/fastdeploy/pipeline/pptinypose/__init__.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
# #
|
||||
# # Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# # you may not use this file except in compliance with the License.
|
||||
# # You may obtain a copy of the License at
|
||||
# #
|
||||
# # http://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# # Unless required by applicable law or agreed to in writing, software
|
||||
# # distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# # See the License for the specific language governing permissions and
|
||||
# # limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from ... import c_lib_wrap as C
|
||||
|
||||
|
||||
class PPTinyPose(object):
|
||||
def __init__(self, det_model=None, pptinypose_model=None):
|
||||
"""Set initialized detection model object and pptinypose model object
|
||||
|
||||
:param det_model: (fastdeploy.vision.detection.PicoDet)Initialized detection model object
|
||||
:param pptinypose_model: (fastdeploy.vision.keypointdetection.PPTinyPose)Initialized pptinypose model object
|
||||
"""
|
||||
assert det_model is not None or pptinypose_model is not None, "The det_model and pptinypose_model cannot be None."
|
||||
self._pipeline = C.pipeline.PPTinyPose(det_model._model,
|
||||
pptinypose_model._model)
|
||||
|
||||
def predict(self, input_image):
|
||||
"""Predict the keypoint detection result for an input image
|
||||
|
||||
:param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format
|
||||
:return: KeyPointDetectionResult
|
||||
"""
|
||||
return self._pipeline.predict(input_image)
|
||||
|
||||
@property
|
||||
def detection_model_score_threshold(self):
|
||||
"""Atrribute of PPTinyPose pipeline model. Stating the score threshold for detectin model to filter bbox before inputting pptinypose model
|
||||
|
||||
:return: value of detection_model_score_threshold(float)
|
||||
"""
|
||||
return self._pipeline.detection_model_score_threshold
|
||||
|
||||
@detection_model_score_threshold.setter
|
||||
def detection_model_score_threshold(self, value):
|
||||
"""Set attribute detection_model_score_threshold of PPTinyPose pipeline model.
|
||||
|
||||
:param value: (float)The value to set use_dark
|
||||
"""
|
||||
assert isinstance(
|
||||
value, float
|
||||
), "The value to set `detection_model_score_threshold` must be type of float."
|
||||
self._pipeline.detection_model_score_threshold = value
|
@@ -16,6 +16,7 @@ from __future__ import absolute_import
|
||||
from . import detection
|
||||
from . import classification
|
||||
from . import segmentation
|
||||
from . import keypointdetection
|
||||
|
||||
from . import matting
|
||||
from . import facedet
|
||||
|
16
python/fastdeploy/vision/keypointdetection/__init__.py
Normal file
16
python/fastdeploy/vision/keypointdetection/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from .pptinypose import PPTinyPose
|
@@ -0,0 +1,69 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
from .... import FastDeployModel, ModelFormat
|
||||
from .... import c_lib_wrap as C
|
||||
|
||||
|
||||
class PPTinyPose(FastDeployModel):
|
||||
def __init__(self,
|
||||
model_file,
|
||||
params_file,
|
||||
config_file,
|
||||
runtime_option=None,
|
||||
model_format=ModelFormat.PADDLE):
|
||||
"""load a PPTinyPose model exported by PaddleDetection.
|
||||
|
||||
:param model_file: (str)Path of model file, e.g pptinypose/model.pdmodel
|
||||
:param params_file: (str)Path of parameters file, e.g pptinypose/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string
|
||||
:param config_file: (str)Path of configuration file for deployment, e.g pptinypose/infer_cfg.yml
|
||||
:param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU
|
||||
:param model_format: (fastdeploy.ModelForamt)Model format of the loaded model
|
||||
"""
|
||||
super(PPTinyPose, self).__init__(runtime_option)
|
||||
|
||||
assert model_format == ModelFormat.PADDLE, "PPTinyPose model only support model format of ModelFormat.Paddle now."
|
||||
self._model = C.vision.keypointdetection.PPTinyPose(
|
||||
model_file, params_file, config_file, self._runtime_option,
|
||||
model_format)
|
||||
assert self.initialized, "PPTinyPose model initialize failed."
|
||||
|
||||
def predict(self, input_image):
|
||||
"""Detect keypoints in an input image
|
||||
|
||||
:param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format
|
||||
:return: KeyPointDetectionResult
|
||||
"""
|
||||
assert input_image is not None, "The input image data is None."
|
||||
return self._model.predict(input_image)
|
||||
|
||||
@property
|
||||
def use_dark(self):
|
||||
"""Atrribute of PPTinyPose model. Stating whether using Distribution-Aware Coordinate Representation for Human Pose Estimation(DARK for short) in postprocess, default is True
|
||||
|
||||
:return: value of use_dark(bool)
|
||||
"""
|
||||
return self._model.use_dark
|
||||
|
||||
@use_dark.setter
|
||||
def use_dark(self, value):
|
||||
"""Set attribute use_dark of PPTinyPose model.
|
||||
|
||||
:param value: (bool)The value to set use_dark
|
||||
"""
|
||||
assert isinstance(
|
||||
value, bool), "The value to set `use_dark` must be type of bool."
|
||||
self._model.use_dark = value
|
@@ -26,6 +26,11 @@ def vis_detection(im_data,
|
||||
line_size, font_size)
|
||||
|
||||
|
||||
def vis_keypoint_detection(im_data, keypoint_det_result, conf_threshold=0.5):
|
||||
return C.vision.Visualize.vis_keypoint_detection(
|
||||
im_data, keypoint_det_result, conf_threshold)
|
||||
|
||||
|
||||
def vis_face_detection(im_data, face_det_result, line_size=1, font_size=0.5):
|
||||
return C.vision.vis_face_detection(im_data, face_det_result, line_size,
|
||||
font_size)
|
||||
|
100
tests/eval_example/test_pptinypose.py
Normal file
100
tests/eval_example/test_pptinypose.py
Normal file
@@ -0,0 +1,100 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import fastdeploy as fd
|
||||
import cv2
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
|
||||
def test_keypointdetection_pptinypose():
|
||||
pp_tinypose_model_url = "https://bj.bcebos.com/fastdeploy/tests/PP_TinyPose_256x192_test.tgz"
|
||||
fd.download_and_decompress(pp_tinypose_model_url, ".")
|
||||
model_path = "./PP_TinyPose_256x192_test"
|
||||
# 配置runtime,加载模型
|
||||
runtime_option = fd.RuntimeOption()
|
||||
model_file = os.path.join(model_path, "model.pdmodel")
|
||||
params_file = os.path.join(model_path, "model.pdiparams")
|
||||
config_file = os.path.join(model_path, "infer_cfg.yml")
|
||||
image_file = os.path.join(model_path, "hrnet_demo.jpg")
|
||||
baseline_file = os.path.join(model_path, "baseline.npy")
|
||||
model = fd.vision.keypointdetection.PPTinyPose(
|
||||
model_file, params_file, config_file, runtime_option=runtime_option)
|
||||
|
||||
# 预测图片关键点
|
||||
im = cv2.imread(image_file)
|
||||
result = model.predict(im)
|
||||
result = np.concatenate(
|
||||
(np.array(result.keypoints), np.array(result.scores)[:, np.newaxis]),
|
||||
axis=1)
|
||||
baseline = np.load(baseline_file)
|
||||
diff = np.fabs(result - np.array(baseline))
|
||||
thres = 1e-05
|
||||
assert diff.max() < thres, "The diff is %f, which is bigger than %f" % (
|
||||
diff.max(), thres)
|
||||
print("No diff")
|
||||
|
||||
|
||||
def test_keypointdetection_det_keypoint_unite():
|
||||
det_keypoint_unite_model_url = "https://bj.bcebos.com/fastdeploy/tests/PicoDet_320x320_TinyPose_256x192_test.tgz"
|
||||
fd.download_and_decompress(det_keypoint_unite_model_url, ".")
|
||||
model_path = "./PicoDet_320x320_TinyPose_256x192_test"
|
||||
# 配置runtime,加载模型
|
||||
runtime_option = fd.RuntimeOption()
|
||||
tinypose_model_file = os.path.join(
|
||||
model_path, "PP_TinyPose_256x192_infer/model.pdmodel")
|
||||
tinypose_params_file = os.path.join(
|
||||
model_path, "PP_TinyPose_256x192_infer/model.pdiparams")
|
||||
tinypose_config_file = os.path.join(
|
||||
model_path, "PP_TinyPose_256x192_infer/infer_cfg.yml")
|
||||
picodet_model_file = os.path.join(
|
||||
model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/model.pdmodel")
|
||||
picodet_params_file = os.path.join(
|
||||
model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/model.pdiparams")
|
||||
picodet_config_file = os.path.join(
|
||||
model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/infer_cfg.yml")
|
||||
image_file = os.path.join(model_path, "000000018491.jpg")
|
||||
# image_file = os.path.join(model_path, "hrnet_demo.jpg")
|
||||
|
||||
baseline_file = os.path.join(model_path, "baseline.npy")
|
||||
|
||||
tinypose_model = fd.vision.keypointdetection.PPTinyPose(
|
||||
tinypose_model_file,
|
||||
tinypose_params_file,
|
||||
tinypose_config_file,
|
||||
runtime_option=runtime_option)
|
||||
|
||||
det_model = fd.vision.detection.PicoDet(
|
||||
picodet_model_file,
|
||||
picodet_params_file,
|
||||
picodet_config_file,
|
||||
runtime_option=runtime_option)
|
||||
|
||||
# 预测图片关键点
|
||||
im = cv2.imread(image_file)
|
||||
pipeline = fd.pipeline.PPTinyPose(det_model, tinypose_model)
|
||||
pipeline.detection_model_score_threshold = 0.5
|
||||
result = pipeline.predict(im)
|
||||
print(result)
|
||||
result = np.concatenate(
|
||||
(np.array(result.keypoints), np.array(result.scores)[:, np.newaxis]),
|
||||
axis=1)
|
||||
print(result)
|
||||
np.save("baseline.npy", result)
|
||||
baseline = np.load(baseline_file)
|
||||
diff = np.fabs(result - np.array(baseline))
|
||||
thres = 1e-05
|
||||
assert diff.max() < thres, "The diff is %f, which is bigger than %f" % (
|
||||
diff.max(), thres)
|
||||
print("No diff")
|
Reference in New Issue
Block a user