From b565c15bf79889dc1b52d2f94c53b8b5b76cdb04 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Fri, 21 Oct 2022 09:28:23 +0800 Subject: [PATCH] [Model] Add tinypose single && pipeline model (#177) * Add tinypose model * Add PPTinypose python API * Fix picodet preprocess bug && Add Tinypose examples * Update tinypose example code * Update ppseg preprocess if condition * Update ppseg backend support type * Update permute.h * Update README.md * Update code with comments * Move files dir * Delete premute.cc * Add single model pptinypose * Delete pptinypose old code in ppdet * Code format * Add ppdet + pptinypose pipeline model * Fix bug for posedetpipeline * Change Frontend to ModelFormat * Change Frontend to ModelFormat in __init__.py * Add python posedetpipeline/ * Update pptinypose example dir name * Update README.md * Update README.md * Update README.md * Update README.md * Create keypointdetection_result.md * Create README.md * Create README.md * Create README.md * Update README.md * Update README.md * Create README.md * Fix det_keypoint_unite_infer.py bug * Create README.md * Update PP-Tinypose by comment * Update by comment * Add pipeline directory * Add pptinypose dir * Update pptinypose to align accuracy * Addd warpAffine processor * Update GetCpuMat to GetOpenCVMat * Add comment for pptinypose && pipline * Update docs/main_page.md * Add README.md for pptinypose * Add README for det_keypoint_unite * Remove ENABLE_PIPELINE option * Remove ENABLE_PIPELINE option * Change pptinypose default backend * PP-TinyPose Pipeline support multi PP-Detection models * Update pp-tinypose comment * Update by comments * Add single test example Co-authored-by: Jason --- CMakeLists.txt | 7 +- docs/api/vision_results/README.md | 3 +- .../keypointdetection_result.md | 45 +++ docs/api_docs/cpp/main_page.md | 1 + docs/api_docs/python/keypoint_detection.md | 8 +- docs/api_docs/python/vision_results_cn.md | 13 + docs/api_docs/python/vision_results_en.md | 13 + examples/vision/README.md | 1 + examples/vision/keypointdetection/README.md | 17 ++ .../det_keypoint_unite/README.md | 38 +++ .../det_keypoint_unite/cpp/CMakeLists.txt | 14 + .../det_keypoint_unite/cpp/README.md | 84 ++++++ .../cpp/det_keypoint_unite_infer.cc | 196 +++++++++++++ .../det_keypoint_unite/python/README.md | 74 +++++ .../python/det_keypoint_unite_infer.py | 91 ++++++ .../keypointdetection/tiny_pose/README.md | 37 +++ .../tiny_pose/cpp/CMakeLists.txt | 13 + .../keypointdetection/tiny_pose/cpp/README.md | 88 ++++++ .../tiny_pose/cpp/pptinypose_infer.cc | 143 ++++++++++ .../tiny_pose/python/README.md | 79 ++++++ .../tiny_pose/python/pptinypose_infer.py | 62 +++++ fastdeploy/backends/paddle/util.cc | 4 + fastdeploy/pipeline.h | 21 ++ fastdeploy/pipeline/pipeline_pybind.cc | 24 ++ fastdeploy/pipeline/pptinypose/pipeline.cc | 70 +++++ fastdeploy/pipeline/pptinypose/pipeline.h | 67 +++++ .../pptinypose/pptinyposepipeline_pybind.cc | 58 ++++ fastdeploy/pybind/main.cc.in | 4 + fastdeploy/pybind/main.h | 2 + fastdeploy/vision.h | 1 + fastdeploy/vision/common/processors/base.cc | 4 +- fastdeploy/vision/common/processors/base.h | 8 +- fastdeploy/vision/common/processors/crop.h | 1 + fastdeploy/vision/common/processors/mat.h | 17 +- .../vision/common/processors/transform.h | 1 + .../vision/common/processors/warp_affine.cc | 51 ++++ .../vision/common/processors/warp_affine.h | 66 +++++ fastdeploy/vision/common/result.cc | 23 ++ fastdeploy/vision/common/result.h | 26 +- fastdeploy/vision/detection/ppdet/ppyoloe.cc | 11 +- .../vision/keypointdet/keypointdet_pybind.cc | 26 ++ .../keypointdet/pptinypose/pptinypose.cc | 262 ++++++++++++++++++ .../keypointdet/pptinypose/pptinypose.h | 90 ++++++ .../pptinypose/pptinypose_pybind.cc | 42 +++ .../pptinypose/pptinypose_utils.cc | 125 +++++++++ .../keypointdet/pptinypose/pptinypose_utils.h | 51 ++++ fastdeploy/vision/segmentation/ppseg/model.cc | 6 +- fastdeploy/vision/utils/crop_image.cc | 61 ++++ fastdeploy/vision/utils/dark_parse.cc | 81 ++++++ fastdeploy/vision/utils/utils.h | 23 ++ fastdeploy/vision/vision_pybind.cc | 11 + fastdeploy/vision/visualize/keypoint.cc | 62 +++++ fastdeploy/vision/visualize/visualize.h | 3 + .../vision/visualize/visualize_pybind.cc | 11 + python/fastdeploy/__init__.py | 1 + python/fastdeploy/pipeline/__init__.py | 16 ++ .../pipeline/pptinypose/__init__.py | 55 ++++ python/fastdeploy/vision/__init__.py | 1 + .../vision/keypointdetection/__init__.py | 16 ++ .../keypointdetection/pptinypose/__init__.py | 69 +++++ .../fastdeploy/vision/visualize/__init__.py | 5 + tests/eval_example/test_pptinypose.py | 100 +++++++ 62 files changed, 2583 insertions(+), 20 deletions(-) mode change 100755 => 100644 CMakeLists.txt create mode 100644 docs/api/vision_results/keypointdetection_result.md mode change 100755 => 100644 docs/api_docs/cpp/main_page.md create mode 100644 examples/vision/keypointdetection/README.md create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/README.md create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/cpp/CMakeLists.txt create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/cpp/det_keypoint_unite_infer.cc create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/python/README.md create mode 100644 examples/vision/keypointdetection/det_keypoint_unite/python/det_keypoint_unite_infer.py create mode 100644 examples/vision/keypointdetection/tiny_pose/README.md create mode 100644 examples/vision/keypointdetection/tiny_pose/cpp/CMakeLists.txt create mode 100644 examples/vision/keypointdetection/tiny_pose/cpp/README.md create mode 100644 examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc create mode 100644 examples/vision/keypointdetection/tiny_pose/python/README.md create mode 100644 examples/vision/keypointdetection/tiny_pose/python/pptinypose_infer.py create mode 100644 fastdeploy/pipeline.h create mode 100644 fastdeploy/pipeline/pipeline_pybind.cc create mode 100644 fastdeploy/pipeline/pptinypose/pipeline.cc create mode 100644 fastdeploy/pipeline/pptinypose/pipeline.h create mode 100644 fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc create mode 100644 fastdeploy/vision/common/processors/warp_affine.cc create mode 100644 fastdeploy/vision/common/processors/warp_affine.h create mode 100644 fastdeploy/vision/keypointdet/keypointdet_pybind.cc create mode 100644 fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc create mode 100644 fastdeploy/vision/keypointdet/pptinypose/pptinypose.h create mode 100644 fastdeploy/vision/keypointdet/pptinypose/pptinypose_pybind.cc create mode 100644 fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc create mode 100644 fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h create mode 100644 fastdeploy/vision/utils/crop_image.cc create mode 100644 fastdeploy/vision/utils/dark_parse.cc create mode 100644 fastdeploy/vision/visualize/keypoint.cc create mode 100644 python/fastdeploy/pipeline/__init__.py create mode 100644 python/fastdeploy/pipeline/pptinypose/__init__.py create mode 100644 python/fastdeploy/vision/keypointdetection/__init__.py create mode 100644 python/fastdeploy/vision/keypointdetection/pptinypose/__init__.py create mode 100644 tests/eval_example/test_pptinypose.py diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100755 new mode 100644 index 042d5645b..c57251d42 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,10 +184,11 @@ file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastde file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) +file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*.cc) file(GLOB_RECURSE DEPLOY_VISION_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cu) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS}) set(DEPEND_LIBS "") @@ -389,6 +390,7 @@ if(ENABLE_VISION) list(APPEND DEPLOY_VISION_SRCS ${DEPLOY_VISION_CUDA_SRCS}) endif() list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_VISION_SRCS}) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PIPELINE_SRCS}) include_directories(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include) include(${PROJECT_SOURCE_DIR}/cmake/opencv.cmake) @@ -586,7 +588,8 @@ if(BUILD_FASTDEPLOY_PYTHON) if(NOT ENABLE_VISION) file(GLOB_RECURSE VISION_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*_pybind.cc) - list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${VISION_PYBIND_SRCS}) + file(GLOB_RECURSE PIPELINE_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*_pybind.cc) + list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${VISION_PYBIND_SRCS} ${PIPELINE_PYBIND_SRCS}) endif() if (NOT ENABLE_TEXT) diff --git a/docs/api/vision_results/README.md b/docs/api/vision_results/README.md index 31076ce24..3df91cc0e 100644 --- a/docs/api/vision_results/README.md +++ b/docs/api/vision_results/README.md @@ -6,8 +6,9 @@ FastDeploy根据视觉模型的任务类型，定义了不同的结构体(`fastd | :----- | :--- | :---- | :------- | | ClassifyResult | [C++/Python文档](./classification_result.md) | 图像分类返回结果 | ResNet50、MobileNetV3等 | | SegmentationResult | [C++/Python文档](./segmentation_result.md) | 图像分割返回结果 | PP-HumanSeg、PP-LiteSeg等 | -| DetectionResult | [C++/Python文档](./detection_result.md) | 目标检测返回结果 | PPYOLOE、YOLOv7系列模型等 | +| DetectionResult | [C++/Python文档](./detection_result.md) | 目标检测返回结果 | PP-YOLOE、YOLOv7系列模型等 | | FaceDetectionResult | [C++/Python文档](./face_detection_result.md) | 目标检测返回结果 | SCRFD、RetinaFace系列模型等 | +| KeyPointDetectionResult | [C++/Python文档](./keypointdetection_result.md) | 关键点检测返回结果 | PP-Tinypose系列模型等 | | FaceRecognitionResult | [C++/Python文档](./face_recognition_result.md) | 目标检测返回结果 | ArcFace、CosFace系列模型等 | | MattingResult | [C++/Python文档](./matting_result.md) | 目标检测返回结果 | MODNet系列模型等 | | OCRResult | [C++/Python文档](./ocr_result.md) | 文本框检测，分类和文本识别返回结果 | OCR系列模型等 | diff --git a/docs/api/vision_results/keypointdetection_result.md b/docs/api/vision_results/keypointdetection_result.md new file mode 100644 index 000000000..a47057ad8 --- /dev/null +++ b/docs/api/vision_results/keypointdetection_result.md @@ -0,0 +1,45 @@ +# KeyPointDetectionResult 目标检测结果 + +KeyPointDetectionResult 代码定义在`fastdeploy/vision/common/result.h`中，用于表明图像中目标行为的各个关键点坐标和置信度。 + +## C++ 定义 + +`fastdeploy::vision::KeyPointDetectionResult` + +```c++ +struct KeyPointDetectionResult { + std::vector> keypoints; + std::vector scores; + int num_joints = -1; + void Clear(); + std::string Str(); +}; +``` + +- **keypoints**: 成员变量，表示识别到的目标行为的关键点坐标。`keypoints.size()= N * J * 2`， + - `N`：图片中的目标数量 + - `J`：num_joints（一个目标的关键点数量） + - `3`:坐标信息[x, y] +- **scores**: 成员变量，表示识别到的目标行为的关键点坐标的置信度。`scores.size()= N * J` + - `N`：图片中的目标数量 + - `J`:num_joints（一个目标的关键点数量） +- **num_joints**: 成员变量，一个目标的关键点数量 + +- **num_joints**: 成员变量，一个目标的关键点数量 +- **Clear()**: 成员函数，用于清除结构体中存储的结果 +- **Str()**: 成员函数，将结构体中的信息以字符串形式输出（用于Debug） + +## Python 定义 + +`fastdeploy.vision.KeyPointDetectionResult` + +- **keypoints**(list of list(float)): 成员变量，表示识别到的目标行为的关键点坐标。 + `keypoints.size()= N * J * 2` + `N`:图片中的目标数量 + `J`:num_joints（关键点数量） + `3`:坐标信息[x, y, conf] +- **scores**(list of float): 成员变量，表示识别到的目标行为的关键点坐标的置信度。 + `scores.size()= N * J` + `N`:图片中的目标数量 + `J`:num_joints（一个目标的关键点数量） +- **num_joints**(int): 成员变量，一个目标的关键点数量 diff --git a/docs/api_docs/cpp/main_page.md b/docs/api_docs/cpp/main_page.md old mode 100755 new mode 100644 index 46a4d0ba1..017ad7b60 --- a/docs/api_docs/cpp/main_page.md +++ b/docs/api_docs/cpp/main_page.md @@ -26,5 +26,6 @@ Currently, FastDeploy supported backends listed as below, | Task | Model | API | Example | | :---- | :---- | :---- | :----- | | object detection | PaddleDetection/PPYOLOE | [fastdeploy::vision::detection::PPYOLOE](./classfastdeploy_1_1vision_1_1detection_1_1PPYOLOE.html) | [C++](./)/[Python](./) | +| keypoint detection | PaddleDetection/PPTinyPose | [fastdeploy::vision::keypointdetection::PPTinyPose](./classfastdeploy_1_1vision_1_1keypointdetection_1_1PPTinyPose.html) | [C++](./)/[Python](./) | | image classification | PaddleClassification serials | [fastdeploy::vision::classification::PaddleClasModel](./classfastdeploy_1_1vision_1_1classification_1_1PaddleClasModel.html) | [C++](./)/[Python](./) | | semantic segmentation | PaddleSegmentation serials | [fastdeploy::vision::classification::PaddleSegModel](./classfastdeploy_1_1vision_1_1segmentation_1_1PaddleSegModel.html) | [C++](./)/[Python](./) | diff --git a/docs/api_docs/python/keypoint_detection.md b/docs/api_docs/python/keypoint_detection.md index d5f2ad135..b2ee83704 100644 --- a/docs/api_docs/python/keypoint_detection.md +++ b/docs/api_docs/python/keypoint_detection.md @@ -1,3 +1,9 @@ # Keypoint Detection API -comming soon... +## fastdeploy.vision.keypointdetection.PPTinyPose + +```{eval-rst} +.. autoclass:: fastdeploy.vision.keypointdetection.PPTinyPose + :members: + :inherited-members: +``` diff --git a/docs/api_docs/python/vision_results_cn.md b/docs/api_docs/python/vision_results_cn.md index 586464a06..dab22e6a5 100644 --- a/docs/api_docs/python/vision_results_cn.md +++ b/docs/api_docs/python/vision_results_cn.md @@ -40,6 +40,19 @@ API:`fastdeploy.vision.FaceDetectionResult` , 该结果返回: - **landmarks**(list of list(float)): 成员变量，表示单张图片检测出来的所有人脸的关键点. - **landmarks_per_face**(int): 成员变量，表示每个人脸框中的关键点的数量. +## KeyPointDetectionResult +KeyPointDetectionResult 代码定义在`fastdeploy/vision/common/result.h`中，用于表明图像中目标行为的各个关键点坐标和置信度。 + +API:`fastdeploy.vision.KeyPointDetectionResult` , 该结果返回: +- **keypoints**(list of list(float)): 成员变量，表示识别到的目标行为的关键点坐标。`keypoints.size()= N * J * 2`， + - `N`：图片中的目标数量 + - `J`：num_joints（一个目标的关键点数量） + - `3`:坐标信息[x, y] +- **scores**(list of float): 成员变量，表示识别到的目标行为的关键点坐标的置信度。`scores.size()= N * J` + - `N`：图片中的目标数量 + - `J`:num_joints（一个目标的关键点数量） +- **num_joints**(int): 成员变量，表示一个目标的关键点数量 + ## FaceRecognitionResult FaceRecognitionResult 代码定义在`fastdeploy/vision/common/result.h`中，用于表明人脸识别模型对图像特征的embedding. diff --git a/docs/api_docs/python/vision_results_en.md b/docs/api_docs/python/vision_results_en.md index 1e97b2e9d..a7ae54a57 100644 --- a/docs/api_docs/python/vision_results_en.md +++ b/docs/api_docs/python/vision_results_en.md @@ -40,6 +40,19 @@ API: `fastdeploy.vision.FaceDetectionResult`, The FaceDetectionResult will retur - **landmarks**(list of list(float)): Member variables that represent the key points of all faces detected by a single image. - **landmarks_per_face**(int):Member variable indicating the number of key points in each face frame. +## KeyPointDetectionResult +The KeyPointDetectionResult code is defined in `fastdeploy/vision/common/result.h` and is used to indicate the coordinates and confidence of each keypoint of the target behavior in the image. + +API:`fastdeploy.vision.KeyPointDetectionResult`, The KeyPointDetectionResult will return: +- **keypoints**(list of list(float)): Member variable, representing the key point coordinates of the identified target behavior. `keypoints.size()= N * J * 2`， + - `N`: number of objects in the picture + - `J`: num_joints（number of keypoints for a target） + - `3`: 坐标信息[x, y] +- **scores**(list of float): Member variable, representing the confidence of the keypoint coordinates of the recognized target behavior. `scores.size()= N * J` + - `N`: number of objects in the picture + - `J`: num_joints（number of keypoints for a target） +- **num_joints**(int): Member variable, representing the number of keypoints for a target + ## FaceRecognitionResult The FaceRecognitionResult code is defined in `fastdeploy/vision/common/result.h` and is used to indicate the embedding of the image features by the face recognition model. diff --git a/examples/vision/README.md b/examples/vision/README.md index 99e8a2140..ca56edd48 100644 --- a/examples/vision/README.md +++ b/examples/vision/README.md @@ -8,6 +8,7 @@ | Segmentation | 语义分割，输入图像，给出图像中每个像素的分类及置信度 | [SegmentationResult](../../docs/api/vision_results/segmentation_result.md) | | Classification | 图像分类，输入图像，给出图像的分类结果和置信度 | [ClassifyResult](../../docs/api/vision_results/classification_result.md) | | FaceDetection | 人脸检测，输入图像，检测图像中人脸位置，并返回检测框坐标及人脸关键点 | [FaceDetectionResult](../../docs/api/vision_results/face_detection_result.md) | +| KeypointDetection | 关键点检测，输入图像，返回图像中人物行为的各个关键点坐标和置信度 | [KeyPointDetectionResult](../../docs/api/vision_results/keypointdetection_result.md) | | FaceRecognition | 人脸识别，输入图像，返回可用于相似度计算的人脸特征的embedding | [FaceRecognitionResult](../../docs/api/vision_results/face_recognition_result.md) | | Matting | 抠图，输入图像，返回图片的前景每个像素点的Alpha值 | [MattingResult](../../docs/api/vision_results/matting_result.md) | | OCR | 文本框检测，分类，文本框内容识别，输入图像，返回文本框坐标，文本框的方向类别以及框内的文本内容 | [OCRResult](../../docs/api/vision_results/ocr_result.md) | diff --git a/examples/vision/keypointdetection/README.md b/examples/vision/keypointdetection/README.md new file mode 100644 index 000000000..bcc368e91 --- /dev/null +++ b/examples/vision/keypointdetection/README.md @@ -0,0 +1,17 @@ +# 关键点检测模型 + +FastDeploy目前支持两种关键点检测任务方式的部署 + +| 任务 | 说明 | 模型格式 | 示例 | 版本 | +| :---| :--- | :--- | :------- | :--- | +| 单人关键点检测 | 部署PP-TinyPose系列模型，输入图像仅包含单人 | Paddle | 参考[tinypose目录](./tiny_pose/) | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | +| 单人/多人关键点检测 | 部署PicoDet + PP-TinyPose的模型串联任务，输入图像先通过检测模型，得到独立的人像子图后，再经过PP-TinyPose模型检测关键点 | Paddle | 参考[det_keypoint_unite目录](./det_keypoint_unite/) |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | + +# 预训练模型准备 +本文档提供了如下预训练模型，开发者可直接下载使用 +| 模型 | 说明 | 模型格式 | 版本 | +| :--- | :--- | :------- | :--- | +| [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 单人关键点检测模型 | Paddle | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | +| [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 单人关键点检测模型 | Paddle | [Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | +| [PicoDet-S-Lcnet-Pedestrian-192x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_192x192_infer.tgz) + [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 单人关键点检测串联配置 | Paddle |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | +| [PicoDet-S-Lcnet-Pedestrian-320x320](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz) + [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 多人关键点检测串联配置 | Paddle |[Release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose) | diff --git a/examples/vision/keypointdetection/det_keypoint_unite/README.md b/examples/vision/keypointdetection/det_keypoint_unite/README.md new file mode 100644 index 000000000..c323a5c1c --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/README.md @@ -0,0 +1,38 @@ +# PP-PicoDet + PP-TinyPose 联合部署(Pipeline) + +## 模型版本说明 + +- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5) + +目前FastDeploy支持如下模型的部署 + +- [PP-PicoDet + PP-TinyPose系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md) + +## 准备PP-TinyPose部署模型 + +PP-TinyPose以及PP-PicoDet模型导出，请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md) + +**注意**:导出的推理模型包含`model.pdmodel`、`model.pdiparams`和`infer_cfg.yml`三个文件，FastDeploy会从yaml文件中获取模型在推理时需要的预处理信息。 + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PP-PicoDet + PP-TinyPose（Pipeline）导出的部分模型，开发者可直接下载使用。 + +| 应用场景 | 模型 | 参数文件大小 | AP(业务数据集) | AP(COCO Val 单人/多人) | 单人/多人推理耗时 (FP32) | 单人/多人推理耗时（FP16) | +|:-------------------------------|:--------------------------------- |:----- |:----- | :----- | :----- | :----- | +| 单人模型配置 |[PicoDet-S-Lcnet-Pedestrian-192x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_192x192_infer.tgz) + [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 4.6MB + 5.3MB | 86.2% | 52.8% | 12.90ms | 9.61ms | +| 多人模型配置 |[PicoDet-S-Lcnet-Pedestrian-320x320](https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz) + [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 4.6M + 5.3MB | 85.7% | 49.9% | 47.63ms | 34.62ms | + +**说明** +- 关键点检测模型的精度指标是基于对应行人检测模型检测得到的检测框。 +- 精度测试中去除了flip操作，且检测置信度阈值要求0.5。 +- 速度测试环境为qualcomm snapdragon 865，采用arm8下4线程推理。 +- Pipeline速度包含模型的预处理、推理及后处理部分。 +- 精度测试中，为了公平比较，多人数据去除了6人以上（不含6人）的图像。 + +更多信息请参考：[PP-TinyPose 官方文档](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md) + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/keypointdetection/det_keypoint_unite/cpp/CMakeLists.txt b/examples/vision/keypointdetection/det_keypoint_unite/cpp/CMakeLists.txt new file mode 100644 index 000000000..56c76c043 --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.12) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/det_keypoint_unite_infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md b/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md new file mode 100644 index 000000000..943427217 --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md @@ -0,0 +1,84 @@ +# PP-PicoDet + PP-TinyPose (Pipeline) C++部署示例 + +本目录下提供`det_keypoint_unite_infer.cc`快速完成多人模型配置 PP-PicoDet + PP-TinyPose 在CPU/GPU，以及GPU上通过TensorRT加速部署的`单图多人关键点检测`示例。执行如下脚本即可完成 +>> **注意**: PP-TinyPose单模型独立部署，请参考[PP-TinyPose 单模型](../../tiny_pose/cpp/README.md) + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + + +以Linux上推理为例，在本目录执行如下命令即可完成编译测试 + +```bash +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.3.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.3.0.tgz +cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/keypointdetection/tiny_pose/cpp/ +mkdir build +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.3.0 +make -j + +# 下载PP-TinyPose和PP-PicoDet模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz +tar -xvf PP_TinyPose_256x192_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz +tar -xvf PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/000000018491.jpg + +# CPU推理 +./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 0 +# GPU推理 +./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 1 +# GPU上TensorRT推理 +./infer_demo PP_PicoDet_V2_S_Pedestrian_320x320_infer PP_TinyPose_256x192_infer 000000018491.jpg 2 +``` + +运行完成可视化结果如下图所示 +
+ +
+ +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## PP-TinyPose C++接口 + +### PP-TinyPose类 + +```c++ +fastdeploy::pipeline::PPTinyPose( + fastdeploy::vision::detection::PPYOLOE* det_model, + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model) +``` + +PPTinyPose Pipeline模型加载和初始化。 + +**参数** + +> * **model_det_modelfile**(fastdeploy::vision::detection): 初始化后的检测模型，参考[PP-TinyPose](../../tiny_pose/README.md) +> * **pptinypose_model**(fastdeploy::vision::keypointdetection): 初始化后的检测模型[Detection](../../../detection/paddledetection/README.md)，暂时只提供PaddleDetection系列 + +#### Predict函数 + +> ```c++ +> PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出关键点检测结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: 关键点检测结果，包括关键点的坐标以及关键点对应的概率值, KeyPointDetectionResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 后处理参数 +> > * **detection_model_score_threshold**(bool): +输入PP-TinyPose模型前，Detectin模型过滤检测框的分数阈值 + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/keypointdetection/det_keypoint_unite/cpp/det_keypoint_unite_infer.cc b/examples/vision/keypointdetection/det_keypoint_unite/cpp/det_keypoint_unite_infer.cc new file mode 100644 index 000000000..f2704104a --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/cpp/det_keypoint_unite_infer.cc @@ -0,0 +1,196 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#include "fastdeploy/pipeline.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& det_model_dir, + const std::string& tinypose_model_dir, + const std::string& image_file) { + auto det_model_file = det_model_dir + sep + "model.pdmodel"; + auto det_params_file = det_model_dir + sep + "model.pdiparams"; + auto det_config_file = det_model_dir + sep + "infer_cfg.yml"; + auto det_model = fastdeploy::vision::detection::PicoDet( + det_model_file, det_params_file, det_config_file); + if (!det_model.Initialized()) { + std::cerr << "Detection Model Failed to initialize." << std::endl; + return; + } + + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + + auto pipeline =fastdeploy::pipeline::PPTinyPose(&det_model, &tinypose_model); + pipeline.detection_model_score_threshold = 0.5; + if (!pipeline.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测框结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.2); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "TinyPose visualized result saved in ./vis_result.jpg" + << std::endl; +} + +void GpuInfer(const std::string& det_model_dir, + const std::string& tinypose_model_dir, + const std::string& image_file) { + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto det_model_file = det_model_dir + sep + "model.pdmodel"; + auto det_params_file = det_model_dir + sep + "model.pdiparams"; + auto det_config_file = det_model_dir + sep + "infer_cfg.yml"; + auto det_model = fastdeploy::vision::detection::PicoDet( + det_model_file, det_params_file, det_config_file, option); + if (!det_model.Initialized()) { + std::cerr << "Detection Model Failed to initialize." << std::endl; + return; + } + + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file, option); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + + auto pipeline = + fastdeploy::pipeline::PPTinyPose( + &det_model, &tinypose_model); + pipeline.detection_model_score_threshold = 0.5; + if (!pipeline.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测框结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.2); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "TinyPose visualized result saved in ./vis_result.jpg" + << std::endl; +} + +void TrtInfer(const std::string& det_model_dir, + const std::string& tinypose_model_dir, + const std::string& image_file) { + auto det_model_file = det_model_dir + sep + "model.pdmodel"; + auto det_params_file = det_model_dir + sep + "model.pdiparams"; + auto det_config_file = det_model_dir + sep + "infer_cfg.yml"; + + auto det_option = fastdeploy::RuntimeOption(); + det_option.UseGpu(); + det_option.UseTrtBackend(); + det_option.SetTrtInputShape("image", {1, 3, 320, 320}); + det_option.SetTrtInputShape("scale_factor", {1, 2}); + auto det_model = fastdeploy::vision::detection::PicoDet( + det_model_file, det_params_file, det_config_file, det_option); + if (!det_model.Initialized()) { + std::cerr << "Detection Model Failed to initialize." << std::endl; + return; + } + + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_option = fastdeploy::RuntimeOption(); + tinypose_option.UseGpu(); + tinypose_option.UseTrtBackend(); + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file, + tinypose_option); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + + auto pipeline = + fastdeploy::pipeline::PPTinyPose( + &det_model, &tinypose_model); + pipeline.detection_model_score_threshold = 0.5; + if (!pipeline.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测关键点结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.2); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "TinyPose visualized result saved in ./vis_result.jpg" + << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 5) { + std::cout << "Usage: infer_demo path/to/detection_model_dir " + "path/to/pptinypose_model_dir path/to/image run_option, " + "e.g ./infer_model ./picodet_model_dir ./pptinypose_model_dir " + "./test.jpeg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + if (std::atoi(argv[4]) == 0) { + CpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 1) { + GpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 2) { + TrtInfer(argv[1], argv[2], argv[3]); + } + return 0; +} diff --git a/examples/vision/keypointdetection/det_keypoint_unite/python/README.md b/examples/vision/keypointdetection/det_keypoint_unite/python/README.md new file mode 100644 index 000000000..e401b655a --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/python/README.md @@ -0,0 +1,74 @@ +# PP-PicoDet + PP-TinyPose (Pipeline) Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`det_keypoint_unite_infer.py`快速完成多人模型配置 PP-PicoDet + PP-TinyPose 在CPU/GPU，以及GPU上通过TensorRT加速部署的`单图多人关键点检测`示例。执行如下脚本即可完成 +>> **注意**: PP-TinyPose单模型独立部署，请参考[PP-TinyPose 单模型](../../tiny_pose//python/README.md) + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/keypointdetection/det_keypoint_unite/python + +# 下载PP-TinyPose模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz +tar -xvf PP_TinyPose_256x192_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz +tar -xvf PP_PicoDet_V2_S_Pedestrian_320x320_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/000000018491.jpg +# CPU推理 +python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device cpu +# GPU推理 +python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python det_keypoint_unite_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --det_model_dir PP_PicoDet_V2_S_Pedestrian_320x320_infer --image 000000018491.jpg --device gpu --use_trt True +``` + +运行完成可视化结果如下图所示 +
+ +
+ +## PPTinyPosePipeline Python接口 + +```python +fd.pipeline.PPTinyPose(det_model=None, pptinypose_model=None) +``` + +PPTinyPosePipeline模型加载和初始化，其中det_model是使用`fd.vision.detection.PicoDet`[参考Detection文档](../../../detection/paddledetection/python/)初始化的检测模型，pptinypose_model是使用`fd.vision.keypointdetection.PPTinyPose`[参考PP-TinyPose文档](../../tiny_pose/python/)初始化的检测模型 + +**参数** + +> * **det_model**(str): 初始化后的检测模型 +> * **pptinypose_model**(str): 初始化后的PP-TinyPose模型 + +### predict函数 + +> ```python +> PPTinyPosePipeline.predict(input_image) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **input_image**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.KeyPointDetectionResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 后处理参数 +> > * **detection_model_score_threshold**(bool): +输入PP-TinyPose模型前，Detectin模型过滤检测框的分数阈值 + +## 其它文档 + +- [Pipeline 模型介绍](..) +- [Pipeline C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/runtime/how_to_change_backend.md) diff --git a/examples/vision/keypointdetection/det_keypoint_unite/python/det_keypoint_unite_infer.py b/examples/vision/keypointdetection/det_keypoint_unite/python/det_keypoint_unite_infer.py new file mode 100644 index 000000000..b0952439f --- /dev/null +++ b/examples/vision/keypointdetection/det_keypoint_unite/python/det_keypoint_unite_infer.py @@ -0,0 +1,91 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--tinypose_model_dir", + required=True, + help="path of paddletinypose model directory") + parser.add_argument( + "--det_model_dir", help="path of paddledetection model directory") + parser.add_argument( + "--image", required=True, help="path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="wether to use tensorrt.") + return parser.parse_args() + + +def build_picodet_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("image", [1, 3, 320, 320]) + option.set_trt_input_shape("scale_factor", [1, 2]) + return option + + +def build_tinypose_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("image", [1, 3, 256, 192]) + return option + + +args = parse_arguments() +picodet_model_file = os.path.join(args.det_model_dir, "model.pdmodel") +picodet_params_file = os.path.join(args.det_model_dir, "model.pdiparams") +picodet_config_file = os.path.join(args.det_model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_picodet_option(args) +det_model = fd.vision.detection.PicoDet( + picodet_model_file, + picodet_params_file, + picodet_config_file, + runtime_option=runtime_option) + +tinypose_model_file = os.path.join(args.tinypose_model_dir, "model.pdmodel") +tinypose_params_file = os.path.join(args.tinypose_model_dir, "model.pdiparams") +tinypose_config_file = os.path.join(args.tinypose_model_dir, "infer_cfg.yml") +# 配置runtime，加载模型 +runtime_option = build_tinypose_option(args) +tinypose_model = fd.vision.keypointdetection.PPTinyPose( + tinypose_model_file, + tinypose_params_file, + tinypose_config_file, + runtime_option=runtime_option) + +# 预测图片检测结果 +im = cv2.imread(args.image) +pipeline = fd.pipeline.PPTinyPose(det_model, tinypose_model) +pipeline.detection_model_score_threshold = 0.5 +pipeline_result = pipeline.predict(im) +print("Paddle TinyPose Result:\n", pipeline_result) + +# 预测结果可视化 +vis_im = fd.vision.vis_keypoint_detection( + im, pipeline_result, conf_threshold=0.2) +cv2.imwrite("visualized_result.jpg", vis_im) +print("TinyPose visualized result save in ./visualized_result.jpg") diff --git a/examples/vision/keypointdetection/tiny_pose/README.md b/examples/vision/keypointdetection/tiny_pose/README.md new file mode 100644 index 000000000..2166a1c03 --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/README.md @@ -0,0 +1,37 @@ +# PP-TinyPose 模型部署 + +## 模型版本说明 + +- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5) + +目前FastDeploy支持如下模型的部署 + +- [PP-TinyPose系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md) + +## 准备PP-TinyPose部署模型 + +PP-TinyPose模型导出，请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md) + +**注意**:PP-TinyPose导出的模型包含`model.pdmodel`、`model.pdiparams`和`infer_cfg.yml`三个文件，FastDeploy会从yaml文件中获取模型在推理时需要的预处理信息。 + + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PP-TinyPose导出的部分模型，开发者可直接下载使用。 + +| 模型 | 参数文件大小 |输入Shape | AP(业务数据集) | AP(COCO Val) | FLOPS | 单人推理耗时 (FP32) | 单人推理耗时（FP16) | +|:---------------------------------------------------------------- |:----- |:----- | :----- | :----- | :----- | :----- | :----- | +| [PP-TinyPose-128x96](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz) | 5.3MB | 128x96 | 84.3% | 58.4% | 81.56 M | 4.57ms | 3.27ms | +| [PP-TinyPose-256x192](https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz) | 5.3M | 256x96 | 91.0% | 68.3% | 326.24M | 14.07ms | 8.33ms | + +**说明** +- 关键点检测模型使用`COCO train2017`和`AI Challenger trainset`作为训练集。使用`COCO person keypoints val2017`作为测试集。 +- 关键点检测模型的精度指标所依赖的检测框为ground truth标注得到。 +- 推理速度测试环境为 Qualcomm Snapdragon 865，采用arm8下4线程推理得到。 + +更多信息请参考：[PP-TinyPose 官方文档](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose/README.md) + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/keypointdetection/tiny_pose/cpp/CMakeLists.txt b/examples/vision/keypointdetection/tiny_pose/cpp/CMakeLists.txt new file mode 100644 index 000000000..0f492f44b --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.12) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_tinypose_demo ${PROJECT_SOURCE_DIR}/pptinypose_infer.cc) +target_link_libraries(infer_tinypose_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/keypointdetection/tiny_pose/cpp/README.md b/examples/vision/keypointdetection/tiny_pose/cpp/README.md new file mode 100644 index 000000000..8b6135b58 --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/cpp/README.md @@ -0,0 +1,88 @@ +# PP-TinyPose C++部署示例 + +本目录下提供`pptinypose_infer.cc`快速完成PP-TinyPose在CPU/GPU，以及GPU上通过TensorRT加速部署的`单图单人关键点检测`示例 +>> **注意**: PP-Tinypose单模型目前只支持单图单人关键点检测，因此输入的图片应只包含一个人或者进行过裁剪的图像。多人关键点检测请参考[PP-TinyPose Pipeline](../../det_keypoint_unite/cpp/README.md) + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + + +以Linux上推理为例，在本目录执行如下命令即可完成编译测试 + +```bash +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.3.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.3.0.tgz +cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/keypointdetection/tiny_pose/cpp/ +mkdir build +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.3.0 +make -j + +# 下载PP-TinyPose模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz +tar -xvf PP_TinyPose_256x192_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/hrnet_demo.jpg + + +# CPU推理 +./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 0 +# GPU推理 +./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 1 +# GPU上TensorRT推理 +./infer_tinypose_demo PP_TinyPose_256x192_infer hrnet_demo.jpg 2 +``` + +运行完成可视化结果如下图所示 +
+ +
+ +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## PP-TinyPose C++接口 + +### PP-TinyPose类 + +```c++ +fastdeploy::vision::keypointdetection::PPTinyPose( + const string& model_file, + const string& params_file = "", + const string& config_file, + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +PPTinyPose模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 推理部署配置文件 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出关键点检测结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: 关键点检测结果，包括关键点的坐标以及关键点对应的概率值, KeyPointDetectionResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 后处理参数 +> > * **use_dark**(bool): 是否使用DARK进行后处理[参考论文](https://arxiv.org/abs/1910.06278) + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc b/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc new file mode 100644 index 000000000..30587dbfb --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc @@ -0,0 +1,143 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& tinypose_model_dir, + const std::string& image_file) { + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + if (!tinypose_model.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测框结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto tinypose_vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.5); + cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im); + std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg" + << std::endl; +} + +void GpuInfer(const std::string& tinypose_model_dir, + const std::string& image_file) { + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file, option); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + if (!tinypose_model.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测框结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto tinypose_vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.5); + cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im); + std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg" + << std::endl; +} + +void TrtInfer(const std::string& tinypose_model_dir, + const std::string& image_file) { + auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; + auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; + auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto tinypose_option = fastdeploy::RuntimeOption(); + tinypose_option.UseGpu(); + tinypose_option.UseTrtBackend(); + auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( + tinypose_model_file, tinypose_params_file, tinypose_config_file, + tinypose_option); + if (!tinypose_model.Initialized()) { + std::cerr << "TinyPose Model Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + fastdeploy::vision::KeyPointDetectionResult res; + if (!tinypose_model.Predict(&im, &res)) { + std::cerr << "TinyPose Prediction Failed." << std::endl; + return; + } else { + std::cout << "TinyPose Prediction Done!" << std::endl; + } + // 输出预测框结果 + std::cout << res.Str() << std::endl; + + // 可视化预测结果 + auto tinypose_vis_im = + fastdeploy::vision::VisKeypointDetection(im, res, 0.5); + cv::imwrite("tinypose_vis_result.jpg", tinypose_vis_im); + std::cout << "TinyPose visualized result saved in ./tinypose_vis_result.jpg" + << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout << "Usage: infer_demo path/to/pptinypose_model_dir path/to/image " + "run_option, " + "e.g ./infer_model ./pptinypose_model_dir ./test.jpeg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2]); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2]); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2]); + } + return 0; +} diff --git a/examples/vision/keypointdetection/tiny_pose/python/README.md b/examples/vision/keypointdetection/tiny_pose/python/README.md new file mode 100644 index 000000000..2d95e7a2f --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/python/README.md @@ -0,0 +1,79 @@ +# PP-TinyPose Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`pptinypose_infer.py`快速完成PP-TinyPose在CPU/GPU，以及GPU上通过TensorRT加速部署的`单图单人关键点检测`示例。执行如下脚本即可完成 + +>> **注意**: PP-Tinypose单模型目前只支持单图单人关键点检测，因此输入的图片应只包含一个人或者进行过裁剪的图像。多人关键点检测请参考[PP-TinyPose Pipeline](../../det_keypoint_unite/python/README.md) + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/keypointdetection/tiny_pose/python + +# 下载PP-TinyPose模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_256x192_infer.tgz +tar -xvf PP_TinyPose_256x192_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/hrnet_demo.jpg + +# CPU推理 +python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device cpu +# GPU推理 +python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python pptinypose_infer.py --tinypose_model_dir PP_TinyPose_256x192_infer --image hrnet_demo.jpg --device gpu --use_trt True +``` + +运行完成可视化结果如下图所示 +
+ +
+ +## PP-TinyPose Python接口 + +```python +fd.vision.keypointdetection.PPTinyPose(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +PP-TinyPose模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/EXPORT_MODEL.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 推理部署配置文件 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> PPTinyPose.predict(input_image) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **input_image**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.KeyPointDetectionResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 后处理参数 +用户可按照自己的实际需求，修改下列后处理参数，从而影响最终的推理和部署效果 + +> > * **use_dark**(bool): 是否使用DARK进行后处理[参考论文](https://arxiv.org/abs/1910.06278) + + +## 其它文档 + +- [PP-TinyPose 模型介绍](..) +- [PP-TinyPose C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/runtime/how_to_change_backend.md) diff --git a/examples/vision/keypointdetection/tiny_pose/python/pptinypose_infer.py b/examples/vision/keypointdetection/tiny_pose/python/pptinypose_infer.py new file mode 100644 index 000000000..29c384b75 --- /dev/null +++ b/examples/vision/keypointdetection/tiny_pose/python/pptinypose_infer.py @@ -0,0 +1,62 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--tinypose_model_dir", + required=True, + help="path of paddletinypose model directory") + parser.add_argument( + "--image", required=True, help="path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="wether to use tensorrt.") + return parser.parse_args() + + +def build_tinypose_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("image", [1, 3, 256, 192]) + return option + + +args = parse_arguments() + +tinypose_model_file = os.path.join(args.tinypose_model_dir, "model.pdmodel") +tinypose_params_file = os.path.join(args.tinypose_model_dir, "model.pdiparams") +tinypose_config_file = os.path.join(args.tinypose_model_dir, "infer_cfg.yml") +# 配置runtime，加载模型 +runtime_option = build_tinypose_option(args) +tinypose_model = fd.vision.keypointdetection.PPTinyPose( + tinypose_model_file, + tinypose_params_file, + tinypose_config_file, + runtime_option=runtime_option) +# 预测图片检测结果 +im = cv2.imread(args.image) +tinypose_result = tinypose_model.predict(im) +print("Paddle TinyPose Result:\n", tinypose_result) + +# 预测结果可视化 +vis_im = fd.vision.vis_keypoint_detection( + im, tinypose_result, conf_threshold=0.5) +cv2.imwrite("visualized_result.jpg", vis_im) +print("TinyPose visualized result save in ./visualized_result.jpg") diff --git a/fastdeploy/backends/paddle/util.cc b/fastdeploy/backends/paddle/util.cc index 216c788b2..005f9966b 100644 --- a/fastdeploy/backends/paddle/util.cc +++ b/fastdeploy/backends/paddle/util.cc @@ -52,6 +52,10 @@ void ShareTensorFromFDTensor(paddle_infer::Tensor* tensor, tensor->CopyFromCpu(static_cast(fd_tensor.Data())); } return; + } else if (fd_tensor.dtype == FDDataType::UINT8) { + tensor->ShareExternalData(static_cast(fd_tensor.Data()), + shape, paddle_infer::PlaceType::kCPU); + return; } FDASSERT(false, "Unexpected data type(%s) while infer with PaddleBackend.", Str(fd_tensor.dtype).c_str()); diff --git a/fastdeploy/pipeline.h b/fastdeploy/pipeline.h new file mode 100644 index 000000000..5d16935cc --- /dev/null +++ b/fastdeploy/pipeline.h @@ -0,0 +1,21 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/core/config.h" +#ifdef ENABLE_VISION +#include "fastdeploy/pipeline/pptinypose/pipeline.h" +#endif + +#include "fastdeploy/vision/visualize/visualize.h" diff --git a/fastdeploy/pipeline/pipeline_pybind.cc b/fastdeploy/pipeline/pipeline_pybind.cc new file mode 100644 index 000000000..9fad1aff5 --- /dev/null +++ b/fastdeploy/pipeline/pipeline_pybind.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { + +void BindPPTinyPosePipeline(pybind11::module& m); + +void BindPipeline(pybind11::module& m) { + BindPPTinyPosePipeline(m); +} +} // namespace fastdeploy diff --git a/fastdeploy/pipeline/pptinypose/pipeline.cc b/fastdeploy/pipeline/pptinypose/pipeline.cc new file mode 100644 index 000000000..6a2f2d4ba --- /dev/null +++ b/fastdeploy/pipeline/pptinypose/pipeline.cc @@ -0,0 +1,70 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pipeline/pptinypose/pipeline.h" + +namespace fastdeploy { +namespace pipeline { +PPTinyPose::PPTinyPose( + fastdeploy::vision::detection::PPYOLOE* det_model, + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model) + : detector_(det_model), pptinypose_model_(pptinypose_model) {} + +bool PPTinyPose::Detect( + cv::Mat* img, fastdeploy::vision::DetectionResult* detection_res) { + if (!detector_->Predict(img, detection_res)) { + FDERROR << "There's a error while detectiong human box in image." + << std::endl; + return false; + } + return true; +} + +bool PPTinyPose::KeypointDetect( + cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result, + fastdeploy::vision::DetectionResult& detection_result) { + if (!pptinypose_model_->Predict(img, result, detection_result)) { + FDERROR << "There's a error while detecting keypoint in image " + << std::endl; + return false; + } + return true; +} + +bool PPTinyPose::Predict( + cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result) { + result->Clear(); + fastdeploy::vision::DetectionResult detection_res; + if (nullptr != detector_ && !Detect(img, &detection_res)) { + FDERROR << "Failed to detect image." << std::endl; + return false; + } + fastdeploy::vision::DetectionResult filter_detection_res; + for (size_t i = 0; i < detection_res.boxes.size(); ++i) { + if (detection_res.scores[i] > detection_model_score_threshold) { + filter_detection_res.boxes.push_back(detection_res.boxes[i]); + filter_detection_res.scores.push_back(detection_res.scores[i]); + filter_detection_res.label_ids.push_back(detection_res.label_ids[i]); + } + } + if (nullptr != pptinypose_model_ && + !KeypointDetect(img, result, filter_detection_res)) { + FDERROR << "Failed to detect keypoint in image " << std::endl; + return false; + } + return true; +}; + +} // namespace pipeline +} // namespace fastdeploy diff --git a/fastdeploy/pipeline/pptinypose/pipeline.h b/fastdeploy/pipeline/pptinypose/pipeline.h new file mode 100644 index 000000000..90d6e21f0 --- /dev/null +++ b/fastdeploy/pipeline/pptinypose/pipeline.h @@ -0,0 +1,67 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/detection/ppdet/model.h" +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h" + +namespace fastdeploy { +/** \brief All pipeline model APIs are defined inside this namespace + * + */ +namespace pipeline { + +/*! @brief PPTinyPose Pipeline object used when to load a detection model + pptinypose model + */ +class FASTDEPLOY_DECL PPTinyPose { + public: + /** \brief Set initialized detection model object and pptinypose model object + * + * \param[in] det_model Initialized detection model object + * \param[in] pptinypose_model Initialized pptinypose model object + */ + PPTinyPose( + fastdeploy::vision::detection::PPYOLOE* det_model, + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model); + + /** \brief Predict the keypoint detection result for an input image + * + * \param[in] img The input image data, comes from cv::imread() + * \param[in] result The output keypoint detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* img, + fastdeploy::vision::KeyPointDetectionResult* result); + + /* \brief The score threshold for detectin model to filter bbox before inputting pptinypose model + */ + float detection_model_score_threshold = 0; + + protected: + fastdeploy::vision::detection::PPYOLOE* detector_ = nullptr; + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model_ = + nullptr; + + virtual bool Detect(cv::Mat* img, + fastdeploy::vision::DetectionResult* result); + virtual bool KeypointDetect( + cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result, + fastdeploy::vision::DetectionResult& detection_result); +}; + +} // namespace pipeline +} // namespace fastdeploy diff --git a/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc b/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc new file mode 100644 index 000000000..8aee0474e --- /dev/null +++ b/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc @@ -0,0 +1,58 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPPTinyPosePipeline(pybind11::module& m) { + pybind11::class_(m, "PPTinyPose") + + // explicitly pybind more kinds of detection models here + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def(pybind11::init()) + + .def("predict", [](pipeline::PPTinyPose& self, + pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::KeyPointDetectionResult res; + self.Predict(&mat, &res); + return res; + }) + + .def_readwrite("detection_model_score_threshold", + &pipeline::PPTinyPose::detection_model_score_threshold); +} + +} // namespace fastdeploy diff --git a/fastdeploy/pybind/main.cc.in b/fastdeploy/pybind/main.cc.in index e93d18e11..222d75fd9 100644 --- a/fastdeploy/pybind/main.cc.in +++ b/fastdeploy/pybind/main.cc.in @@ -20,6 +20,7 @@ void BindRuntime(pybind11::module&); void BindFDModel(pybind11::module&); void BindVision(pybind11::module&); void BindText(pybind11::module&); +void BindPipeline(pybind11::module&); pybind11::dtype FDDataTypeToNumpyDataType(const FDDataType& fd_dtype) { pybind11::dtype dt; @@ -154,6 +155,9 @@ PYBIND11_MODULE(@PY_LIBRARY_NAME@, m) { auto vision_module = m.def_submodule("vision", "Vision module of FastDeploy."); BindVision(vision_module); + auto pipeline_module = + m.def_submodule("pipeline", "Pipeline module of FastDeploy."); + BindPipeline(pipeline_module); #endif #ifdef ENABLE_TEXT auto text_module = diff --git a/fastdeploy/pybind/main.h b/fastdeploy/pybind/main.h index b31c49f07..4cd21b7ea 100644 --- a/fastdeploy/pybind/main.h +++ b/fastdeploy/pybind/main.h @@ -24,6 +24,7 @@ #ifdef ENABLE_VISION #include "fastdeploy/vision.h" +#include "fastdeploy/pipeline.h" #endif #ifdef ENABLE_TEXT @@ -37,6 +38,7 @@ namespace fastdeploy { void BindBackend(pybind11::module&); void BindVision(pybind11::module&); void BindText(pybind11::module& m); +void BindPipeline(pybind11::module& m); pybind11::dtype FDDataTypeToNumpyDataType(const FDDataType& fd_dtype); diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index d01e2e15c..51fc393e5 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -38,6 +38,7 @@ #include "fastdeploy/vision/faceid/contrib/insightface_rec.h" #include "fastdeploy/vision/faceid/contrib/partial_fc.h" #include "fastdeploy/vision/faceid/contrib/vpl.h" +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h" #include "fastdeploy/vision/matting/contrib/modnet.h" #include "fastdeploy/vision/matting/ppmatting/ppmatting.h" #include "fastdeploy/vision/ocr/ppocr/classifier.h" diff --git a/fastdeploy/vision/common/processors/base.cc b/fastdeploy/vision/common/processors/base.cc index a2359c16e..e6a945dac 100644 --- a/fastdeploy/vision/common/processors/base.cc +++ b/fastdeploy/vision/common/processors/base.cc @@ -32,5 +32,5 @@ bool Processor::operator()(Mat* mat, ProcLib lib) { return ret; } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h index d066a2f94..b67205636 100644 --- a/fastdeploy/vision/common/processors/base.h +++ b/fastdeploy/vision/common/processors/base.h @@ -35,7 +35,9 @@ class Processor { virtual bool ImplByOpenCV(Mat* mat) = 0; virtual bool ImplByFalconCV(Mat* mat) { - FDASSERT(false, "%s is not implemented with FalconCV, please use OpenCV instead.", Name().c_str()); + FDASSERT(false, + "%s is not implemented with FalconCV, please use OpenCV instead.", + Name().c_str()); return false; } @@ -43,5 +45,5 @@ class Processor { ProcLib lib = ProcLib::OPENCV); }; -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/crop.h b/fastdeploy/vision/common/processors/crop.h index 8e583adca..681e1fca2 100644 --- a/fastdeploy/vision/common/processors/crop.h +++ b/fastdeploy/vision/common/processors/crop.h @@ -27,6 +27,7 @@ class Crop : public Processor { width_ = width; height_ = height; } + bool ImplByOpenCV(Mat* mat); std::string Name() { return "Crop"; } diff --git a/fastdeploy/vision/common/processors/mat.h b/fastdeploy/vision/common/processors/mat.h index ab1a1f35e..9cd1a4164 100644 --- a/fastdeploy/vision/common/processors/mat.h +++ b/fastdeploy/vision/common/processors/mat.h @@ -23,8 +23,8 @@ #endif namespace fcv { - class Mat; -} +class Mat; +} // namespace fcv namespace fastdeploy { namespace vision { @@ -46,8 +46,8 @@ struct FASTDEPLOY_DECL Mat { // this only used if you don't want to write // the original data, and write to a new cv::Mat // then replace the old cv::Mat of this structure - void SetMat(const cv::Mat& mat) { - cpu_mat = mat; + void SetMat(const cv::Mat& mat) { + cpu_mat = mat; mat_type = ProcLib::OPENCV; } @@ -56,9 +56,14 @@ struct FASTDEPLOY_DECL Mat { return &cpu_mat; } + inline const cv::Mat* GetOpenCVMat() const { + FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure."); + return &cpu_mat; + } + #ifdef ENABLE_FALCONCV - void SetMat(const fcv::Mat& mat) { - fcv_mat = mat; + void SetMat(const fcv::Mat& mat) { + fcv_mat = mat; mat_type = Proclib::FALCONCV; } diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h index 5522c138c..b16b95b38 100644 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -30,3 +30,4 @@ #include "fastdeploy/vision/common/processors/resize_by_long.h" #include "fastdeploy/vision/common/processors/resize_by_short.h" #include "fastdeploy/vision/common/processors/stride_pad.h" +#include "fastdeploy/vision/common/processors/warp_affine.h" diff --git a/fastdeploy/vision/common/processors/warp_affine.cc b/fastdeploy/vision/common/processors/warp_affine.cc new file mode 100644 index 000000000..da5c6377f --- /dev/null +++ b/fastdeploy/vision/common/processors/warp_affine.cc @@ -0,0 +1,51 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/warp_affine.h" + +namespace fastdeploy { +namespace vision { + +bool WarpAffine::ImplByOpenCV(Mat* mat) { + if (mat->layout != Layout::HWC) { + FDERROR << "WarpAffine: The format of input is not HWC." << std::endl; + return false; + } + cv::Mat* im = mat->GetOpenCVMat(); + if (width_ > 0 && height_ > 0) { + cv::warpAffine(*im, *im, trans_matrix_, cv::Size(width_, height_), interp_, border_mode_, borderValue_); + } else { + FDERROR << "WarpAffine: the parameters must satisfy (width > 0 && height > 0) ." + << std::endl; + return false; + } + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + + return true; +} + +bool WarpAffine::Run(Mat* mat, + const cv::Mat& trans_matrix, + int width, int height, + int interp, + int border_mode, + const cv::Scalar& borderValue, + ProcLib lib) { + auto r = WarpAffine(trans_matrix, width, height, interp, border_mode, borderValue); + return r(mat, lib); +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/warp_affine.h b/fastdeploy/vision/common/processors/warp_affine.h new file mode 100644 index 000000000..5548d495f --- /dev/null +++ b/fastdeploy/vision/common/processors/warp_affine.h @@ -0,0 +1,66 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class WarpAffine : public Processor { + public: + WarpAffine(const cv::Mat& trans_matrix, + int width, int height, + int interp = 1, + int border_mode = 0, + const cv::Scalar& borderValue = cv::Scalar()) { + trans_matrix_ = trans_matrix; + width_ = width; + height_ = height; + interp_ = interp; + border_mode_ = border_mode; + borderValue_ = borderValue; + } + + bool ImplByOpenCV(Mat* mat); + std::string Name() { return "WarpAffine"; } + + bool SetTransformMatrix(const cv::Mat &trans_matrix) { + trans_matrix_ = trans_matrix; + return true; + } + + std::tuple GetWidthAndHeight() { + return std::make_tuple(width_, height_); + } + + static bool Run(Mat* mat, + const cv::Mat& trans_matrix, + int width, int height, + int interp = 1, + int border_mode = 0, + const cv::Scalar& borderValue = cv::Scalar(), + ProcLib lib = ProcLib::OPENCV); + + private: + cv::Mat trans_matrix_; + int width_; + int height_; + int interp_; + int border_mode_; + cv::Scalar borderValue_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index d4f586d6a..c2e770083 100644 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -117,6 +117,29 @@ std::string DetectionResult::Str() { return out; } +void KeyPointDetectionResult::Clear() { + std::vector>().swap(keypoints); + std::vector().swap(scores); + num_joints = -1; +} + +void KeyPointDetectionResult::Reserve(int size) { keypoints.reserve(size); } + +void KeyPointDetectionResult::Resize(int size) { keypoints.resize(size); } + +std::string KeyPointDetectionResult::Str() { + std::string out; + + out = "KeyPointDetectionResult: [x, y, conf]\n"; + for (size_t i = 0; i < keypoints.size(); ++i) { + out = out + std::to_string(keypoints[i][0]) + "," + + std::to_string(keypoints[i][1]) + ", " + + std::to_string(scores[i]) + "\n"; + } + out += "num_joints:" + std::to_string(num_joints) + "\n"; + return out; +} + void OCRResult::Clear() { boxes.clear(); text.clear(); diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index b3dc922cb..e37538280 100644 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -29,7 +29,8 @@ enum FASTDEPLOY_DECL ResultType { FACE_DETECTION, FACE_RECOGNITION, MATTING, - MASK + MASK, + KEYPOINT_DETECTION }; struct FASTDEPLOY_DECL BaseResult { @@ -114,6 +115,29 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult { std::string Str(); }; +/*! @brief KeyPoint Detection result structure for all the keypoint detection models + */ +struct FASTDEPLOY_DECL KeyPointDetectionResult : public BaseResult { + /** \brief All the coordinates of detected keypoints for an input image, the size of `keypoints` is num_detected_objects * num_joints, and the element of `keypoint` is a array of 2 float values, means [x, y] + */ + std::vector> keypoints; + //// The confidence for all the detected points + std::vector scores; + //// Number of joints for a detected object + int num_joints = -1; + + ResultType type = ResultType::KEYPOINT_DETECTION; + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + struct FASTDEPLOY_DECL OCRResult : public BaseResult { std::vector> boxes; diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc index ee28dfe38..299b1b665 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -89,6 +89,7 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() { processors_.push_back(std::make_shared()); + bool has_permute = false; for (const auto& op : cfg["Preprocess"]) { std::string op_name = op["type"].as(); if (op_name == "NormalizeImage") { @@ -132,6 +133,7 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() { } } else if (op_name == "Permute") { // Do nothing, do permute as the last operation + has_permute = true; continue; // processors_.push_back(std::make_shared()); } else if (op_name == "Pad") { @@ -150,7 +152,14 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() { return false; } } - processors_.push_back(std::make_shared()); + if (has_permute) { + // permute = cast + HWC2CHW + processors_.push_back(std::make_shared("float")); + processors_.push_back(std::make_shared()); + } else { + processors_.push_back(std::make_shared()); + } + return true; } diff --git a/fastdeploy/vision/keypointdet/keypointdet_pybind.cc b/fastdeploy/vision/keypointdet/keypointdet_pybind.cc new file mode 100644 index 000000000..35fffdcd4 --- /dev/null +++ b/fastdeploy/vision/keypointdet/keypointdet_pybind.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { + +void BindPPTinyPose(pybind11::module& m); + +void BindKeyPointDetection(pybind11::module& m) { + auto keypointdetection_module = m.def_submodule( + "keypointdetection", "Image object keypoint detection models."); + BindPPTinyPose(keypointdetection_module); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc new file mode 100644 index 000000000..9bca72532 --- /dev/null +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc @@ -0,0 +1,262 @@ +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h" +#include "fastdeploy/vision/utils/utils.h" +#include "yaml-cpp/yaml.h" +#ifdef ENABLE_PADDLE_FRONTEND +#include "paddle2onnx/converter.h" +#endif +#include "fastdeploy/vision.h" + +namespace fastdeploy { +namespace vision { +namespace keypointdetection { + +PPTinyPose::PPTinyPose(const std::string& model_file, + const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { + config_file_ = config_file; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool PPTinyPose::Initialize() { + if (!BuildPreprocessPipelineFromConfig()) { + FDERROR << "Failed to build preprocess pipeline from configuration file." + << std::endl; + return false; + } + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + + return true; +} + +bool PPTinyPose::BuildPreprocessPipelineFromConfig() { + processors_.clear(); + YAML::Node cfg; + try { + cfg = YAML::LoadFile(config_file_); + } catch (YAML::BadFile& e) { + FDERROR << "Failed to load yaml file " << config_file_ + << ", maybe you should check this file." << std::endl; + return false; + } + + std::string arch = cfg["arch"].as(); + if (arch != "HRNet" && arch != "HigherHRNet") { + FDERROR << "Require the arch of model is HRNet or HigherHRNet, but arch " + << "defined in " + << "config file is " << arch << "." << std::endl; + return false; + } + + processors_.push_back(std::make_shared()); + + for (const auto& op : cfg["Preprocess"]) { + std::string op_name = op["type"].as(); + if (op_name == "NormalizeImage") { + auto mean = op["mean"].as>(); + auto std = op["std"].as>(); + bool is_scale = op["is_scale"].as(); + processors_.push_back(std::make_shared(mean, std, is_scale)); + } else if (op_name == "Permute") { + // permute = cast + HWC2CHW + processors_.push_back(std::make_shared("float")); + processors_.push_back(std::make_shared()); + } else if (op_name == "TopDownEvalAffine") { + auto trainsize = op["trainsize"].as>(); + int height = trainsize[1]; + int width = trainsize[0]; + cv::Mat trans_matrix(2, 3, CV_64FC1); + processors_.push_back( + std::make_shared(trans_matrix, width, height, 1)); + } else { + FDERROR << "Unexcepted preprocess operator: " << op_name << "." + << std::endl; + return false; + } + } + return true; +} + +bool PPTinyPose::Preprocess(Mat* mat, std::vector* outputs) { + for (size_t i = 0; i < processors_.size(); ++i) { + if (processors_[i]->Name().compare("WarpAffine") == 0) { + auto processor = dynamic_cast(processors_[i].get()); + float origin_width = static_cast(mat->Width()); + float origin_height = static_cast(mat->Height()); + std::vector center = {origin_width / 2.0f, origin_height / 2.0f}; + std::vector scale = {origin_width, origin_height}; + int resize_width = -1; + int resize_height = -1; + std::tie(resize_width, resize_height) = processor->GetWidthAndHeight(); + cv::Mat trans_matrix(2, 3, CV_64FC1); + GetAffineTransform(center, scale, 0, {resize_width, resize_height}, &trans_matrix, 0); + if (!(processor->SetTransformMatrix(trans_matrix))) { + FDERROR << "Failed to set transform matrix of " + << processors_[i]->Name() + << " processor." << std::endl; + } + } + if (!(*(processors_[i].get()))(mat)) { + FDERROR << "Failed to process image data in " << processors_[i]->Name() + << "." << std::endl; + return false; + } + } + + outputs->resize(1); + (*outputs)[0].name = InputInfoOfRuntime(0).name; + mat->ShareWithTensor(&((*outputs)[0])); + + // reshape to [1, c, h, w] + (*outputs)[0].ExpandDim(0); + + return true; +} + +bool PPTinyPose::Postprocess(std::vector& infer_result, + KeyPointDetectionResult* result, + const std::vector& center, + const std::vector& scale) { + FDASSERT(infer_result[1].shape[0] == 1, + "Only support batch = 1 in FastDeploy now."); + result->Clear(); + + // Calculate output length + int outdata_size = + std::accumulate(infer_result[0].shape.begin(), + infer_result[0].shape.end(), 1, std::multiplies()); + int idxdata_size = + std::accumulate(infer_result[1].shape.begin(), + infer_result[1].shape.end(), 1, std::multiplies()); + + if (outdata_size < 6) { + FDWARNING << "PPTinyPose No object detected." << std::endl; + } + float* out_data = static_cast(infer_result[0].Data()); + void* idx_data = infer_result[1].Data(); + int idx_dtype = infer_result[1].dtype; + std::vector out_data_shape(infer_result[0].shape.begin(), + infer_result[0].shape.end()); + std::vector idx_data_shape(infer_result[1].shape.begin(), + infer_result[1].shape.end()); + std::vector preds(out_data_shape[1] * 3, 0); + std::vector heatmap(out_data, out_data + outdata_size); + std::vector idxout(idxdata_size); + if (idx_dtype == FDDataType::INT32) { + std::copy(static_cast(idx_data), + static_cast(idx_data) + idxdata_size, idxout.begin()); + } else if (idx_dtype == FDDataType::INT64) { + std::copy(static_cast(idx_data), + static_cast(idx_data) + idxdata_size, idxout.begin()); + } else { + FDERROR << "Only support process inference result with INT32/INT64 data type, but now it's " << idx_dtype << "." << std::endl; + } + GetFinalPredictions(heatmap, out_data_shape, idxout, center, scale, &preds, + this->use_dark); + result->Reserve(outdata_size); + result->num_joints = out_data_shape[1]; + result->keypoints.clear(); + for (int i = 0; i < out_data_shape[1]; i++) { + result->keypoints.push_back({preds[i * 3 + 1], preds[i * 3 + 2]}); + result->scores.push_back(preds[i * 3]); + } + return true; +} + +bool PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result) { + std::vector center = {round(im->cols / 2.0f), round(im->rows / 2.0f)}; + std::vector scale = {static_cast(im->cols), static_cast(im->rows)}; + Mat mat(*im); + std::vector processed_data; + if (!Preprocess(&mat, &processed_data)) { + FDERROR << "Failed to preprocess input data while using model:" + << ModelName() << "." << std::endl; + return false; + } + std::vector infer_result; + if (!Infer(processed_data, &infer_result)) { + FDERROR << "Failed to inference while using model:" << ModelName() << "." + << std::endl; + return false; + } + if (!Postprocess(infer_result, result, center, scale)) { + FDERROR << "Failed to postprocess while using model:" << ModelName() << "." + << std::endl; + return false; + } + + return true; +} + +bool PPTinyPose::Predict(cv::Mat* im, KeyPointDetectionResult* result, + const DetectionResult& detection_result) { + std::vector crop_imgs; + std::vector> center_bs; + std::vector> scale_bs; + int crop_imgs_num = 0; + int box_num = detection_result.boxes.size(); + for (int i = 0; i < box_num; i++) { + auto box = detection_result.boxes[i]; + auto label_id = detection_result.label_ids[i]; + int channel = im->channels(); + cv::Mat cv_crop_img(0, 0, CV_32SC(channel)); + Mat crop_img(cv_crop_img); + std::vector rect(box.begin(), box.end()); + std::vector center; + std::vector scale; + if (label_id == 0) { + Mat mat(*im); + utils::CropImageByBox(mat, &crop_img, rect, ¢er, &scale); + center_bs.emplace_back(center); + scale_bs.emplace_back(scale); + crop_imgs.emplace_back(crop_img); + crop_imgs_num += 1; + } + } + for (int i = 0; i < crop_imgs_num; i++) { + std::vector processed_data; + if (!Preprocess(&crop_imgs[i], &processed_data)) { + FDERROR << "Failed to preprocess input data while using model:" + << ModelName() << "." << std::endl; + return false; + } + std::vector infer_result; + if (!Infer(processed_data, &infer_result)) { + FDERROR << "Failed to inference while using model:" << ModelName() << "." + << std::endl; + return false; + } + KeyPointDetectionResult one_cropimg_result; + if (!Postprocess(infer_result, &one_cropimg_result, center_bs[i], + scale_bs[i])) { + FDERROR << "Failed to postprocess while using model:" << ModelName() + << "." << std::endl; + return false; + } + if (result->num_joints == -1) { + result->num_joints = one_cropimg_result.num_joints; + } + std::copy(one_cropimg_result.keypoints.begin(), + one_cropimg_result.keypoints.end(), + std::back_inserter(result->keypoints)); + std::copy(one_cropimg_result.scores.begin(), + one_cropimg_result.scores.end(), + std::back_inserter(result->scores)); + } + + return true; +} + +} // namespace keypointdetection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose.h b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.h new file mode 100644 index 000000000..bfa8a8e60 --- /dev/null +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.h @@ -0,0 +1,90 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h" + +namespace fastdeploy { +namespace vision { +/** \brief All keypoint detection model APIs are defined inside this namespace + * + */ +namespace keypointdetection { + +/*! @brief PPTinyPose model object used when to load a PPTinyPose model exported by PaddleDetection + */ +class FASTDEPLOY_DECL PPTinyPose : public FastDeployModel { + public: + /** \brief Set path of model file and configuration file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g pptinypose/model.pdmodel + * \param[in] params_file Path of parameter file, e.g pptinypose/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] config_file Path of configuration file for deployment, e.g pptinypose/infer_cfg.yml + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ + PPTinyPose(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + + /// Get model's name + std::string ModelName() const { return "PaddleDetection/PPTinyPose"; } + + /** \brief Predict the keypoint detection result for an input image + * + * \param[in] im The input image data, comes from cv::imread() + * \param[in] result The output keypoint detection result will be writen to this structure + * \return true if the keypoint prediction successed, otherwise false + */ + bool Predict(cv::Mat* im, KeyPointDetectionResult* result); + + /** \brief Predict the keypoint detection result with given detection result for an input image + * + * \param[in] im The input image data, comes from cv::imread() + * \param[in] result The output keypoint detection result will be writen to this structure + * \param[in] detection_result The structure strores pedestrian detection result, which is used to crop image for multi-persons keypoint detection + * \return true if the keypoint prediction successed, otherwise false + */ + bool Predict(cv::Mat* im, KeyPointDetectionResult* result, + const DetectionResult& detection_result); + + /** \brief Whether using Distribution-Aware Coordinate Representation for Human Pose Estimation(DARK for short) in postprocess, default is true + */ + bool use_dark = true; + + protected: + bool Initialize(); + /// Build the preprocess pipeline from the loaded model + bool BuildPreprocessPipelineFromConfig(); + /// Preprocess an input image, and set the preprocessed results to `outputs` + bool Preprocess(Mat* mat, std::vector* outputs); + + /// Postprocess the inferenced results, and set the final result to `result` + bool Postprocess(std::vector& infer_result, + KeyPointDetectionResult* result, + const std::vector& center, + const std::vector& scale); + + private: + std::vector> processors_; + std::string config_file_; +}; +} // namespace keypointdetection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose_pybind.cc b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_pybind.cc new file mode 100644 index 000000000..7fc6a2aab --- /dev/null +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_pybind.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPPTinyPose(pybind11::module& m) { + pybind11::class_( + m, "PPTinyPose") + .def(pybind11::init()) + .def("predict", + [](vision::keypointdetection::PPTinyPose& self, + pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::KeyPointDetectionResult res; + self.Predict(&mat, &res); + return res; + }) + .def( + "predict", + [](vision::keypointdetection::PPTinyPose& self, pybind11::array& data, + vision::DetectionResult& detection_result) { + auto mat = PyArrayToCvMat(data); + vision::KeyPointDetectionResult res; + self.Predict(&mat, &res, detection_result); + return res; + }) + .def_readwrite("use_dark", + &vision::keypointdetection::PPTinyPose::use_dark); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc new file mode 100644 index 000000000..c0d80a4a6 --- /dev/null +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.cc @@ -0,0 +1,125 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h" +#define PI 3.1415926535 +#define HALF_CIRCLE_DEGREE 180 + +namespace fastdeploy { +namespace vision { +namespace keypointdetection { + +cv::Point2f Get3dPoint(const cv::Point2f& a, const cv::Point2f& b) { + cv::Point2f direct{a.x - b.x, a.y - b.y}; + return cv::Point2f(a.x - direct.y, a.y + direct.x); +} + +std::vector GetDir(const float src_point_x, const float src_point_y, + const float rot_rad) { + float sn = sin(rot_rad); + float cs = cos(rot_rad); + std::vector src_result{0.0, 0.0}; + src_result[0] = src_point_x * cs - src_point_y * sn; + src_result[1] = src_point_x * sn + src_point_y * cs; + return src_result; +} + +void AffineTransform(const float pt_x, const float pt_y, const cv::Mat& trans, + std::vector* preds, const int p) { + double new1[3] = {pt_x, pt_y, 1.0}; + cv::Mat new_pt(3, 1, trans.type(), new1); + cv::Mat w = trans * new_pt; + (*preds)[p * 3 + 1] = static_cast(w.at(0, 0)); + (*preds)[p * 3 + 2] = static_cast(w.at(1, 0)); +} + +void GetAffineTransform(const std::vector& center, + const std::vector& scale, const float rot, + const std::vector& output_size, cv::Mat* trans, + const int inv) { + float src_w = scale[0]; + float dst_w = static_cast(output_size[0]); + float dst_h = static_cast(output_size[1]); + float rot_rad = rot * PI / HALF_CIRCLE_DEGREE; + std::vector src_dir = GetDir(-0.5 * src_w, 0, rot_rad); + std::vector dst_dir{-0.5f * dst_w, 0.0}; + cv::Point2f srcPoint2f[3], dstPoint2f[3]; + srcPoint2f[0] = cv::Point2f(center[0], center[1]); + srcPoint2f[1] = cv::Point2f(center[0] + src_dir[0], center[1] + src_dir[1]); + srcPoint2f[2] = Get3dPoint(srcPoint2f[0], srcPoint2f[1]); + + dstPoint2f[0] = cv::Point2f(dst_w * 0.5, dst_h * 0.5); + dstPoint2f[1] = + cv::Point2f(dst_w * 0.5 + dst_dir[0], dst_h * 0.5 + dst_dir[1]); + dstPoint2f[2] = Get3dPoint(dstPoint2f[0], dstPoint2f[1]); + if (inv == 0) { + (*trans) = cv::getAffineTransform(srcPoint2f, dstPoint2f); + } else { + (*trans) = cv::getAffineTransform(dstPoint2f, srcPoint2f); + } +} + +void TransformPreds(std::vector& coords, + const std::vector& center, + const std::vector& scale, + const std::vector& output_size, + const std::vector& dim, + std::vector* target_coords) { + cv::Mat trans(2, 3, CV_64FC1); + GetAffineTransform(center, scale, 0, output_size, &trans, 1); + for (int p = 0; p < dim[1]; ++p) { + AffineTransform(coords[p * 2], coords[p * 2 + 1], trans, target_coords, p); + } +} + +void GetFinalPredictions(const std::vector& heatmap, + const std::vector& dim, + const std::vector& idxout, + const std::vector& center, + const std::vector scale, + std::vector* preds, const bool DARK) { + std::vector coords(dim[1] * 2); + + int heatmap_height = dim[2]; + int heatmap_width = dim[3]; + for (int j = 0; j < dim[1]; ++j) { + int index = j * dim[2] * dim[3]; + int idx = idxout[j]; + (*preds)[j * 3] = heatmap[index + idx]; + coords[j * 2] = idx % heatmap_width; + coords[j * 2 + 1] = idx / heatmap_width; + int px = int(coords[j * 2] + 0.5); + int py = int(coords[j * 2 + 1] + 0.5); + if (DARK && px > 1 && px < heatmap_width - 2) { + utils::DarkParse(heatmap, dim, &coords, px, py, index, j); + } else { + if (px > 0 && px < heatmap_width - 1) { + float diff_x = heatmap[index + py * dim[3] + px + 1] - + heatmap[index + py * dim[3] + px - 1]; + coords[j * 2] += diff_x > 0 ? 1 : -1 * 0.25; + } + if (py > 0 && py < heatmap_height - 1) { + float diff_y = heatmap[index + (py + 1) * dim[3] + px] - + heatmap[index + (py - 1) * dim[3] + px]; + coords[j * 2 + 1] += diff_y > 0 ? 1 : -1 * 0.25; + } + } + } + std::vector img_size{heatmap_width, heatmap_height}; + TransformPreds(coords, center, scale, img_size, dim, preds); +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h new file mode 100644 index 000000000..1298d66be --- /dev/null +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose_utils.h @@ -0,0 +1,51 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace keypointdetection { + +cv::Point2f Get3dPoint(const cv::Point2f& a, const cv::Point2f& b); + +std::vector GetDir(const float src_point_x, const float src_point_y, + const float rot_rad); + +void GetAffineTransform(const std::vector& center, + const std::vector& scale, const float rot, + const std::vector& output_size, cv::Mat* trans, + const int inv); + +void AffineTransform(const float pt_x, const float pt_y, const cv::Mat& trans, + std::vector* preds, const int p); + +void TransformPreds(std::vector& coords, + const std::vector& center, + const std::vector& scale, + const std::vector& output_size, + const std::vector& dim, + std::vector* target_coords); + +void GetFinalPredictions(const std::vector& heatmap, + const std::vector& dim, + const std::vector& idxout, + const std::vector& center, + const std::vector scale, + std::vector* preds, const bool DARK); + +} // namespace keypointdetection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 9622ab56c..cd28836fb 100644 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -143,9 +143,9 @@ bool PaddleSegModel::Preprocess(Mat* mat, FDTensor* output) { int resize_height = -1; std::tie(resize_width, resize_height) = processor->GetWidthAndHeight(); if (is_vertical_screen && (resize_width > resize_height)) { - if (processor->SetWidthAndHeight(resize_height, resize_width)) { - FDERROR << "Failed to set Resize processor width and height " - << processors_[i]->Name() << "." << std::endl; + if (!(processor->SetWidthAndHeight(resize_height, resize_width))) { + FDERROR << "Failed to set width and height of " + << processors_[i]->Name() << " processor." << std::endl; } } } diff --git a/fastdeploy/vision/utils/crop_image.cc b/fastdeploy/vision/utils/crop_image.cc new file mode 100644 index 000000000..5d15844b5 --- /dev/null +++ b/fastdeploy/vision/utils/crop_image.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace utils { + +bool CropImageByBox(const Mat& src_im, Mat* dst_im, + const std::vector& box, std::vector* center, + std::vector* scale, const float expandratio) { + const cv::Mat* img = src_im.GetOpenCVMat(); + cv::Mat* crop_img = dst_im->GetOpenCVMat(); + int xmin = static_cast(box[0]); + int ymin = static_cast(box[1]); + int xmax = static_cast(box[2]); + int ymax = static_cast(box[3]); + float centerx = (xmin + xmax) / 2.0f; + float centery = (ymin + ymax) / 2.0f; + float half_h = (ymax - ymin) * (1 + expandratio) / 2.0f; + float half_w = (xmax - xmin) * (1 + expandratio) / 2.0f; + // adjust h or w to keep image ratio, expand the shorter edge + if (half_h * 3 > half_w * 4) { + half_w = half_h * 0.75; + } + int crop_xmin =std::max(0, static_cast(centerx - half_w)); + int crop_ymin =std::max(0, static_cast(centery - half_h)); + int crop_xmax = std::min(img->cols - 1, static_cast(centerx + half_w)); + int crop_ymax = std::min(img->rows - 1, static_cast(centery + half_h)); + + crop_img->create(crop_ymax - crop_ymin, crop_xmax - crop_xmin, img->type()); + *crop_img = + (*img)(cv::Range(crop_ymin, crop_ymax), cv::Range(crop_xmin, crop_xmax)); + center->clear(); + center->emplace_back((crop_xmin + crop_xmax) / 2.0f); + center->emplace_back((crop_ymin + crop_ymax) / 2.0f); + + scale->clear(); + scale->emplace_back((crop_xmax - crop_xmin)); + scale->emplace_back((crop_ymax - crop_ymin)); + + dst_im->SetWidth(crop_img->cols); + dst_im->SetHeight(crop_img->rows); + return true; +} + +} // namespace utils +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/utils/dark_parse.cc b/fastdeploy/vision/utils/dark_parse.cc new file mode 100644 index 000000000..807065404 --- /dev/null +++ b/fastdeploy/vision/utils/dark_parse.cc @@ -0,0 +1,81 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace utils { + +void DarkParse(const std::vector& heatmap, const std::vector& dim, + std::vector* coords, const int px, const int py, + const int index, const int ch) { + /*DARK postpocessing, Zhang et al. Distribution-Aware Coordinate + Representation for Human Pose Estimation (CVPR 2020). + 1) offset = - hassian.inv() * derivative + 2) dx = (heatmap[x+1] - heatmap[x-1])/2. + 3) dxx = (dx[x+1] - dx[x-1])/2. + 4) derivative = Mat([dx, dy]) + 5) hassian = Mat([[dxx, dxy], [dxy, dyy]]) + */ + std::vector::const_iterator first1 = heatmap.begin() + index; + std::vector::const_iterator last1 = + heatmap.begin() + index + dim[2] * dim[3]; + std::vector heatmap_ch(first1, last1); + cv::Mat heatmap_mat = cv::Mat(heatmap_ch).reshape(0, dim[2]); + heatmap_mat.convertTo(heatmap_mat, CV_32FC1); + cv::GaussianBlur(heatmap_mat, heatmap_mat, cv::Size(3, 3), 0, 0); + heatmap_mat = heatmap_mat.reshape(1, 1); + heatmap_ch = std::vector(heatmap_mat.reshape(1, 1)); + + float epsilon = 1e-10; + // sample heatmap to get values in around target location + float xy = log(fmax(heatmap_ch[py * dim[3] + px], epsilon)); + float xr = log(fmax(heatmap_ch[py * dim[3] + px + 1], epsilon)); + float xl = log(fmax(heatmap_ch[py * dim[3] + px - 1], epsilon)); + + float xr2 = log(fmax(heatmap_ch[py * dim[3] + px + 2], epsilon)); + float xl2 = log(fmax(heatmap_ch[py * dim[3] + px - 2], epsilon)); + float yu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px], epsilon)); + float yd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px], epsilon)); + float yu2 = log(fmax(heatmap_ch[(py + 2) * dim[3] + px], epsilon)); + float yd2 = log(fmax(heatmap_ch[(py - 2) * dim[3] + px], epsilon)); + float xryu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px + 1], epsilon)); + float xryd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px + 1], epsilon)); + float xlyu = log(fmax(heatmap_ch[(py + 1) * dim[3] + px - 1], epsilon)); + float xlyd = log(fmax(heatmap_ch[(py - 1) * dim[3] + px - 1], epsilon)); + + // compute dx/dy and dxx/dyy with sampled values + float dx = 0.5 * (xr - xl); + float dy = 0.5 * (yu - yd); + float dxx = 0.25 * (xr2 - 2 * xy + xl2); + float dxy = 0.25 * (xryu - xryd - xlyu + xlyd); + float dyy = 0.25 * (yu2 - 2 * xy + yd2); + + // finally get offset by derivative and hassian, which combined by dx/dy and + // dxx/dyy + if (dxx * dyy - dxy * dxy != 0) { + float M[2][2] = {dxx, dxy, dxy, dyy}; + float D[2] = {dx, dy}; + cv::Mat hassian(2, 2, CV_32F, M); + cv::Mat derivative(2, 1, CV_32F, D); + cv::Mat offset = -hassian.inv() * derivative; + (*coords)[ch * 2] += offset.at(0, 0); + (*coords)[ch * 2 + 1] += offset.at(1, 0); + } +} + +} // namespace utils +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/utils/utils.h b/fastdeploy/vision/utils/utils.h index 89fc6a17e..4d6a006c3 100644 --- a/fastdeploy/vision/utils/utils.h +++ b/fastdeploy/vision/utils/utils.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include "fastdeploy/core/fd_tensor.h" @@ -24,6 +25,7 @@ #include "fastdeploy/function/reduce.h" #include "fastdeploy/function/softmax.h" #include "fastdeploy/function/transpose.h" +#include "fastdeploy/vision/common/processors/mat.h" namespace fastdeploy { namespace vision { @@ -73,6 +75,27 @@ FASTDEPLOY_DECL float CosineSimilarity(const std::vector& a, const std::vector& b, bool normalized = true); +bool CropImageByBox(const Mat& src_im, Mat* dst_im, + const std::vector& box, std::vector* center, + std::vector* scale, const float expandratio = 0.3); + +/** + * Function: for keypoint detection model, fine positioning of keypoints in postprocess + * Parameters: + * heatmap: model inference results for keypoint detection models + * dim: shape information of the inference result + * coords: coordinates after refined positioning + * px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions + * py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions + * index: index information of heatmap pixels + * ch: channel + * Paper reference: DARK postpocessing, Zhang et al. Distribution-Aware Coordinate + * Representation for Human Pose Estimation (CVPR 2020). + */ +void DarkParse(const std::vector& heatmap, const std::vector& dim, + std::vector* coords, const int px, const int py, + const int index, const int ch); + } // namespace utils } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/vision_pybind.cc b/fastdeploy/vision/vision_pybind.cc index dbade6c82..70f1990a4 100644 --- a/fastdeploy/vision/vision_pybind.cc +++ b/fastdeploy/vision/vision_pybind.cc @@ -23,6 +23,7 @@ void BindMatting(pybind11::module& m); void BindFaceDet(pybind11::module& m); void BindFaceId(pybind11::module& m); void BindOcr(pybind11::module& m); +void BindKeyPointDetection(pybind11::module& m); #ifdef ENABLE_VISION_VISUALIZE void BindVisualize(pybind11::module& m); #endif @@ -95,6 +96,15 @@ void BindVision(pybind11::module& m) { .def("__repr__", &vision::MattingResult::Str) .def("__str__", &vision::MattingResult::Str); + pybind11::class_(m, + "KeyPointDetectionResult") + .def(pybind11::init()) + .def_readwrite("keypoints", &vision::KeyPointDetectionResult::keypoints) + .def_readwrite("scores", &vision::KeyPointDetectionResult::scores) + .def_readwrite("num_joints", &vision::KeyPointDetectionResult::num_joints) + .def("__repr__", &vision::KeyPointDetectionResult::Str) + .def("__str__", &vision::KeyPointDetectionResult::Str); + BindDetection(m); BindClassification(m); BindSegmentation(m); @@ -102,6 +112,7 @@ void BindVision(pybind11::module& m) { BindFaceId(m); BindMatting(m); BindOcr(m); + BindKeyPointDetection(m); #ifdef ENABLE_VISION_VISUALIZE BindVisualize(m); #endif diff --git a/fastdeploy/vision/visualize/keypoint.cc b/fastdeploy/vision/visualize/keypoint.cc new file mode 100644 index 000000000..21be32561 --- /dev/null +++ b/fastdeploy/vision/visualize/keypoint.cc @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef ENABLE_VISION_VISUALIZE + +#include "fastdeploy/vision/visualize/visualize.h" + +namespace fastdeploy { +namespace vision { + +cv::Mat VisKeypointDetection(const cv::Mat& im, + const KeyPointDetectionResult& results, + float conf_threshold) { + const int edge[][2] = {{0, 1}, {0, 2}, {1, 3}, {2, 4}, {3, 5}, + {4, 6}, {5, 7}, {6, 8}, {7, 9}, {8, 10}, + {5, 11}, {6, 12}, {11, 13}, {12, 14}, {13, 15}, + {14, 16}, {11, 12}}; + auto colormap = GenerateColorMap(); + cv::Mat vis_img = im.clone(); + int detection_nums = results.keypoints.size() / 17; + for (int i = 0; i < detection_nums; i++){ + int index = i * 17; + bool is_over_threshold = true; + for (int j = 0; j < results.num_joints; j++) { + if (results.scores[index + j] < conf_threshold) { + is_over_threshold = false; + break; + } + } + if (is_over_threshold) { + for (int k = 0; k < results.num_joints; k++) { + int x_coord = int(results.keypoints[index + k][0]); + int y_coord = int(results.keypoints[index + k][1]); + cv::circle(vis_img, cv::Point2d(x_coord, y_coord), 1, + cv::Scalar(0, 0, 255), 2); + int x_start = int(results.keypoints[index + edge[k][0]][0]); + int y_start = int(results.keypoints[index + edge[k][0]][1]); + int x_end = int(results.keypoints[index + edge[k][1]][0]); + int y_end = int(results.keypoints[index + edge[k][1]][1]); + cv::line(vis_img, cv::Point2d(x_start, y_start), cv::Point2d(x_end, y_end), + colormap[k], 1); + } + } + + } + return vis_img; +} + +} // namespace vision +} // namespace fastdeploy +#endif diff --git a/fastdeploy/vision/visualize/visualize.h b/fastdeploy/vision/visualize/visualize.h index d8c7526d0..256ffba70 100644 --- a/fastdeploy/vision/visualize/visualize.h +++ b/fastdeploy/vision/visualize/visualize.h @@ -84,6 +84,9 @@ FASTDEPLOY_DECL cv::Mat SwapBackground(const cv::Mat& im, const cv::Mat& background, const SegmentationResult& result, int background_label); +FASTDEPLOY_DECL cv::Mat VisKeypointDetection(const cv::Mat& im, + const KeyPointDetectionResult& results, + float conf_threshold = 0.5f); } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/visualize/visualize_pybind.cc b/fastdeploy/vision/visualize/visualize_pybind.cc index 7b429a629..83ccb34f4 100644 --- a/fastdeploy/vision/visualize/visualize_pybind.cc +++ b/fastdeploy/vision/visualize/visualize_pybind.cc @@ -98,6 +98,17 @@ void BindVisualize(pybind11::module& m) { vision::Mat(vis_im).ShareWithTensor(&out); return TensorToPyArray(out); }) + .def_static( + "vis_keypoint_detection", + [](pybind11::array& im_data, vision::KeyPointDetectionResult& result, + float conf_threshold) { + auto im = PyArrayToCvMat(im_data); + auto vis_im = vision::VisKeypointDetection( + im, result, conf_threshold); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) .def_static( "vis_face_detection", [](pybind11::array& im_data, vision::FaceDetectionResult& result, diff --git a/python/fastdeploy/__init__.py b/python/fastdeploy/__init__.py index ccb740abd..e8a84d285 100644 --- a/python/fastdeploy/__init__.py +++ b/python/fastdeploy/__init__.py @@ -25,5 +25,6 @@ from .runtime import Runtime, RuntimeOption from .model import FastDeployModel from . import c_lib_wrap as C from . import vision +from . import pipeline from . import text from .download import download, download_and_decompress diff --git a/python/fastdeploy/pipeline/__init__.py b/python/fastdeploy/pipeline/__init__.py new file mode 100644 index 000000000..818b682d5 --- /dev/null +++ b/python/fastdeploy/pipeline/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import + +from .pptinypose import PPTinyPose diff --git a/python/fastdeploy/pipeline/pptinypose/__init__.py b/python/fastdeploy/pipeline/pptinypose/__init__.py new file mode 100644 index 000000000..36041f914 --- /dev/null +++ b/python/fastdeploy/pipeline/pptinypose/__init__.py @@ -0,0 +1,55 @@ +# # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. + +from __future__ import absolute_import +from ... import c_lib_wrap as C + + +class PPTinyPose(object): + def __init__(self, det_model=None, pptinypose_model=None): + """Set initialized detection model object and pptinypose model object + + :param det_model: (fastdeploy.vision.detection.PicoDet)Initialized detection model object + :param pptinypose_model: (fastdeploy.vision.keypointdetection.PPTinyPose)Initialized pptinypose model object + """ + assert det_model is not None or pptinypose_model is not None, "The det_model and pptinypose_model cannot be None." + self._pipeline = C.pipeline.PPTinyPose(det_model._model, + pptinypose_model._model) + + def predict(self, input_image): + """Predict the keypoint detection result for an input image + + :param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: KeyPointDetectionResult + """ + return self._pipeline.predict(input_image) + + @property + def detection_model_score_threshold(self): + """Atrribute of PPTinyPose pipeline model. Stating the score threshold for detectin model to filter bbox before inputting pptinypose model + + :return: value of detection_model_score_threshold(float) + """ + return self._pipeline.detection_model_score_threshold + + @detection_model_score_threshold.setter + def detection_model_score_threshold(self, value): + """Set attribute detection_model_score_threshold of PPTinyPose pipeline model. + + :param value: (float)The value to set use_dark + """ + assert isinstance( + value, float + ), "The value to set `detection_model_score_threshold` must be type of float." + self._pipeline.detection_model_score_threshold = value diff --git a/python/fastdeploy/vision/__init__.py b/python/fastdeploy/vision/__init__.py index 86f4f978f..28cd0564f 100644 --- a/python/fastdeploy/vision/__init__.py +++ b/python/fastdeploy/vision/__init__.py @@ -16,6 +16,7 @@ from __future__ import absolute_import from . import detection from . import classification from . import segmentation +from . import keypointdetection from . import matting from . import facedet diff --git a/python/fastdeploy/vision/keypointdetection/__init__.py b/python/fastdeploy/vision/keypointdetection/__init__.py new file mode 100644 index 000000000..11bc6185a --- /dev/null +++ b/python/fastdeploy/vision/keypointdetection/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from .pptinypose import PPTinyPose diff --git a/python/fastdeploy/vision/keypointdetection/pptinypose/__init__.py b/python/fastdeploy/vision/keypointdetection/pptinypose/__init__.py new file mode 100644 index 000000000..49692acd3 --- /dev/null +++ b/python/fastdeploy/vision/keypointdetection/pptinypose/__init__.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import logging +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class PPTinyPose(FastDeployModel): + def __init__(self, + model_file, + params_file, + config_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """load a PPTinyPose model exported by PaddleDetection. + + :param model_file: (str)Path of model file, e.g pptinypose/model.pdmodel + :param params_file: (str)Path of parameters file, e.g pptinypose/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param config_file: (str)Path of configuration file for deployment, e.g pptinypose/infer_cfg.yml + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPTinyPose, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "PPTinyPose model only support model format of ModelFormat.Paddle now." + self._model = C.vision.keypointdetection.PPTinyPose( + model_file, params_file, config_file, self._runtime_option, + model_format) + assert self.initialized, "PPTinyPose model initialize failed." + + def predict(self, input_image): + """Detect keypoints in an input image + + :param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: KeyPointDetectionResult + """ + assert input_image is not None, "The input image data is None." + return self._model.predict(input_image) + + @property + def use_dark(self): + """Atrribute of PPTinyPose model. Stating whether using Distribution-Aware Coordinate Representation for Human Pose Estimation(DARK for short) in postprocess, default is True + + :return: value of use_dark(bool) + """ + return self._model.use_dark + + @use_dark.setter + def use_dark(self, value): + """Set attribute use_dark of PPTinyPose model. + + :param value: (bool)The value to set use_dark + """ + assert isinstance( + value, bool), "The value to set `use_dark` must be type of bool." + self._model.use_dark = value diff --git a/python/fastdeploy/vision/visualize/__init__.py b/python/fastdeploy/vision/visualize/__init__.py index 60912d66e..95b8e816a 100644 --- a/python/fastdeploy/vision/visualize/__init__.py +++ b/python/fastdeploy/vision/visualize/__init__.py @@ -26,6 +26,11 @@ def vis_detection(im_data, line_size, font_size) +def vis_keypoint_detection(im_data, keypoint_det_result, conf_threshold=0.5): + return C.vision.Visualize.vis_keypoint_detection( + im_data, keypoint_det_result, conf_threshold) + + def vis_face_detection(im_data, face_det_result, line_size=1, font_size=0.5): return C.vision.vis_face_detection(im_data, face_det_result, line_size, font_size) diff --git a/tests/eval_example/test_pptinypose.py b/tests/eval_example/test_pptinypose.py new file mode 100644 index 000000000..0da899293 --- /dev/null +++ b/tests/eval_example/test_pptinypose.py @@ -0,0 +1,100 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np + + +def test_keypointdetection_pptinypose(): + pp_tinypose_model_url = "https://bj.bcebos.com/fastdeploy/tests/PP_TinyPose_256x192_test.tgz" + fd.download_and_decompress(pp_tinypose_model_url, ".") + model_path = "./PP_TinyPose_256x192_test" + # 配置runtime，加载模型 + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + image_file = os.path.join(model_path, "hrnet_demo.jpg") + baseline_file = os.path.join(model_path, "baseline.npy") + model = fd.vision.keypointdetection.PPTinyPose( + model_file, params_file, config_file, runtime_option=runtime_option) + + # 预测图片关键点 + im = cv2.imread(image_file) + result = model.predict(im) + result = np.concatenate( + (np.array(result.keypoints), np.array(result.scores)[:, np.newaxis]), + axis=1) + baseline = np.load(baseline_file) + diff = np.fabs(result - np.array(baseline)) + thres = 1e-05 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) + print("No diff") + + +def test_keypointdetection_det_keypoint_unite(): + det_keypoint_unite_model_url = "https://bj.bcebos.com/fastdeploy/tests/PicoDet_320x320_TinyPose_256x192_test.tgz" + fd.download_and_decompress(det_keypoint_unite_model_url, ".") + model_path = "./PicoDet_320x320_TinyPose_256x192_test" + # 配置runtime，加载模型 + runtime_option = fd.RuntimeOption() + tinypose_model_file = os.path.join( + model_path, "PP_TinyPose_256x192_infer/model.pdmodel") + tinypose_params_file = os.path.join( + model_path, "PP_TinyPose_256x192_infer/model.pdiparams") + tinypose_config_file = os.path.join( + model_path, "PP_TinyPose_256x192_infer/infer_cfg.yml") + picodet_model_file = os.path.join( + model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/model.pdmodel") + picodet_params_file = os.path.join( + model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/model.pdiparams") + picodet_config_file = os.path.join( + model_path, "PP_PicoDet_V2_S_Pedestrian_320x320_infer/infer_cfg.yml") + image_file = os.path.join(model_path, "000000018491.jpg") + # image_file = os.path.join(model_path, "hrnet_demo.jpg") + + baseline_file = os.path.join(model_path, "baseline.npy") + + tinypose_model = fd.vision.keypointdetection.PPTinyPose( + tinypose_model_file, + tinypose_params_file, + tinypose_config_file, + runtime_option=runtime_option) + + det_model = fd.vision.detection.PicoDet( + picodet_model_file, + picodet_params_file, + picodet_config_file, + runtime_option=runtime_option) + + # 预测图片关键点 + im = cv2.imread(image_file) + pipeline = fd.pipeline.PPTinyPose(det_model, tinypose_model) + pipeline.detection_model_score_threshold = 0.5 + result = pipeline.predict(im) + print(result) + result = np.concatenate( + (np.array(result.keypoints), np.array(result.scores)[:, np.newaxis]), + axis=1) + print(result) + np.save("baseline.npy", result) + baseline = np.load(baseline_file) + diff = np.fabs(result - np.array(baseline)) + thres = 1e-05 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) + print("No diff")