From c6f615e3711990f40aa28ad9f273fd8479d29e4b Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 16 Sep 2022 19:06:43 +0800 Subject: [PATCH] Add doc shows how to change inference backend (#244) * Add doc shows how to change inference backend * Update README.md --- examples/vision/README.md | 2 + .../classification/paddleclas/cpp/README.md | 1 + .../paddleclas/python/README.md | 1 + .../detection/paddledetection/cpp/README.md | 1 + .../paddledetection/python/README.md | 1 + .../detection/scaledyolov4/cpp/README.md | 1 + .../detection/scaledyolov4/python/README.md | 1 + examples/vision/detection/yolor/cpp/README.md | 1 + .../vision/detection/yolov5/cpp/README.md | 1 + .../vision/detection/yolov5/python/README.md | 1 + .../vision/detection/yolov5lite/cpp/README.md | 1 + .../detection/yolov5lite/python/README.md | 1 + .../vision/detection/yolov6/cpp/README.md | 1 + .../vision/detection/yolov6/python/README.md | 1 + .../vision/detection/yolov7/cpp/README.md | 1 + .../vision/detection/yolov7/python/README.md | 1 + examples/vision/detection/yolox/cpp/README.md | 1 + .../vision/detection/yolox/python/README.md | 1 + examples/vision/facedet/scrfd/cpp/README.md | 1 + .../vision/facedet/scrfd/python/README.md | 1 + .../vision/faceid/insightface/cpp/README.md | 1 + .../faceid/insightface/python/README.md | 1 + examples/vision/how_to_change_backend.md | 45 +++++++++++++++++++ examples/vision/matting/modnet/cpp/README.md | 1 + .../vision/matting/modnet/python/README.md | 1 + .../vision/matting/ppmatting/cpp/README.md | 1 + .../vision/matting/ppmatting/python/README.md | 1 + .../vision/ocr/PPOCRSystemv2/cpp/README.md | 1 + .../vision/ocr/PPOCRSystemv2/python/README.md | 3 +- .../segmentation/paddleseg/cpp/README.md | 1 + .../segmentation/paddleseg/python/README.md | 1 + 31 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 examples/vision/how_to_change_backend.md diff --git a/examples/vision/README.md b/examples/vision/README.md index 03303bcd2..553817d78 100644 --- a/examples/vision/README.md +++ b/examples/vision/README.md @@ -24,3 +24,5 @@ FastDeploy针对飞桨的视觉套件，以及外部热门模型，提供端到 - 加载模型 - 调用`predict`接口 + +FastDeploy在各视觉模型部署时，也支持一键切换后端推理引擎，详情参阅[如何切换模型推理引擎](how_to_change_backend.md)。 diff --git a/examples/vision/classification/paddleclas/cpp/README.md b/examples/vision/classification/paddleclas/cpp/README.md index 80ebad983..27dccaab8 100644 --- a/examples/vision/classification/paddleclas/cpp/README.md +++ b/examples/vision/classification/paddleclas/cpp/README.md @@ -77,3 +77,4 @@ PaddleClas模型加载和初始化，其中model_file, params_file为训练模 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/classification/paddleclas/python/README.md b/examples/vision/classification/paddleclas/python/README.md index 6a00fb9fb..c25f96f81 100644 --- a/examples/vision/classification/paddleclas/python/README.md +++ b/examples/vision/classification/paddleclas/python/README.md @@ -72,3 +72,4 @@ PaddleClas模型加载和初始化，其中model_file, params_file为训练模 - [PaddleClas 模型介绍](..) - [PaddleClas C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/paddledetection/cpp/README.md b/examples/vision/detection/paddledetection/cpp/README.md index 444fa829a..f34a64d16 100644 --- a/examples/vision/detection/paddledetection/cpp/README.md +++ b/examples/vision/detection/paddledetection/cpp/README.md @@ -77,3 +77,4 @@ PaddleDetection PPYOLOE模型加载和初始化，其中model_file为导出的ON - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/paddledetection/python/README.md b/examples/vision/detection/paddledetection/python/README.md index ad25f11d5..c0da76ab9 100644 --- a/examples/vision/detection/paddledetection/python/README.md +++ b/examples/vision/detection/paddledetection/python/README.md @@ -74,3 +74,4 @@ PaddleDetection中各个模型，包括PPYOLOE/PicoDet/PaddleYOLOX/YOLOv3/PPYOLO - [PaddleDetection 模型介绍](..) - [PaddleDetection C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/scaledyolov4/cpp/README.md b/examples/vision/detection/scaledyolov4/cpp/README.md index 713c5a69a..6bf3a5e7b 100644 --- a/examples/vision/detection/scaledyolov4/cpp/README.md +++ b/examples/vision/detection/scaledyolov4/cpp/README.md @@ -88,3 +88,4 @@ ScaledYOLOv4模型加载和初始化，其中model_file为导出的ONNX模型格 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/scaledyolov4/python/README.md b/examples/vision/detection/scaledyolov4/python/README.md index 0db010447..9c38e68c4 100644 --- a/examples/vision/detection/scaledyolov4/python/README.md +++ b/examples/vision/detection/scaledyolov4/python/README.md @@ -81,3 +81,4 @@ ScaledYOLOv4模型加载和初始化，其中model_file为导出的ONNX模型格 - [ScaledYOLOv4 模型介绍](..) - [ScaledYOLOv4 C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolor/cpp/README.md b/examples/vision/detection/yolor/cpp/README.md index d8b7a5b25..4564c78b7 100644 --- a/examples/vision/detection/yolor/cpp/README.md +++ b/examples/vision/detection/yolor/cpp/README.md @@ -88,3 +88,4 @@ YOLOR模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov5/cpp/README.md b/examples/vision/detection/yolov5/cpp/README.md index 2102f013e..67dfa4a10 100644 --- a/examples/vision/detection/yolov5/cpp/README.md +++ b/examples/vision/detection/yolov5/cpp/README.md @@ -88,3 +88,4 @@ YOLOv5模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov5/python/README.md b/examples/vision/detection/yolov5/python/README.md index 0e3b125be..25a32bddb 100644 --- a/examples/vision/detection/yolov5/python/README.md +++ b/examples/vision/detection/yolov5/python/README.md @@ -78,3 +78,4 @@ YOLOv5模型加载和初始化，其中model_file为导出的ONNX模型格式 - [YOLOv5 模型介绍](..) - [YOLOv5 C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov5lite/cpp/README.md b/examples/vision/detection/yolov5lite/cpp/README.md index 3eab6dd48..df889b6d2 100644 --- a/examples/vision/detection/yolov5lite/cpp/README.md +++ b/examples/vision/detection/yolov5lite/cpp/README.md @@ -88,3 +88,4 @@ YOLOv5Lite模型加载和初始化，其中model_file为导出的ONNX模型格 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov5lite/python/README.md b/examples/vision/detection/yolov5lite/python/README.md index ad81b4d5e..8270b596d 100644 --- a/examples/vision/detection/yolov5lite/python/README.md +++ b/examples/vision/detection/yolov5lite/python/README.md @@ -78,3 +78,4 @@ YOLOv5Lite模型加载和初始化，其中model_file为导出的ONNX模型格 - [YOLOv5Lite 模型介绍](..) - [YOLOv5Lite C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov6/cpp/README.md b/examples/vision/detection/yolov6/cpp/README.md index af526a907..b335ad068 100644 --- a/examples/vision/detection/yolov6/cpp/README.md +++ b/examples/vision/detection/yolov6/cpp/README.md @@ -88,3 +88,4 @@ YOLOv6模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov6/python/README.md b/examples/vision/detection/yolov6/python/README.md index 23fce1283..5da487f3b 100644 --- a/examples/vision/detection/yolov6/python/README.md +++ b/examples/vision/detection/yolov6/python/README.md @@ -80,3 +80,4 @@ YOLOv6模型加载和初始化，其中model_file为导出的ONNX模型格式 - [YOLOv6 模型介绍](..) - [YOLOv6 C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov7/cpp/README.md b/examples/vision/detection/yolov7/cpp/README.md index dd8ec5186..02137cd2b 100644 --- a/examples/vision/detection/yolov7/cpp/README.md +++ b/examples/vision/detection/yolov7/cpp/README.md @@ -88,3 +88,4 @@ YOLOv7模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolov7/python/README.md b/examples/vision/detection/yolov7/python/README.md index 2e2e0deb3..e9b796576 100644 --- a/examples/vision/detection/yolov7/python/README.md +++ b/examples/vision/detection/yolov7/python/README.md @@ -78,3 +78,4 @@ YOLOv7模型加载和初始化，其中model_file为导出的ONNX模型格式 - [YOLOv7 模型介绍](..) - [YOLOv7 C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolox/cpp/README.md b/examples/vision/detection/yolox/cpp/README.md index e7bf5bd8c..7523d6582 100644 --- a/examples/vision/detection/yolox/cpp/README.md +++ b/examples/vision/detection/yolox/cpp/README.md @@ -87,3 +87,4 @@ YOLOX模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/detection/yolox/python/README.md b/examples/vision/detection/yolox/python/README.md index 961bd51ef..59316356e 100644 --- a/examples/vision/detection/yolox/python/README.md +++ b/examples/vision/detection/yolox/python/README.md @@ -76,3 +76,4 @@ YOLOX模型加载和初始化，其中model_file为导出的ONNX模型格式 - [YOLOX 模型介绍](..) - [YOLOX C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/facedet/scrfd/cpp/README.md b/examples/vision/facedet/scrfd/cpp/README.md index f014f26b2..a7cb17ff5 100644 --- a/examples/vision/facedet/scrfd/cpp/README.md +++ b/examples/vision/facedet/scrfd/cpp/README.md @@ -92,3 +92,4 @@ SCRFD模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/facedet/scrfd/python/README.md b/examples/vision/facedet/scrfd/python/README.md index 7256fe9f4..e4ec92b32 100644 --- a/examples/vision/facedet/scrfd/python/README.md +++ b/examples/vision/facedet/scrfd/python/README.md @@ -81,3 +81,4 @@ SCRFD模型加载和初始化，其中model_file为导出的ONNX模型格式 - [SCRFD 模型介绍](..) - [SCRFD C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/faceid/insightface/cpp/README.md b/examples/vision/faceid/insightface/cpp/README.md index fe0f184b8..7ea48bac5 100644 --- a/examples/vision/faceid/insightface/cpp/README.md +++ b/examples/vision/faceid/insightface/cpp/README.md @@ -130,3 +130,4 @@ VPL模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/faceid/insightface/python/README.md b/examples/vision/faceid/insightface/python/README.md index 8fbfcd12e..079238de6 100644 --- a/examples/vision/faceid/insightface/python/README.md +++ b/examples/vision/faceid/insightface/python/README.md @@ -97,3 +97,4 @@ ArcFace模型加载和初始化，其中model_file为导出的ONNX模型格式 - [InsightFace 模型介绍](..) - [InsightFace C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/how_to_change_backend.md b/examples/vision/how_to_change_backend.md new file mode 100644 index 000000000..6a90ed095 --- /dev/null +++ b/examples/vision/how_to_change_backend.md @@ -0,0 +1,45 @@ +# 如何切换模型推理后端 + +FastDeploy中各视觉模型可支持多种后端，包括 +- OpenVINO (支持Paddle/ONNX两种格式模型, 仅支持CPU上推理) +- ONNX Runtime (支持Paddle/ONNX两种格式模型， 支持CPU/GPU） +- TensorRT (支持Paddle/ONNX两种格式模型，仅支持GPU上推理) +- Paddle Inference(支持Paddle格式模型， 支持CPU/GPU) + +所有模型切换后端方式均通过RuntimeOption进行切换， + +**Python** +``` +import fastdeploy as fd +option = fd.RuntimeOption() + +# 切换使用CPU/GPU +option.use_cpu() +option.use_gpu() + +# 切换不同后端 +option.use_paddle_backend() # Paddle Inference +option.use_trt_backend() # TensorRT +option.use_openvino_backend() # OpenVINO +option.use_ort_backend() # ONNX Runtime + +``` + +**C++** +``` +fastdeploy::RuntimeOption option; + +# 切换使用CPU/GPU +option.UseCpu(); +option.UseGpu(); + +# 切换不同后端 +option.UsePaddleBackend(); // Paddle Inference +option.UseTrtBackend(); // TensorRT +option.UseOpenVINOBackend(); // OpenVINO +option.UseOrtBackend(); // ONNX Runtime +``` + +具体示例可参阅`FastDeploy/examples/vision`下不同模型的python或c++推理代码 + +更多`RuntimeOption`的配置方式参考[RuntimeOption API文档](../../docs/api/runtime/runtime_option.md) diff --git a/examples/vision/matting/modnet/cpp/README.md b/examples/vision/matting/modnet/cpp/README.md index 3ae36c99b..ea6d243b8 100644 --- a/examples/vision/matting/modnet/cpp/README.md +++ b/examples/vision/matting/modnet/cpp/README.md @@ -95,3 +95,4 @@ MODNet模型加载和初始化，其中model_file为导出的ONNX模型格式。 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/matting/modnet/python/README.md b/examples/vision/matting/modnet/python/README.md index f22b77982..65c5f0649 100644 --- a/examples/vision/matting/modnet/python/README.md +++ b/examples/vision/matting/modnet/python/README.md @@ -84,3 +84,4 @@ MODNet模型加载和初始化，其中model_file为导出的ONNX模型格式 - [MODNet 模型介绍](..) - [MODNet C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/matting/ppmatting/cpp/README.md b/examples/vision/matting/ppmatting/cpp/README.md index 726d8fcb3..4d79bcde8 100644 --- a/examples/vision/matting/ppmatting/cpp/README.md +++ b/examples/vision/matting/ppmatting/cpp/README.md @@ -88,3 +88,4 @@ PPMatting模型加载和初始化，其中model_file为导出的Paddle模型格 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/matting/ppmatting/python/README.md b/examples/vision/matting/ppmatting/python/README.md index 427a3dcd4..b156f3675 100644 --- a/examples/vision/matting/ppmatting/python/README.md +++ b/examples/vision/matting/ppmatting/python/README.md @@ -75,3 +75,4 @@ PPMatting模型加载和初始化，其中model_file, params_file以及config_fi - [PPMatting 模型介绍](..) - [PPMatting C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/ocr/PPOCRSystemv2/cpp/README.md b/examples/vision/ocr/PPOCRSystemv2/cpp/README.md index ed88a7a1d..2c4aae5fc 100644 --- a/examples/vision/ocr/PPOCRSystemv2/cpp/README.md +++ b/examples/vision/ocr/PPOCRSystemv2/cpp/README.md @@ -147,3 +147,4 @@ Recognizer类初始化时,需要在label_path参数中,输入识别模型所需 - [PPOCR 系列模型介绍](../../) - [PPOCRv2 Python部署](../python) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/ocr/PPOCRSystemv2/python/README.md b/examples/vision/ocr/PPOCRSystemv2/python/README.md index 02dde9cfc..7917b02a9 100644 --- a/examples/vision/ocr/PPOCRSystemv2/python/README.md +++ b/examples/vision/ocr/PPOCRSystemv2/python/README.md @@ -124,4 +124,5 @@ Recognizer类初始化时,需要在rec_label_file参数中,输入识别模型所 - [PPOCR 系列模型介绍](../../) - [PPOCRv2 C++部署](../cpp) -- [模型预测结果说明](../../../../../docs/api/vision_results/) \ No newline at end of file +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/segmentation/paddleseg/cpp/README.md b/examples/vision/segmentation/paddleseg/cpp/README.md index 24497fef6..345a708e8 100644 --- a/examples/vision/segmentation/paddleseg/cpp/README.md +++ b/examples/vision/segmentation/paddleseg/cpp/README.md @@ -88,3 +88,4 @@ PaddleSegModel模型加载和初始化，其中model_file为导出的Paddle模 - [模型介绍](../../) - [Python部署](../python) - [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md) diff --git a/examples/vision/segmentation/paddleseg/python/README.md b/examples/vision/segmentation/paddleseg/python/README.md index 786127695..5aaa4f5a6 100644 --- a/examples/vision/segmentation/paddleseg/python/README.md +++ b/examples/vision/segmentation/paddleseg/python/README.md @@ -76,3 +76,4 @@ PaddleSeg模型加载和初始化，其中model_file, params_file以及config_fi - [PaddleSeg 模型介绍](..) - [PaddleSeg C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../how_to_change_backend.md)