[Serving] Simple serving YOLOv5 and PP-OCRv3 example, add uvicorn to fastdeploy tools (#986)

* ppocrv3 simple serving

* add uvicorn to fd tools

* update ppdet simple serving readme

* yolov5 simple serving

* not import simple serving by default

* remove config from envs

* update comment

examples/vision/detection/paddledetection/python/serving/README_CN.md

@@ -17,16 +17,9 @@ cd FastDeploy/examples/vision/detection/paddledetection/python/serving
 wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
 tar xvf ppyoloe_crn_l_300e_coco.tgz
 
-# 安装uvicorn
+# 启动服务，可修改server.py中的配置项来指定硬件、后端等
-pip install uvicorn
+# 可通过--host、--port指定IP和端口号
-
+fastdeploy simple_serving --app server:app
-# 启动服务，可选择是否使用GPU和TensorRT，可根据uvicorn --help配置IP、端口号等
-# CPU
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=cpu uvicorn server:app
-# GPU
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu uvicorn server:app
-# GPU上使用TensorRT （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待）
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu USE_TRT=true uvicorn server:app
 ```
 
 客户端：

examples/vision/detection/paddledetection/python/serving/README_EN.md

@@ -17,17 +17,9 @@ cd FastDeploy/examples/vision/detection/paddledetection/python/serving
 wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
 tar xvf ppyoloe_crn_l_300e_coco.tgz
 
-# Install uvicorn
+# Launch server, change the configurations in server.py to select hardware, backend, etc.
-pip install uvicorn
+# and use --host, --port to specify IP and port
-
+fastdeploy simple_serving --app server:app
-# Launch server, it's configurable to use GPU and TensorRT,
-# and run 'uvicorn --help' to check how to specify IP and port, etc.
-# CPU
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=cpu uvicorn server:app
-# GPU
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu uvicorn server:app
-# GPU and TensorRT
-MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu USE_TRT=true uvicorn server:app
 ```
 
 Client:

examples/vision/detection/paddledetection/python/serving/client.py

@@ -1,20 +1,15 @@
 import requests
 import json
 import cv2
-import base64
 import fastdeploy as fd
+from fastdeploy.serving.utils import cv2_to_base64
 
 if __name__ == '__main__':
     url = "http://127.0.0.1:8000/fd/ppyoloe"
     headers = {"Content-Type": "application/json"}
 
     im = cv2.imread("000000014439.jpg")
-    data = {
+    data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}}
-        "data": {
-            "image": fd.serving.utils.cv2_to_base64(im)
-        },
-        "parameters": {}
-    }
 
     resp = requests.post(url=url, headers=headers, data=json.dumps(data))
     if resp.status_code == 200:

examples/vision/detection/paddledetection/python/serving/server.py

@@ -1,18 +1,16 @@
 import fastdeploy as fd
+from fastdeploy.serving.server import SimpleServer
 import os
 import logging
 
 logging.getLogger().setLevel(logging.INFO)
 
-# Get arguments from envrionment variables
+# Configurations
-model_dir = os.environ.get('MODEL_DIR')
+model_dir = 'ppyoloe_crn_l_300e_coco'
-device = os.environ.get('DEVICE', 'cpu')
+device = 'cpu'
-use_trt = os.environ.get('USE_TRT', False)
+use_trt = False
-
-# Prepare model, download from hub or use local dir
-if model_dir is None:
-    model_dir = fd.download_model(name='ppyoloe_crn_l_300e_coco')
 
+# Prepare model
 model_file = os.path.join(model_dir, "model.pdmodel")
 params_file = os.path.join(model_dir, "model.pdiparams")
 config_file = os.path.join(model_dir, "infer_cfg.yml")
@@ -33,7 +31,7 @@ model_instance = fd.vision.detection.PPYOLOE(
     runtime_option=option)
 
 # Create server, setup REST API
-app = fd.serving.SimpleServer()
+app = SimpleServer()
 app.register(
     task_name="fd/ppyoloe",
     model_handler=fd.serving.handler.VisionModelHandler,

examples/vision/detection/yolov5/python/serving/README.md

@@ -0,0 +1 @@
+README_CN.md

examples/vision/detection/yolov5/python/serving/README_CN.md

@@ -0,0 +1,36 @@
+简体中文 | [English](README_EN.md)
+
+# YOLOv5 Python轻量服务化部署示例
+
+在部署前，需确认以下两个步骤
+
+- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)  
+- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
+
+服务端：
+```bash
+# 下载部署示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/detection/yolov5/python/serving
+
+# 下载模型文件
+wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s_infer.tar
+tar xvf yolov5s_infer.tar
+
+# 启动服务，可修改server.py中的配置项来指定硬件、后端等
+# 可通过--host、--port指定IP和端口号
+fastdeploy simple_serving --app server:app
+```
+
+客户端：
+```bash
+# 下载部署示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/detection/yolov5/python/serving
+
+# 下载测试图片
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+
+# 请求服务，获取推理结果（如有必要，请修改脚本中的IP和端口号）
+python client.py
+```

examples/vision/detection/yolov5/python/serving/README_EN.md

@@ -0,0 +1,36 @@
+English | [简体中文](README_CN.md)
+
+# YOLOv5 Python Simple Serving Demo
+
+
+## Environment
+
+- 1. Prepare environment and install FastDeploy Python whl, refer to [download_prebuilt_libraries](../../../../../../docs/en/build_and_install/download_prebuilt_libraries.md)
+
+Server:
+```bash
+# Download demo code
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/detection/yolov5/python/serving
+
+# Download model
+wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s_infer.tar
+tar xvf yolov5s_infer.tar
+
+# Launch server, change the configurations in server.py to select hardware, backend, etc.
+# and use --host, --port to specify IP and port
+fastdeploy simple_serving --app server:app
+```
+
+Client:
+```bash
+# Download demo code
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/detection/yolov5/python/serving
+
+# Download test image
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+
+# Send request and get inference result (Please adapt the IP and port if necessary)
+python client.py
+```

examples/vision/detection/yolov5/python/serving/client.py

@@ -0,0 +1,23 @@
+import requests
+import json
+import cv2
+import fastdeploy as fd
+from fastdeploy.serving.utils import cv2_to_base64
+
+if __name__ == '__main__':
+    url = "http://127.0.0.1:8000/fd/yolov5s"
+    headers = {"Content-Type": "application/json"}
+
+    im = cv2.imread("000000014439.jpg")
+    data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}}
+
+    resp = requests.post(url=url, headers=headers, data=json.dumps(data))
+    if resp.status_code == 200:
+        r_json = json.loads(resp.json()["result"])
+        det_result = fd.vision.utils.json_to_detection(r_json)
+        vis_im = fd.vision.vis_detection(im, det_result, score_threshold=0.5)
+        cv2.imwrite("visualized_result.jpg", vis_im)
+        print("Visualized result save in ./visualized_result.jpg")
+    else:
+        print("Error code:", resp.status_code)
+        print(resp.text)

examples/vision/detection/yolov5/python/serving/server.py

@@ -0,0 +1,38 @@
+import fastdeploy as fd
+from fastdeploy.serving.server import SimpleServer
+import os
+import logging
+
+logging.getLogger().setLevel(logging.INFO)
+
+# Configurations
+model_dir = 'yolov5s_infer'
+device = 'cpu'
+use_trt = False
+
+# Prepare model
+model_file = os.path.join(model_dir, "model.pdmodel")
+params_file = os.path.join(model_dir, "model.pdiparams")
+
+# Setup runtime option to select hardware, backend, etc.
+option = fd.RuntimeOption()
+if device.lower() == 'gpu':
+    option.use_gpu()
+if use_trt:
+    option.use_trt_backend()
+    option.set_trt_input_shape("images", [1, 3, 640, 640])
+    option.set_trt_cache_file('yolov5s.trt')
+
+# Create model instance
+model_instance = fd.vision.detection.YOLOv5(
+    model_file,
+    params_file,
+    runtime_option=option,
+    model_format=fd.ModelFormat.PADDLE)
+
+# Create server, setup REST API
+app = SimpleServer()
+app.register(
+    task_name="fd/yolov5s",
+    model_handler=fd.serving.handler.VisionModelHandler,
+    predictor=model_instance)

examples/vision/ocr/PP-OCRv3/python/serving/README.md

@@ -0,0 +1 @@
+README_CN.md

examples/vision/ocr/PP-OCRv3/python/serving/README_CN.md

@@ -0,0 +1,44 @@
+简体中文 | [English](README_EN.md)
+
+# PP-OCRv3 Python轻量服务化部署示例
+
+在部署前，需确认以下两个步骤
+
+- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)  
+- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
+
+服务端：
+```bash
+# 下载部署示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving
+
+# 下载模型和字典文件
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar
+tar xvf ch_PP-OCRv3_det_infer.tar
+
+wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar
+tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar
+
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar
+tar xvf ch_PP-OCRv3_rec_infer.tar
+
+wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt
+
+# 启动服务，可修改server.py中的配置项来指定硬件、后端等
+# 可通过--host、--port指定IP和端口号
+fastdeploy simple_serving --app server:app
+```
+
+客户端：
+```bash
+# 下载部署示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving
+
+# 下载测试图片
+wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg
+
+# 请求服务，获取推理结果（如有必要，请修改脚本中的IP和端口号）
+python client.py
+```

examples/vision/ocr/PP-OCRv3/python/serving/README_EN.md

@@ -0,0 +1,43 @@
+English | [简体中文](README_CN.md)
+
+# PP-OCRv3 Python Simple Serving Demo
+
+## Environment
+
+- 1. Prepare environment and install FastDeploy Python whl, refer to [download_prebuilt_libraries](../../../../../../docs/en/build_and_install/download_prebuilt_libraries.md)
+
+Server:
+```bash
+# Download demo code
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving
+
+# Download models and labels
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar
+tar xvf ch_PP-OCRv3_det_infer.tar
+
+wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar
+tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar
+
+wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar
+tar xvf ch_PP-OCRv3_rec_infer.tar
+
+wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt
+
+# Launch server, change the configurations in server.py to select hardware, backend, etc.
+# and use --host, --port to specify IP and port
+fastdeploy simple_serving --app server:app
+```
+
+Client:
+```bash
+# Download demo code
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving
+
+# Download test image
+wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg
+
+# Send request and get inference result (Please adapt the IP and port if necessary)
+python client.py
+```

examples/vision/ocr/PP-OCRv3/python/serving/client.py

@@ -0,0 +1,24 @@
+import requests
+import json
+import cv2
+import fastdeploy as fd
+from fastdeploy.serving.utils import cv2_to_base64
+
+if __name__ == '__main__':
+    url = "http://127.0.0.1:8000/fd/ppocrv3"
+    headers = {"Content-Type": "application/json"}
+
+    im = cv2.imread("12.jpg")
+    data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}}
+
+    resp = requests.post(url=url, headers=headers, data=json.dumps(data))
+    if resp.status_code == 200:
+        r_json = json.loads(resp.json()["result"])
+        print(r_json)
+        ocr_result = fd.vision.utils.json_to_ocr(r_json)
+        vis_im = fd.vision.vis_ppocr(im, ocr_result)
+        cv2.imwrite("visualized_result.jpg", vis_im)
+        print("Visualized result save in ./visualized_result.jpg")
+    else:
+        print("Error code:", resp.status_code)
+        print(resp.text)

examples/vision/ocr/PP-OCRv3/python/serving/server.py

@@ -0,0 +1,80 @@
+import fastdeploy as fd
+from fastdeploy.serving.server import SimpleServer
+import os
+import logging
+
+logging.getLogger().setLevel(logging.INFO)
+
+# Configurations
+det_model_dir = 'ch_PP-OCRv3_det_infer'
+cls_model_dir = 'ch_ppocr_mobile_v2.0_cls_infer'
+rec_model_dir = 'ch_PP-OCRv3_rec_infer'
+rec_label_file = 'ppocr_keys_v1.txt'
+device = 'cpu'
+# backend: ['paddle', 'trt'], you can also use other backends, but need to modify
+# the runtime option below
+backend = 'paddle'
+
+# Prepare models
+# Detection model
+det_model_file = os.path.join(det_model_dir, "inference.pdmodel")
+det_params_file = os.path.join(det_model_dir, "inference.pdiparams")
+# Classification model
+cls_model_file = os.path.join(cls_model_dir, "inference.pdmodel")
+cls_params_file = os.path.join(cls_model_dir, "inference.pdiparams")
+# Recognition model
+rec_model_file = os.path.join(rec_model_dir, "inference.pdmodel")
+rec_params_file = os.path.join(rec_model_dir, "inference.pdiparams")
+
+# Setup runtime option to select hardware, backend, etc.
+option = fd.RuntimeOption()
+if device.lower() == 'gpu':
+    option.use_gpu()
+if backend == 'trt':
+    option.use_trt_backend()
+else:
+    option.use_paddle_infer_backend()
+
+det_option = option
+det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640],
+                               [1, 3, 960, 960])
+
+# det_option.set_trt_cache_file("det_trt_cache.trt")
+print(det_model_file, det_params_file)
+det_model = fd.vision.ocr.DBDetector(
+    det_model_file, det_params_file, runtime_option=det_option)
+
+cls_batch_size = 1
+rec_batch_size = 6
+
+cls_option = option
+cls_option.set_trt_input_shape("x", [1, 3, 48, 10],
+                               [cls_batch_size, 3, 48, 320],
+                               [cls_batch_size, 3, 48, 1024])
+
+# cls_option.set_trt_cache_file("cls_trt_cache.trt")
+cls_model = fd.vision.ocr.Classifier(
+    cls_model_file, cls_params_file, runtime_option=cls_option)
+
+rec_option = option
+rec_option.set_trt_input_shape("x", [1, 3, 48, 10],
+                               [rec_batch_size, 3, 48, 320],
+                               [rec_batch_size, 3, 48, 2304])
+
+# rec_option.set_trt_cache_file("rec_trt_cache.trt")
+rec_model = fd.vision.ocr.Recognizer(
+    rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option)
+
+# Create PPOCRv3 pipeline
+ppocr_v3 = fd.vision.ocr.PPOCRv3(
+    det_model=det_model, cls_model=cls_model, rec_model=rec_model)
+
+ppocr_v3.cls_batch_size = cls_batch_size
+ppocr_v3.rec_batch_size = rec_batch_size
+
+# Create server, setup REST API
+app = SimpleServer()
+app.register(
+    task_name="fd/ppocrv3",
+    model_handler=fd.serving.handler.VisionModelHandler,
+    predictor=ppocr_v3)

python/fastdeploy/serving/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from __future__ import absolute_import
-
-from .server import SimpleServer

python/requirements.txt

@@ -3,6 +3,6 @@ requests
 tqdm
 numpy
 opencv-python
-fastdeploy-tools==0.0.1
+fastdeploy-tools>=0.0.1
 pyyaml
 fastapi

tools/common_tools/common_tools.py

@@ -1,10 +1,12 @@
 import argparse
 import ast
+import uvicorn
 
 
 def argsparser():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('tools', choices=['compress', 'convert'])
+    parser.add_argument(
+        'tools', choices=['compress', 'convert', 'simple_serving'])
     ## argumentments for auto compression
     parser.add_argument(
         '--config_path',
@@ -69,6 +71,19 @@ def argsparser():
         type=ast.literal_eval,
         default=False,
         help="Turn on code optimization")
+    ## arguments for simple serving
+    parser.add_argument(
+        "--app",
+        type=str,
+        default="server:app",
+        help="Simple serving app string")
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="127.0.0.1",
+        help="Simple serving host IP address")
+    parser.add_argument(
+        "--port", type=int, default=8000, help="Simple serving host port")
     ## arguments for other tools
     return parser
 
@@ -116,6 +131,8 @@ def main():
         except ImportError:
             print(
                 "Model convert failed! Please check if you have installed it!")
+    if args.tools == "simple_serving":
+        uvicorn.run(args.app, host=args.host, port=args.port, app_dir='.')
 
 
 if __name__ == '__main__':

tools/setup.py

@@ -3,13 +3,16 @@ import setuptools
 long_description = "fastdeploy-tools is a toolkit for FastDeploy, including auto compression .etc.\n\n"
 long_description += "Usage of auto compression: fastdeploy compress --config_path=./yolov7_tiny_qat_dis.yaml --method='QAT' --save_dir='./v7_qat_outmodel/' \n"
 
+install_requires = ['uvicorn==0.16.0']
+
 setuptools.setup(
     name="fastdeploy-tools",  # name of package
-    version="0.0.1",  #version of package
+    version="0.0.2",  #version of package
     description="A toolkit for FastDeploy.",
     long_description=long_description,
     long_description_content_type="text/plain",
     packages=setuptools.find_packages(),
+    install_requires=install_requires,
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: Apache Software License",