diff --git a/examples/vision/detection/paddledetection/python/serving/README_CN.md b/examples/vision/detection/paddledetection/python/serving/README_CN.md index b9bfe008d..f73206ba3 100644 --- a/examples/vision/detection/paddledetection/python/serving/README_CN.md +++ b/examples/vision/detection/paddledetection/python/serving/README_CN.md @@ -17,16 +17,9 @@ cd FastDeploy/examples/vision/detection/paddledetection/python/serving wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz tar xvf ppyoloe_crn_l_300e_coco.tgz -# 安装uvicorn -pip install uvicorn - -# 启动服务,可选择是否使用GPU和TensorRT,可根据uvicorn --help配置IP、端口号等 -# CPU -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=cpu uvicorn server:app -# GPU -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu uvicorn server:app -# GPU上使用TensorRT (注意:TensorRT推理第一次运行,有序列化模型的操作,有一定耗时,需要耐心等待) -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu USE_TRT=true uvicorn server:app +# 启动服务,可修改server.py中的配置项来指定硬件、后端等 +# 可通过--host、--port指定IP和端口号 +fastdeploy simple_serving --app server:app ``` 客户端: diff --git a/examples/vision/detection/paddledetection/python/serving/README_EN.md b/examples/vision/detection/paddledetection/python/serving/README_EN.md index 7ccf087c0..56049981d 100644 --- a/examples/vision/detection/paddledetection/python/serving/README_EN.md +++ b/examples/vision/detection/paddledetection/python/serving/README_EN.md @@ -17,17 +17,9 @@ cd FastDeploy/examples/vision/detection/paddledetection/python/serving wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz tar xvf ppyoloe_crn_l_300e_coco.tgz -# Install uvicorn -pip install uvicorn - -# Launch server, it's configurable to use GPU and TensorRT, -# and run 'uvicorn --help' to check how to specify IP and port, etc. -# CPU -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=cpu uvicorn server:app -# GPU -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu uvicorn server:app -# GPU and TensorRT -MODEL_DIR=ppyoloe_crn_l_300e_coco DEVICE=gpu USE_TRT=true uvicorn server:app +# Launch server, change the configurations in server.py to select hardware, backend, etc. +# and use --host, --port to specify IP and port +fastdeploy simple_serving --app server:app ``` Client: diff --git a/examples/vision/detection/paddledetection/python/serving/client.py b/examples/vision/detection/paddledetection/python/serving/client.py index c10f61976..50538a1a0 100644 --- a/examples/vision/detection/paddledetection/python/serving/client.py +++ b/examples/vision/detection/paddledetection/python/serving/client.py @@ -1,20 +1,15 @@ import requests import json import cv2 -import base64 import fastdeploy as fd +from fastdeploy.serving.utils import cv2_to_base64 if __name__ == '__main__': url = "http://127.0.0.1:8000/fd/ppyoloe" headers = {"Content-Type": "application/json"} im = cv2.imread("000000014439.jpg") - data = { - "data": { - "image": fd.serving.utils.cv2_to_base64(im) - }, - "parameters": {} - } + data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}} resp = requests.post(url=url, headers=headers, data=json.dumps(data)) if resp.status_code == 200: diff --git a/examples/vision/detection/paddledetection/python/serving/server.py b/examples/vision/detection/paddledetection/python/serving/server.py index 5127cbd44..9abd2ed56 100644 --- a/examples/vision/detection/paddledetection/python/serving/server.py +++ b/examples/vision/detection/paddledetection/python/serving/server.py @@ -1,18 +1,16 @@ import fastdeploy as fd +from fastdeploy.serving.server import SimpleServer import os import logging logging.getLogger().setLevel(logging.INFO) -# Get arguments from envrionment variables -model_dir = os.environ.get('MODEL_DIR') -device = os.environ.get('DEVICE', 'cpu') -use_trt = os.environ.get('USE_TRT', False) - -# Prepare model, download from hub or use local dir -if model_dir is None: - model_dir = fd.download_model(name='ppyoloe_crn_l_300e_coco') +# Configurations +model_dir = 'ppyoloe_crn_l_300e_coco' +device = 'cpu' +use_trt = False +# Prepare model model_file = os.path.join(model_dir, "model.pdmodel") params_file = os.path.join(model_dir, "model.pdiparams") config_file = os.path.join(model_dir, "infer_cfg.yml") @@ -33,7 +31,7 @@ model_instance = fd.vision.detection.PPYOLOE( runtime_option=option) # Create server, setup REST API -app = fd.serving.SimpleServer() +app = SimpleServer() app.register( task_name="fd/ppyoloe", model_handler=fd.serving.handler.VisionModelHandler, diff --git a/examples/vision/detection/yolov5/python/serving/README.md b/examples/vision/detection/yolov5/python/serving/README.md new file mode 120000 index 000000000..bacd3186b --- /dev/null +++ b/examples/vision/detection/yolov5/python/serving/README.md @@ -0,0 +1 @@ +README_CN.md \ No newline at end of file diff --git a/examples/vision/detection/yolov5/python/serving/README_CN.md b/examples/vision/detection/yolov5/python/serving/README_CN.md new file mode 100644 index 000000000..28963fd3f --- /dev/null +++ b/examples/vision/detection/yolov5/python/serving/README_CN.md @@ -0,0 +1,36 @@ +简体中文 | [English](README_EN.md) + +# YOLOv5 Python轻量服务化部署示例 + +在部署前,需确认以下两个步骤 + +- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装,参考[FastDeploy Python安装](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +服务端: +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/python/serving + +# 下载模型文件 +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s_infer.tar +tar xvf yolov5s_infer.tar + +# 启动服务,可修改server.py中的配置项来指定硬件、后端等 +# 可通过--host、--port指定IP和端口号 +fastdeploy simple_serving --app server:app +``` + +客户端: +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/python/serving + +# 下载测试图片 +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg + +# 请求服务,获取推理结果(如有必要,请修改脚本中的IP和端口号) +python client.py +``` diff --git a/examples/vision/detection/yolov5/python/serving/README_EN.md b/examples/vision/detection/yolov5/python/serving/README_EN.md new file mode 100644 index 000000000..b0cb92244 --- /dev/null +++ b/examples/vision/detection/yolov5/python/serving/README_EN.md @@ -0,0 +1,36 @@ +English | [简体中文](README_CN.md) + +# YOLOv5 Python Simple Serving Demo + + +## Environment + +- 1. Prepare environment and install FastDeploy Python whl, refer to [download_prebuilt_libraries](../../../../../../docs/en/build_and_install/download_prebuilt_libraries.md) + +Server: +```bash +# Download demo code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/python/serving + +# Download model +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s_infer.tar +tar xvf yolov5s_infer.tar + +# Launch server, change the configurations in server.py to select hardware, backend, etc. +# and use --host, --port to specify IP and port +fastdeploy simple_serving --app server:app +``` + +Client: +```bash +# Download demo code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/python/serving + +# Download test image +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg + +# Send request and get inference result (Please adapt the IP and port if necessary) +python client.py +``` diff --git a/examples/vision/detection/yolov5/python/serving/client.py b/examples/vision/detection/yolov5/python/serving/client.py new file mode 100644 index 000000000..bb310dff6 --- /dev/null +++ b/examples/vision/detection/yolov5/python/serving/client.py @@ -0,0 +1,23 @@ +import requests +import json +import cv2 +import fastdeploy as fd +from fastdeploy.serving.utils import cv2_to_base64 + +if __name__ == '__main__': + url = "http://127.0.0.1:8000/fd/yolov5s" + headers = {"Content-Type": "application/json"} + + im = cv2.imread("000000014439.jpg") + data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}} + + resp = requests.post(url=url, headers=headers, data=json.dumps(data)) + if resp.status_code == 200: + r_json = json.loads(resp.json()["result"]) + det_result = fd.vision.utils.json_to_detection(r_json) + vis_im = fd.vision.vis_detection(im, det_result, score_threshold=0.5) + cv2.imwrite("visualized_result.jpg", vis_im) + print("Visualized result save in ./visualized_result.jpg") + else: + print("Error code:", resp.status_code) + print(resp.text) diff --git a/examples/vision/detection/yolov5/python/serving/server.py b/examples/vision/detection/yolov5/python/serving/server.py new file mode 100644 index 000000000..138260b05 --- /dev/null +++ b/examples/vision/detection/yolov5/python/serving/server.py @@ -0,0 +1,38 @@ +import fastdeploy as fd +from fastdeploy.serving.server import SimpleServer +import os +import logging + +logging.getLogger().setLevel(logging.INFO) + +# Configurations +model_dir = 'yolov5s_infer' +device = 'cpu' +use_trt = False + +# Prepare model +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") + +# Setup runtime option to select hardware, backend, etc. +option = fd.RuntimeOption() +if device.lower() == 'gpu': + option.use_gpu() +if use_trt: + option.use_trt_backend() + option.set_trt_input_shape("images", [1, 3, 640, 640]) + option.set_trt_cache_file('yolov5s.trt') + +# Create model instance +model_instance = fd.vision.detection.YOLOv5( + model_file, + params_file, + runtime_option=option, + model_format=fd.ModelFormat.PADDLE) + +# Create server, setup REST API +app = SimpleServer() +app.register( + task_name="fd/yolov5s", + model_handler=fd.serving.handler.VisionModelHandler, + predictor=model_instance) diff --git a/examples/vision/ocr/PP-OCRv3/python/serving/README.md b/examples/vision/ocr/PP-OCRv3/python/serving/README.md new file mode 120000 index 000000000..bacd3186b --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/serving/README.md @@ -0,0 +1 @@ +README_CN.md \ No newline at end of file diff --git a/examples/vision/ocr/PP-OCRv3/python/serving/README_CN.md b/examples/vision/ocr/PP-OCRv3/python/serving/README_CN.md new file mode 100644 index 000000000..8b7fc2fa3 --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/serving/README_CN.md @@ -0,0 +1,44 @@ +简体中文 | [English](README_EN.md) + +# PP-OCRv3 Python轻量服务化部署示例 + +在部署前,需确认以下两个步骤 + +- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装,参考[FastDeploy Python安装](../../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +服务端: +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving + +# 下载模型和字典文件 +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar +tar xvf ch_PP-OCRv3_det_infer.tar + +wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar +tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar + +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar +tar xvf ch_PP-OCRv3_rec_infer.tar + +wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt + +# 启动服务,可修改server.py中的配置项来指定硬件、后端等 +# 可通过--host、--port指定IP和端口号 +fastdeploy simple_serving --app server:app +``` + +客户端: +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving + +# 下载测试图片 +wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg + +# 请求服务,获取推理结果(如有必要,请修改脚本中的IP和端口号) +python client.py +``` diff --git a/examples/vision/ocr/PP-OCRv3/python/serving/README_EN.md b/examples/vision/ocr/PP-OCRv3/python/serving/README_EN.md new file mode 100644 index 000000000..fb3d6c9b8 --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/serving/README_EN.md @@ -0,0 +1,43 @@ +English | [简体中文](README_CN.md) + +# PP-OCRv3 Python Simple Serving Demo + +## Environment + +- 1. Prepare environment and install FastDeploy Python whl, refer to [download_prebuilt_libraries](../../../../../../docs/en/build_and_install/download_prebuilt_libraries.md) + +Server: +```bash +# Download demo code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving + +# Download models and labels +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar +tar xvf ch_PP-OCRv3_det_infer.tar + +wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar +tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar + +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar +tar xvf ch_PP-OCRv3_rec_infer.tar + +wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt + +# Launch server, change the configurations in server.py to select hardware, backend, etc. +# and use --host, --port to specify IP and port +fastdeploy simple_serving --app server:app +``` + +Client: +```bash +# Download demo code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/ocr/PP-OCRv3/python/serving + +# Download test image +wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg + +# Send request and get inference result (Please adapt the IP and port if necessary) +python client.py +``` diff --git a/examples/vision/ocr/PP-OCRv3/python/serving/client.py b/examples/vision/ocr/PP-OCRv3/python/serving/client.py new file mode 100644 index 000000000..6849c2204 --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/serving/client.py @@ -0,0 +1,24 @@ +import requests +import json +import cv2 +import fastdeploy as fd +from fastdeploy.serving.utils import cv2_to_base64 + +if __name__ == '__main__': + url = "http://127.0.0.1:8000/fd/ppocrv3" + headers = {"Content-Type": "application/json"} + + im = cv2.imread("12.jpg") + data = {"data": {"image": cv2_to_base64(im)}, "parameters": {}} + + resp = requests.post(url=url, headers=headers, data=json.dumps(data)) + if resp.status_code == 200: + r_json = json.loads(resp.json()["result"]) + print(r_json) + ocr_result = fd.vision.utils.json_to_ocr(r_json) + vis_im = fd.vision.vis_ppocr(im, ocr_result) + cv2.imwrite("visualized_result.jpg", vis_im) + print("Visualized result save in ./visualized_result.jpg") + else: + print("Error code:", resp.status_code) + print(resp.text) diff --git a/examples/vision/ocr/PP-OCRv3/python/serving/server.py b/examples/vision/ocr/PP-OCRv3/python/serving/server.py new file mode 100644 index 000000000..0078b7112 --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/serving/server.py @@ -0,0 +1,80 @@ +import fastdeploy as fd +from fastdeploy.serving.server import SimpleServer +import os +import logging + +logging.getLogger().setLevel(logging.INFO) + +# Configurations +det_model_dir = 'ch_PP-OCRv3_det_infer' +cls_model_dir = 'ch_ppocr_mobile_v2.0_cls_infer' +rec_model_dir = 'ch_PP-OCRv3_rec_infer' +rec_label_file = 'ppocr_keys_v1.txt' +device = 'cpu' +# backend: ['paddle', 'trt'], you can also use other backends, but need to modify +# the runtime option below +backend = 'paddle' + +# Prepare models +# Detection model +det_model_file = os.path.join(det_model_dir, "inference.pdmodel") +det_params_file = os.path.join(det_model_dir, "inference.pdiparams") +# Classification model +cls_model_file = os.path.join(cls_model_dir, "inference.pdmodel") +cls_params_file = os.path.join(cls_model_dir, "inference.pdiparams") +# Recognition model +rec_model_file = os.path.join(rec_model_dir, "inference.pdmodel") +rec_params_file = os.path.join(rec_model_dir, "inference.pdiparams") + +# Setup runtime option to select hardware, backend, etc. +option = fd.RuntimeOption() +if device.lower() == 'gpu': + option.use_gpu() +if backend == 'trt': + option.use_trt_backend() +else: + option.use_paddle_infer_backend() + +det_option = option +det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], + [1, 3, 960, 960]) + +# det_option.set_trt_cache_file("det_trt_cache.trt") +print(det_model_file, det_params_file) +det_model = fd.vision.ocr.DBDetector( + det_model_file, det_params_file, runtime_option=det_option) + +cls_batch_size = 1 +rec_batch_size = 6 + +cls_option = option +cls_option.set_trt_input_shape("x", [1, 3, 48, 10], + [cls_batch_size, 3, 48, 320], + [cls_batch_size, 3, 48, 1024]) + +# cls_option.set_trt_cache_file("cls_trt_cache.trt") +cls_model = fd.vision.ocr.Classifier( + cls_model_file, cls_params_file, runtime_option=cls_option) + +rec_option = option +rec_option.set_trt_input_shape("x", [1, 3, 48, 10], + [rec_batch_size, 3, 48, 320], + [rec_batch_size, 3, 48, 2304]) + +# rec_option.set_trt_cache_file("rec_trt_cache.trt") +rec_model = fd.vision.ocr.Recognizer( + rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option) + +# Create PPOCRv3 pipeline +ppocr_v3 = fd.vision.ocr.PPOCRv3( + det_model=det_model, cls_model=cls_model, rec_model=rec_model) + +ppocr_v3.cls_batch_size = cls_batch_size +ppocr_v3.rec_batch_size = rec_batch_size + +# Create server, setup REST API +app = SimpleServer() +app.register( + task_name="fd/ppocrv3", + model_handler=fd.serving.handler.VisionModelHandler, + predictor=ppocr_v3) diff --git a/python/fastdeploy/serving/__init__.py b/python/fastdeploy/serving/__init__.py index e40cbb900..7d175762c 100644 --- a/python/fastdeploy/serving/__init__.py +++ b/python/fastdeploy/serving/__init__.py @@ -12,5 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import - -from .server import SimpleServer diff --git a/python/requirements.txt b/python/requirements.txt index 3fa7f18d1..3c463f4bb 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,6 +3,6 @@ requests tqdm numpy opencv-python -fastdeploy-tools==0.0.1 +fastdeploy-tools>=0.0.1 pyyaml fastapi diff --git a/tools/common_tools/common_tools.py b/tools/common_tools/common_tools.py index d945f905a..3a0f1f6c6 100755 --- a/tools/common_tools/common_tools.py +++ b/tools/common_tools/common_tools.py @@ -1,10 +1,12 @@ import argparse import ast +import uvicorn def argsparser(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('tools', choices=['compress', 'convert']) + parser.add_argument( + 'tools', choices=['compress', 'convert', 'simple_serving']) ## argumentments for auto compression parser.add_argument( '--config_path', @@ -69,6 +71,19 @@ def argsparser(): type=ast.literal_eval, default=False, help="Turn on code optimization") + ## arguments for simple serving + parser.add_argument( + "--app", + type=str, + default="server:app", + help="Simple serving app string") + parser.add_argument( + "--host", + type=str, + default="127.0.0.1", + help="Simple serving host IP address") + parser.add_argument( + "--port", type=int, default=8000, help="Simple serving host port") ## arguments for other tools return parser @@ -116,6 +131,8 @@ def main(): except ImportError: print( "Model convert failed! Please check if you have installed it!") + if args.tools == "simple_serving": + uvicorn.run(args.app, host=args.host, port=args.port, app_dir='.') if __name__ == '__main__': diff --git a/tools/setup.py b/tools/setup.py index de312e02e..20ec6489a 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -3,13 +3,16 @@ import setuptools long_description = "fastdeploy-tools is a toolkit for FastDeploy, including auto compression .etc.\n\n" long_description += "Usage of auto compression: fastdeploy compress --config_path=./yolov7_tiny_qat_dis.yaml --method='QAT' --save_dir='./v7_qat_outmodel/' \n" +install_requires = ['uvicorn==0.16.0'] + setuptools.setup( name="fastdeploy-tools", # name of package - version="0.0.1", #version of package + version="0.0.2", #version of package description="A toolkit for FastDeploy.", long_description=long_description, long_description_content_type="text/plain", packages=setuptools.find_packages(), + install_requires=install_requires, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License",