mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-06 00:57:33 +08:00
[Other] PPOCR models support model clone function (#1072)
* Refactor PaddleSeg with preprocessor && postprocessor * Fix bugs * Delete redundancy code * Modify by comments * Refactor according to comments * Add batch evaluation * Add single test script * Add ppliteseg single test script && fix eval(raise) error * fix bug * Fix evaluation segmentation.py batch predict * Fix segmentation evaluation bug * Fix evaluation segmentation bugs * Update segmentation result docs * Update old predict api and DisableNormalizeAndPermute * Update resize segmentation label map with cv::INTER_NEAREST * Add Model Clone function for PaddleClas && PaddleDet && PaddleSeg * Add multi thread demo * Add python model clone function * Add multi thread python && C++ example * Fix bug * Update python && cpp multi_thread examples * Add cpp && python directory * Add README.md for examples * Delete redundant code * Create README_CN.md * Rename README_CN.md to README.md * Update README.md * Update README.md * Update VERSION_NUMBER * Update requirements.txt * Update README.md * update version in doc: * [Serving]Update Dockerfile (#1037) Update Dockerfile * Add license notice for RVM onnx model file (#1060) * [Model] Add GPL-3.0 license (#1065) Add GPL-3.0 license * PPOCR model support model clone * Update README.md * Update PPOCRv2 && PPOCRv3 clone code * Update PPOCR python __init__ * Add multi thread ocr example code * Update README.md * Update README.md * Update ResNet50_vd_infer multi process code * Add PPOCR multi process && thread example * Update README.md * Update README.md * Update multi-thread docs Co-authored-by: Jason <jiangjiajun@baidu.com> Co-authored-by: leiqing <54695910+leiqing1@users.noreply.github.com> Co-authored-by: heliqi <1101791222@qq.com> Co-authored-by: WJJ1995 <wjjisloser@163.com>
This commit is contained in:
huangjianhui
60
tutorials/multi_thread/python/pipeline/README.md
Normal file
60
tutorials/multi_thread/python/pipeline/README.md
Normal file
@@ -0,0 +1,60 @@
|
||||
English | [简体中文](README_CN.md)
|
||||
# PPOCRv3 Python multi-thread/multi-process Deployment Example
|
||||
|
||||
Two steps before deployment
|
||||
|
||||
- 1. Software and hardware should meet the requirements. Please refer to [FastDeploy Environment Requirements](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. Install FastDeploy Python whl package. Refer to [FastDeploy Python Installation](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
This directory provides example file `multi_thread_process_ocr.py` to fast deploy multi-thread/multi-process ResNet50_vd on CPU/GPU and GPU accelerated by TensorRT. The script is as follows
|
||||
|
||||
|
||||
```bash
|
||||
# Download deployment example code
|
||||
git clone https://github.com/PaddlePaddle/FastDeploy.git
|
||||
cd FastDeploy/tutorials/multi_thread/python/pipeline
|
||||
|
||||
# Download model, image, and dictionary files
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar
|
||||
tar xvf ch_PP-OCRv3_det_infer.tar
|
||||
|
||||
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar
|
||||
tar xvf ch_PP-OCRv3_rec_infer.tar
|
||||
|
||||
wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg
|
||||
|
||||
wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt
|
||||
|
||||
# CPU multi-thread inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device cpu --thread_num 1
|
||||
# CPU multi-process inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device cpu --use_multi_process True --process_num 1
|
||||
|
||||
# GPU multi-thread inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --thread_num 1
|
||||
# GPU multi-process inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --use_multi_process True --process_num 1
|
||||
|
||||
# Use TensorRT multi-thread inference on GPU （Attention: It is somewhat time-consuming for the operation of model serialization when running TensorRT inference for the first time. Please be patient.）
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --backend trt --thread_num 1
|
||||
# Use TensorRT multi-process inference on GPU （Attention: It is somewhat time-consuming for the operation of model serialization when running TensorRT inference for the first time. Please be patient.）
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --backend trt --use_multi_process True --process_num 1
|
||||
|
||||
# KunlunXin XPU multi-thread inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device kunlunxin --thread_num 1
|
||||
# KunlunXin XPU multi-process inference
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device kunlunxin --use_multi_process True --process_num 1
|
||||
|
||||
```
|
||||
>> **Notice**: `--image_path` can be the path of the pictures folder
|
||||
|
||||
The result returned after running is as follows
|
||||
```
|
||||
thread: 0 , result: det boxes: [[42,413],[483,391],[484,428],[43,450]]rec text: 上海斯格威铂尔大酒店 rec score:0.949773 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[187,456],[399,448],[400,480],[188,488]]rec text: 打浦路15号 rec score:0.910265 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[23,507],[513,488],[515,529],[24,548]]rec text: 绿洲仕格维花园公寓 rec score:0.934239 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[74,553],[427,542],[428,571],[75,582]]rec text: 打浦路252935号 rec score:0.872207 cls label: 0 cls score: 1.000000
|
||||
```
|
||||
60
tutorials/multi_thread/python/pipeline/README_CN.md
Normal file
60
tutorials/multi_thread/python/pipeline/README_CN.md
Normal file
@@ -0,0 +1,60 @@
|
||||
[English](README.md) | 简体中文
|
||||
# PPOCR模型 Python多线程/进程部署示例
|
||||
|
||||
在部署前，需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
本目录下提供`multi_thread_process_ocr.py`快速完成PPOCRv3在CPU/GPU，以及GPU上通过TensorRT加速部署的多线程/进程示例。执行如下脚本即可完成
|
||||
|
||||
|
||||
```bash
|
||||
#下载部署示例代码
|
||||
git clone https://github.com/PaddlePaddle/FastDeploy.git
|
||||
cd FastDeploy/tutorials/multi_thread/python/pipeline
|
||||
|
||||
# 下载模型,图片和字典文件
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar
|
||||
tar xvf ch_PP-OCRv3_det_infer.tar
|
||||
|
||||
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
tar -xvf ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar
|
||||
tar xvf ch_PP-OCRv3_rec_infer.tar
|
||||
|
||||
wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/doc/imgs/12.jpg
|
||||
|
||||
wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt
|
||||
|
||||
# CPU多线程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device cpu --thread_num 1
|
||||
# CPU多进程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device cpu --use_multi_process True --process_num 1
|
||||
|
||||
# GPU多线程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --thread_num 1
|
||||
# GPU多进程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --use_multi_process True --process_num 1
|
||||
|
||||
# GPU上使用TensorRT多线程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --backend trt --thread_num 1
|
||||
# GPU上使用TensorRT多进程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device gpu --backend trt --use_multi_process True --process_num 1
|
||||
|
||||
# 昆仑芯XPU多线程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device kunlunxin --thread_num 1
|
||||
# 昆仑芯XPU多进程推理
|
||||
python multi_thread_process_ocr.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image_path 12.jpg --device kunlunxin --use_multi_process True --process_num 1
|
||||
|
||||
```
|
||||
>> **注意**: `--image_path` 可以输入图片文件夹的路径
|
||||
|
||||
运行完成后返回结果如下所示
|
||||
```
|
||||
thread: 0 , result: det boxes: [[42,413],[483,391],[484,428],[43,450]]rec text: 上海斯格威铂尔大酒店 rec score:0.949773 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[187,456],[399,448],[400,480],[188,488]]rec text: 打浦路15号 rec score:0.910265 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[23,507],[513,488],[515,529],[24,548]]rec text: 绿洲仕格维花园公寓 rec score:0.934239 cls label: 0 cls score: 1.000000
|
||||
det boxes: [[74,553],[427,542],[428,571],[75,582]]rec text: 打浦路252935号 rec score:0.872207 cls label: 0 cls score: 1.000000
|
||||
```
|
||||
279
tutorials/multi_thread/python/pipeline/multi_thread_process_ocr.py
Executable file
279
tutorials/multi_thread/python/pipeline/multi_thread_process_ocr.py
Executable file
@@ -0,0 +1,279 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from threading import Thread
|
||||
import fastdeploy as fd
|
||||
import cv2
|
||||
import os
|
||||
from multiprocessing import Pool
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
import argparse
|
||||
import ast
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--det_model", required=True, help="Path of Detection model of PPOCR.")
|
||||
parser.add_argument(
|
||||
"--cls_model",
|
||||
required=True,
|
||||
help="Path of Classification model of PPOCR.")
|
||||
parser.add_argument(
|
||||
"--rec_model",
|
||||
required=True,
|
||||
help="Path of Recognization model of PPOCR.")
|
||||
parser.add_argument(
|
||||
"--rec_label_file",
|
||||
required=True,
|
||||
help="Path of Recognization model of PPOCR.")
|
||||
parser.add_argument(
|
||||
"--image_path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="The directory or path or file list of the images to be predicted."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default='cpu',
|
||||
help="Type of inference device, support 'cpu', 'kunlunxin' or 'gpu'.")
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
type=str,
|
||||
default="default",
|
||||
help="Type of inference backend, support ort/trt/paddle/openvino, default 'openvino' for cpu, 'tensorrt' for gpu"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device_id",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Define which GPU card used to run model.")
|
||||
parser.add_argument(
|
||||
"--cpu_thread_num",
|
||||
type=int,
|
||||
default=9,
|
||||
help="Number of threads while inference on CPU.")
|
||||
parser.add_argument(
|
||||
"--cls_bs",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Classification model inference batch size.")
|
||||
parser.add_argument(
|
||||
"--rec_bs",
|
||||
type=int,
|
||||
default=6,
|
||||
help="Recognition model inference batch size")
|
||||
parser.add_argument("--thread_num", type=int, default=1, help="thread num")
|
||||
parser.add_argument(
|
||||
"--use_multi_process",
|
||||
type=ast.literal_eval,
|
||||
default=False,
|
||||
help="Wether to use multi process.")
|
||||
parser.add_argument(
|
||||
"--process_num", type=int, default=1, help="process num")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def get_image_list(image_path):
|
||||
image_list = []
|
||||
if os.path.isfile(image_path):
|
||||
image_list.append(image_path)
|
||||
# load image in a directory
|
||||
elif os.path.isdir(image_path):
|
||||
for root, dirs, files in os.walk(image_path):
|
||||
for f in files:
|
||||
image_list.append(os.path.join(root, f))
|
||||
else:
|
||||
raise FileNotFoundError(
|
||||
'{} is not found. it should be a path of image, or a directory including images.'.
|
||||
format(image_path))
|
||||
|
||||
if len(image_list) == 0:
|
||||
raise RuntimeError(
|
||||
'There are not image file in `--image_path`={}'.format(image_path))
|
||||
|
||||
return image_list
|
||||
|
||||
|
||||
def build_option(args):
|
||||
option = fd.RuntimeOption()
|
||||
if args.device.lower() == "gpu":
|
||||
option.use_gpu(args.device_id)
|
||||
|
||||
option.set_cpu_thread_num(args.cpu_thread_num)
|
||||
|
||||
if args.device.lower() == "kunlunxin":
|
||||
option.use_kunlunxin()
|
||||
return option
|
||||
|
||||
if args.backend.lower() == "trt":
|
||||
assert args.device.lower(
|
||||
) == "gpu", "TensorRT backend require inference on device GPU."
|
||||
option.use_trt_backend()
|
||||
elif args.backend.lower() == "pptrt":
|
||||
assert args.device.lower(
|
||||
) == "gpu", "Paddle-TensorRT backend require inference on device GPU."
|
||||
option.use_trt_backend()
|
||||
option.enable_paddle_trt_collect_shape()
|
||||
option.enable_paddle_to_trt()
|
||||
elif args.backend.lower() == "ort":
|
||||
option.use_ort_backend()
|
||||
elif args.backend.lower() == "paddle":
|
||||
option.use_paddle_infer_backend()
|
||||
elif args.backend.lower() == "openvino":
|
||||
assert args.device.lower(
|
||||
) == "cpu", "OpenVINO backend require inference on device CPU."
|
||||
option.use_openvino_backend()
|
||||
return option
|
||||
|
||||
|
||||
def load_model(args, runtime_option):
|
||||
# Detection模型, 检测文字框
|
||||
det_model_file = os.path.join(args.det_model, "inference.pdmodel")
|
||||
det_params_file = os.path.join(args.det_model, "inference.pdiparams")
|
||||
# Classification模型，方向分类，可选
|
||||
cls_model_file = os.path.join(args.cls_model, "inference.pdmodel")
|
||||
cls_params_file = os.path.join(args.cls_model, "inference.pdiparams")
|
||||
# Recognition模型，文字识别模型
|
||||
rec_model_file = os.path.join(args.rec_model, "inference.pdmodel")
|
||||
rec_params_file = os.path.join(args.rec_model, "inference.pdiparams")
|
||||
rec_label_file = args.rec_label_file
|
||||
|
||||
# PPOCR的cls和rec模型现在已经支持推理一个Batch的数据
|
||||
# 定义下面两个变量后, 可用于设置trt输入shape, 并在PPOCR模型初始化后, 完成Batch推理设置
|
||||
cls_batch_size = 1
|
||||
rec_batch_size = 6
|
||||
|
||||
# 当使用TRT时，分别给三个模型的runtime设置动态shape,并完成模型的创建.
|
||||
# 注意: 需要在检测模型创建完成后，再设置分类模型的动态输入并创建分类模型, 识别模型同理.
|
||||
# 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数.
|
||||
det_option = runtime_option
|
||||
det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640],
|
||||
[1, 3, 960, 960])
|
||||
# 用户可以把TRT引擎文件保存至本地
|
||||
#det_option.set_trt_cache_file(args.det_model + "/det_trt_cache.trt")
|
||||
global det_model
|
||||
det_model = fd.vision.ocr.DBDetector(
|
||||
det_model_file, det_params_file, runtime_option=det_option)
|
||||
|
||||
cls_option = runtime_option
|
||||
cls_option.set_trt_input_shape("x", [1, 3, 48, 10],
|
||||
[cls_batch_size, 3, 48, 320],
|
||||
[cls_batch_size, 3, 48, 1024])
|
||||
# 用户可以把TRT引擎文件保存至本地
|
||||
#cls_option.set_trt_cache_file(args.cls_model + "/cls_trt_cache.trt")
|
||||
global cls_model
|
||||
cls_model = fd.vision.ocr.Classifier(
|
||||
cls_model_file, cls_params_file, runtime_option=cls_option)
|
||||
|
||||
rec_option = runtime_option
|
||||
rec_option.set_trt_input_shape("x", [1, 3, 48, 10],
|
||||
[rec_batch_size, 3, 48, 320],
|
||||
[rec_batch_size, 3, 48, 2304])
|
||||
# 用户可以把TRT引擎文件保存至本地
|
||||
#rec_option.set_trt_cache_file(args.rec_model + "/rec_trt_cache.trt")
|
||||
global rec_model
|
||||
rec_model = fd.vision.ocr.Recognizer(
|
||||
rec_model_file,
|
||||
rec_params_file,
|
||||
rec_label_file,
|
||||
runtime_option=rec_option)
|
||||
|
||||
# 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None
|
||||
global ppocr_v3
|
||||
ppocr_v3 = fd.vision.ocr.PPOCRv3(
|
||||
det_model=det_model, cls_model=cls_model, rec_model=rec_model)
|
||||
|
||||
# 给cls和rec模型设置推理时的batch size
|
||||
# 此值能为-1, 和1到正无穷
|
||||
# 当此值为-1时, cls和rec模型的batch size将默认和det模型检测出的框的数量相同
|
||||
ppocr_v3.cls_batch_size = cls_batch_size
|
||||
ppocr_v3.rec_batch_size = rec_batch_size
|
||||
|
||||
|
||||
def predict(model, img_list):
|
||||
result_list = []
|
||||
# predict ppocr result
|
||||
for image in img_list:
|
||||
im = cv2.imread(image)
|
||||
result = model.predict(im)
|
||||
result_list.append(result)
|
||||
return result_list
|
||||
|
||||
|
||||
def process_predict(image):
|
||||
# predict ppocr result
|
||||
im = cv2.imread(image)
|
||||
result = ppocr_v3.predict(im)
|
||||
print(result)
|
||||
|
||||
|
||||
class WrapperThread(Thread):
|
||||
def __init__(self, func, args):
|
||||
super(WrapperThread, self).__init__()
|
||||
self.func = func
|
||||
self.args = args
|
||||
|
||||
def run(self):
|
||||
self.result = self.func(*self.args)
|
||||
|
||||
def get_result(self):
|
||||
return self.result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_arguments()
|
||||
|
||||
imgs_list = get_image_list(args.image_path)
|
||||
# 对于三个模型，均采用同样的部署配置
|
||||
# 用户也可根据自行需求分别配置
|
||||
runtime_option = build_option(args)
|
||||
|
||||
if args.use_multi_process:
|
||||
process_num = args.process_num
|
||||
with Pool(
|
||||
process_num,
|
||||
initializer=load_model,
|
||||
initargs=(args, runtime_option)) as pool:
|
||||
pool.map(process_predict, imgs_list)
|
||||
else:
|
||||
load_model(args, runtime_option)
|
||||
threads = []
|
||||
thread_num = args.thread_num
|
||||
image_num_each_thread = int(len(imgs_list) / thread_num)
|
||||
# unless you want independent model in each thread, actually model.clone()
|
||||
# is the same as model when creating thead because of the existence of
|
||||
# GIL(Global Interpreter Lock) in python. In addition, model.clone() will consume
|
||||
# additional memory to store independent member variables
|
||||
for i in range(thread_num):
|
||||
if i == thread_num - 1:
|
||||
t = WrapperThread(
|
||||
predict,
|
||||
args=(ppocr_v3.clone(),
|
||||
imgs_list[i * image_num_each_thread:]))
|
||||
else:
|
||||
t = WrapperThread(
|
||||
predict,
|
||||
args=(ppocr_v3.clone(),
|
||||
imgs_list[i * image_num_each_thread:(i + 1) *
|
||||
image_num_each_thread - 1], args.topk))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for i in range(thread_num):
|
||||
threads[i].join()
|
||||
|
||||
for i in range(thread_num):
|
||||
for result in threads[i].get_result():
|
||||
print('thread:', i, ', result: ', result)
|
||||
51
tutorials/multi_thread/python/single_model/README.md
Normal file
51
tutorials/multi_thread/python/single_model/README.md
Normal file
@@ -0,0 +1,51 @@
|
||||
English | [简体中文](README_CN.md)
|
||||
# Example of PaddleClas models Python multi-thread/multi-process Deployment
|
||||
|
||||
Before deployment, two steps require confirmation
|
||||
|
||||
- 1. Software and hardware should meet the requirements. Please refer to [FastDeploy Environment Requirements](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. Install the FastDeploy Python whl package. Please refer to [FastDeploy Python Installation](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
This directory provides example file `multi_thread_process.py` to fast deploy multi-thread/multi-process ResNet50_vd on CPU/GPU and GPU accelerated by TensorRT. The script is as follows
|
||||
|
||||
|
||||
```bash
|
||||
# Download deployment example code
|
||||
git clone https://github.com/PaddlePaddle/FastDeploy.git
|
||||
cd FastDeploy/tutorials/multi_thread/python
|
||||
|
||||
# Download the ResNet50_vd model file and test images
|
||||
wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz
|
||||
tar -xvf ResNet50_vd_infer.tgz
|
||||
wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg
|
||||
|
||||
|
||||
# CPU multi-thread inference
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --thread_num 1
|
||||
# CPU multi-process inference
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# GPU multi-thread inference
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --thread_num 1
|
||||
# GPU multi-process inference
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# Use TensorRT multi-thread inference on GPU （Attention: It is somewhat time-consuming for the operation of model serialization when running TensorRT inference for the first time. Please be patient.）
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --thread_num 1
|
||||
# Use TensorRT multi-process inference on GPU （Attention: It is somewhat time-consuming for the operation of model serialization when running TensorRT inference for the first time. Please be patient.）
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# IPU multi-thread inference（Attention: It is somewhat time-consuming for the operation of model serialization when running IPU inference for the first time. Please be patient.）
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --thread_num 1
|
||||
# IPU multi-process inference（Attention: It is somewhat time-consuming for the operation of model serialization when running IPU inference for the first time. Please be patient.）
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --use_multi_process True --process_num 1
|
||||
```
|
||||
>> **Notice**: `--image_path` can be the path of the pictures folder
|
||||
|
||||
The result returned after running is as follows
|
||||
```bash
|
||||
ClassifyResult(
|
||||
label_ids: 153,
|
||||
scores: 0.686229,
|
||||
)
|
||||
```
|
||||
@@ -1,9 +1,10 @@
|
||||
[English](README.md) | 简体中文
|
||||
# PaddleClas模型 Python多线程/进程部署示例
|
||||
|
||||
在部署前，需确认以下两个步骤
|
||||
|
||||
- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../docs/cn/build_and_install/download_prebuilt_libraries.md)
|
||||
|
||||
本目录下提供`multi_thread_process.py`快速完成ResNet50_vd在CPU/GPU，以及GPU上通过TensorRT加速部署的多线程/进程示例。执行如下脚本即可完成
|
||||
|
||||
@@ -20,24 +21,24 @@ wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/Ima
|
||||
|
||||
|
||||
# CPU多线程推理
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --thread_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --thread_num 1
|
||||
# CPU多进程推理
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --use_multi_process True --process_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device cpu --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# GPU多线程推理
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --thread_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --thread_num 1
|
||||
# GPU多进程推理
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --use_multi_process True --process_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# GPU上使用TensorRT多线程推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待）
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --thread_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --thread_num 1
|
||||
# GPU上使用TensorRT多进程推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待）
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --use_multi_process True --process_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device gpu --use_trt True --topk 1 --use_multi_process True --process_num 1
|
||||
|
||||
# IPU多线程推理（注意：IPU推理首次运行会有序列化模型的操作，有一定耗时，需要耐心等待）
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --thread_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --thread_num 1
|
||||
# IPU多进程推理（注意：IPU推理首次运行会有序列化模型的操作，有一定耗时，需要耐心等待）
|
||||
python infer.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --use_multi_process True --process_num 1
|
||||
python multi_thread_process.py --model ResNet50_vd_infer --image_path ILSVRC2012_val_00000010.jpeg --device ipu --topk 1 --use_multi_process True --process_num 1
|
||||
```
|
||||
>> **注意**: `--image_path` 可以输入图片文件夹的路径
|
||||
|
||||
@@ -47,4 +48,4 @@ ClassifyResult(
|
||||
label_ids: 153,
|
||||
scores: 0.686229,
|
||||
)
|
||||
```
|
||||
```
|
||||
@@ -1,4 +1,17 @@
|
||||
import numpy as np
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from threading import Thread
|
||||
import fastdeploy as fd
|
||||
import cv2
|
||||
@@ -67,6 +80,7 @@ def build_option(args):
|
||||
option = fd.RuntimeOption()
|
||||
|
||||
if args.device.lower() == "gpu":
|
||||
option.use_paddle_backend()
|
||||
option.use_gpu()
|
||||
|
||||
if args.device.lower() == "ipu":
|
||||
@@ -77,6 +91,16 @@ def build_option(args):
|
||||
return option
|
||||
|
||||
|
||||
def load_model(args, runtime_option):
|
||||
model_file = os.path.join(args.model, "inference.pdmodel")
|
||||
params_file = os.path.join(args.model, "inference.pdiparams")
|
||||
config_file = os.path.join(args.model, "inference_cls.yaml")
|
||||
global model
|
||||
model = fd.vision.classification.PaddleClasModel(
|
||||
model_file, params_file, config_file, runtime_option=runtime_option)
|
||||
#return model
|
||||
|
||||
|
||||
def predict(model, img_list, topk):
|
||||
result_list = []
|
||||
# predict classification result
|
||||
@@ -91,7 +115,7 @@ def process_predict(image):
|
||||
# predict classification result
|
||||
im = cv2.imread(image)
|
||||
result = model.predict(im, args.topk)
|
||||
return result
|
||||
print(result)
|
||||
|
||||
|
||||
class WrapperThread(Thread):
|
||||
@@ -114,19 +138,15 @@ if __name__ == '__main__':
|
||||
# configure runtime and load model
|
||||
runtime_option = build_option(args)
|
||||
|
||||
model_file = os.path.join(args.model, "inference.pdmodel")
|
||||
params_file = os.path.join(args.model, "inference.pdiparams")
|
||||
config_file = os.path.join(args.model, "inference_cls.yaml")
|
||||
model = fd.vision.classification.PaddleClasModel(
|
||||
model_file, params_file, config_file, runtime_option=runtime_option)
|
||||
if args.use_multi_process:
|
||||
results = []
|
||||
process_num = args.process_num
|
||||
with Pool(process_num) as pool:
|
||||
results = pool.map(process_predict, imgs_list)
|
||||
for result in results:
|
||||
print(result)
|
||||
with Pool(
|
||||
process_num,
|
||||
initializer=load_model,
|
||||
initargs=(args, runtime_option)) as pool:
|
||||
pool.map(process_predict, imgs_list)
|
||||
else:
|
||||
load_model(args, runtime_option)
|
||||
threads = []
|
||||
thread_num = args.thread_num
|
||||
image_num_each_thread = int(len(imgs_list) / thread_num)
|
||||
Reference in New Issue
Block a user