mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-12 03:50:39 +08:00
1860d3ab78
* add onnx_ort_runtime demo * rm in requirements * support batch eval * fixed MattingResults bug * move assignment for DetectionResult * integrated x2paddle * add model convert readme * update readme * re-lint * add processor api * Add MattingResult Free * change valid_cpu_backends order * add ppocr benchmark * mv bs from 64 to 32 Co-authored-by: Jason <jiangjiajun@baidu.com>
285 lines
11 KiB
Python
285 lines
11 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import fastdeploy as fd
|
|
import cv2
|
|
import os
|
|
import numpy as np
|
|
import time
|
|
|
|
|
|
def parse_arguments():
|
|
import argparse
|
|
import ast
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--model_dir", required=True, help="Model dir of PPOCR.")
|
|
parser.add_argument(
|
|
"--det_model", required=True, help="Path of Detection model of PPOCR.")
|
|
parser.add_argument(
|
|
"--cls_model",
|
|
required=True,
|
|
help="Path of Classification model of PPOCR.")
|
|
parser.add_argument(
|
|
"--rec_model",
|
|
required=True,
|
|
help="Path of Recognization model of PPOCR.")
|
|
parser.add_argument(
|
|
"--rec_label_file",
|
|
required=True,
|
|
help="Path of Recognization model of PPOCR.")
|
|
parser.add_argument(
|
|
"--image", type=str, required=False, help="Path of test image file.")
|
|
parser.add_argument(
|
|
"--cpu_num_thread",
|
|
type=int,
|
|
default=8,
|
|
help="default number of cpu thread.")
|
|
parser.add_argument(
|
|
"--device_id", type=int, default=0, help="device(gpu) id")
|
|
parser.add_argument(
|
|
"--iter_num",
|
|
required=True,
|
|
type=int,
|
|
default=300,
|
|
help="number of iterations for computing performace.")
|
|
parser.add_argument(
|
|
"--device",
|
|
default="cpu",
|
|
help="Type of inference device, support 'cpu' or 'gpu'.")
|
|
parser.add_argument(
|
|
"--backend",
|
|
type=str,
|
|
default="default",
|
|
help="inference backend, default, ort, ov, trt, paddle, paddle_trt.")
|
|
parser.add_argument(
|
|
"--enable_trt_fp16",
|
|
type=ast.literal_eval,
|
|
default=False,
|
|
help="whether enable fp16 in trt backend")
|
|
parser.add_argument(
|
|
"--enable_collect_memory_info",
|
|
type=ast.literal_eval,
|
|
default=False,
|
|
help="whether enable collect memory info")
|
|
args = parser.parse_args()
|
|
return args
|
|
|
|
|
|
def build_option(args):
|
|
option = fd.RuntimeOption()
|
|
device = args.device
|
|
backend = args.backend
|
|
enable_trt_fp16 = args.enable_trt_fp16
|
|
option.set_cpu_thread_num(args.cpu_num_thread)
|
|
if device == "gpu":
|
|
option.use_gpu()
|
|
if backend == "ort":
|
|
option.use_ort_backend()
|
|
elif backend == "paddle":
|
|
option.use_paddle_backend()
|
|
elif backend in ["trt", "paddle_trt"]:
|
|
option.use_trt_backend()
|
|
if backend == "paddle_trt":
|
|
option.enable_paddle_to_trt()
|
|
if enable_trt_fp16:
|
|
option.enable_trt_fp16()
|
|
elif backend == "default":
|
|
return option
|
|
else:
|
|
raise Exception(
|
|
"While inference with GPU, only support default/ort/paddle/trt/paddle_trt now, {} is not supported.".
|
|
format(backend))
|
|
elif device == "cpu":
|
|
if backend == "ort":
|
|
option.use_ort_backend()
|
|
elif backend == "ov":
|
|
option.use_openvino_backend()
|
|
elif backend == "paddle":
|
|
option.use_paddle_backend()
|
|
elif backend == "default":
|
|
return option
|
|
else:
|
|
raise Exception(
|
|
"While inference with CPU, only support default/ort/ov/paddle now, {} is not supported.".
|
|
format(backend))
|
|
else:
|
|
raise Exception(
|
|
"Only support device CPU/GPU now, {} is not supported.".format(
|
|
device))
|
|
|
|
return option
|
|
|
|
|
|
def get_current_memory_mb(gpu_id=None):
|
|
import pynvml
|
|
import psutil
|
|
pid = os.getpid()
|
|
p = psutil.Process(pid)
|
|
info = p.memory_full_info()
|
|
cpu_mem = info.uss / 1024. / 1024.
|
|
gpu_mem = 0
|
|
if gpu_id is not None:
|
|
pynvml.nvmlInit()
|
|
handle = pynvml.nvmlDeviceGetHandleByIndex(0)
|
|
meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
gpu_mem = meminfo.used / 1024. / 1024.
|
|
return cpu_mem, gpu_mem
|
|
|
|
|
|
def get_current_gputil(gpu_id):
|
|
import GPUtil
|
|
GPUs = GPUtil.getGPUs()
|
|
gpu_load = GPUs[gpu_id].load
|
|
return gpu_load
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
args = parse_arguments()
|
|
option = build_option(args)
|
|
# Detection Model
|
|
det_model_file = os.path.join(args.model_dir, args.det_model,
|
|
"inference.pdmodel")
|
|
det_params_file = os.path.join(args.model_dir, args.det_model,
|
|
"inference.pdiparams")
|
|
# Classification Model
|
|
cls_model_file = os.path.join(args.model_dir, args.cls_model,
|
|
"inference.pdmodel")
|
|
cls_params_file = os.path.join(args.model_dir, args.cls_model,
|
|
"inference.pdiparams")
|
|
# Recognition Model
|
|
rec_model_file = os.path.join(args.model_dir, args.rec_model,
|
|
"inference.pdmodel")
|
|
rec_params_file = os.path.join(args.model_dir, args.rec_model,
|
|
"inference.pdiparams")
|
|
rec_label_file = os.path.join(args.model_dir, args.rec_label_file)
|
|
|
|
gpu_id = args.device_id
|
|
enable_collect_memory_info = args.enable_collect_memory_info
|
|
end2end_statis = list()
|
|
cpu_mem = list()
|
|
gpu_mem = list()
|
|
gpu_util = list()
|
|
if args.device == "cpu":
|
|
file_path = args.model_dir + "_model_" + args.backend + "_" + \
|
|
args.device + "_" + str(args.cpu_num_thread) + ".txt"
|
|
else:
|
|
if args.enable_trt_fp16:
|
|
file_path = args.model_dir + "_model_" + args.backend + "_fp16_" + args.device + ".txt"
|
|
else:
|
|
file_path = args.model_dir + "_model_" + args.backend + "_" + args.device + ".txt"
|
|
f = open(file_path, "w")
|
|
f.writelines("===={}====: \n".format(os.path.split(file_path)[-1][:-4]))
|
|
|
|
try:
|
|
rec_option = option
|
|
if "OCRv2" in args.model_dir:
|
|
det_option = option
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
det_option.set_trt_input_shape(
|
|
"x", [1, 3, 64, 64], [1, 3, 640, 640], [1, 3, 960, 960])
|
|
det_model = fd.vision.ocr.DBDetector(
|
|
det_model_file, det_params_file, runtime_option=det_option)
|
|
cls_option = option
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
cls_option.set_trt_input_shape(
|
|
"x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 1024])
|
|
cls_model = fd.vision.ocr.Classifier(
|
|
cls_model_file, cls_params_file, runtime_option=cls_option)
|
|
rec_option = option
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
rec_option.set_trt_input_shape(
|
|
"x", [1, 3, 32, 10], [10, 3, 32, 320], [32, 3, 32, 2304])
|
|
rec_model = fd.vision.ocr.Recognizer(
|
|
rec_model_file,
|
|
rec_params_file,
|
|
rec_label_file,
|
|
runtime_option=rec_option)
|
|
model = fd.vision.ocr.PPOCRv2(
|
|
det_model=det_model, cls_model=cls_model, rec_model=rec_model)
|
|
elif "OCRv3" in args.model_dir:
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
det_option.set_trt_input_shape(
|
|
"x", [1, 3, 64, 64], [1, 3, 640, 640], [1, 3, 960, 960])
|
|
det_model = fd.vision.ocr.DBDetector(
|
|
det_model_file, det_params_file, runtime_option=det_option)
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
cls_option.set_trt_input_shape(
|
|
"x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 1024])
|
|
cls_model = fd.vision.ocr.Classifier(
|
|
cls_model_file, cls_params_file, runtime_option=cls_option)
|
|
if args.backend in ["trt", "paddle_trt"]:
|
|
rec_option.set_trt_input_shape(
|
|
"x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 2304])
|
|
rec_model = fd.vision.ocr.Recognizer(
|
|
rec_model_file,
|
|
rec_params_file,
|
|
rec_label_file,
|
|
runtime_option=rec_option)
|
|
model = fd.vision.ocr.PPOCRv3(
|
|
det_model=det_model, cls_model=cls_model, rec_model=rec_model)
|
|
else:
|
|
raise Exception("model {} not support now in ppocr series".format(
|
|
args.model_dir))
|
|
det_model.enable_record_time_of_runtime()
|
|
cls_model.enable_record_time_of_runtime()
|
|
rec_model.enable_record_time_of_runtime()
|
|
im_ori = cv2.imread(args.image)
|
|
for i in range(args.iter_num):
|
|
im = im_ori
|
|
start = time.time()
|
|
result = model.predict(im)
|
|
end2end_statis.append(time.time() - start)
|
|
if enable_collect_memory_info:
|
|
gpu_util.append(get_current_gputil(gpu_id))
|
|
cm, gm = get_current_memory_mb(gpu_id)
|
|
cpu_mem.append(cm)
|
|
gpu_mem.append(gm)
|
|
|
|
runtime_statis_det = det_model.print_statis_info_of_runtime()
|
|
runtime_statis_cls = cls_model.print_statis_info_of_runtime()
|
|
runtime_statis_rec = rec_model.print_statis_info_of_runtime()
|
|
|
|
warmup_iter = args.iter_num // 5
|
|
end2end_statis_repeat = end2end_statis[warmup_iter:]
|
|
if enable_collect_memory_info:
|
|
cpu_mem_repeat = cpu_mem[warmup_iter:]
|
|
gpu_mem_repeat = gpu_mem[warmup_iter:]
|
|
gpu_util_repeat = gpu_util[warmup_iter:]
|
|
|
|
dump_result = dict()
|
|
dump_result["runtime"] = (
|
|
runtime_statis_det["avg_time"] + runtime_statis_cls["avg_time"] +
|
|
runtime_statis_rec["avg_time"]) * 1000
|
|
dump_result["end2end"] = np.mean(end2end_statis_repeat) * 1000
|
|
if enable_collect_memory_info:
|
|
dump_result["cpu_rss_mb"] = np.mean(cpu_mem_repeat)
|
|
dump_result["gpu_rss_mb"] = np.mean(gpu_mem_repeat)
|
|
dump_result["gpu_util"] = np.mean(gpu_util_repeat)
|
|
|
|
f.writelines("Runtime(ms): {} \n".format(str(dump_result["runtime"])))
|
|
f.writelines("End2End(ms): {} \n".format(str(dump_result["end2end"])))
|
|
if enable_collect_memory_info:
|
|
f.writelines("cpu_rss_mb: {} \n".format(
|
|
str(dump_result["cpu_rss_mb"])))
|
|
f.writelines("gpu_rss_mb: {} \n".format(
|
|
str(dump_result["gpu_rss_mb"])))
|
|
f.writelines("gpu_util: {} \n".format(
|
|
str(dump_result["gpu_util"])))
|
|
except:
|
|
f.writelines("!!!!!Infer Failed\n")
|
|
|
|
f.close()
|