From cf8f53e36d376dc0f6235d494cd08fb81225c749 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Sat, 5 Nov 2022 07:54:16 +0000 Subject: [PATCH] Modify model tests --- tests/models/README.md | 14 +++++ tests/models/test_pfld.py | 25 +++++---- tests/models/test_ppmatting.py | 98 +++++++++++++++++---------------- tests/models/test_pptinypose.py | 10 ++-- tests/models/test_pptracking.py | 10 ++-- tests/models/test_rvm.py | 9 ++- tests/models/test_yolov5cls.py | 37 +++++++------ 7 files changed, 111 insertions(+), 92 deletions(-) create mode 100644 tests/models/README.md diff --git a/tests/models/README.md b/tests/models/README.md new file mode 100644 index 000000000..c80ccc9dd --- /dev/null +++ b/tests/models/README.md @@ -0,0 +1,14 @@ +# 添加模型单测 + + +所有模型统一使用`runtime_config.py`中的RuntimeOption进行配置 + +``` +import runtime_config as rc + + +model = fd.vision.XXX(..., runtime_option=rc.test_option) +``` + + +验证For循环跑2+次与Baseline结果符合预期 diff --git a/tests/models/test_pfld.py b/tests/models/test_pfld.py index ef1ba448e..8d455b165 100644 --- a/tests/models/test_pfld.py +++ b/tests/models/test_pfld.py @@ -22,19 +22,20 @@ def test_facealignment_pfld(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/pfld-106-lite.onnx" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png" output_url = "https://bj.bcebos.com/paddlehub/fastdeploy/result_landmarks.npy" - fd.download(model_url, ".") - fd.download(input_url, ".") - fd.download(output_url, ".") - model_path = "pfld-106-lite.onnx" + fd.download(model_url, "resources") + fd.download(input_url, "resources") + fd.download(output_url, "resources") + model_path = "resources/pfld-106-lite.onnx" # use ORT model = fd.vision.facealign.PFLD(model_path, runtime_option=rc.test_option) # compare diff - im = cv2.imread("./facealign_input.png") - result = model.predict(im.copy()) - expect = np.load("./result_landmarks.npy") - - diff = np.fabs(np.array(result.landmarks) - expect) - thres = 1e-04 - assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( - diff.max(), thres) + im = cv2.imread("resources/facealign_input.png") + for i in range(2): + result = model.predict(im) + expect = np.load("resources/result_landmarks.npy") + + diff = np.fabs(np.array(result.landmarks) - expect) + thres = 1e-04 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) diff --git a/tests/models/test_ppmatting.py b/tests/models/test_ppmatting.py index 78a085a5f..8021f5b22 100644 --- a/tests/models/test_ppmatting.py +++ b/tests/models/test_ppmatting.py @@ -22,9 +22,9 @@ import runtime_config as rc def test_matting_ppmatting(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "./PP-Matting-512" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "./resources/PP-Matting-512" model_file = os.path.join(model_path, "model.pdmodel") params_file = os.path.join(model_path, "model.pdiparams") config_file = os.path.join(model_path, "deploy.yaml") @@ -32,26 +32,27 @@ def test_matting_ppmatting(): model_file, params_file, config_file, runtime_option=rc.test_option) # 预测图片抠图结果 - im = cv2.imread("./matting_input.jpg") - result = model.predict(im.copy()) - pkl_url = "https://bj.bcebos.com/fastdeploy/tests/ppmatting_result.pkl" - if pkl_url: - fd.download(pkl_url, ".") - with open("./ppmatting_result.pkl", "rb") as f: - baseline = pickle.load(f) - - diff = np.fabs(np.array(result.alpha) - np.array(baseline)) - thres = 1e-05 - assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( - diff.max(), thres) + im = cv2.imread("./resources/matting_input.jpg") + for i in range(2): + result = model.predict(im) + pkl_url = "https://bj.bcebos.com/fastdeploy/tests/ppmatting_result.pkl" + if pkl_url: + fd.download(pkl_url, "resources") + with open("./resources/ppmatting_result.pkl", "rb") as f: + baseline = pickle.load(f) + + diff = np.fabs(np.array(result.alpha) - np.array(baseline)) + thres = 1e-05 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) def test_matting_ppmodnet(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/PPModnet_MobileNetV2.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "./PPModnet_MobileNetV2" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "./resources/PPModnet_MobileNetV2" model_file = os.path.join(model_path, "model.pdmodel") params_file = os.path.join(model_path, "model.pdiparams") config_file = os.path.join(model_path, "deploy.yaml") @@ -59,27 +60,29 @@ def test_matting_ppmodnet(): model_file, params_file, config_file, runtime_option=rc.test_option) # 预测图片抠图结果 - im = cv2.imread("./matting_input.jpg") - result = model.predict(im.copy()) + im = cv2.imread("./resources/matting_input.jpg") - pkl_url = "https://bj.bcebos.com/fastdeploy/tests/ppmodnet_result.pkl" - if pkl_url: - fd.download(pkl_url, ".") - with open("./ppmodnet_result.pkl", "rb") as f: - baseline = pickle.load(f) - - diff = np.fabs(np.array(result.alpha) - np.array(baseline)) - thres = 1e-05 - assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( - diff.max(), thres) + for i in range(2): + result = model.predict(im) + + pkl_url = "https://bj.bcebos.com/fastdeploy/tests/ppmodnet_result.pkl" + if pkl_url: + fd.download(pkl_url, "resources") + with open("./resources/ppmodnet_result.pkl", "rb") as f: + baseline = pickle.load(f) + + diff = np.fabs(np.array(result.alpha) - np.array(baseline)) + thres = 1e-05 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) def test_matting_pphumanmatting(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/PPHumanMatting.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "./PPHumanMatting" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "./resources/PPHumanMatting" # 配置runtime，加载模型 runtime_option = fd.RuntimeOption() model_file = os.path.join(model_path, "model.pdmodel") @@ -89,17 +92,18 @@ def test_matting_pphumanmatting(): model_file, params_file, config_file, runtime_option=rc.test_option) # 预测图片抠图结果 - im = cv2.imread("./matting_input.jpg") - result = model.predict(im.copy()) - - pkl_url = "https://bj.bcebos.com/fastdeploy/tests/pphumanmatting_result.pkl" - if pkl_url: - fd.download(pkl_url, ".") - - with open("./pphumanmatting_result.pkl", "rb") as f: - baseline = pickle.load(f) - - diff = np.fabs(np.array(result.alpha) - np.array(baseline)) - thres = 1e-05 - assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( - diff.max(), thres) + im = cv2.imread("./resources/matting_input.jpg") + for i in range(2): + result = model.predict(im) + + pkl_url = "https://bj.bcebos.com/fastdeploy/tests/pphumanmatting_result.pkl" + if pkl_url: + fd.download(pkl_url, "resources") + + with open("./resources/pphumanmatting_result.pkl", "rb") as f: + baseline = pickle.load(f) + + diff = np.fabs(np.array(result.alpha) - np.array(baseline)) + thres = 1e-05 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) diff --git a/tests/models/test_pptinypose.py b/tests/models/test_pptinypose.py index 95cacdd5e..fe838c490 100644 --- a/tests/models/test_pptinypose.py +++ b/tests/models/test_pptinypose.py @@ -20,8 +20,8 @@ import runtime_config as rc def test_keypointdetection_pptinypose(): pp_tinypose_model_url = "https://bj.bcebos.com/fastdeploy/tests/PP_TinyPose_256x192_test.tgz" - fd.download_and_decompress(pp_tinypose_model_url, ".") - model_path = "./PP_TinyPose_256x192_test" + fd.download_and_decompress(pp_tinypose_model_url, "resources") + model_path = "./resources/PP_TinyPose_256x192_test" # 配置runtime，加载模型 runtime_option = fd.RuntimeOption() model_file = os.path.join(model_path, "model.pdmodel") @@ -48,8 +48,8 @@ def test_keypointdetection_pptinypose(): def test_keypointdetection_det_keypoint_unite(): det_keypoint_unite_model_url = "https://bj.bcebos.com/fastdeploy/tests/PicoDet_320x320_TinyPose_256x192_test.tgz" - fd.download_and_decompress(det_keypoint_unite_model_url, ".") - model_path = "./PicoDet_320x320_TinyPose_256x192_test" + fd.download_and_decompress(det_keypoint_unite_model_url, "resources") + model_path = "./resources/PicoDet_320x320_TinyPose_256x192_test" # 配置runtime，加载模型 runtime_option = fd.RuntimeOption() tinypose_model_file = os.path.join( @@ -91,7 +91,7 @@ def test_keypointdetection_det_keypoint_unite(): (np.array(result.keypoints), np.array(result.scores)[:, np.newaxis]), axis=1) print(result) - np.save("baseline.npy", result) + np.save("resources/baseline.npy", result) baseline = np.load(baseline_file) diff = np.fabs(result - np.array(baseline)) thres = 1e-05 diff --git a/tests/models/test_pptracking.py b/tests/models/test_pptracking.py index 42010705c..b8842c73f 100644 --- a/tests/models/test_pptracking.py +++ b/tests/models/test_pptracking.py @@ -23,16 +23,16 @@ import runtime_config as rc def test_pptracking(): model_url = "https://bj.bcebos.com/fastdeploy/tests/pptracking.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "pptracking/fairmot_hrnetv2_w18_dlafpn_30e_576x320" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/pptracking/fairmot_hrnetv2_w18_dlafpn_30e_576x320" # use default backend runtime_option = fd.RuntimeOption() model_file = os.path.join(model_path, "model.pdmodel") params_file = os.path.join(model_path, "model.pdiparams") config_file = os.path.join(model_path, "infer_cfg.yml") model = fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=rc.test_option) - cap = cv2.VideoCapture("./person.mp4") + cap = cv2.VideoCapture("./resources/person.mp4") frame_id = 0 while True: _, frame = cap.read() @@ -40,7 +40,7 @@ def test_pptracking(): break result = model.predict(frame) # compare diff - expect = pickle.load(open("pptracking/frame" + str(frame_id) + ".pkl", "rb")) + expect = pickle.load(open("resources/pptracking/frame" + str(frame_id) + ".pkl", "rb")) diff_boxes = np.fabs(np.array(expect["boxes"]) - np.array(result.boxes)) diff_scores = np.fabs(np.array(expect["scores"]) - np.array(result.scores)) diff = max(diff_boxes.max(), diff_scores.max()) diff --git a/tests/models/test_rvm.py b/tests/models/test_rvm.py index 23fd544c6..10d680948 100644 --- a/tests/models/test_rvm.py +++ b/tests/models/test_rvm.py @@ -22,11 +22,10 @@ import runtime_config as rc def test_matting_rvm_cpu(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/rvm.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/video.mp4" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "rvm/rvm_mobilenetv3_fp32.onnx" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/rvm/rvm_mobilenetv3_fp32.onnx" # use ORT - runtime_option.use_ort_backend() model = fd.vision.matting.RobustVideoMatting( model_path, runtime_option=rc.test_option) @@ -39,7 +38,7 @@ def test_matting_rvm_cpu(): break result = model.predict(frame) # compare diff - expect_alpha = np.load("rvm/result_alpha_" + str(frame_id) + ".npy") + expect_alpha = np.load("resources/rvm/result_alpha_" + str(frame_id) + ".npy") result_alpha = np.array(result.alpha).reshape(1920, 1080) diff = np.fabs(expect_alpha - result_alpha) thres = 1e-05 diff --git a/tests/models/test_yolov5cls.py b/tests/models/test_yolov5cls.py index aeafad519..a7070de4d 100755 --- a/tests/models/test_yolov5cls.py +++ b/tests/models/test_yolov5cls.py @@ -22,9 +22,9 @@ import runtime_config as rc def test_classification_yolov5cls(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5n-cls.tgz" input_url = "https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg" - fd.download_and_decompress(model_url, ".") - fd.download(input_url, ".") - model_path = "yolov5n-cls/yolov5n-cls.onnx" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/yolov5n-cls/yolov5n-cls.onnx" # use ORT runtime_option = fd.RuntimeOption() runtime_option.use_ort_backend() @@ -32,18 +32,19 @@ def test_classification_yolov5cls(): model_path, runtime_option=rc.test_option) # compare diff - im = cv2.imread("./ILSVRC2012_val_00000010.jpeg") - result = model.predict(im.copy(), topk=5) - with open("yolov5n-cls/result.pkl", "rb") as f: - expect = pickle.load(f) - - diff_label = np.fabs( - np.array(result.label_ids) - np.array(expect["labels"])) - diff_score = np.fabs(np.array(result.scores) - np.array(expect["scores"])) - thres = 1e-05 - assert diff_label.max( - ) < thres, "The label diff is %f, which is bigger than %f" % ( - diff_label.max(), thres) - assert diff_score.max( - ) < thres, "The score diff is %f, which is bigger than %f" % ( - diff_score.max(), thres) + im = cv2.imread("./resources/ILSVRC2012_val_00000010.jpeg") + for i in range(2): + result = model.predict(im, topk=5) + with open("resources/yolov5n-cls/result.pkl", "rb") as f: + expect = pickle.load(f) + + diff_label = np.fabs( + np.array(result.label_ids) - np.array(expect["labels"])) + diff_score = np.fabs(np.array(result.scores) - np.array(expect["scores"])) + thres = 1e-05 + assert diff_label.max( + ) < thres, "The label diff is %f, which is bigger than %f" % ( + diff_label.max(), thres) + assert diff_score.max( + ) < thres, "The score diff is %f, which is bigger than %f" % ( + diff_score.max(), thres)