Update release task (#289)

* Update precision diff regulation

* Update diff precision regulation

* Add judgement when python scripts run error

* Fix bug

* Fix bug

* Fix bug

* Add -DCMAKE_CXX_COMPILER

Co-authored-by: root <root@bjyz-sys-gpu-kongming2.bjyz.baidu.com>

tests/release_task/CMakeLists.txt

@@ -20,4 +20,4 @@ if(WIN32)
     target_link_libraries(infer_ppyoloe_demo ${FASTDEPLOY_LIBS} gflags ${CMAKE_THREAD_LIBS_INIT} Shlwapi.lib)
 else()
     target_link_libraries(infer_ppyoloe_demo ${FASTDEPLOY_LIBS} gflags ${CMAKE_THREAD_LIBS_INIT})
-endif
+endif()

tests/release_task/compare_with_gt.py

@@ -44,18 +44,37 @@ def write2file(error_file):
         f.write(args.platform + " " + py_version + " " +
                 args.result_path.split(".")[0] + "\n")
 
+def save_numpy_result(file_path, error_msg):
+    np.savetxt(file_path, error_msg, fmt='%f',delimiter=',')
 
 def check_result(gt_result, infer_result, args):
     if len(gt_result) != len(infer_result):
         infer_result = infer_result[-len(gt_result):]
     diff = np.abs(gt_result - infer_result)
-    if (diff > 1e-5).any():
+    label_diff = diff[:, -1]
-        print(args.platform, args.device, "diff ", diff)
+    score_diff = diff[:, -2]
+    boxes_diff = diff[:, :-2]
+    boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6)
+    is_diff = False
+    if (label_diff > 0).any(): 
+        print(args.platform, args.device, "label diff ", label_diff)        
+        is_diff = True
+        save_numpy_result("label_diff_bool.txt", label_diff > 0)
+    if (score_diff > 1e-4).any():
+        print(args.platform, args.device, "score diff ", score_diff)
+        is_diff = True
+        save_numpy_result("score_diff_bool.txt", score_diff > 1e-4)
+    if (boxes_diff_ratio > 1e-4).any() and (boxes_diff > 1e-3).any():
+        print(args.platform, args.device, "boxes diff ", boxes_diff_ratio)
+        is_diff = True
+        save_numpy_result("boxes_diff_bool.txt", boxes_diff > 1e-3)
+        save_numpy_result("boxes_diff_ratio.txt", boxes_diff_ratio)
+        save_numpy_result("boxes_diff_ratio_bool.txt", boxes_diff_ratio > 1e-4)
+    if is_diff:
         write2file("result.txt")
     else:
         print(args.platform, args.device, "No diff")
 
-
 if __name__ == '__main__':
     args = parse_arguments()
 

tests/release_task/cpp_run.bat

@@ -38,7 +38,7 @@ wget -q  https://fastdeploy.bj.bcebos.com/dev/cpp/$CPP_FASTDEPLOY_PACKAGE.zip
 tar -xf %cd%\%CPP_FASTDEPLOY_PACKAGE%.zip
 
 mkdir build && cd build
-cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2"
+cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2"  -DCMAKE_CXX_COMPILER=%CMAKE_CXX_COMPILER%
 
 msbuild infer_demo.sln /m:4 /p:Configuration=Release /p:Platform=x64
 
@@ -80,12 +80,16 @@ for %%b in (%RUN_CASES%) do (
     ) 
 )
 
+
 set res_file=%cd%\result.txt
 
 if exist %res_file% (
     for /f "delims=" %%i in (%res_file%) do echo %%i
     exit -1
 ) else (
+    if %errorlevel% NEQ 0 (
+        exit -1
+    )
     exit 0
 )
 :END

tests/release_task/cpp_run.sh

@@ -43,7 +43,7 @@ wget -q  https://fastdeploy.bj.bcebos.com/dev/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz
 
 tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz
 mkdir build && cd build
-cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../$CPP_FASTDEPLOY_PACKAGE
+cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../$CPP_FASTDEPLOY_PACKAGE -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER
 make -j
 
 for((i=0;i<case_number;i+=1))
@@ -66,10 +66,16 @@ do
        fi
 done
 
-res_file="*result.txt"
+ret=$?
+
+res_file="result.txt"
 if [ ! -f $res_file ];then
+       if [ $ret -ne 0 ];then
+               exit -1
+       fi
        exit 0
 else
        cat $res_file
        exit -1
 fi
+

tests/release_task/py_run.bat

@@ -54,12 +54,16 @@ for %%b in (%RUN_CASES%) do (
     ) 
 )
 
+
 set res_file=%cd%\result.txt
 
 if exist %res_file% (
     for /f "delims=" %%i in (%res_file%) do echo %%i
     exit -1
 ) else (
+    if %errorlevel% NEQ 0 (
+        exit -1
+    )
     exit 0
 )
 :END

tests/release_task/py_run.sh

@@ -45,8 +45,8 @@ do
        backend=${RUN_CASE[j]}
        echo "Python Backend:" $backend
        if [ "$backend" != "trt" ];then
-               python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_cpu_result.txt
+               python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt
-               python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_cpu_result.txt --platform $PLATFORM --device cpu
+               python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu
        fi
        if [ "$DEVICE" = "gpu" ];then
 
@@ -54,14 +54,19 @@ do
                        python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt
                        python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt
 	       else
-		       python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_gpu_result.txt
+		       python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt
-                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_gpu_result.txt --platform $PLATFORM --device gpu
+                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu
                fi
        fi
 done
 
-res_file="*result.txt"
+ret=$? 
+
+res_file="result.txt"
 if [ ! -f $res_file ];then
+       if [ $ret -ne 0 ];then
+	       exit -1
+       fi
        exit 0
 else
        cat $res_file