Optimize coverage jobs (#3683)

.github/workflows/_unit_test_coverage.yml

@@ -164,8 +164,8 @@ jobs:
 
           git config --global --add safe.directory /workspace/FastDeploy
           cd FastDeploy
+          git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
           python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
-
           pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
           python -m pip install coverage
@@ -173,6 +173,10 @@ jobs:
           python -m pip install pytest-cov
           python -m pip install jsonschema aistudio_sdk==0.3.5
           python -m pip install ${fd_wheel_url}
+          rm -rf fastdeploy
+          # coverage subprocess use
+          python -m pip install ${fd_wheel_url} --no-deps --target=/workspace/FastDeploy
+          export PYTHONPATH=/workspace/FastDeploy/
           if [ -d "tests/plugins" ]; then
               cd tests/plugins
               python setup.py install
@@ -182,12 +186,11 @@ jobs:
           fi
           export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
           export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
-          export COVERAGE_PROCESS_START=/workspace/FastDeploy/scripts/.coveragerc
           TEST_EXIT_CODE=0
           bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
-          git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
           echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
           coverage combine coveragedata/ || echo "No data to combine"
+          coverage report
           coverage xml -o python_coverage_all.xml
           COVERAGE_EXIT_CODE=0
           if [[ "$IS_PR" == "true" ]]; then
@@ -228,7 +231,7 @@ jobs:
             echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT
             echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV
           fi
-          unittest_result="tests/failed_tests.log"
+          unittest_result="failed_tests.log"
           if [ -s ${unittest_result} ];then
             python ${push_file} ${unittest_result} ${target_path}/UnitTestResult
             target_path_stripped="${target_path#paddle-github-action/}"

scripts/.coveragerc

@@ -1,8 +1,9 @@
 [run]
 branch = True
 source = fastdeploy
-parallel = True
 concurrency = multiprocessing
+patch = subprocess
+parallel = True
 
 [paths]
 source =

scripts/coverage_run.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-run_path="$DIR/../tests/"
+tests_path="$DIR/../tests/"
 export PYTEST_INI="$DIR/../tests/cov_pytest.ini"
+run_path=$( realpath "$DIR/../")
 
 export COVERAGE_FILE=${COVERAGE_FILE:-$DIR/../coveragedata/.coverage}
 export COVERAGE_RCFILE=${COVERAGE_RCFILE:-$DIR/../scripts/.coveragerc}
-export COVERAGE_PROCESS_START=${COVERAGE_PROCESS_START:-$DIR/../scripts/.coveragerc}
+
-cd "$run_path" || exit 1
 
 failed_tests_file="failed_tests.log"
 > "$failed_tests_file"
@@ -16,7 +16,7 @@ failed_tests_file="failed_tests.log"
 # 执行 pytest，每个文件单独跑
 ##################################
 # 收集 pytest 文件
-TEST_FILES=$(python -m pytest --collect-only -q -c ${PYTEST_INI} --disable-warnings | grep -Eo '^.*test_.*\.py' | sort | uniq)
+TEST_FILES=$(python -m pytest --collect-only -q -c ${PYTEST_INI} ${tests_path} --rootdir=${run_path} --disable-warnings | grep -Eo '^.*test_.*\.py' | sort | uniq)
 
 
 failed_pytest=0
@@ -24,7 +24,7 @@ success_pytest=0
 
 for file in $TEST_FILES; do
     echo "Running pytest file: $file"
-    python -m pytest -c ${PYTEST_INI} --cov-config=${COVERAGE_RCFILE} "$file" -vv -s
+    python -m coverage run -m pytest -c ${PYTEST_INI} "$file" -vv -s
     status=$?
     if [ "$status" -ne 0 ]; then
         echo "$file" >> "$failed_tests_file"

tests/cov_pytest.ini

@@ -1,31 +1,24 @@
 [pytest]
 # 跳过目录
 addopts =
-    --ignore=ci_use
+    --ignore=tests/ci_use
-    --ignore=ce
+    --ignore=tests/ce
-    --ignore=layers/test_append_attention.py
+    --ignore=tests/layers/test_append_attention.py
-    --ignore=layers/test_attention.py
+    --ignore=tests/layers/test_attention.py
-    --ignore=operators/test_rejection_top_p_sampling.py
+    --ignore=tests/operators/test_rejection_top_p_sampling.py
-    --ignore=operators/test_perchannel_gemm.py
+    --ignore=tests/operators/test_perchannel_gemm.py
-    --ignore=operators/test_scaled_gemm_f8_i4_f16.py
+    --ignore=tests/operators/test_scaled_gemm_f8_i4_f16.py
-    --ignore=operators/test_topp_sampling.py
+    --ignore=tests/operators/test_topp_sampling.py
-    --ignore=operators/test_stop_generation.py
+    --ignore=tests/operators/test_stop_generation.py
-    --ignore=operators/test_air_topp_sampling.py
+    --ignore=tests/operators/test_air_topp_sampling.py
-    --ignore=operators/test_fused_moe.py
+    --ignore=tests/operators/test_fused_moe.py
-    --ignore=operators/test_stop_generation_multi_ends.py
+    --ignore=tests/operators/test_stop_generation_multi_ends.py
-    --ignore=graph_optimization/test_cuda_graph.py
+    --ignore=tests/graph_optimization/test_cuda_graph.py
-    --ignore=graph_optimization/test_cuda_graph_dynamic_subgraph.py
+    --ignore=tests/graph_optimization/test_cuda_graph_dynamic_subgraph.py
-    --ignore=graph_optimization/test_cuda_graph_spec_decode
+    --ignore=tests/graph_optimization/test_cuda_graph_spec_decode
-    --ignore=layers/test_quant_layer.py
+    --ignore=tests/layers/test_quant_layer.py
-    --ignore=operators/test_token_penalty.py
+    --ignore=tests/operators/test_token_penalty.py
-    --ignore=operators/test_split_fuse.py
+    --ignore=tests/operators/test_split_fuse.py
-    --ignore=operators/test_flash_mask_attn.py
+    --ignore=tests/operators/test_flash_mask_attn.py
-    --ignore=operators/test_w4afp8_gemm.py
+    --ignore=tests/operators/test_w4afp8_gemm.py
-    --ignore=operators/test_tree_mask.py
+    --ignore=tests/operators/test_tree_mask.py
-    --cov=fastdeploy
-    --cov-branch
-    --cov-append
-    --cov-report=
-
-# 输出更详细的结果
-console_output_style = progress