Optimize CI execution workflow (#3371)

* Optimize CI execution workflow

* fix

.github/workflows/_base_test.yml

@@ -70,96 +70,109 @@ jobs:
           CACHE_DIR: ${{ inputs.CACHE_DIR }}
           MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
         run: |
-            runner_name="${{ runner.name }}"
+          runner_name="${{ runner.name }}"
-            last_char="${runner_name: -1}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
 
-            if [[ "$last_char" =~ [0-7] ]]; then
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
-              DEVICES="$last_char"
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
-            else
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
-              DEVICES="0"
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
-            fi
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
 
-            FLASK_PORT=$((42068 + DEVICES * 100))
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
-            FD_API_PORT=$((42088 + DEVICES * 100))
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
-            FD_ENGINE_QUEUE_PORT=$((42058 + DEVICES * 100))
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
-            FD_METRICS_PORT=$((42078 + DEVICES * 100))
+            touch "${CACHE_DIR}/gitconfig"
-            echo "Test ENV Parameter:"
+          fi
-            echo "========================================================="
+          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
-            echo "FLASK_PORT=${FLASK_PORT}"
+            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
-            echo "FD_API_PORT=${FD_API_PORT}"
+            exit 1
-            echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          fi
-            echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
-            echo "DEVICES=${DEVICES}"
-            echo "========================================================="
 
-            CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
-            echo "CACHE_DIR is set to ${CACHE_DIR}"
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
-            if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
-              touch "${CACHE_DIR}/gitconfig"
-            fi
-            if [ ! -d "${MODEL_CACHE_DIR}" ]; then
-              echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
-              exit 1
-            fi
 
-            PARENT_DIR=$(dirname "$WORKSPACE")
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
 
-            docker run --rm --ipc=host --pid=host --net=host \
+          for port in "${PORTS[@]}"; do
-            -v $(pwd):/workspace \
+              PIDS=$(lsof -t -i :$port || true)
-            -w /workspace \
+              if [ -n "$PIDS" ]; then
-            -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
-            -e "FD_API_PORT=${FD_API_PORT}" \
+                  echo "$PIDS" | xargs -r kill -9
-            -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+                  echo "Port $port cleared" | tee -a $LOG_FILE
-            -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+              else
-            -e "FLASK_PORT=${FLASK_PORT}" \
+                  echo "Port $port is free" | tee -a $LOG_FILE
-            -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+              fi
-            -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          done
-            -v "${CACHE_DIR}/.cache:/root/.cache" \
-            -v "${CACHE_DIR}/ConfigDir:/root/.config" \
-            -e TZ="Asia/Shanghai" \
-            --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
-            python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
-            pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
 
-            python -m pip install ${fastdeploy_wheel_url}
+          docker run --rm --ipc=host --pid=host --net=host \
-            python -m pip install pytest
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -e TZ="Asia/Shanghai" \
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
-            wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
-            chmod +x ./llm-deploy-linux-amd64
-            ./llm-deploy-linux-amd64 -python python3.10 \
-            -model_name ERNIE-4.5-0.3B-Paddle \
-            -model_path /MODELDATA \
-            --skip install
 
-            git config --global --add safe.directory /workspace/FastDeploy
+          python -m pip install ${fastdeploy_wheel_url}
-            cd FastDeploy
+          python -m pip install pytest
-            pushd test/ce/deploy
-            python3.10 deploy.py > dd.log 2>&1 &
-            sleep 3
-            curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
-              -H "Content-Type: application/json" \
-              -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
 
-            curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
+          wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
-            popd
+          chmod +x ./llm-deploy-linux-amd64
+          ./llm-deploy-linux-amd64 -python python3.10 \
+          -model_name ERNIE-4.5-0.3B-Paddle \
+          -model_path /MODELDATA \
+          --skip install
 
-            pushd test/ce/server
+          git config --global --add safe.directory /workspace/FastDeploy
-            export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
+          cd FastDeploy
-            export TEMPLATE=TOKEN_LOGPROB
+          pushd test/ce/deploy
-            TEST_EXIT_CODE=0
+          python3.10 deploy.py > dd.log 2>&1 &
-            python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py || TEST_EXIT_CODE=1
+          sleep 3
-            curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
-              -H "Content-Type: application/json" \
+            -H "Content-Type: application/json" \
-              -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
+            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
-            curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
+
-            python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1
+          curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
-            popd
+          popd
-            echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
+
-            '
+          pushd test/ce/server
-            if [ -f ./FastDeploy/exit_code.env ]; then
+          export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
-              source ./FastDeploy/exit_code.env
+          export TEMPLATE=TOKEN_LOGPROB
-              cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
+          TEST_EXIT_CODE=0
-            fi
+          python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py || TEST_EXIT_CODE=1
-            echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
-            exit ${TEST_EXIT_CODE}
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
+          curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
+          python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1
+          popd
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
+          '
+          if [ -f ./FastDeploy/exit_code.env ]; then
+            source ./FastDeploy/exit_code.env
+            cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
+          exit ${TEST_EXIT_CODE}

.github/workflows/_logprob_test_linux.yml

@@ -62,18 +62,14 @@ jobs:
           MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
         run: |
           runner_name="${{ runner.name }}"
-          last_char="${runner_name: -1}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
 
-          if [[ "$last_char" =~ [0-7] ]]; then
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
-            DEVICES="$last_char"
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
-          else
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
-            DEVICES="0"
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
-          fi
-
-          FLASK_PORT=$((42068 + DEVICES * 100))
-          FD_API_PORT=$((42088 + DEVICES * 100))
-          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICES * 100))
-          FD_METRICS_PORT=$((42078 + DEVICES * 100))
           echo "Test ENV Parameter:"
           echo "========================================================="
           echo "FLASK_PORT=${FLASK_PORT}"
@@ -93,9 +89,24 @@ jobs:
             exit 1
           fi
 
-          PARENT_DIR=$(dirname "$WORKSPACE")
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
-          unset http_proxy
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
-          unset https_proxy
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
 
           docker run --rm --ipc=host --pid=host --net=host \
           -v $(pwd):/workspace \

.github/workflows/_pre_ce_test.yml

@@ -99,6 +99,25 @@ jobs:
             touch "${CACHE_DIR}/gitconfig"
           fi
 
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
           docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
           -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
           -v "${CACHE_DIR}/.cache:/root/.cache" \
@@ -115,8 +134,5 @@ jobs:
           cd FastDeploy
           python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           python -m pip install ${fd_wheel_url}
-          for port in $FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT; do
-            lsof -t -i :$port | xargs -r kill -9 || true
-          done
           bash scripts/run_pre_ce.sh
           '

.github/workflows/_unit_test_coverage.yml

@@ -74,81 +74,99 @@ jobs:
           BASE_REF: ${{ github.event.pull_request.base.ref }}
           MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
         run: |
-            set -x
+          set -x
-            runner_name="${{ runner.name }}"
+          runner_name="${{ runner.name }}"
-            CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
-            DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
-            DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
 
-            FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
-            FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
-            FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
-            FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
-            echo "Test ENV Parameter:"
+          echo "Test ENV Parameter:"
-            echo "========================================================="
+          echo "========================================================="
-            echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FLASK_PORT=${FLASK_PORT}"
-            echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
-            echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
-            echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
-            echo "DEVICES=${DEVICES}"
+          echo "DEVICES=${DEVICES}"
-            echo "========================================================="
+          echo "========================================================="
 
-            CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
-            echo "CACHE_DIR is set to ${CACHE_DIR}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
-            if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
-              touch "${CACHE_DIR}/gitconfig"
+            touch "${CACHE_DIR}/gitconfig"
-            fi
+          fi
-            PARENT_DIR=$(dirname "$WORKSPACE")
-            echo "PARENT_DIR:$PARENT_DIR"
-            docker run --rm --net=host \
-            --cap-add=SYS_PTRACE --privileged --shm-size=64G \
-            -v $(pwd):/workspace -w /workspace \
-            -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
-            -v "${CACHE_DIR}/.cache:/root/.cache" \
-            -v "${CACHE_DIR}/ConfigDir:/root/.config" \
-            -v "${MODEL_CACHE_DIR}:/ModelData:ro" \
-            -e "MODEL_PATH=/ModelData" \
-            -e "FD_API_PORT=${FD_API_PORT}" \
-            -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
-            -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
-            -e "FLASK_PORT=${FLASK_PORT}" \
-            -e TZ="Asia/Shanghai" \
-            -e "fd_wheel_url=${fd_wheel_url}" \
-            -e "BASE_REF=${BASE_REF}" \
-            --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
 
-            git config --global --add safe.directory /workspace/FastDeploy
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
-            cd FastDeploy
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
-            python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
 
-            pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
 
-            python -m pip install coverage
+          for port in "${PORTS[@]}"; do
-            python -m pip install diff-cover
+              PIDS=$(lsof -t -i :$port || true)
-            python -m pip install ${fd_wheel_url}
+              if [ -n "$PIDS" ]; then
-            if [ -d "test/plugins" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
-                cd test/plugins
+                  echo "$PIDS" | xargs -r kill -9
-                python setup.py install
+                  echo "Port $port cleared" | tee -a $LOG_FILE
-                cd ../..
+              else
-            else
+                  echo "Port $port is free" | tee -a $LOG_FILE
-                echo "Warning: test/plugins directory not found, skipping setup.py install"
+              fi
-            fi
+          done
-            export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
+
-            export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
-            TEST_EXIT_CODE=0
+
-            bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
+          docker run --rm --net=host \
-            git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
+          --cap-add=SYS_PTRACE --shm-size=64G \
-            echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
+          -v $(pwd):/workspace -w /workspace \
-            coverage combine coveragedata/
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
-            coverage xml -o python_coverage_all.xml
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
-            COVERAGE_EXIT_CODE=0
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
-            diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
+          -v "${MODEL_CACHE_DIR}:/ModelData:ro" \
-            echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
+          -e "MODEL_PATH=/ModelData" \
-            python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
+          -e "FD_API_PORT=${FD_API_PORT}" \
-            '
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
-            if [ -f FastDeploy/exit_code.env ]; then
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
-              cat FastDeploy/exit_code.env >> $GITHUB_ENV
+          -e "FLASK_PORT=${FLASK_PORT}" \
-            fi
+          -e TZ="Asia/Shanghai" \
+          -e "fd_wheel_url=${fd_wheel_url}" \
+          -e "BASE_REF=${BASE_REF}" \
+          --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
+
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+
+          pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install coverage
+          python -m pip install diff-cover
+          python -m pip install ${fd_wheel_url}
+          if [ -d "test/plugins" ]; then
+              cd test/plugins
+              python setup.py install
+              cd ../..
+          else
+              echo "Warning: test/plugins directory not found, skipping setup.py install"
+          fi
+          export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
+          export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
+          TEST_EXIT_CODE=0
+          bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
+          git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
+          coverage combine coveragedata/
+          coverage xml -o python_coverage_all.xml
+          COVERAGE_EXIT_CODE=0
+          diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
+          echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
+          python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
+          '
+          if [ -f FastDeploy/exit_code.env ]; then
+            cat FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
       - name: Upload unit resule and diff coverage to bos
         id: cov_upload
         shell: bash

scripts/coverage_run.sh

@@ -65,11 +65,25 @@ for dir in "${dirs[@]}"; do
         echo "Skipping disabled test: $test_file"
         continue
       fi
-
+      # TODO: Add a framework to manage unit test execution time
-      python -m coverage run "$test_file"
+      timeout 600 python -m coverage run "$test_file"
       if [ $? -ne 0 ]; then
         echo "$test_file" >> "$failed_tests_file"
         fail=$((fail + 1))
+
+        PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+        echo "==== PORT CLEAN AFTER UT FAILED ===="
+
+        for port in "${PORTS[@]}"; do
+            PIDS=$(lsof -t -i :$port)
+            if [ -n "$PIDS" ]; then
+                echo "Port $port is occupied by PID(s): $PIDS"
+                echo "$PIDS" | xargs -r kill -9
+                echo "Port $port cleared"
+            else
+                echo "Port $port is free"
+            fi
+        done
       else
         success=$((success + 1))
       fi