Optimize CI execution workflow. (#3371) (#3384)

* fix

.github/workflows/_logprob_test_linux.yml

@@ -62,18 +62,22 @@ jobs:
           MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
         run: |
           runner_name="${{ runner.name }}"
-          last_char="${runner_name: -1}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
 
-          if [[ "$last_char" =~ [0-7] ]]; then
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
-            DEVICES="$last_char"
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
-          else
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
-            DEVICES="0"
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
-          fi
+          echo "Test ENV Parameter:"
-
+          echo "========================================================="
-          FLASK_PORT=$((9160 + DEVICES * 100))
+          echo "FLASK_PORT=${FLASK_PORT}"
-          FD_API_PORT=$((9180 + DEVICES * 100))
+          echo "FD_API_PORT=${FD_API_PORT}"
-          FD_ENGINE_QUEUE_PORT=$((9150 + DEVICES * 100))
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
-          FD_METRICS_PORT=$((9170 + DEVICES * 100))
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
 
           CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
           echo "CACHE_DIR is set to ${CACHE_DIR}"
@@ -85,7 +89,24 @@ jobs:
             exit 1
           fi
 
-          PARENT_DIR=$(dirname "$WORKSPACE")
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
 
           docker run --ipc=host --pid=host --net=host \
           -v $(pwd):/workspace \

.github/workflows/_pre_ce_test.yml

@@ -99,6 +99,25 @@ jobs:
             touch "${CACHE_DIR}/gitconfig"
           fi
 
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
           docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
           -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
           -v "${CACHE_DIR}/.cache:/root/.cache" \
@@ -115,8 +134,5 @@ jobs:
           cd FastDeploy
           python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           python -m pip install ${fd_wheel_url}
-          for port in $FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT; do
-            lsof -t -i :$port | xargs -r kill -9 || true
-          done
           bash scripts/run_pre_ce.sh
           '

.github/workflows/_unit_test_coverage.yml

@@ -98,10 +98,28 @@ jobs:
           if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
             touch "${CACHE_DIR}/gitconfig"
           fi
-            PARENT_DIR=$(dirname "$WORKSPACE")
+
-            echo "PARENT_DIR:$PARENT_DIR"
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
           docker run --rm --net=host \
-            --cap-add=SYS_PTRACE --privileged --shm-size=64G \
+          --cap-add=SYS_PTRACE --shm-size=64G \
           -v $(pwd):/workspace -w /workspace \
           -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
           -v "${CACHE_DIR}/.cache:/root/.cache" \
@@ -126,6 +144,13 @@ jobs:
           python -m pip install coverage
           python -m pip install diff-cover
           python -m pip install ${fd_wheel_url}
+          if [ -d "test/plugins" ]; then
+              cd test/plugins
+              python setup.py install
+              cd ../..
+          else
+              echo "Warning: test/plugins directory not found, skipping setup.py install"
+          fi
           export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
           export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
           TEST_EXIT_CODE=0
@@ -142,6 +167,7 @@ jobs:
           if [ -f FastDeploy/exit_code.env ]; then
             cat FastDeploy/exit_code.env >> $GITHUB_ENV
           fi
+
       - name: Upload unit resule and diff coverage to bos
         id: cov_upload
         shell: bash

scripts/coverage_run.sh

@@ -66,11 +66,25 @@ for dir in "${dirs[@]}"; do
         echo "Skipping disabled test: $test_file"
         continue
       fi
-
+      # TODO: Add a framework to manage unit test execution time
-      python -m coverage run "$test_file"
+      timeout 600 python -m coverage run "$test_file"
       if [ $? -ne 0 ]; then
         echo "$test_file" >> "$failed_tests_file"
         fail=$((fail + 1))
+
+        PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+        echo "==== PORT CLEAN AFTER UT FAILED ===="
+
+        for port in "${PORTS[@]}"; do
+            PIDS=$(lsof -t -i :$port)
+            if [ -n "$PIDS" ]; then
+                echo "Port $port is occupied by PID(s): $PIDS"
+                echo "$PIDS" | xargs -r kill -9
+                echo "Port $port cleared"
+            else
+                echo "Port $port is free"
+            fi
+        done
       else
         success=$((success + 1))
       fi