Merge branch 'release/2.2' into fix-gpu-memory-oom

* [fix]Modify follow-up push parameters and Modify the verification method for thinking length (#4086)

* 续推参数  generated_token_ids 修改成 completion_token_ids;修改思考长度校验方式

* 续推参数  generated_token_ids 修改成 completion_token_ids;修改思考长度校验方式

* 续推参数  generated_token_ids 修改成 completion_token_ids;修改思考长度校验方式

* 续推参数  generated_token_ids 修改成 completion_token_ids;修改思考长度校验方式

* add completion_token_ids

* add logger

* fix reasoning_max_tokens ParameterError

* add unittest

* add unittest

* add unittest

* add unittest

* add unittest

* add unit test

* fix

.clang-format

@@ -1,180 +1,29 @@
+# This file is used by clang-format to autoformat paddle source code
+#
+# The clang-format is part of llvm toolchain.
+# It need to install llvm and clang to format source code style.
+#
+# The basic usage is,
+#   clang-format -i -style=file PATH/TO/SOURCE/CODE
+#
+# The -style=file implicit use ".clang-format" file located in one of
+# parent directory.
+# The -i means inplace change.
+#
+# The document of clang-format is
+#   http://clang.llvm.org/docs/ClangFormat.html
+#   http://clang.llvm.org/docs/ClangFormatStyleOptions.html
 ---
 Language:        Cpp
-# BasedOnStyle:  LLVM
-AccessModifierOffset: -1
-AlignAfterOpenBracket: Align
-AlignArrayOfStructures: None
-AlignConsecutiveMacros: None
-AlignConsecutiveAssignments: None
-AlignConsecutiveBitFields: None
-AlignConsecutiveDeclarations: None
-AlignEscapedNewlines: Right
-AlignOperands:   Align
-AlignTrailingComments: true
-AllowAllArgumentsOnNextLine: true
-AllowAllConstructorInitializersOnNextLine: true
-AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortEnumsOnASingleLine: true
-AllowShortBlocksOnASingleLine: Never
-AllowShortCaseLabelsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: All
-AllowShortLambdasOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: Never
-AllowShortLoopsOnASingleLine: false
-AlwaysBreakAfterDefinitionReturnType: None
-AlwaysBreakAfterReturnType: None
-AlwaysBreakBeforeMultilineStrings: false
-AlwaysBreakTemplateDeclarations: MultiLine
-AttributeMacros:
-  - __capability
-BinPackArguments: true
-BinPackParameters: true
-BraceWrapping:
-  AfterCaseLabel:  false
-  AfterClass:      false
-  AfterControlStatement: Never
-  AfterEnum:       false
-  AfterFunction:   false
-  AfterNamespace:  false
-  AfterObjCDeclaration: false
-  AfterStruct:     false
-  AfterUnion:      false
-  AfterExternBlock: false
-  BeforeCatch:     false
-  BeforeElse:      false
-  BeforeLambdaBody: false
-  BeforeWhile:     false
-  IndentBraces:    false
-  SplitEmptyFunction: true
-  SplitEmptyRecord: true
-  SplitEmptyNamespace: true
-BreakBeforeBinaryOperators: None
-BreakBeforeConceptDeclarations: true
-BreakBeforeBraces: Attach
-BreakBeforeInheritanceComma: false
-BreakInheritanceList: BeforeColon
-BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
-BreakConstructorInitializers: BeforeColon
-BreakAfterJavaFieldAnnotations: false
-BreakStringLiterals: true
-ColumnLimit:     80
-# CommentPragmas:  '^ IWYU pragma:'
-# CommentPragmas:  '^[^ ]'
-CommentPragmas:  '^\\.+'
-CompactNamespaces: false
-ConstructorInitializerAllOnOneLineOrOnePerLine: false
-ConstructorInitializerIndentWidth: 4
+BasedOnStyle:  Google
+IndentWidth:     4
+TabWidth:        2
 ContinuationIndentWidth: 4
-Cpp11BracedListStyle: true
-DeriveLineEnding: true
-DerivePointerAlignment: false
-DisableFormat:   false
-EmptyLineAfterAccessModifier: Never
-EmptyLineBeforeAccessModifier: LogicalBlock
-ExperimentalAutoDetectBinPacking: false
-FixNamespaceComments: true
-ForEachMacros:
-  - foreach
-  - Q_FOREACH
-  - BOOST_FOREACH
-IfMacros:
-  - KJ_IF_MAYBE
+AccessModifierOffset: -1  # The private/protected/public has no indent in class
+Standard:  Cpp11
+AllowAllParametersOfDeclarationOnNextLine: true
+BinPackParameters: false
+BinPackArguments: false
 IncludeBlocks: Preserve
-IncludeCategories:
-  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
-    Priority:        2
-    SortPriority:    0
-    CaseSensitive:   false
-  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
-    Priority:        3
-    SortPriority:    0
-    CaseSensitive:   false
-  - Regex:           '.*'
-    Priority:        1
-    SortPriority:    0
-    CaseSensitive:   false
-IncludeIsMainRegex: '(Test)?$'
-IncludeIsMainSourceRegex: ''
-IndentAccessModifiers: false
-IndentCaseLabels: false
-IndentCaseBlocks: false
-IndentGotoLabels: true
-IndentPPDirectives: None
-IndentExternBlock: AfterExternBlock
-IndentRequires:  false
-IndentWidth:     2
-IndentWrappedFunctionNames: false
-InsertTrailingCommas: None
-JavaScriptQuotes: Leave
-JavaScriptWrapImports: true
-KeepEmptyLinesAtTheStartOfBlocks: true
-LambdaBodyIndentation: Signature
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
-MaxEmptyLinesToKeep: 1
-NamespaceIndentation: None
-ObjCBinPackProtocolList: Auto
-ObjCBlockIndentWidth: 2
-ObjCBreakBeforeNestedBlockParam: true
-ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: true
-PenaltyBreakAssignment: 2
-PenaltyBreakBeforeFirstCallParameter: 19
-PenaltyBreakComment: 300
-PenaltyBreakFirstLessLess: 120
-PenaltyBreakString: 1000
-PenaltyBreakTemplateDeclaration: 10
-PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 60
-PenaltyIndentedWhitespace: 0
-PointerAlignment: Left
-PPIndentWidth:   -1
-ReferenceAlignment: Pointer
-ReflowComments:  false
-ShortNamespaceLines: 1
-SortIncludes:    CaseSensitive
-SortJavaStaticImport: Before
-SortUsingDeclarations: true
-SpaceAfterCStyleCast: false
-SpaceAfterLogicalNot: false
-SpaceAfterTemplateKeyword: true
-SpaceBeforeAssignmentOperators: true
-SpaceBeforeCaseColon: false
-SpaceBeforeCpp11BracedList: false
-SpaceBeforeCtorInitializerColon: true
-SpaceBeforeInheritanceColon: true
-SpaceBeforeParens: ControlStatements
-SpaceAroundPointerQualifiers: Default
-SpaceBeforeRangeBasedForLoopColon: true
-SpaceInEmptyBlock: false
-SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 2
-SpacesInAngles:  Never
-SpacesInConditionalStatement: false
-SpacesInContainerLiterals: true
-SpacesInCStyleCastParentheses: false
-SpacesInLineCommentPrefix:
-  Minimum:         1
-  Maximum:         -1
-SpacesInParentheses: false
-SpacesInSquareBrackets: false
-SpaceBeforeSquareBrackets: false
-BitFieldColonSpacing: Both
-Standard:        Latest
-StatementAttributeLikeMacros:
-  - Q_EMIT
-StatementMacros:
-  - Q_UNUSED
-  - QT_REQUIRE_VERSION
-TabWidth:        8
-UseCRLF:         false
-UseTab:          Never
-WhitespaceSensitiveMacros:
-  - STRINGIZE
-  - PP_STRINGIZE
-  - BOOST_PP_STRINGIZE
-  - NS_SWIFT_NAME
-  - CF_SWIFT_NAME
+IncludeIsMainSourceRegex: (\.cu)$
 ...
-

.clang_format.hook

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-readonly VERSION="3.8"
-
-version=$(clang-format -version)
-
-if ! [[ version==∗"VERSION"* ]]; then
-    echo "clang-format version check failed."
-    echo "a version contains 'VERSION′isneeded,butget′version'"
-    echo "you can install the right version, and make an soft-link to '$PATH' env"
-    exit -1
-fi
-
-clang-format -style=google $@

.cpplint_pre_commit.hook

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-#TOTAL_ERRORS=0
-#echo "HAHAHAHAHHA"
-#exit 5
-#
-#files=$(
-#
-#if [[ ! $TRAVIS_BRANCH ]]; then
-#  # install cpplint on local machine.
-#  if [[ ! $(which cpplint) ]]; then
-#    pip install cpplint
-#  fi
-#  # diff files on local machine. 
-#  files=$(git diff --cached --name-status | awk 'Extra open brace or missing close brace2}')
-#else
-#  # diff files between PR and latest commit on Travis CI. 
-#  branch_ref=(gitrev−parse"TRAVIS_BRANCH")
-#  head_ref=$(git rev-parse HEAD)
-#  files=(gitdiff−−name−statusbranch_ref $head_ref | awk 'Extra open brace or missing close brace2}')
-#fi
-## The trick to remove deleted files: https://stackoverflow.com/a/2413151
-#for file in $files; do
-#    echo $file
-#    if [[ $file =~ ^(patches/.*) ]]; then
-#        continue;
-#    else
-#        cpplint --filter=-readability/fn_size $file;
-#        TOTAL_ERRORS=(exprTOTAL_ERRORS + $?);
-#    fi
-#done
-#
-#exit $TOTAL_ERRORS
-
-if git rev-parse --verify HEAD >/dev/null 2>&1
-then
-    against=HEAD
-else
-    # Initial commit: diff against an empty tree object
-    against=4b825dc642cb6eb9a060e54bf8d69288fbee4904
-fi
-
-# Redirect output to stderr.
-exec 1>&2
-
-cpplint=cpplint
-sum=0
-filters='-build/include_order,-build/namespaces,-legal/copyright,-runtime/references,-build/include_what_you_use'
-        
-# for cpp
-for file in $(git diff-index --name-status $against -- | grep -E '\.[ch](pp)?$' | awk '{print $2}'); do
-    $cpplint --filter=$filters $file
-    sum=$(expr ${sum} + $?)
-done
-    
-if [ ${sum} -eq 0 ]; then
-    exit 0
-else
-    exit 1
-fi

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ignore = E203, E402, E501, E731, E741, W503, W605, E722, E231, W604, E702, E226, E221, E713, E271
+max-line-length = 119
+
+# E402: module level import not at top of file
+per-file-ignores =
+    __init__.py:F401,F403,E402

.github/ISSUE_TEMPLATE/english.md

@@ -1,18 +0,0 @@
----
-name: English
-about: Report issue in English
-title: ''
-labels: ''
-assignees: ''
-
----
-
-## Environment
-
-FastDeploy version: e.g 0.8.0 or the latest code in develop branch
-OS Platform: e.g. Linux x64 / Windows x64 / Mac OSX 12.1(arm or intel)
-Hardware: e.g. Nvidia GPU 3080Ti  CUDA 11.2 CUDNN 8.3
-Program Language: e.g. Python 3.8
-
-## Problem description
-Please attach the log file if there's problem happend.

.github/ISSUE_TEMPLATE/other.md

@@ -1,10 +0,0 @@
----
-name: Other
-about: Other issues, e.g feature/model request
-title: ''
-labels: ''
-assignees: ''
-
----
-
-

.github/ISSUE_TEMPLATE/报告issue.md

@@ -1,35 +0,0 @@
----
-name: 报告issue
-about: 反馈使用中遇到的问题
-title: ''
-labels: ''
-assignees: ''
-
----
-
-*********************************************
-温馨提示：根据社区不完全统计，按照模板提问，可以加快回复和解决问题的速度
-*********************************************
-
-## 环境
-
-- 【FastDeploy版本】： 说明具体的版本，如fastdeploy-linux-gpu-0.8.0
-- 【编译命令】如果您是自行编译的FastDeploy，请说明您的编译方式（参数命令）
-- 【系统平台】: Linux x64(Ubuntu 18.04) / Windows x64(Windows10) / Mac OSX arm(12.0) / Mac OSX intel(12.0)
-- 【硬件】： 说明具体硬件型号，如 Nvidia GPU 3080TI， CUDA 11.2 CUDNN 8.3
-- 【编译语言】： C++ / Python(3.7或3.8等）
-
-## 问题日志及出现问题的操作流程
-- 附上详细的问题日志有助于快速定位分析
-- 【模型跑不通】
-- - 先执行`examples`下的部署示例，包括使用examples提供的模型，确认是否可以正确执行
-- - 如若`examples`下的代码可以运行，但自己的模型，或自己的代码不能运行
-- - - 提供复现问题的 代码+模型+错误log，供工程师快速定位问题
-- 【模型精度问题】
-- - 先执行`examples`下的部署示例，包括使用examples提供的模型，确认是否可以正确执行
-- - 如若`examples`下的代码可以运行，但自己的模型，或自己的代码不能运行
-- - - 提供复现问题的 代码+模型+错误log，供工程师快速定位问题
-- 【性能问题】描述清楚对比的方式
-- - 注意性能测试，循环跑N次，取后80%的用时平均（模型启动时，刚开始受限于资源分配，速度会较慢）
-- - FastDeploy的Predict包含模型本身之外的数据前后处理用时
-- - - 提供复现问题的 代码+模型+错误log，供工程师快速定位问题

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,10 +0,0 @@
-<!-- Demo: https://github.com/PaddlePaddle/Paddle/pull/24810 -->
-
-
-### PR types(PR类型)
-<!-- One of PR types [ Model | Backend | Serving | Quantization | Doc | Bug Fix | Other] -->
-
-### Description
-<!-- Describe what this PR does -->
-
-

.github/workflows/Codestyle-Check.yml

@@ -0,0 +1,50 @@
+name: Codestyle-Check
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+
+jobs:
+  pre-commit:
+    name: Pre Commit
+    if: ${{ github.repository_owner == 'PaddlePaddle' }}
+    runs-on: ubuntu-latest
+    env:
+      PR_ID: ${{ github.event.pull_request.number }}
+      BRANCH: ${{ github.event.pull_request.base.ref }}
+
+    steps:
+      - name: Cleanup
+        run: |
+          rm -rf * .[^.]*
+
+      - name: Checkout base repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          fetch-depth: 1000
+
+      - name: Merge PR to test branch
+        run: |
+          git fetch origin pull/${PR_ID}/merge
+          git checkout -b test FETCH_HEAD
+
+      - name: Setup python3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install pre-commit==4.2.0 cpplint==1.6.0 clang-format==13.0.0
+
+      - name: Check pre-commit
+        env:
+          SKIP_CLANG_TIDY_CHECK: "ON"
+        run: |
+          set +e
+          bash -x tools/codestyle/pre_commit.sh;EXCODE=$?
+          exit $EXCODE

.github/workflows/_accuracy_test.yml

@@ -0,0 +1,186 @@
+name: Accuracy Test
+description: "Run Accuracy Tests"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  accuracy_tests:
+    runs-on: [self-hosted, GPU-h20-1Cards]
+    timeout-minutes: 60
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+
+      - name: Run FastDeploy Base Tests
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fastdeploy_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+        run: |
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
+            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
+            exit 1
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --ipc=host --pid=host --net=host \
+          --name ${runner_name} \
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -e TZ="Asia/Shanghai" \
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install ${fastdeploy_wheel_url}
+          python -m pip install pytest
+
+          wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
+          chmod +x ./llm-deploy-linux-amd64
+          ./llm-deploy-linux-amd64 -python python3.10 \
+          -model_name ERNIE-4.5-0.3B-Paddle \
+          -model_path /MODELDATA \
+          --skip install
+
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          pushd tests/ce/deploy
+          python3.10 deploy.py > dd.log 2>&1 &
+          sleep 3
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
+
+          curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
+          popd
+
+          pushd tests/ce/accuracy_cases
+          export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
+          export TEMPLATE=TOKEN_LOGPROB
+          export MODEL_SIZE=0.3B
+          TEST_EXIT_CODE=0
+          python gsm8k.py || TEST_EXIT_CODE=1
+          popd
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
+          '
+          if [ -f ./FastDeploy/exit_code.env ]; then
+            source ./FastDeploy/exit_code.env
+            cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
+          exit ${TEST_EXIT_CODE}

.github/workflows/_base_test.yml

@@ -0,0 +1,229 @@
+name: Base Test
+description: "Run Base Tests"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  base_tests:
+    runs-on: [self-hosted, GPU-h20-1Cards]
+    timeout-minutes: 60
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+
+      - name: Run FastDeploy Base Tests
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fastdeploy_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+        run: |
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
+            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
+            exit 1
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --ipc=host --pid=host --net=host \
+          --name ${runner_name} \
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -e TZ="Asia/Shanghai" \
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install ${fastdeploy_wheel_url}
+          python -m pip install pytest
+
+          wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
+          chmod +x ./llm-deploy-linux-amd64
+          ./llm-deploy-linux-amd64 -python python3.10 \
+          -model_name ERNIE-4.5-0.3B-Paddle \
+          -model_path /MODELDATA \
+          --skip install
+
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          pushd tests/ce/deploy
+          python3.10 deploy.py > dd.log 2>&1 &
+          sleep 3
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
+
+          check_service() {
+            local timeout=${1:-90}
+            local url="http://localhost:${FLASK_PORT}/wait_for_infer?timeout=${timeout}"
+            local resp
+
+            resp=$(curl -s -X POST "$url")
+
+            if echo "$resp" | grep -q "服务启动超时"; then
+              exit 8
+            fi
+          }
+
+          check_service 90
+          popd
+
+          pushd tests/ce/server
+          export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
+          export TEMPLATE=TOKEN_LOGPROB
+          TEST_EXIT_CODE=0
+          python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py test_completions.py test_return_token_ids.py || TEST_EXIT_CODE=1
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
+          check_service 90
+          python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1
+
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+            -H "Content-Type: application/json" \
+            -d "{ \"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--max-concurrency\": 5, \"--max-waiting-time\": 1 }"
+          check_service 90
+          python -m pytest -sv test_max_concurrency.py || TEST_EXIT_CODE=1
+
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+            -H "Content-Type: application/json" \
+            -d "{ \"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--max-concurrency\": 5000, \"--max-waiting-time\": 1 }"
+          check_service 90
+          python -m pytest -sv test_max_waiting_time.py || TEST_EXIT_CODE=1
+
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ernie-4_5-21b-a3b-bf16-paddle\", \"--config\": \"21b_mtp.yaml\", \"--enable-logprob\": \"False\"}"
+          check_service 180
+          export TEMPLATE=TOKEN_NORMAL
+          python -m pytest -sv test_seed_usage.py -k "not test_seed_stream" || TEST_EXIT_CODE=1
+
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
+            -H "Content-Type: application/json" \
+            -d "{\"--model\": \"/MODELDATA/ernie-4_5-21b-a3b-bf16-paddle\", \"--config\": \"21b_sot.yaml\", \"--enable-logprob\": \"False\"}"
+          check_service 360
+          export TEMPLATE=TOKEN_NORMAL
+          python -m pytest -sv test_seed_usage.py -k "not test_seed_stream" || TEST_EXIT_CODE=1
+
+          popd
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
+          '
+          if [ -f ./FastDeploy/exit_code.env ]; then
+            source ./FastDeploy/exit_code.env
+            cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
+          exit ${TEST_EXIT_CODE}

.github/workflows/_build_linux.yml

@@ -0,0 +1,200 @@
+name: FastDeploy Linux GPU Build Task
+description: "FastDeploy packages build and upload"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      COMPILE_ARCH:
+        description: "Build GPU Archs"
+        required: true
+        type: string
+        default: "80,90"
+      WITH_NIGHTLY_BUILD:
+        description: "Enable nightly build mode (e.g. add date suffix to version)"
+        required: false
+        type: string
+        default: "OFF"
+      FD_VERSION:
+        description: "FastDeploy Package Version"
+        required: false
+        type: string
+        default: ""
+      PADDLEVERSION:
+        description: "Paddle Version Build Use"
+        required: false
+        type: string
+        default: ""
+      PADDLE_WHL_URL:
+        description: "Paddle Wheel Package URL"
+        required: false
+        type: string
+        default: ""
+      UPLOAD:
+        description: "Upload Package"
+        required: false
+        type: string
+        default: "ON"
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+    outputs:
+      wheel_path:
+        description: "Output path of the generated wheel"
+        value: ${{ jobs.fd-build.outputs.wheel_path }}
+jobs:
+  fd-build:
+    runs-on: [self-hosted, GPU-Build]
+    timeout-minutes: 240
+    outputs:
+      wheel_path: ${{ steps.set_output.outputs.wheel_path }}
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+          IS_PR: ${{ github.event_name == 'pull_request' }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+      - name: FastDeploy Build
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          compile_arch: ${{ inputs.COMPILE_ARCH }}
+          fd_version: ${{ inputs.FD_VERSION }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          BRANCH_REF: ${{ github.ref_name }}
+          PADDLEVERSION: ${{ inputs.PADDLEVERSION }}
+          PADDLE_WHL_URL: ${{ inputs.PADDLE_WHL_URL }}
+          WITH_NIGHTLY_BUILD: ${{ inputs.WITH_NIGHTLY_BUILD }}
+        run: |
+            set -x
+            runner_name="${{ runner.name }}"
+            CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+            gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+
+            CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+            echo "CACHE_DIR is set to ${CACHE_DIR}"
+            if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+              touch "${CACHE_DIR}/gitconfig"
+            fi
+            PARENT_DIR=$(dirname "$WORKSPACE")
+            echo "PARENT_DIR:$PARENT_DIR"
+            docker run --rm --net=host \
+            --cap-add=SYS_PTRACE --privileged --shm-size=64G \
+            -v $(pwd):/workspace -w /workspace \
+            -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+            -v "${CACHE_DIR}/.cache:/root/.cache" \
+            -v "${CACHE_DIR}/.ccache:/root/.ccache" \
+            -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+            -e TZ="Asia/Shanghai" \
+            -e "COMPILE_ARCH=${compile_arch}" \
+            -e "FD_VERSION=${fd_version}" \
+            -e "WITH_NIGHTLY_BUILD=${WITH_NIGHTLY_BUILD}" \
+            -e "PADDLEVERSION=${PADDLEVERSION}" \
+            -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \
+            -e "BRANCH_REF=${BRANCH_REF}" \
+            --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c '
+            if [[ -n "${FD_VERSION}" ]]; then
+              export FASTDEPLOY_VERSION=${FD_VERSION}
+              echo "Custom FastDeploy version: ${FASTDEPLOY_VERSION}"
+            fi
+
+            git config --global --add safe.directory /workspace/FastDeploy
+            chown -R $(whoami) /workspace/FastDeploy
+            cd FastDeploy
+            if [[ "${WITH_NIGHTLY_BUILD}" == "ON" ]];then
+              GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD)
+              DATE_ONLY=$(echo $GIT_COMMIT_TIME | sed "s/ .*//;s/-//g")
+              echo "Git Commit Time: $GIT_COMMIT_TIME"
+              echo "Date Only: $DATE_ONLY"
+              export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}"
+            fi
+            # 针对不同分支和tag使用不同的PaddlePaddle安装包
+            if [[ "${PADDLE_WHL_URL}" != "" ]];then
+              python -m pip install ${PADDLE_WHL_URL}
+            elif [[ "${PADDLEVERSION}" != "" ]];then
+              python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+            else
+              python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+            fi
+
+            pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            python -m pip install wheel
+            # 编译RDMA
+            export ENABLE_FD_RDMA=1
+            bash build.sh 1 python false [${COMPILE_ARCH}]
+            ls ./dist/*.whl
+            '
+      - name: Package Upload
+        id: set_output
+        env:
+          compile_arch: ${{ inputs.COMPILE_ARCH }}
+        run: |
+            set -x
+            if [[ "${{ github.event_name }}" == "pull_request" ]];then
+              commit_id=${{ github.event.pull_request.head.sha }}
+              pr_num=${{ github.event.pull_request.number }}
+              target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_}
+            elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+              commit_id=${{ github.sha }}
+              tag_name=${{ github.ref_name }}
+              target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id}/SM${compile_arch//,/_}
+            else
+              commit_id=${{ github.sha }}
+              branch_name=${{ github.ref_name }}
+              target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id}/SM${compile_arch//,/_}
+            fi
+            wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+            push_file=$(realpath bos_tools.py)
+            python --version
+            python -m pip install bce-python-sdk==0.9.29
+            cd FastDeploy/dist/
+            matches=($(ls fastdeploy*.whl))
+            if [ ${#matches[@]} -ne 1 ]; then
+              echo "Error: Found ${#matches[@]} matching files, expected exactly 1"
+              exit 1
+            fi
+            fd_wheel_name=${matches[0]}
+            echo "Found: $fd_wheel_name"
+            tree -L 3
+            python ${push_file} fastdeploy*.whl ${target_path}
+            target_path_stripped="${target_path#paddle-github-action/}"
+            WHEEL_PATH=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
+            echo "wheel_path=${WHEEL_PATH}" >> $GITHUB_OUTPUT

.github/workflows/_clone_linux.yml

@@ -0,0 +1,78 @@
+name: FastDeploy Code Clone
+description: "FastDeploy clone and upload"
+
+on:
+  workflow_call:
+    inputs:
+        bos_dir:
+          type: string
+          required: false
+          default: 'FastDeploy'
+    outputs:
+      repo_archive_url:
+        description: "Compressed source code archive."
+        value: ${{ jobs.code-clone.outputs.repo_archive_url }}
+jobs:
+  code-clone:
+    runs-on:
+      group: HK-Clone
+    outputs:
+      repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
+    steps:
+      - name: Clone FastDeploy
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request'
+                && github.event.pull_request.base.ref
+                || github.ref_name }}
+          submodules: 'recursive'
+          fetch-depth: 1000
+
+      - name: Merge PR (if needed)
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          git config --global user.name "FastDeployCI"
+          git config --global user.email "fastdeploy_ci@example.com"
+          echo "Fetching and merging PR..."
+          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr/${{ github.event.pull_request.number }}
+          git merge --no-ff pr/${{ github.event.pull_request.number }}
+          echo "PR Branch log "
+          git log --oneline -n 5 pr/${{ github.event.pull_request.number }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Code Info Show and Upload
+        id: set_output
+        env:
+          AK: paddle
+          SK: paddle
+        run: |
+          git config --unset http.https://github.com/.extraheader
+          git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
+          git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
+          echo "Current HEAD Log:"
+          git log --oneline -n 5
+          ls
+          cd ..
+          tar -zcf FastDeploy.tar.gz FastDeploy
+          if [[ "${{ github.event_name }}" == "pull_request" ]];then
+            commit_id=${{ github.event.pull_request.head.sha }}
+            pr_num=${{ github.event.pull_request.number }}
+            target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}
+          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+            commit_id=${{ github.sha }}
+            tag_name=${{ github.ref_name }}
+            target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id}
+          else
+            commit_id=${{ github.sha }}
+            branch_name=${{ github.ref_name }}
+            target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id}
+          fi
+          wget -O bos_tools.py -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} FastDeploy.tar.gz ${target_path}
+          target_path_stripped="${target_path#paddle-github-action/}"
+          REPO_ARCHIVE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
+          echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT

.github/workflows/_logprob_test_linux.yml

@@ -0,0 +1,184 @@
+name: Run FastDeploy LogProb Tests
+description: "Run FastDeploy LogProb Tests"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      PADDLETEST_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+        default: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz"
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run_tests_logprob:
+    runs-on: [self-hosted, GPU-h20-1Cards]
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          paddletest_archive_url: ${{ inputs.PADDLETEST_ARCHIVE_URL }}
+        run: |
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            -e "BASE_BRANCH=${BASE_BRANCH}" \
+            ${docker_image} /bin/bash -c '
+            rm -rf /workspace/*
+            '
+            wget -q ${paddletest_archive_url}
+            tar -xf PaddleTest.tar.gz
+            rm -rf PaddleTest.tar.gz
+            cd PaddleTest
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+      - name: logprob test
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fastdeploy_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+        run: |
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
+            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
+            exit 1
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --ipc=host --pid=host --net=host \
+          --name ${runner_name} \
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -e TZ="Asia/Shanghai" \
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install ${fastdeploy_wheel_url}
+
+          wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
+          chmod +x ./llm-deploy-linux-amd64
+          ./llm-deploy-linux-amd64 -python python3.10 \
+          -model_name ERNIE-4.5-0.3B-Paddle \
+          -model_path /MODELDATA \
+          --skip install
+
+          cd PaddleTest/framework/ServeTest
+          python3.10 deploy.py > dd.log 2>&1 &
+          sleep 3
+          curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
+              -H "Content-Type: application/json" \
+              -d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
+
+          curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
+          curl -s -o /dev/null -w "%{http_code}" -m 2 "http://0.0.0.0:${FD_API_PORT}/health"
+          curl -X POST "http://0.0.0.0:${FD_API_PORT}/v1/chat/completions" \
+            -H "Content-Type: application/json" \
+            -d "{\"messages\": [{\"role\": \"user\", \"content\": \"1+1=?\"}], \"logprobs\": true}"
+          set +e
+          rm -rf ./baseline_output
+          cp -r baseline/ERNIE-4.5-0.3B-Paddle ./baseline_output
+          LOGPROB_EXIT_CODE=0
+          python3.10 lanucher.py --request_template TOKEN_LOGPROB --url http://localhost:${FD_API_PORT}/v1/chat/completions  --case ./cases/demo.yaml  --concurrency 1 --name demo --exe logprob || LOGPROB_EXIT_CODE=$?
+          echo "LOGPROB_EXIT_CODE=${LOGPROB_EXIT_CODE}" > /workspace/exit_code.env
+          curl -X POST http://localhost:${FLASK_PORT}/stop
+          sleep 10s
+          cat *result.log
+          exit 0
+          '
+          if [ $? -ne 0 ];then
+            exit 1
+          fi
+
+          if [ -f exit_code.env ]; then
+            cat exit_code.env >> $GITHUB_ENV
+          fi
+      - name: logprob test result
+        if: ${{ env.LOGPROB_EXIT_CODE != 0 }}
+        shell: bash
+        run: |
+          echo "logprob test failed with exit code ${{ env.LOGPROB_EXIT_CODE }}"
+          exit 8

.github/workflows/_pre_ce_test.yml

@@ -0,0 +1,148 @@
+name: Pre-CE-Test
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run_ce_cases:
+    runs-on: [self-hosted, PRE_CE_RUN_2Card]
+    timeout-minutes: 60
+    steps:
+      - name: Print current runner name
+        run: |
+          echo "Current runner name: ${{ runner.name }}"
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+
+      - name: Run CI unittest
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+        run: |
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --net=host \
+          --name ${runner_name} \
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -v "${MODEL_CACHE_DIR}:/ModelData:ro" \
+          -e "MODEL_PATH=/ModelData" \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -e "fd_wheel_url=${fd_wheel_url}" \
+          --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+          python -m pip install ${fd_wheel_url}
+          bash scripts/run_pre_ce.sh
+          '

.github/workflows/_stable_test.yml

@@ -0,0 +1,170 @@
+name: Stable Test
+description: "Run Stable Tests"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  stable_tests:
+    runs-on: [self-hosted, GPU-h1z1-2Cards]
+    timeout-minutes: 60
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+
+      - name: Run FastDeploy Stable Tests
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fastdeploy_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+        run: |
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42038 + DEVICE_PORT * 100))
+          FD_INFERENCE_MSG_QUEUE_ID=$(( 42048 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_INFERENCE_MSG_QUEUE_ID=${FD_INFERENCE_MSG_QUEUE_ID}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+          if [ ! -d "${MODEL_CACHE_DIR}" ]; then
+            echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
+            exit 1
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --ipc=host --pid=host --net=host \
+          --name ${runner_name} \
+          -v $(pwd):/workspace \
+          -w /workspace \
+          -e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -e "FD_INFERENCE_MSG_QUEUE_ID=${FD_INFERENCE_MSG_QUEUE_ID}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -v "${MODEL_CACHE_DIR}:/MODELDATA" \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -e TZ="Asia/Shanghai" \
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install ${fastdeploy_wheel_url}
+          python -m pip install pytest
+
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          TEST_EXIT_CODE=0
+          pushd tests/ce/stable_cases
+          bash launch_model.sh /MODELDATA
+          bash run.sh || TEST_EXIT_CODE=1
+          popd
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
+          '
+          if [ -f ./FastDeploy/exit_code.env ]; then
+            source ./FastDeploy/exit_code.env
+            cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
+          exit ${TEST_EXIT_CODE}

.github/workflows/_unit_test_coverage.yml

@@ -0,0 +1,322 @@
+name: Coverage Check
+description: "Run FastDeploy Unit Tests and Coverage"
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the FastDeploy Wheel."
+        required: true
+        type: string
+      CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+      MODEL_CACHE_DIR:
+        description: "Cache Dir Use"
+        required: false
+        type: string
+        default: ""
+    secrets:
+      github-token:
+        required: true
+
+jobs:
+  check_cov_skip:
+    uses: ./.github/workflows/check-bypass.yml
+    secrets:
+      github-token: ${{ secrets.github-token }}
+    with:
+      workflow-name: coverage
+
+  run_tests_with_coverage:
+    runs-on: [self-hosted, GPU-h1z1-2Cards]
+    timeout-minutes: 60
+    needs: check_cov_skip
+    if: needs.check_cov_skip.outputs.can-skip != 'true'
+    outputs:
+      diff_cov_file_url: ${{ steps.cov_upload.outputs.diff_cov_file_url }}
+      unittest_failed_url: ${{ steps.cov_upload.outputs.unittest_failed_url }}
+      diff_cov_result_json_url: ${{ steps.cov_upload.outputs.diff_cov_result_json_url }}
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              if [ -d ${REPO_NAME} ]; then
+                echo "Directory ${REPO_NAME} exists, removing it..."
+                rm -rf ${REPO_NAME}*
+              fi
+            '
+
+            wget -q ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "fastdeploy_ci@example.com"
+            git log -n 3 --oneline
+      - name: Run FastDeploy Unit Tests and Coverage
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          CACHE_DIR: ${{ inputs.CACHE_DIR }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
+          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
+          IS_PR: ${{ github.event_name == 'pull_request' }}
+        run: |
+          if [[ "$IS_PR" == "true" ]]; then
+            echo "Running on PR"
+          else
+            echo "Not a PR"
+          fi
+          runner_name="${{ runner.name }}"
+          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
+
+          FLASK_PORT=$((42068 + DEVICE_PORT * 100))
+          FD_API_PORT=$((42088 + DEVICE_PORT * 100))
+          FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
+          FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
+          FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
+          echo "Test ENV Parameter:"
+          echo "========================================================="
+          echo "FLASK_PORT=${FLASK_PORT}"
+          echo "FD_API_PORT=${FD_API_PORT}"
+          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
+          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
+          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
+          echo "DEVICES=${DEVICES}"
+          echo "========================================================="
+
+          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
+          echo "CACHE_DIR is set to ${CACHE_DIR}"
+          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
+            touch "${CACHE_DIR}/gitconfig"
+          fi
+
+          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
+          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
+          echo "==== LOG_FILE is ${LOG_FILE} ===="
+
+          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
+
+          for port in "${PORTS[@]}"; do
+              PIDS=$(lsof -t -i :$port || true)
+              if [ -n "$PIDS" ]; then
+                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
+                  echo "$PIDS" | xargs -r kill -9
+                  echo "Port $port cleared" | tee -a $LOG_FILE
+              else
+                  echo "Port $port is free" | tee -a $LOG_FILE
+              fi
+          done
+
+          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
+
+          echo "========================================================="
+          echo "Ensuring no stale container named ${runner_name} ..."
+          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
+            echo "Removing stale container: ${runner_name}"
+            docker rm -f ${runner_name} || true
+          fi
+
+          docker run --rm --net=host \
+          --name ${runner_name} \
+          --cap-add=SYS_PTRACE --shm-size=64G \
+          -v $(pwd):/workspace -w /workspace \
+          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
+          -v "${CACHE_DIR}/.cache:/root/.cache" \
+          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
+          -v "${MODEL_CACHE_DIR}:/ModelData:ro" \
+          -e "MODEL_PATH=/ModelData" \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          -e "FLASK_PORT=${FLASK_PORT}" \
+          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
+          -e TZ="Asia/Shanghai" \
+          -e "fd_wheel_url=${fd_wheel_url}" \
+          -e "BASE_REF=${BASE_REF}" \
+          -e "IS_PR=${IS_PR}" \
+          --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
+
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
+          python -m pip install paddlepaddle-gpu==3.2.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
+          pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+          python -m pip install coverage
+          python -m pip install diff-cover
+          python -m pip install pytest-cov
+          python -m pip install jsonschema aistudio_sdk==0.3.5
+          python -m pip install ${fd_wheel_url}
+          rm -rf fastdeploy
+          # coverage subprocess use
+          python -m pip install ${fd_wheel_url} --no-deps --target=/workspace/FastDeploy
+          export PYTHONPATH=/workspace/FastDeploy/
+          if [ -d "tests/plugins" ]; then
+              cd tests/plugins
+              python setup.py install
+              cd ../..
+          else
+              echo "Warning: tests/plugins directory not found, skipping setup.py install"
+          fi
+          export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
+          export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
+          TEST_EXIT_CODE=0
+          bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
+          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
+          coverage combine coveragedata/ || echo "No data to combine"
+          coverage report
+          coverage xml -o python_coverage_all.xml
+          COVERAGE_EXIT_CODE=0
+          if [[ "$IS_PR" == "true" ]]; then
+            diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
+            python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
+          else
+            echo "Not a PR, skipping diff-cover"
+          fi
+          echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
+          '
+          if [ -f FastDeploy/exit_code.env ]; then
+            cat FastDeploy/exit_code.env >> $GITHUB_ENV
+          fi
+      - name: Upload unit resule and diff coverage to bos
+        id: cov_upload
+        shell: bash
+        run: |
+          cd FastDeploy
+          commit_id=${{ github.event.pull_request.head.sha }}
+          pr_num=${{ github.event.pull_request.number }}
+          target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_}
+          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          diff_cov_file="diff_coverage.xml"
+          if [ -f ${diff_cov_file} ];then
+            python ${push_file} ${diff_cov_file} ${target_path}/CoverageData
+            target_path_stripped="${target_path#paddle-github-action/}"
+            DIFF_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_file}
+            echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_OUTPUT
+            echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_ENV
+          fi
+          diff_cov_result_json="diff_coverage.json"
+          if [ -f ${diff_cov_result_json} ];then
+            python ${push_file} ${diff_cov_result_json} ${target_path}/CoverageData
+            target_path_stripped="${target_path#paddle-github-action/}"
+            DIFF_COV_JSON_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_result_json}
+            echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT
+            echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV
+          fi
+          unittest_result="failed_tests.log"
+          if [ -s ${unittest_result} ];then
+            python ${push_file} ${unittest_result} ${target_path}/UnitTestResult
+            target_path_stripped="${target_path#paddle-github-action/}"
+            UNIT_TEST_RESULT_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/UnitTestResult/${unittest_result}
+            echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_OUTPUT
+            echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_ENV
+          fi
+      - name: Check Unit Test Success
+        shell: bash
+        run: |
+          cd FastDeploy
+          if [ "$TEST_EXIT_CODE" -eq 8 ]; then
+            filename=$(basename "$unittest_failed_url")
+            if [ -z "${unittest_failed_url}" ]; then
+              echo "No diff unit failed file URL provided."
+            else
+              rm -rf "${filename}"
+              wget -O ${filename} ${unittest_failed_url} || echo "Download unittest file failed, but continuing..."
+            fi
+            echo "Unit tests failed (exit code 8)"
+            if [ -f "${filename}" ];then
+              echo "Failed test cases:"
+              cat "${filename}"
+            fi
+            exit "$TEST_EXIT_CODE"
+          fi
+          echo "All tests passed"
+
+      - name: Verify Code Coverage Threshold (80%)
+        if: ${{ github.event_name == 'pull_request' }}
+        shell: bash
+        run: |
+          cd FastDeploy
+          if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then
+            echo "Coverage generation failed (exit code 9)"
+            filename=$(basename "$diff_cov_result_json_url")
+            if [ -z "${diff_cov_result_json_url}" ]; then
+              echo "No diff cov result file URL provided."
+            else
+              rm -rf "${filename}"
+              wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..."
+            fi
+            if [ -f "${filename}" ];then
+              echo "Failed test cases:"
+              if command -v jq >/dev/null 2>&1; then
+                  jq . "${filename}"
+              else
+                  cat "${filename}"
+              fi
+            fi
+            exit "$COVERAGE_EXIT_CODE"
+          fi
+          echo "coverage passed"
+          exit 0
+
+  diff_coverage_report:
+    needs: run_tests_with_coverage
+    if: always()
+    runs-on: ubuntu-latest
+    env:
+      fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+    steps:
+      - name: coverage diff file download
+        shell: bash
+        env:
+          diff_cov_file_url: ${{ needs.run_tests_with_coverage.outputs.diff_cov_file_url }}
+        run: |
+          wget ${fd_archive_url}
+          tar -xf FastDeploy.tar.gz
+          cd FastDeploy
+          if [ -z "${diff_cov_file_url}" ]; then
+            echo "No diff coverage file URL provided."
+            exit 0
+          fi
+          wget "${diff_cov_file_url}" -O ./diff_coverage.xml || echo "Download cov file failed, but continuing..."
+      - name: Upload diff coverage report
+        if: ${{ needs.run_tests_with_coverage.outputs.diff_cov_file_url != null && needs.run_tests_with_coverage.outputs.diff_cov_file_url != '' }}
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./FastDeploy/diff_coverage.xml
+          name: python diff coverage
+          verbose: true
+          disable_search: true
+          commit_parent: false
+          flags: diff

.github/workflows/approve.yml

@@ -0,0 +1,42 @@
+name: Approval
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  Approval:
+    name: Approval
+    if: ${{ github.repository_owner == 'PaddlePaddle' }}
+    runs-on: ubuntu-latest
+    env:
+      PR_ID: ${{ github.event.pull_request.number }}
+      BRANCH: ${{ github.event.pull_request.base.ref }}
+    steps:
+      - name: Checkout base repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          fetch-depth: 1000
+
+      - name: Merge PR to test branch
+        run: |
+          git fetch origin pull/${PR_ID}/merge
+          git checkout -b test FETCH_HEAD
+          git log -n 3 --oneline
+          git remote add upstream https://github.com/PaddlePaddle/FastDeploy.git
+          git fetch upstream $BRANCH
+
+      - name: Setup python3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Run approval check script
+        run: |
+          bash scripts/check_approval.sh

.github/workflows/build.yml

@@ -1,31 +0,0 @@
-name: Build
-on: [pull_request]
-
-jobs:
-  macOS-latest-py:
-    runs-on: macos-latest
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v1
-
-      - name: Get CMake
-        uses: lukka/get-cmake@latest
-
-      - name: Get Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Build FastDeploy
-        working-directory: ./python
-        run: |
-          export ENABLE_ORT_BACKEND=ON
-          export ENABLE_PADDLE_BACKEND=OFF
-          export ENABLE_OPENVINO_BACKEND=OFF
-          export ENABLE_VISION=ON
-          export ENABLE_TEXT=ON
-          python -m pip install wheel
-          python setup.py build
-          python setup.py bdist_wheel
-          ls -l

.github/workflows/ce_job.yml

@@ -0,0 +1,248 @@
+name: CE Compile Job
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - develop
+      - 'release/*'
+permissions: read-all
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  ce_job_pre_check:
+    runs-on: ubuntu-latest
+    env:
+      COMPILE_BRANCH: ${{ vars.COMPILE_BRANCH }}
+      CE_COMPILE_SELECTION: ${{ vars.CE_COMPILE_SELECTION }}
+      COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
+    outputs:
+      branch_match: ${{ steps.set_output.outputs.branch_match }}
+      compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }}
+      sm8689_match: ${{ steps.set_output.outputs.sm8689_match }}
+      sm8090_match: ${{ steps.set_output.outputs.sm8090_match }}
+
+    steps:
+      - name: Set Version
+        id: set_output
+        env:
+          COMPILE_BRANCH: ${{ env.COMPILE_BRANCH }}
+          CE_COMPILE_SELECTION: ${{ env.CE_COMPILE_SELECTION }}
+          COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ env.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+        run: |
+          # 选择要触发编译任务的分支 done
+          # 选择指定分支要编译的任务 8090或者8689
+          # 指定分支编译要使用的Paddle的安装包,默认使用nightly最新的
+
+          IFS=',' read -ra BRANCHES <<< "$COMPILE_BRANCH"
+          MATCH=false
+          for b in "${BRANCHES[@]}"; do
+            if [[ "$b" == "${GITHUB_REF_NAME}" ]]; then
+              MATCH=true
+              break
+            fi
+          done
+          echo "branch_match=$MATCH" >> $GITHUB_OUTPUT
+
+          # 通过变量CE_COMPILE_SELECTION中的映射关系,决定分支是编译sm8090还是sm8689
+          for pair in $(echo "$CE_COMPILE_SELECTION" | tr ';' ' '); do
+            branch=$(echo "$pair" | cut -d',' -f1)
+            compile_task_list=$(echo "$pair" | cut -d',' -f2)
+
+            if [[ "$branch" == "$GITHUB_REF_NAME" ]]; then
+
+              # 判断里面是否包含 sm8090 或 sm8689
+              if [[ "$compile_task_list" == *"sm8090"* ]]; then
+                echo "sm8090_match=true" >> $GITHUB_OUTPUT
+              fi
+              if [[ "$compile_task_list" == *"sm8689"* ]]; then
+                echo "sm8689_match=true" >> $GITHUB_OUTPUT
+              fi
+              break
+            fi
+          done
+
+          # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL
+          for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do
+            branch=$(echo "$pair" | cut -d',' -f1)
+            paddle_whl_url=$(echo "$pair" | cut -d',' -f2)
+            if [[ "$branch" == "${{ github.ref_name }}" ]]; then
+              FOUND_PADDLE_URL="$paddle_whl_url"
+              echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT
+              break
+            fi
+          done
+
+  print_ce_job_pre_check_outputs:
+    runs-on: ubuntu-latest
+    needs: ce_job_pre_check
+    steps:
+      - name: Print outputs as JSON
+        run: |
+          echo '${{ toJSON(needs.ce_job_pre_check.outputs) }}'
+
+
+  clone:
+    environment: CodeSync
+    name: FD-Clone-Linux
+    runs-on: ubuntu-latest
+    needs: ce_job_pre_check
+    if: ${{ needs.ce_job_pre_check.outputs.branch_match == 'true' }}
+    outputs:
+      repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
+    steps:
+      - name: Clone FastDeploy
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request'
+                && github.event.pull_request.base.ref
+                || github.ref_name }}
+          submodules: 'recursive'
+          fetch-depth: 1000
+
+      - name: Python Setup
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Code Info Show and Upload
+        id: set_output
+        env:
+          AK: ${{ secrets.BOS_AK }}
+          SK: ${{ secrets.BOS_SK }}
+        run: |
+          git config --unset http.https://github.com/.extraheader
+          git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
+          git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
+          echo "Current HEAD Log:"
+          git log --oneline -n 5
+          ls
+          cd ..
+          tar -zcf FastDeploy.tar.gz FastDeploy
+          commit_id=${{ github.sha }}
+          branch_name=${{ github.ref_name }}
+          target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}
+          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} FastDeploy.tar.gz ${target_path}
+          target_path_stripped="${target_path#paddle-qa/}"
+          REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
+          echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT
+
+  resultshow:
+    name: Show Code Archive Output
+    needs: clone
+    runs-on: ubuntu-latest
+    steps:
+      - name: Print wheel path
+        run: |
+          echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"
+
+  build_sm8090:
+    name: BUILD_SM8090
+    needs: [clone, ce_job_pre_check]
+    if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
+    uses: ./.github/workflows/_build_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      COMPILE_ARCH: "80,90"
+      WITH_NIGHTLY_BUILD: OFF
+      FD_VERSION: 0.0.0
+      PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }}
+
+  build_sm8689:
+    name: BUILD_SM8689
+    needs: [clone, ce_job_pre_check]
+    if: ${{ needs.ce_job_pre_check.outputs.sm8689_match == 'true' }}
+    uses: ./.github/workflows/_build_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      COMPILE_ARCH: "86,89"
+      WITH_NIGHTLY_BUILD: OFF
+      FD_VERSION: 0.0.0
+      PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }}
+
+  ce_upload_sm8090:
+    environment: CodeSync
+    name: CE_UPLOAD
+    needs: build_sm8090
+    runs-on: ubuntu-latest
+    env:
+      AK: ${{ secrets.BOS_AK }}
+      SK: ${{ secrets.BOS_SK }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      COMPILE_ARCH: "80,90"
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Wheel Info Show and Upload
+        run: |
+          echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}"
+          wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }}
+          filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }})
+
+          commit_id=${{ github.sha }}
+          branch_name=${{ github.ref_name }}
+          target_path=paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}
+
+          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} ${filename} ${target_path}
+          target_path_stripped="${target_path#paddle-qa/}"
+          WHEEL_PATH=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
+          echo "commit wheel url is ${WHEEL_PATH}"
+
+          target_path_latest=paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest
+          python ${push_file} ${filename} ${target_path_latest}
+          target_path_stripped_latest="${target_path_latest#paddle-qa/}"
+          WHEEL_PATH_LATEST=https://paddle-qa.bj.bcebos.com/${target_path_stripped_latest}/${fd_wheel_name}
+          echo "latest wheel url is ${WHEEL_PATH_LATEST}"
+
+  ce_upload_sm8689:
+    environment: CodeSync
+    name: CE_UPLOAD
+    needs: build_sm8689
+    runs-on: ubuntu-latest
+    env:
+      AK: ${{ secrets.BOS_AK }}
+      SK: ${{ secrets.BOS_SK }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }}
+      COMPILE_ARCH: "86,89"
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Wheel Info Show and Upload
+        run: |
+          echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}"
+          wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }}
+          filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }})
+
+          commit_id=${{ github.sha }}
+          branch_name=${{ github.ref_name }}
+          target_path=paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}
+
+          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} ${filename} ${target_path}
+          target_path_stripped="${target_path#paddle-qa/}"
+          WHEEL_PATH=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
+          echo "commit wheel url is ${WHEEL_PATH}"
+
+          target_path_latest=paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest
+          python ${push_file} ${filename} ${target_path_latest}
+          target_path_stripped_latest="${target_path_latest#paddle-qa/}"
+          WHEEL_PATH_LATEST=https://paddle-qa.bj.bcebos.com/${target_path_stripped_latest}/${fd_wheel_name}
+          echo "latest wheel url is ${WHEEL_PATH_LATEST}"

.github/workflows/check-bypass.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_call:
+    inputs:
+      workflow-name:
+        required: true
+        type: string
+    secrets:
+      github-token:
+        required: true
+    outputs:
+      can-skip:
+        description: "Whether the workflow can be skipped."
+        value: ${{ jobs.check-bypass.outputs.can-skip }}
+
+jobs:
+  check-bypass:
+    name: Check bypass
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CI_TEAM_MEMBERS: '["yuanlehome","YuanRisheng","Jiang-Jia-Jun","DDDivano","XieYunshen"]'
+    outputs:
+      can-skip: ${{ steps.check-bypass.outputs.can-skip }}
+    steps:
+      - name: Cleanup
+        run: |
+          rm -rf * .[^.]*
+
+      - id: check-bypass
+        name: Check Bypass
+        uses: PFCCLab/ci-bypass@v1
+        with:
+          github-token: ${{ secrets.github-token }}
+          non-pull-request-event-strategy: 'never-skipped'
+          type: 'composite'
+          composite-rule: |
+            {
+              "any": [
+                {
+                  "type": "labeled",
+                  "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
+                  "username": ${{ env.CI_TEAM_MEMBERS }}
+                },
+                {
+                  "type": "commented",
+                  "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
+                  "username": ${{ env.CI_TEAM_MEMBERS }}
+                }
+              ]
+            }

.github/workflows/ci_gcu.yml

@@ -0,0 +1,98 @@
+name: CI_GCU
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-gcu-ci
+  cancel-in-progress: true
+
+jobs:
+  CI_GCU:
+    runs-on:
+      group: GCU
+    steps:
+      - name: Print current runner name
+        run: |
+          echo "Current runner name: ${{ runner.name }}"
+
+      - name: Code Checkout
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-gcu:topsrider3.5.102-ubuntu20-x86_64-gcc84
+        run: |
+          REPO="https://github.com/${{ github.repository }}.git"
+          FULL_REPO="${{ github.repository }}"
+          REPO_NAME="${FULL_REPO##*/}"
+          BASE_BRANCH="${{ github.base_ref }}"
+          # Clean the repository directory before starting
+          docker run --rm --net=host -v $(pwd):/workspace \
+          -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
+          -w /workspace \
+          -e "REPO_NAME=${REPO_NAME}" \
+          -e "BASE_BRANCH=${BASE_BRANCH}" \
+          ${docker_image} /bin/bash -c '
+            if [ -d ${REPO_NAME} ]; then
+              echo "Directory ${REPO_NAME} exists, removing it..."
+              rm -rf ${REPO_NAME}
+            fi
+          '
+          git config --global user.name "FastDeployCI"
+          git config --global user.email "fastdeploy_ci@example.com"
+          source ${{ github.workspace }}/../../../proxy
+          git clone ${REPO} ${REPO_NAME} -b ${BASE_BRANCH}
+          cd FastDeploy
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            git fetch origin pull/${{ github.event.pull_request.number }}/head:pr/${{ github.event.pull_request.number }}
+            git merge pr/${{ github.event.pull_request.number }}
+            git log -n 3 --oneline
+          else
+            git checkout ${{ github.sha }}
+            git log -n 3 --oneline
+          fi
+          echo "Copy models..."
+          sudo mkdir -p ci_models && sudo cp -r /work/deps/ERNIE-4.5-21B-A3B-Paddle ci_models
+          echo "Copy models done."
+
+      - name: Run CI unittest
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-gcu:topsrider3.5.102-ubuntu20-x86_64-gcc84
+        run: |
+          runner_name="${{ runner.name }}"
+          last_char="${runner_name: -1}"
+
+          if [[ "$last_char" =~ [0-3] ]]; then
+            gcu_id="$last_char"
+          else
+            gcu_id="0"
+          fi
+          FD_API_PORT=$((9180 + gcu_id * 100))
+          FD_ENGINE_QUEUE_PORT=$((9150 + gcu_id * 100))
+          FD_METRICS_PORT=$((9170 + gcu_id * 100))
+
+          PARENT_DIR=$(dirname "$WORKSPACE")
+          echo "PARENT_DIR:$PARENT_DIR"
+          echo "Install drivers..."
+          cd /work/deps
+          sudo bash TopsRider_i3x_*_deb_amd64.run --driver --no-auto-load -y
+          cd -
+          echo "Create docker..."
+          docker run --rm --network=host --ipc=host --privileged \
+          -v $(pwd):/workspace \
+          -v /home:/home \
+          -v /work:/work \
+          -w /workspace \
+          -e "MODEL_PATH=./ci_models" \
+          -e "http_proxy=$(git config --global --get http.proxy)" \
+          -e "https_proxy=$(git config --global --get https.proxy)" \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+          ${docker_image} /bin/bash -c "
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          bash scripts/run_ci_gcu.sh
+          "

.github/workflows/ci_iluvatar.yml

@@ -0,0 +1,85 @@
+name: CI_ILUVATAR
+
+on:
+  pull_request:
+    branches: [ develop ]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-iluvatar-ci
+  cancel-in-progress: true
+
+jobs:
+  CI_ILUVATAR:
+    runs-on:
+      group: IXUCA
+    steps:
+      - name: Print current runner name
+        run: |
+          echo "Current runner name: ${{ runner.name }}"
+      # Because the system version is lower than 2.23, the checkout cannot be used.
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+
+      - name: Code Checkout
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:latest
+        run: |
+          REPO="https://github.com/${{ github.repository }}.git"
+          FULL_REPO="${{ github.repository }}"
+          REPO_NAME="${FULL_REPO##*/}"
+          # Clean the repository directory before starting
+          docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+          -e "REPO_NAME=${REPO_NAME}" \
+          ${docker_image} /bin/bash -c '
+            if [ -d ${REPO_NAME} ]; then
+              echo "Directory ${REPO_NAME} exists, removing it..."
+              rm -rf ${REPO_NAME}
+            fi
+          '
+          git config --global user.name "FastDeployCI"
+          git config --global user.email "fastdeploy_ci@example.com"
+          git clone ${REPO} ${REPO_NAME}
+          cd FastDeploy
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            git fetch origin pull/${{ github.event.pull_request.number }}/head:pr/${{ github.event.pull_request.number }}
+            git merge pr/${{ github.event.pull_request.number }}
+            git log -n 3 --oneline
+          else
+            git checkout ${{ github.sha }}
+            git log -n 3 --oneline
+          fi
+
+      - name: Run CI unittest
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:latest
+        run: |
+          runner_name="${{ runner.name }}"
+          last_char="${runner_name: -1}"
+
+          if [[ "$last_char" =~ [0-3] ]]; then
+            gpu_id="$last_char"
+          else
+            gpu_id="0"
+          fi
+          FD_API_PORT=$((9180 + gpu_id * 100))
+          FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100))
+          FD_METRICS_PORT=$((9170 + gpu_id * 100))
+
+          PARENT_DIR=$(dirname "$WORKSPACE")
+          echo "PARENT_DIR:$PARENT_DIR"
+          docker run --rm --net=host --pid=host --cap-add=ALL --privileged --shm-size=64G  \
+          -v /usr/src:/usr/src -v /lib/modules:/lib/modules -v /dev:/dev \
+          -v $(pwd):/workspace -w /workspace \
+          -v "/data1/fastdeploy:/data1/fastdeploy" \
+          -e "MODEL_PATH=/ssd3/model" \
+          -e "http_proxy=$(git config --global --get http.proxy)" \
+          -e "https_proxy=$(git config --global --get https.proxy)" \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+           ${docker_image} /bin/bash -c "
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          bash scripts/run_ci_iluvatar.sh
+          "

.github/workflows/ci_xpu.yml

@@ -0,0 +1,88 @@
+name: CI_XPU
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-xpu-ci
+  cancel-in-progress: true
+
+jobs:
+  CI_XPU:
+    runs-on: [self-hosted, XPU-P800-8Card]
+    steps:
+      - name: Print current runner name
+        run: |
+          echo "Current runner name: ${{ runner.name }}"
+      # Because the system version is lower than 2.23, the checkout cannot be used.
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+
+      - name: Code Checkout
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:2.1.0
+        run: |
+          REPO="https://github.com/${{ github.repository }}.git"
+          FULL_REPO="${{ github.repository }}"
+          REPO_NAME="${FULL_REPO##*/}"
+          BASE_BRANCH="${{ github.base_ref }}"
+          # Clean the repository directory before starting
+          docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+          -e "REPO_NAME=${REPO_NAME}" \
+          -e "BASE_BRANCH=${BASE_BRANCH}" \
+          ${docker_image} /bin/bash -c '
+            if [ -d ${REPO_NAME} ]; then
+              echo "Directory ${REPO_NAME} exists, removing it..."
+              rm -rf ${REPO_NAME}
+            fi
+          '
+          git config --global user.name "FastDeployCI"
+          git config --global user.email "fastdeploy_ci@example.com"
+          git clone ${REPO} ${REPO_NAME} -b ${BASE_BRANCH}
+          cd FastDeploy
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            git fetch origin pull/${{ github.event.pull_request.number }}/head:pr/${{ github.event.pull_request.number }}
+            git merge pr/${{ github.event.pull_request.number }}
+            git log -n 3 --oneline
+          else
+            git checkout ${{ github.sha }}
+            git log -n 3 --oneline
+          fi
+
+      - name: Run CI unittest
+        env:
+          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:2.1.0
+        run: |
+          runner_name="${{ runner.name }}"
+          last_char="${runner_name: -1}"
+
+          if [[ "$last_char" =~ [0-3] ]]; then
+            gpu_id="$last_char"
+          else
+            gpu_id="0"
+          fi
+          FD_API_PORT=$((9180 + gpu_id * 100))
+          FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100))
+          FD_METRICS_PORT=$((9170 + gpu_id * 100))
+
+          PARENT_DIR=$(dirname "$WORKSPACE")
+          echo "PARENT_DIR:$PARENT_DIR"
+          docker run --rm --net=host --cap-add=SYS_PTRACE --privileged --shm-size=64G  \
+          -v $(pwd):/workspace -w /workspace \
+          -v "/ssd3:/ssd3" \
+          -e "MODEL_PATH=/ssd3/model" \
+          -e "http_proxy=$(git config --global --get http.proxy)" \
+          -e "https_proxy=$(git config --global --get https.proxy)" \
+          -e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
+          -e "FD_API_PORT=${FD_API_PORT}" \
+          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
+          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
+           ${docker_image} /bin/bash -c "
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          bash scripts/run_ci_xpu.sh
+          "

.github/workflows/gh-pages.yml

@@ -0,0 +1,24 @@
+name: Deploy GitHub Pages
+
+on:
+  push:
+    branches: [ develop ]
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: pip install mkdocs-material mkdocs-get-deps mkdocs-material-extensions mkdocs-multilang mkdocs-static-i18n
+      - name: Deploy to GitHub Pages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
+          mkdocs gh-deploy --force --remote-name origin

.github/workflows/pr_build_and_test.yml

@@ -0,0 +1,97 @@
+name: PR Build and Test
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches: [develop, release/**]
+permissions: read-all
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  clone:
+    name: FD-Clone-Linux
+    uses: ./.github/workflows/_clone_linux.yml
+
+  build:
+    name: FD-Build-Linux
+    needs: clone
+    uses: ./.github/workflows/_build_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      COMPILE_ARCH: "89,90"
+      WITH_NIGHTLY_BUILD: "OFF"
+      FD_VERSION: "0.0.0"
+
+  resultshow:
+    name: Use Build Output
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Print wheel path
+        run: |
+          echo "The built wheel is located at: ${{ needs.build.outputs.wheel_path }}"
+
+  unittest_coverage:
+    name: Run FastDeploy Unit Tests and Coverage
+    needs: [clone,build]
+    uses: ./.github/workflows/_unit_test_coverage.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+    secrets:
+      github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  logprob_test:
+    name: Run FastDeploy LogProb Tests
+    needs: [build]
+    uses: ./.github/workflows/_logprob_test_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz"
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  pre_ce_test:
+    name: Extracted partial CE model tasks to run in CI.
+    needs: [clone,build]
+    uses: ./.github/workflows/_pre_ce_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  base_test:
+    name: Run Base Tests
+    needs: [clone,build]
+    uses: ./.github/workflows/_base_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  accuracy_test:
+    name: Run Accuracy Tests
+    needs: [clone,build]
+    uses: ./.github/workflows/_accuracy_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  stable_test:
+    name: Run Stable Tests
+    needs: [clone,build]
+    uses: ./.github/workflows/_stable_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

.github/workflows/publish_job.yml

@@ -0,0 +1,321 @@
+name: Publish Job
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 18 * * *'   # 2:00 AM China Standard Time (UTC+8)
+  push:
+    # branches:
+    #   - develop
+    tags:
+      - '*'
+
+permissions: read-all
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.sha }}
+  cancel-in-progress: true
+
+
+jobs:
+  publish_pre_check:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.repository.fork == false &&
+      (
+        (github.event_name == 'schedule' && github.ref_name == 'develop') ||
+        (github.event_name == 'push' && github.ref_type == 'tag') ||
+        ((github.event_name == 'workflow_dispatch') &&
+          (github.ref_name == 'develop' || github.ref_type == 'tag'))
+      )
+    env:
+      TAG_VERSION_MAPPINGS: ${{ vars.TAG_VERSION_MAPPINGS }}
+      FD_VERSION_DEV: ${{ vars.FD_VERSION_DEV }}
+      COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
+    outputs:
+      compile_use_paddle_version: ${{ steps.set_output.outputs.compile_use_paddle_version }}
+      compile_continue: ${{ steps.set_output.outputs.compile_continue }}
+      fd_version: ${{ steps.set_output.outputs.fd_version }}
+      with_nightly_build: ${{ steps.set_output.outputs.with_nightly_build }}
+      compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }}
+
+    steps:
+      - name: Get tag version
+        if: github.ref_type == 'tag'
+        run: |
+          TAG_NAME="${GITHUB_REF##*/}"       # 提取 tag 名称，比如 v2.1.0
+          TAG_VERSION="${TAG_NAME#v}"            # 去掉前缀 v
+          echo "FD_VERSION=$TAG_VERSION" >> $GITHUB_ENV
+
+      - name: Check FD version to Paddle version mapping
+        if: github.ref_type == 'tag'
+        env:
+          TARGET_FD: ${{ env.FD_VERSION }}
+        run: |
+          FOUND_PADDLE=""
+          # 遍历映射
+          for pair in $(echo $TAG_VERSION_MAPPINGS | tr ';' ' '); do
+            fd=$(echo "$pair" | cut -d',' -f1)
+            paddle=$(echo "$pair" | cut -d',' -f2)
+            if [[ "$fd" == "$TARGET_FD" ]]; then
+              FOUND_PADDLE="$paddle"
+              break
+            fi
+          done
+
+          if [[ -z "$FOUND_PADDLE" ]]; then
+            echo "No Paddle version found for FD $TARGET_FD"
+          else
+            echo "FD $TARGET_FD maps to Paddle $FOUND_PADDLE"
+            echo "PADDLE_VERSION=$FOUND_PADDLE" >> $GITHUB_ENV
+          fi
+      - name: Set Version
+        id: set_output
+        env:
+          PADDLE_VERSION: ${{ env.PADDLE_VERSION }}
+          FD_VERSION: ${{ env.FD_VERSION }}
+        run: |
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            if [[ -z "$PADDLE_VERSION" ]]; then
+              compile_continue=false
+            else
+              compile_use_paddle_version=$PADDLE_VERSION
+              compile_continue=true
+            fi
+            fd_version=$FD_VERSION
+          fi
+          if [[ "${{ github.ref_name }}" == "develop" ]];then
+            compile_continue=true
+            compile_use_paddle_version=""
+            fd_version=${FD_VERSION_DEV}
+            with_nightly_build=ON
+          fi
+          # Todo
+          # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL
+          for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do
+            branch=$(echo "$pair" | cut -d',' -f1)
+            paddle_whl_url=$(echo "$pair" | cut -d',' -f2)
+            if [[ "$branch" == "${{ github.ref_name }}" ]]; then
+              FOUND_PADDLE_URL="$paddle_whl_url"
+              echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT
+              compile_continue=true
+              break
+            fi
+          done
+          echo "compile_continue=${compile_continue}" >> $GITHUB_OUTPUT
+          echo "compile_use_paddle_version=${compile_use_paddle_version}" >> $GITHUB_OUTPUT
+          echo "fd_version=${fd_version}" >> $GITHUB_OUTPUT
+          echo "with_nightly_build=${with_nightly_build:-OFF}" >> $GITHUB_OUTPUT
+
+  print_publish_pre_check_outputs:
+    runs-on: ubuntu-latest
+    needs: publish_pre_check
+    steps:
+      - name: Print outputs as JSON
+        run: |
+          echo '${{ toJSON(needs.publish_pre_check.outputs) }}'
+
+  clone:
+    environment: CodeSync
+    name: FD-Clone-Linux
+    runs-on: ubuntu-latest
+    needs: publish_pre_check
+    if: ${{ needs.publish_pre_check.outputs.compile_continue == 'true' }}
+    outputs:
+      repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
+    steps:
+      - name: Clone FastDeploy
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+          submodules: 'recursive'
+          fetch-depth: 1000
+
+      - name: Python Setup
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Code Info Show and Upload
+        id: set_output
+        env:
+          AK: ${{ secrets.BOS_AK }}
+          SK: ${{ secrets.BOS_SK }}
+        run: |
+          git config --unset http.https://github.com/.extraheader
+          git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
+          git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
+          echo "Current HEAD Log:"
+          git log --oneline -n 5
+          ls
+          cd ..
+          tar -zcf FastDeploy.tar.gz FastDeploy
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            commit_id=${{ github.sha }}
+            tag_name=${{ github.ref_name }}
+            target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id}
+          else
+            commit_id=${{ github.sha }}
+            branch_name=${{ github.ref_name }}
+            target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}
+          fi
+          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} FastDeploy.tar.gz ${target_path}
+          target_path_stripped="${target_path#paddle-qa/}"
+          REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
+          echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT
+
+  resultshow:
+    name: Show Code Archive Output
+    needs: clone
+    runs-on: ubuntu-latest
+    steps:
+      - name: Print wheel path
+        run: |
+          echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"
+
+  build_sm8090:
+    name: BUILD_SM8090
+    needs: [clone, publish_pre_check]
+    uses: ./.github/workflows/_build_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      COMPILE_ARCH: "80,90"
+      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
+      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
+      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
+      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}
+
+  build_sm8689:
+    name: BUILD_SM8689
+    needs: [clone, publish_pre_check]
+    uses: ./.github/workflows/_build_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      COMPILE_ARCH: "86,89"
+      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
+      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
+      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
+      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}
+
+  paddle_pypi_upload_sm8090:
+    environment: PaddleSourceUpload
+    name: PADDLE_PYPI_UPLOAD_8090
+    needs: build_sm8090
+    runs-on: ubuntu-latest
+    env:
+      AK: ${{ secrets.BOS_AK }}
+      SK: ${{ secrets.BOS_SK }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      COMPILE_ARCH: "80,90"
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Wheel Info Show and Upload
+        if: github.ref_name == 'develop' || github.ref_type == 'tag'
+        run: |
+          echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}"
+          wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL}
+          filename=$(basename ${FASTDEPLOY_WHEEL_URL})
+          if [[ "${{ github.ref_name }}" == "develop" ]];then
+            target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
+          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+            target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
+          else
+            echo "Not develop or tag, do nothing"
+          fi
+          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} ${filename} ${target_path}
+
+  paddle_pypi_upload_sm8689:
+    environment: PaddleSourceUpload
+    name: PADDLE_PYPI_UPLOAD_8689
+    needs: build_sm8689
+    runs-on: ubuntu-latest
+    env:
+      AK: ${{ secrets.BOS_AK }}
+      SK: ${{ secrets.BOS_SK }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }}
+      COMPILE_ARCH: "86,89"
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Wheel Info Show and Upload
+        if: github.ref_name == 'develop' || github.ref_type == 'tag'
+        run: |
+          echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}"
+          wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL}
+          filename=$(basename ${FASTDEPLOY_WHEEL_URL})
+          if [[ "${{ github.ref_name }}" == "develop" ]];then
+            target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
+          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+            target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
+          else
+            echo "Not develop or tag, do nothing"
+          fi
+          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+          push_file=$(realpath bos_tools.py)
+          python -m pip install bce-python-sdk==0.9.29
+          ls
+          python ${push_file} ${filename} ${target_path}
+
+  unittest_coverage:
+    name: Run FastDeploy Unit Tests and Coverage
+    needs: [clone,build_sm8090]
+    uses: ./.github/workflows/_unit_test_coverage.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+    secrets:
+      github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  logprob_test:
+    name: Run FastDeploy LogProb Tests
+    needs: [build_sm8090]
+    uses: ./.github/workflows/_logprob_test_linux.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz"
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  pre_ce_test:
+    name: Extracted partial CE model tasks to run in CI.
+    needs: [clone,build_sm8090]
+    uses: ./.github/workflows/_pre_ce_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  base_test:
+    name: Run Base Tests
+    needs: [clone,build_sm8090]
+    uses: ./.github/workflows/_base_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
+
+  accuracy_test:
+    name: Run Accuracy Tests
+    needs: [clone,build_sm8090]
+    uses: ./.github/workflows/_accuracy_test.yml
+    with:
+      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
+      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
+      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
+      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

.gitignore

@@ -1,69 +1,178 @@
-build
-cmake-build-debug
-cmake-build-release
-.vscode
-FastDeploy.cmake
-build-debug.sh
-*dist
-fastdeploy.egg-info
-fastdeploy_python.egg-info
-fastdeploy_gpu_python.egg-info
-.setuptools-cmake-build
-fastdeploy/version.py
-fastdeploy/core/config.h
-python/fastdeploy/c_lib_wrap.py
-python/fastdeploy/LICENSE*
-python/build_cpu.sh
-python/fastdeploy/ThirdPartyNotices*
-*.so*
-fpython/astdeploy/libs/third_libs
-fastdeploy/core/config.h
-fastdeploy/pybind/main.cc
-python/fastdeploy/libs/lib*
-python/fastdeploy/libs/third_libs
-__pycache__
-build_fd_android.sh
-python/scripts/process_libraries.py
-.vs
-.idea
-.DS_Store
-miniprogram_npm
-node_modules
-.DS_Store
-dist
-etc
-lib
-dist-ssr
-coverage
-*.local
-yalc.*
-.yalc
-examples/vision/collect_quantize_cc.sh
-examples/vision/tests_quantize
-fastdeploy/LICENSE
-fastdeploy/ThirdPartyNotices.txt
-FastDeployCSharp.cmake
-python/fastdeploy/code_version.py
-*.pdmodel
-*.pdiparams
-*.pdiparams.info
-log.txt
-serving/build
-serving/build.encrypt
-serving/build.encrypt.auth
-output
-res
-tmp
+# Virtualenv
+/.venv/
+/venv/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+/bin/
+/build/
+/develop-eggs/
+dist/
+/eggs/
+/lib/
+/lib64/
+/output/
+/parts/
+/sdist/
+/var/
+*.egg-info/
+.installed.cfg
+*.egg
+.eggs
+
+# AUTHORS and ChangeLog will be generated while packaging
+/AUTHORS
+/ChangeLog
+
+# BCloud / BuildSubmitter
+/build_submitter.*
+/logger_client_log
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+.tox/
+.coverage
+.cache
+.pytest_cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+*.doctree
+
+# Sphinx documentation
+/docs/_build/
+
+.env
 log
-nohup.out
-llm/server/__pycache__
-llm/server/data/__pycache__
-llm/server/engine/__pycache__
-llm/server/http_server/__pycache__
-llm/server/log/
-llm/client/build/
-llm/client/dist/
-llm/client/fastdeploy_client.egg-info/
-llm/client/fastdeploy_client/tests/log/
-*.pyc
+.vscode
+.idea
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Django stuff:
 *.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pycharm
+.DS_Store
+.idea/
+FETCH_HEAD
+
+#log
+log/
+
+checkpoints/
+checkpoints_origin/
+result/
+result_lora/
+
+# npu kernel cache
+kernel_meta*
+
+# building custom ops cache and auto-generated codes
+*.o
+fastdeploy_ops.py
+version.txt
+EGG-INFO/
+
+# fp8 generated codes
+autogen/
+fp8_fp8_gemm_scale_bias_act.cu
+fp8_fp8_dual_gemm_scale_bias_act.cu
+visitor_fp8_gemm_fused.cu
+
+# third party
+custom_ops/third_party
+
+fastdeploy/model_executor/ops/base
+fastdeploy/model_executor/ops/gpu/deep_gemm
+
+gemm_profiles.json
+nohup.out
+
+#fp8_deep_gemm
+custom_ops/gpu_ops/fp8_deep_gemm/deep_gemm/include/cutlass
+custom_ops/gpu_ops/fp8_deep_gemm/deep_gemm/include/cute
+
+#marlin_kernel
+custom_ops/gpu_ops/moe/moe_wna16_marlin_utils/kernel_*.cu
+
+#machete_kernel
+custom_ops/gpu_ops/machete/generated
+
+# buff
+custom_ops/tmp*
+
+build
+
+.ccls-cache
+
+third_party
+
+custom_ops/gpu_ops/w4afp8_gemm/w4afp8_gemm_*.cu
+custom_ops/gpu_ops/w4afp8_gemm/w4afp8_gemm_template.h
+
+custom_ops/gpu_ops/wfp8afp8_sparse_gemm/wfp8Afp8_sparse_gemm_*.cu
+custom_ops/gpu_ops/wfp8afp8_sparse_gemm/wfp8Afp8_sparse_gemm_template.h

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "custom_ops/third_party/DeepGEMM"]
+	path = custom_ops/third_party/DeepGEMM
+	url = https://github.com/deepseek-ai/DeepGEMM.git
+[submodule "custom_ops/third_party/cutlass"]
+	path = custom_ops/third_party/cutlass
+	url = https://github.com/NVIDIA/cutlass.git
+[submodule "custom_ops/third_party/nlohmann_json"]
+	path = custom_ops/third_party/nlohmann_json
+	url = https://github.com/nlohmann/json.git

.pre-commit-config.yaml

@@ -1,6 +1,48 @@
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+default_stages:
+  - pre-commit # Run locally
+  - commit-msg
+#   - manual # Run in CI
 repos:
+-   repo: https://github.com/psf/black.git
+    rev: 25.1.0
+    hooks:
+    -   id: black
+        files: \.(py|pyi)$
+        additional_dependencies: [toml]
+# 自动排序
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+    -   id: isort
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+    -   id: flake8
+# 代码检查
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.11.7
+  hooks:
+  - id: ruff
+    args: [--output-format, github, --fix, --line-length=120, --config, pyproject.toml]
+# # 拼写检查
+# - repo: https://github.com/codespell-project/codespell
+#   rev: v2.4.1
+#   hooks:
+#   - id: codespell
+#     additional_dependencies: ['tomli']
+#     args: ['--toml', 'pyproject.toml']
+
+# markdown
+- repo: https://github.com/jackdewinter/pymarkdown
+  rev: v0.9.29
+  hooks:
+  - id: pymarkdown
+    args: ["-d", "MD029,MD031", fix]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: ed714747d7acbc5790b171702bb012af3b0fe145
+  rev: v5.0.0
   hooks:
   - id: check-merge-conflict
   - id: check-symlinks
@@ -9,30 +51,3 @@ repos:
   - id: detect-private-key
   - id: check-symlinks
   - id: check-added-large-files
-
--   repo: local
-    hooks:
-    -   id: copyright_checker
-        name: copyright_checker
-        entry: python ./.copyright.hook
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$
-        exclude: (?!.*third_party)^.*$
-
--   repo: local
-    hooks:
-    -   id: clang-format-with-version-check
-        name: clang-format
-        description: Format files with ClangFormat.
-        entry: bash .clang_format.hook -i
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|hxx|proto)$
-
--   repo: local
-    hooks:
-    -   id: cpplint-cpp-source
-        name: cpplint
-        description: Check C++ code style using cpplint.py.
-        entry: bash .cpplint_pre_commit.hook
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx)$

CMakeLists.txt

@@ -1,772 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PROJECT(fastdeploy C CXX)
-CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
-
-
-option(CSRCS_DIR_NAME "Name of source code directory")
-option(LIBRARY_NAME "Name of build library name")
-option(PY_LIBRARY_NAME "Name of build python library name")
-if(NOT CSRCS_DIR_NAME)
-  set(CSRCS_DIR_NAME ".")
-endif()
-if(NOT LIBRARY_NAME)
-  set(LIBRARY_NAME "fastdeploy")
-endif()
-if(NOT PY_LIBRARY_NAME)
-  set(PY_LIBRARY_NAME "fastdeploy_main")
-endif()
-include(ExternalProject)
-set(THIRD_PARTY_PATH ${CMAKE_CURRENT_BINARY_DIR}/third_libs)
-
-add_subdirectory(${CSRCS_DIR_NAME}/fastdeploy)
-include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake)
-
-# Set C++11 as standard for the whole project
-if(NOT MSVC)
-  if(NOT DEFINED CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD 11)
-  endif()
-  set(CMAKE_CXX_FLAGS "-Wno-format -g0 -O3")
-  if(NEED_ABI0)
-    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
-  else()
-    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
-  endif()
-endif(NOT MSVC)
-
-include(${PROJECT_SOURCE_DIR}/cmake/build_tools.cmake)
-if(UNIX AND (NOT APPLE) AND (NOT ANDROID) AND (NOT WITH_TIMVX))
-  download_patchelf()
-  set(PATCHELF_EXE ${THIRD_PARTY_PATH}/patchelf/bin/patchelf)
-endif()
-
-
-############################# Basic Options for FastDeploy ################################
-option(WITH_GPU "Whether WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernce-gpu/poros-gpu" OFF)
-option(WITH_IPU "Whether WITH_IPU=ON, will enable paddle-infernce-ipu" OFF)
-option(WITH_OPENCL "Whether WITH_OPENCL=ON, will enable paddle-lite-gpu" OFF)
-option(ENABLE_ORT_BACKEND "Whether to enable onnxruntime backend." OFF)
-option(ENABLE_TRT_BACKEND "Whether to enable tensorrt backend." OFF)
-option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF)
-option(ENABLE_POROS_BACKEND "Whether to enable poros backend." OFF)
-option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF)
-option(ENABLE_RKNPU2_BACKEND "Whether to enable RKNPU2 backend." OFF)
-option(ENABLE_SOPHGO_BACKEND "Whether to enable SOPHON backend." OFF)
-option(ENABLE_TVM_BACKEND "Whether to enable TVM backend." OFF)
-option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF)
-option(ENABLE_HORIZON_BACKEND "Whether to enable HORIZON backend." OFF)
-option(ENABLE_VISION "Whether to enable vision models usage." OFF)
-option(ENABLE_TEXT "Whether to enable text models usage." OFF)
-option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF)
-option(ENABLE_CVCUDA "Whether to enable NVIDIA CV-CUDA to boost image preprocess." OFF)
-option(ENABLE_ENCRYPTION "Whether to enable ENCRYPTION." OFF)
-option(ENABLE_BENCHMARK "Whether to enable Benchmark mode." OFF)
-option(WITH_ASCEND "Whether to compile for Huawei Ascend deploy." OFF)
-option(WITH_DIRECTML "Whether to compile for onnxruntime DirectML deploy." OFF)
-option(WITH_TIMVX "Whether to compile for TIMVX deploy." OFF)
-option(WITH_KUNLUNXIN "Whether to compile for KunlunXin XPU deploy." OFF)
-option(WITH_TESTING "Whether to compile with unittest." OFF)
-option(WITH_CAPI "Whether to compile with c api." OFF)
-option(WITH_CSHARPAPI "Whether to compile with c# api" OFF)
-
-option(BUILD_EXAMPLES "Whether to build fastdeploy with vision examples" OFF)
-option(BUILD_PADDLE2ONNX "Whether to build paddle2onnx from sources" OFF)
-
-######################### Paths to user's custom libraries directory #####################
-set(CUDA_DIRECTORY "" CACHE PATH "If build tensorrt backend, need to define path of cuda library.")
-set(TRT_DIRECTORY "" CACHE PATH "If build tensorrt backend, need to define path of tensorrt library.")
-set(ORT_DIRECTORY "" CACHE PATH "User can specify the installed onnxruntime directory.")
-set(OPENCV_DIRECTORY "" CACHE PATH "User can specify the installed opencv directory.")
-set(OPENVINO_DIRECTORY "" CACHE PATH "User can specify the installed openvino directory.")
-
-# Whether to build fastdeploy on device Nvidia Jetson
-# Only support CPU Inference & GPU(TensorRT) Inference Now
-option(BUILD_ON_JETSON "Whether to build fastdeploy on Nvidia Jetson" OFF)
-if(BUILD_ON_JETSON)
-  set(WITH_GPU ON)
-  set(ENABLE_TRT_BACKEND ON)
-  set(ENABLE_ORT_BACKEND ON)
-endif()
-
-# config GIT_URL with github mirrors to speed up dependent repos clone
-option(GIT_URL "Git URL to clone dependent repos" ${GIT_URL})
-if(NOT GIT_URL)
-    set(GIT_URL "https://github.com")
-endif()
-
-# check build options
-include(${PROJECT_SOURCE_DIR}/cmake/check.cmake)
-
-if(WIN32 AND ENABLE_VISION)
-  add_definitions(-DYAML_CPP_DLL)
-  set(YAML_BUILD_SHARED_LIBS ON)
-  set(YAML_CPP_INSTALL ON)
-  set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
-endif()
-
-if(NOT CUDA_DIRECTORY)
-  set(CUDA_DIRECTORY "/usr/local/cuda")
-endif()
-
-option(BUILD_FASTDEPLOY_PYTHON "if build python lib for fastdeploy." OFF)
-
-set(HEAD_DIR "${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}")
-include_directories(${HEAD_DIR})
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
-if (WITH_TIMVX)
-  include(${PROJECT_SOURCE_DIR}/cmake/timvx.cmake)
-endif()
-
-if (WITH_ASCEND)
-  include(${PROJECT_SOURCE_DIR}/cmake/ascend.cmake)
-endif()
-
-if (WITH_KUNLUNXIN)
-  include(${PROJECT_SOURCE_DIR}/cmake/kunlunxin.cmake)
-endif()
-
-if(WITH_IPU)
-  if(NOT ENABLE_PADDLE_BACKEND)
-    message("Will force to set ENABLE_PADDLE_BACKEND when build with GraphCore IPU.")
-    set(ENABLE_PADDLE_BACKEND ON)
-  endif()
-  add_definitions(-DWITH_IPU)
-endif()
-
-if(ANDROID)
-  include(${PROJECT_SOURCE_DIR}/cmake/android.cmake)
-  check_android_options_policy()
-  set_android_cxx_complie_flags()
-endif()
-
-# Check for macOS architecture
-get_osx_architecture()
-
-##################################### Building: FastDeploy C++ SDK #######################################
-add_definitions(-DFASTDEPLOY_LIB)
-# set CMAKE_BUILD_TYPE to Release
-add_definitions(-DCMAKE_BUILD_TYPE=Release)
-# configure files before glob sources.
-configure_file(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/core/config.h.in ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/core/config.h)
-configure_file(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/main.cc.in ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/main.cc)
-file(GLOB_RECURSE ALL_DEPLOY_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*.cc)
-file(GLOB_RECURSE DEPLOY_ORT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/ort/*.cc)
-file(GLOB_RECURSE DEPLOY_PADDLE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/paddle/*.cc)
-file(GLOB_RECURSE DEPLOY_POROS_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/poros/*.cc)
-file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tensorrt/*.cpp)
-file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/openvino/*.cc)
-file(GLOB_RECURSE DEPLOY_RKNPU2_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/rknpu2/*.cc)
-file(GLOB_RECURSE DEPLOY_HORIZON_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/horizon/*.cc)
-file(GLOB_RECURSE DEPLOY_SOPHGO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/sophgo/*.cc)
-file(GLOB_RECURSE DEPLOY_TVM_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tvm/*.cc)
-file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/lite/*.cc)
-file(GLOB_RECURSE DEPLOY_ENCRYPTION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/encryption/*.cc)
-file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*.cc)
-file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc)
-file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc)
-file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc)
-file(GLOB_RECURSE DEPLOY_PADDLE_CUSTOM_OP_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/paddle/ops/*.cc)
-if(WITH_GPU)
-  file(GLOB_RECURSE DEPLOY_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*.cu)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_CUDA_SRCS})
-  file(GLOB_RECURSE DEPLOY_PADDLE_CUSTOM_OP_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/paddle/ops/*.cu)
-  list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_CUSTOM_OP_CUDA_SRCS})
-  file(GLOB_RECURSE DEPLOY_VISION_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cu)
-  list(APPEND DEPLOY_VISION_SRCS ${DEPLOY_VISION_CUDA_SRCS})
-  file(GLOB_RECURSE DEPLOY_TEXT_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cu)
-  list(APPEND DEPLOY_TEXT_SRCS ${DEPLOY_TEXT_CUDA_SRCS})
-endif()
-list(REMOVE_ITEM DEPLOY_PADDLE_SRCS ${DEPLOY_PADDLE_CUSTOM_OP_SRCS})
-list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS}
-                                 ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS}
-                                 ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS}
-                                 ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}
-                                 ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS}
-                                 ${DEPLOY_SOPHGO_SRCS} ${DEPLOY_ENCRYPTION_SRCS}
-                                 ${DEPLOY_HORIZON_SRCS} ${DEPLOY_TVM_SRCS}
-                                 ${DEPLOY_PADDLE_CUSTOM_OP_SRCS})
-
-set(DEPEND_LIBS "")
-
-file(READ "${PROJECT_SOURCE_DIR}/VERSION_NUMBER" FASTDEPLOY_VERSION)
-string(STRIP "${FASTDEPLOY_VERSION}" FASTDEPLOY_VERSION)
-
-# Add eigen lib
-include_directories(${PROJECT_SOURCE_DIR}/third_party/eigen)
-if(WIN32)
-  add_definitions(-DEIGEN_STRONG_INLINE=inline)
-endif()
-
-if(ANDROID)
-  # Set tensor function/openmp compile policy after
-  # ALL_DEPLOY_SRCS/DEPEND_LIBS defined
-  set_android_tensor_funcs_compile_policy()
-  set_android_openmp_compile_policy()
-endif()
-
-# sw(sunway) not support thread_local semantic
-if(WITH_SW)
-  add_definitions(-DEIGEN_AVOID_THREAD_LOCAL)
-endif()
-
-if(ENABLE_ORT_BACKEND)
-  set(ENABLE_PADDLE2ONNX ON)
-  add_definitions(-DENABLE_ORT_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/onnxruntime.cmake)
-  list(APPEND DEPEND_LIBS external_onnxruntime)
-endif()
-
-if(ENABLE_LITE_BACKEND)
-  add_definitions(-DENABLE_LITE_BACKEND)
-  include(${PROJECT_SOURCE_DIR}/cmake/paddlelite.cmake)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_LITE_SRCS})
-  list(APPEND DEPEND_LIBS external_paddle_lite)
-endif()
-
-if(ENABLE_PADDLE_BACKEND)
-  set(ENABLE_PADDLE2ONNX ON)
-  add_definitions(-DENABLE_PADDLE_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/paddle_inference.cmake)
-  list(APPEND DEPEND_LIBS external_paddle_inference)
-  if(external_dnnl_FOUND)
-    list(APPEND DEPEND_LIBS external_dnnl external_omp)
-  endif()
-  if(external_ort_FOUND)
-    list(APPEND DEPEND_LIBS external_p2o external_ort)
-  endif()
-  if(PADDLEINFERENCE_API_CUSTOM_OP)
-    set_paddle_custom_ops_compatible_policy()
-    list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_CUSTOM_OP_SRCS})
-    if(WITH_GPU)
-      list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_CUSTOM_OP_CUDA_SRCS})
-    endif()
-  endif()
-endif()
-
-if(ENABLE_OPENVINO_BACKEND)
-  set(ENABLE_PADDLE2ONNX ON)
-  add_definitions(-DENABLE_OPENVINO_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_OPENVINO_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/openvino.cmake)
-endif()
-
-if(ENABLE_RKNPU2_BACKEND)
-  add_definitions(-DENABLE_RKNPU2_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_RKNPU2_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/rknpu2.cmake)
-  list(APPEND DEPEND_LIBS ${RKNN_RT_LIB})
-endif()
-
-if(ENABLE_HORIZON_BACKEND)
-  add_definitions(-DENABLE_HORIZON_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_HORIZON_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/horizon.cmake)
-  list(APPEND DEPEND_LIBS ${BPU_libs})
-endif()
-
-if(ENABLE_TVM_BACKEND)
-  set(CMAKE_CXX_STANDARD 17)
-  add_definitions(-DENABLE_TVM_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_TVM_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/tvm.cmake)
-  list(APPEND DEPEND_LIBS ${TVM_RUNTIME_LIB})
-endif()
-
-if(ENABLE_SOPHGO_BACKEND)
-  add_definitions(-DENABLE_SOPHGO_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_SOPHGO_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/sophgo.cmake)
-  list(APPEND DEPEND_LIBS ${SOPHGO_RT_LIB})
-endif()
-
-if(ENABLE_POROS_BACKEND)
-  set(CMAKE_CXX_STANDARD 14)
-  add_definitions(-DENABLE_POROS_BACKEND)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_POROS_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/poros.cmake)
-  list(APPEND DEPEND_LIBS external_poros)
-  set(PYTHON_MINIMUM_VERSION 3.6)
-  set(PYTORCH_MINIMUM_VERSION 1.9)
-  set(TENSORRT_MINIMUM_VERSION 8.0)
-  # find python3
-  find_package(Python3 ${PYTHON_MINIMUM_VERSION} REQUIRED COMPONENTS Interpreter Development)
-  message(STATUS "Found Python: ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}.${Python3_VERSION_PATCH}")
-
-  if (NOT Python3_SITELIB)
-    message(FATAL_ERROR "site-packages not found. ")
-  else ()
-    message(STATUS "site-packages: ${Python3_SITELIB}")
-  endif ()
-  include_directories(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/poros/common)
-  # find trt
-  if(NOT WITH_GPU)
-  message(FATAL_ERROR "While -DENABLE_POROS_BACKEND=ON, must set -DWITH_GPU=ON, but now it's OFF")
-  endif()
-  if(NOT TRT_DIRECTORY)
-    message(FATAL_ERROR "While -DENABLE_POROS_BACKEND=ON, must define -DTRT_DIRECTORY, e.g -DTRT_DIRECTORY=/Downloads/TensorRT-8.4")
-  endif()
-  include_directories(${TRT_DIRECTORY}/include)
-  find_library(TRT_INFER_LIB nvinfer ${TRT_DIRECTORY}/lib)
-  find_library(TRT_ONNX_LIB nvonnxparser ${TRT_DIRECTORY}/lib)
-  find_library(TRT_PLUGIN_LIB nvinfer_plugin ${TRT_DIRECTORY}/lib)
-  list(APPEND DEPEND_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_PLUGIN_LIB})
-  if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
-    file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
-  endif()
-  if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-    file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-  endif()
-  find_package(Python COMPONENTS Interpreter Development REQUIRED)
-  message(STATUS "Copying ${TRT_DIRECTORY}/lib to ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib ...")
-  execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib)
-endif()
-
-if(WITH_GPU)
-  add_definitions(-DWITH_GPU)
-  include_directories(${CUDA_DIRECTORY}/include)
-  if(WIN32)
-    find_library(CUDA_LIB cudart ${CUDA_DIRECTORY}/lib/x64)
-    find_library(NVJPEG_LIB nvjpeg ${CUDA_DIRECTORY}/lib/x64)
-    add_definitions(-DENABLE_NVJPEG)
-  else()
-    find_library(CUDA_LIB cudart ${CUDA_DIRECTORY}/lib64)
-    if(NOT BUILD_ON_JETSON)
-      find_library(NVJPEG_LIB nvjpeg ${CUDA_DIRECTORY}/lib64)
-      add_definitions(-DENABLE_NVJPEG)
-    endif()
-  endif()
-  list(APPEND DEPEND_LIBS ${CUDA_LIB} ${NVJPEG_LIB})
-
-  # build CUDA source files in fastdeploy, CUDA source files include CUDA preprocessing, TRT plugins, etc.
-  enable_language(CUDA)
-  message(STATUS "CUDA compiler: ${CMAKE_CUDA_COMPILER}, version: "
-                  "${CMAKE_CUDA_COMPILER_ID} ${CMAKE_CUDA_COMPILER_VERSION}")
-  include(${PROJECT_SOURCE_DIR}/cmake/cuda.cmake)
-endif()
-
-if(WITH_OPENCL)
-  add_definitions(-DWITH_OPENCL)
-endif()
-
-if(ENABLE_TRT_BACKEND)
-  set(ENABLE_PADDLE2ONNX ON)
-  if(APPLE OR ANDROID OR IOS)
-    message(FATAL_ERROR "Cannot enable tensorrt backend in mac/ios/android os, please set -DENABLE_TRT_BACKEND=OFF.")
-  endif()
-  if(NOT WITH_GPU)
-    message(FATAL_ERROR "While -DENABLE_TRT_BACKEND=ON, must set -DWITH_GPU=ON, but now it's OFF")
-  endif()
-  if(NOT BUILD_ON_JETSON)
-    if(NOT TRT_DIRECTORY)
-      set(TRT_INC_DIR /usr/include/x86_64-linux-gnu/)
-      set(TRT_LIB_DIR /usr/lib/x86_64-linux-gnu/)
-    endif()
-  endif()
-  if(BUILD_ON_JETSON)
-    set(TRT_INC_DIR /usr/include/aarch64-linux-gnu/)
-    set(TRT_LIB_DIR /usr/lib/aarch64-linux-gnu/)
-  else()
-    set(TRT_INC_DIR /usr/include/x86_64-linux-gnu/)
-    set(TRT_LIB_DIR /usr/lib/x86_64-linux-gnu/)
-    if(TRT_DIRECTORY)
-      set(TRT_INC_DIR ${TRT_DIRECTORY}/include)
-      set(TRT_LIB_DIR ${TRT_DIRECTORY}/lib)
-    endif()
-  endif()
-
-  add_definitions(-DENABLE_TRT_BACKEND)
-  include_directories(${TRT_INC_DIR})
-  include_directories(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tensorrt/common)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_TRT_SRCS})
-  find_library(TRT_INFER_LIB nvinfer ${TRT_LIB_DIR} NO_DEFAULT_PATH)
-  find_library(TRT_ONNX_LIB nvonnxparser ${TRT_LIB_DIR} NO_DEFAULT_PATH)
-  find_library(TRT_PLUGIN_LIB nvinfer_plugin ${TRT_LIB_DIR} NO_DEFAULT_PATH)
-  list(APPEND DEPEND_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_PLUGIN_LIB})
-
-  if(NOT BUILD_ON_JETSON AND TRT_DIRECTORY)
-    if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
-      file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
-    endif()
-    if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-      file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-    endif()
-    
-    if (NOT Python_EXECUTABLE)
-      find_package(Python COMPONENTS Interpreter Development REQUIRED)
-    endif()
-
-    message(STATUS "Copying ${TRT_DIRECTORY}/lib to ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib ...")
-    execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib)
-    file(GLOB_RECURSE TRT_STATIC_LIBS ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib/*.a)
-    if(TRT_STATIC_LIBS)
-      file(REMOVE ${TRT_STATIC_LIBS})
-    endif()
-    if(UNIX AND (NOT APPLE) AND (NOT ANDROID))
-      execute_process(COMMAND sh -c "ls *.so*" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
-	      COMMAND sh -c "xargs ${PATCHELF_EXE} --force-rpath --set-rpath '$ORIGIN'" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
-    	          RESULT_VARIABLE result
-                      OUTPUT_VARIABLE curr_out
-                      ERROR_VARIABLE  curr_out)
-      if(ret EQUAL "1")
-	     message(FATAL_ERROR "Failed to patchelf tensorrt libraries.")
-      endif()
-      message(STATUS "result:${result} out:${curr_out}")
-    endif()
-  endif()
-endif()
-
-if(ENABLE_VISION)
-  add_definitions(-DENABLE_VISION)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp)
-  list(APPEND DEPEND_LIBS yaml-cpp)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_VISION_SRCS})
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PIPELINE_SRCS})
-  include_directories(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include)
-  include(${PROJECT_SOURCE_DIR}/cmake/opencv.cmake)
-
-  if(ENABLE_FLYCV)
-    add_definitions(-DENABLE_FLYCV)
-    include(${PROJECT_SOURCE_DIR}/cmake/flycv.cmake)
-    list(APPEND DEPEND_LIBS ${FLYCV_LIBRARIES})
-  endif()
-
-  if(ENABLE_CVCUDA)
-    include(${PROJECT_SOURCE_DIR}/cmake/cvcuda.cmake)
-    add_definitions(-DENABLE_CVCUDA)
-    list(APPEND DEPEND_LIBS nvcv_types cvcuda)
-  endif()
-endif()
-
-if(ENABLE_TEXT)
-  add_definitions(-DENABLE_TEXT)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_TEXT_SRCS})
-  include(${PROJECT_SOURCE_DIR}/cmake/fast_tokenizer.cmake)
-endif()
-
-if(ENABLE_ENCRYPTION)
-  add_definitions(-DENABLE_ENCRYPTION)
-  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_ENCRYPTION_SRCS})
-  # include(${PROJECT_SOURCE_DIR}/cmake/gflags.cmake)
-  include(${PROJECT_SOURCE_DIR}/cmake/openssl.cmake)
-  list(APPEND DEPEND_LIBS ${OPENSSL_LIBRARIES})
-endif()
-
-if(ENABLE_PADDLE2ONNX)
-  add_definitions(-DENABLE_PADDLE2ONNX)
-  if(BUILD_PADDLE2ONNX)
-    download_protobuf()
-    include(${PROJECT_SOURCE_DIR}/cmake/build_paddle2onnx.cmake)
-    list(APPEND ALL_DEPLOY_SRCS ${PADDLE2ONNX_ALL_SRCS})
-    list(APPEND DEPEND_LIBS p2o_paddle_proto onnx)
-  else()
-    include(${PROJECT_SOURCE_DIR}/cmake/paddle2onnx.cmake)
-    list(APPEND DEPEND_LIBS external_paddle2onnx)
-  endif()
-endif(ENABLE_PADDLE2ONNX)
-
-if(WITH_CAPI)
-  include(${PROJECT_SOURCE_DIR}/c_api/CMakeLists.txt)
-  if(MSVC)
-  add_definitions(-DFD_CAPI)
-  endif()
-endif()
-
-if(WITH_CSHARPAPI)
-  if(MSVC)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/csharp)
-  endif()
-endif()
-
-
-configure_file(${PROJECT_SOURCE_DIR}/FastDeploy.cmake.in ${PROJECT_SOURCE_DIR}/FastDeploy.cmake @ONLY)
-configure_file(${PROJECT_SOURCE_DIR}/FastDeployCSharp.cmake.in ${PROJECT_SOURCE_DIR}/FastDeployCSharp.cmake @ONLY)
-configure_file(${PROJECT_SOURCE_DIR}/python/fastdeploy/c_lib_wrap.py.in ${PROJECT_SOURCE_DIR}/python/fastdeploy/c_lib_wrap.py)
-configure_file(${PROJECT_SOURCE_DIR}/python/scripts/process_libraries.py.in ${PROJECT_SOURCE_DIR}/python/scripts/process_libraries.py)
-
-list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_PYBIND_SRCS})
-
-add_library(${LIBRARY_NAME} SHARED ${ALL_DEPLOY_SRCS})
-
-redefine_file_macro(${LIBRARY_NAME})
-
-file(READ "${PROJECT_SOURCE_DIR}/VERSION_NUMBER" FASTDEPLOY_VERSION)
-string(STRIP "${FASTDEPLOY_VERSION}" FASTDEPLOY_VERSION)
-if (APPLE)
-  set_target_properties(${LIBRARY_NAME} PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
-elseif(ANDROID)
-  set_android_library_cxx_link_flags()
-elseif(MSVC)
-else()
-  if(WITH_GPU)
-    set_target_properties(${LIBRARY_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-    set_target_properties(${LIBRARY_NAME} PROPERTIES INTERFACE_COMPILE_OPTIONS
-       "$<$<BUILD_INTERFACE:$<COMPILE_LANGUAGE:CXX>>:-fvisibility=hidden>$<$<BUILD_INTERFACE:$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-fvisibility=hidden>")
-  else()
-    set_target_properties(${LIBRARY_NAME} PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
-  endif()
-  set_target_properties(${LIBRARY_NAME} PROPERTIES LINK_FLAGS "-Wl,--exclude-libs,ALL")
-  set_target_properties(${LIBRARY_NAME} PROPERTIES LINK_FLAGS_RELEASE -s)
-endif()
-
-set_target_properties(${LIBRARY_NAME} PROPERTIES VERSION ${FASTDEPLOY_VERSION})
-if(MSVC)
-  # disable warnings for dll export
-  target_compile_options(${LIBRARY_NAME} PRIVATE "$<$<BUILD_INTERFACE:$<COMPILE_LANGUAGE:CXX>>:/wd4251>$<$<BUILD_INTERFACE:$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=/wd4251>")
-  file(GLOB FD_FILES_REQUIRE_BIGOBJ ${CSRCS_DIR_NAME}/fastdeploy/function/reduce.cc)
-  set_source_files_properties(${FD_FILES_REQUIRE_BIGOBJ} PROPERTIES COMPILE_FLAGS "/bigobj")
-endif()
-
-target_link_libraries(${LIBRARY_NAME} ${DEPEND_LIBS})
-
-if(ENABLE_PADDLE_BACKEND)
-  set_paddle_encrypt_auth_compatible_policy(${LIBRARY_NAME})
-endif()
-
-if(ANDROID)
-  set_android_extra_libraries_target()
-endif()
-
-##################################### Examples ####################################
-if(WIN32)
-  if(ENABLE_VISION)
-    if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
-      add_custom_target(copy_yaml_library ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/third_party/yaml-cpp  ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/yaml-cpp/lib DEPENDS ${LIBRARY_NAME})
-    else()
-      add_custom_target(copy_yaml_library ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/third_party/yaml-cpp/Release  ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/yaml-cpp/lib DEPENDS ${LIBRARY_NAME})
-      add_custom_target(copy_yaml_include ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include  ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/yaml-cpp/include DEPENDS ${LIBRARY_NAME})
-    endif()
-  endif()
-endif()
-
-# add examples after prepare include paths for third-parties
-if(BUILD_EXAMPLES AND EXISTS ${PROJECT_SOURCE_DIR}/examples)
-  add_definitions(-DBUILD_EXAMPLES)
-  if(NOT EXECUTABLE_OUTPUT_PATH STREQUAL ${CMAKE_CURRENT_BINARY_DIR}/bin)
-    set(EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/bin)
-  endif()
-  include(${PROJECT_SOURCE_DIR}/cmake/gflags.cmake)
-  add_subdirectory(examples)
-endif()
-
-if (WITH_TESTING AND EXISTS ${PROJECT_SOURCE_DIR}/tests)
-  add_definitions(-DWITH_TESTING)
-  include(${PROJECT_SOURCE_DIR}/cmake/gtest.cmake)
-  if(NOT BUILD_EXAMPLES)
-    include(${PROJECT_SOURCE_DIR}/cmake/gflags.cmake)
-  endif()
-  include(${PROJECT_SOURCE_DIR}/cmake/glog.cmake)
-  add_subdirectory(tests)
-endif()
-
-include(${PROJECT_SOURCE_DIR}/cmake/summary.cmake)
-fastdeploy_summary()
-
-################################ Installation: FastDeploy C++ SDK ###############################
-if(WIN32)
-  install(
-    TARGETS ${LIBRARY_NAME}
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib
-    RUNTIME DESTINATION lib
-  )
-elseif(ANDROID)
-  set_android_libraries_installation()
-else()
-  install(
-    TARGETS ${LIBRARY_NAME}
-    LIBRARY DESTINATION lib)
-endif()
-
-install(
-  DIRECTORY ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy
-  DESTINATION ${CMAKE_INSTALL_PREFIX}/include
-  FILES_MATCHING
-  PATTERN "*.h"
-  PATTERN "${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/*/*.h"
-)
-
-if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/")
-  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/")
-endif()
-
-if(NOT ANDROID)
-  install(
-    DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/third_libs
-  )
-else()
-  set_android_third_libs_installation()
-endif()
-
-install(
-  FILES
-  ${PROJECT_SOURCE_DIR}/LICENSE
-  ${PROJECT_SOURCE_DIR}/ThirdPartyNotices.txt
-  ${PROJECT_SOURCE_DIR}/VERSION_NUMBER
-  ${PROJECT_SOURCE_DIR}/FastDeploy.cmake
-  ${PROJECT_SOURCE_DIR}/FastDeployCSharp.cmake
-  ${PROJECT_SOURCE_DIR}/cmake/FastDeployConfig.cmake
-  ${PROJECT_SOURCE_DIR}/cmake/utils.cmake
-  ${PROJECT_SOURCE_DIR}/cmake/summary.cmake
-  ${PROJECT_SOURCE_DIR}/cmake/openmp.cmake
-  DESTINATION ${CMAKE_INSTALL_PREFIX}
-)
-
-install(
-  FILES ${PROJECT_SOURCE_DIR}/cmake/gflags.cmake
-  DESTINATION ${CMAKE_INSTALL_PREFIX}/utils
-)
-
-if(NOT WIN32)
-  install(
-    FILES ${PROJECT_SOURCE_DIR}/scripts/fastdeploy_init.sh
-    DESTINATION ${CMAKE_INSTALL_PREFIX}
-  )
-else()
-  install(
-    FILES ${PROJECT_SOURCE_DIR}/scripts/fastdeploy_init.bat
-    DESTINATION ${CMAKE_INSTALL_PREFIX}
-  )
-endif()
-
-if(WITH_ASCEND)
-  install(
-    FILES ${PROJECT_SOURCE_DIR}/scripts/ascend_init.sh
-    DESTINATION ${CMAKE_INSTALL_PREFIX}
-  )
-endif()
-
-if(WITH_CAPI)
-  install(
-    DIRECTORY ${PROJECT_SOURCE_DIR}/c_api/fastdeploy_capi
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/include
-    FILES_MATCHING
-    PATTERN "*.h"
-    PATTERN "*/types_internal.h" EXCLUDE
-  )
-endif()
-
-include(${PROJECT_SOURCE_DIR}/cmake/config_cpack.cmake)
-
-if(WIN32 AND BUILD_EXAMPLES)
-  get_windows_path(_tmp_install_dir ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install)
-  get_windows_path(_publish_exe_dir ${EXECUTABLE_OUTPUT_PATH}/Release)
-  list(GET CMAKE_CONFIGURATION_TYPES 0 _CONFIG_TYPE)
-  if((${CMAKE_BUILD_TYPE} MATCHES "Release") OR (${_CONFIG_TYPE} MATCHES "Release"))
-    install(TARGETS ${LIBRARY_NAME} RUNTIME DESTINATION ${EXECUTABLE_OUTPUT_PATH}/Release)
-    add_custom_target(
-      copy_fd_third_dlls_examples ALL COMMAND
-      cmd /C ${PROJECT_SOURCE_DIR}/scripts/fastdeploy_init.bat install ${_tmp_install_dir} ${_publish_exe_dir} noconfirm)
-    add_dependencies(copy_fd_third_dlls_examples ${LIBRARY_NAME} copy_yaml_library)
-  endif()
-endif()
-
-############################### Building: FastDeploy Python Wheel #############################
-if(BUILD_FASTDEPLOY_PYTHON)
-  add_definitions(-DBUILD_FASTDEPLOY_PYTHON)
-  if("${PY_EXT_SUFFIX}" STREQUAL "")
-    if(MSVC)
-      set(PY_EXT_SUFFIX ".pyd")
-    else()
-      set(PY_EXT_SUFFIX ".so")
-    endif()
-  endif()
-
-  # find_package Python has replaced PythonInterp and PythonLibs since cmake 3.12
-  # Use the following command in the future; now this is only compatible with the latest pybind11
-  # find_package(Python ${PY_VERSION} COMPONENTS Interpreter Development REQUIRED)
-  find_package(PythonInterp ${PY_VERSION} REQUIRED)
-  find_package(PythonLibs ${PY_VERSION})
-  if(CMAKE_SYSTEM_NAME STREQUAL "AIX")
-    set(CMAKE_NO_SYSTEM_FROM_IMPORTED 1)
-  endif()
-
-  if(NOT ENABLE_VISION)
-    file(GLOB_RECURSE VISION_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*_pybind.cc)
-    file(GLOB_RECURSE PIPELINE_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pipeline/*_pybind.cc)
-    list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${VISION_PYBIND_SRCS} ${PIPELINE_PYBIND_SRCS})
-  endif()
-
-  if(NOT ENABLE_ENCRYPTION)
-    file(GLOB_RECURSE ENCRYPTION_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/encryption/*_pybind.cc)
-    list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${ENCRYPTION_PYBIND_SRCS})
-  endif()
-
-  if (NOT ENABLE_TEXT)
-    file(GLOB_RECURSE TEXT_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*_pybind.cc)
-    list(REMOVE_ITEM DEPLOY_PYBIND_SRCS ${TEXT_PYBIND_SRCS})
-  endif()
-
-  add_library(${PY_LIBRARY_NAME} MODULE ${DEPLOY_PYBIND_SRCS})
-  redefine_file_macro(${PY_LIBRARY_NAME})
-  set_target_properties(${PY_LIBRARY_NAME} PROPERTIES PREFIX "")
-  set_target_properties(${PY_LIBRARY_NAME}
-                        PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
-  set_target_properties(${PY_LIBRARY_NAME} PROPERTIES SUFFIX ${PY_EXT_SUFFIX})
-  set_target_properties(${PY_LIBRARY_NAME}
-                        PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-  target_include_directories(${PY_LIBRARY_NAME} PRIVATE
-                             $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-                             $<INSTALL_INTERFACE:include>
-                             ${PYTHON_INCLUDE_DIR})
-
-  target_include_directories(${PY_LIBRARY_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/third_party/pybind11/include)
-  target_include_directories(${PY_LIBRARY_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/third_party/dlpack/include)
-
-  if(APPLE)
-    set_target_properties(${PY_LIBRARY_NAME}
-                          PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  endif()
-
-  target_link_libraries(${PY_LIBRARY_NAME} PUBLIC ${LIBRARY_NAME})
-
-  if(MSVC)
-    target_link_libraries(${PY_LIBRARY_NAME} PRIVATE ${PYTHON_LIBRARIES})
-    target_compile_options(${PY_LIBRARY_NAME}
-                           PRIVATE /MP
-                                   /wd4244 # 'argument': conversion from 'google::
-                                           # protobuf::uint64' to 'int', possible
-                                           # loss of data
-                                   /wd4267 # Conversion from 'size_t' to 'int',
-                                           # possible loss of data
-                                   /wd4996 # The second parameter is ignored.
-                                   ${EXTRA_FLAGS})
-    target_compile_options(${PY_LIBRARY_NAME} PRIVATE $<$<NOT:$<CONFIG:Debug>>:/MT> $<$<CONFIG:Debug>:/MTd>)
-  endif()
-
-  file(REMOVE_RECURSE ${PROJECT_SOURCE_DIR}/fastdeploy/libs)
-  file(MAKE_DIRECTORY ${PROJECT_SOURCE_DIR}/fastdeploy/libs)
-
-  if(WIN32)
-    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy_directory   ${CMAKE_CURRENT_BINARY_DIR}/Release ${PROJECT_SOURCE_DIR}/python/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
-  elseif(APPLE)
-    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/*.so** ${CMAKE_CURRENT_BINARY_DIR}/*.dylib** ${PROJECT_SOURCE_DIR}/python/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
-  else()
-    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/*.so* ${PROJECT_SOURCE_DIR}/python/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
-  endif()
-  add_custom_target(copy_third_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install ${PROJECT_SOURCE_DIR}/python/fastdeploy/libs/third_libs DEPENDS ${PY_LIBRARY_NAME})
-endif(BUILD_FASTDEPLOY_PYTHON)
-
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4.0")
-    string(STRIP "${CMAKE_CXX_COMPILER_VERSION}" CMAKE_CXX_COMPILER_VERSION)
-    message(FATAL_ERROR "[ERROR] FastDeploy require g++ version >= 5.4.0, but now your g++ version is ${CMAKE_CXX_COMPILER_VERSION}, this may cause failure! Use -DCMAKE_CXX_COMPILER to define path of your compiler.")
-  endif()
-endif()
-

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-fastdeploy@baidu.com.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

FastDeploy.cmake.in

@@ -1,497 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.8)
-
-# FastDeploy basic infos
-set(FASTDEPLOY_VERSION @FASTDEPLOY_VERSION@)
-set(LIBRARY_NAME @LIBRARY_NAME@)
-
-# If compile with GLIBC_CXX_ABI=0
-set(NEED_ABI0 @NEED_ABI0@)
-
-# Hardware and Language API
-set(WITH_GPU @WITH_GPU@)
-set(WITH_IPU @WITH_IPU@)
-set(WITH_OPENCL @WITH_OPENCL@)
-set(WITH_ASCEND @WITH_ASCEND@)
-set(WITH_DIRECTML @WITH_DIRECTML@)
-set(WITH_TIMVX @WITH_TIMVX@)
-set(WITH_KUNLUNXIN @WITH_KUNLUNXIN@)
-set(WITH_CAPI @WITH_CAPI@)
-set(WITH_CSHARPAPI @WITH_CSHARPAPI@)
-set(WITH_TESTING @WITH_TESTING@)
-set(BUILD_ON_JETSON @BUILD_ON_JETSON@)
-set(RKNN2_TARGET_SOC "@RKNN2_TARGET_SOC@")
-
-# Inference backend and FastDeploy Moudle
-set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@)
-set(ENABLE_RKNPU2_BACKEND @ENABLE_RKNPU2_BACKEND@)
-set(ENABLE_TVM_BACKEND @ENABLE_TVM_BACKEND@)
-set(ENABLE_HORIZON_BACKEND @ENABLE_HORIZON_BACKEND@)
-set(ENABLE_SOPHGO_BACKEND @ENABLE_SOPHGO_BACKEND@)
-set(ENABLE_LITE_BACKEND @ENABLE_LITE_BACKEND@)
-set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@)
-set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@)
-set(ENABLE_POROS_BACKEND @ENABLE_POROS_BACKEND@)
-set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@)
-set(ENABLE_PADDLE2ONNX @ENABLE_PADDLE2ONNX@)
-set(BUILD_PADDLE2ONNX @BUILD_PADDLE2ONNX@)
-
-set(ENABLE_VISION @ENABLE_VISION@)
-set(ENABLE_FLYCV @ENABLE_FLYCV@)
-set(ENABLE_CVCUDA @ENABLE_CVCUDA@)
-set(ENABLE_TEXT @ENABLE_TEXT@)
-set(ENABLE_ENCRYPTION @ENABLE_ENCRYPTION@)
-set(ENABLE_BENCHMARK @ENABLE_BENCHMARK@)
-
-# Version infos and custom settings for third libs
-set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@)
-set(POROS_VERSION @POROS_VERSION@)
-set(OPENVINO_VERSION @OPENVINO_VERSION@)
-set(OPENCV_FILENAME @OPENCV_FILENAME@)
-set(OPENVINO_FILENAME @OPENVINO_FILENAME@)
-set(PADDLELITE_FILENAME @PADDLELITE_FILENAME@)
-set(OPENCV_DIRECTORY "@OPENCV_DIRECTORY@")
-set(ORT_DIRECTORY "@ORT_DIRECTORY@")
-set(OPENVINO_DIRECTORY "@OPENVINO_DIRECTORY@")
-
-# Android: specific option for Android OS
-set(WITH_ANDROID_STATIC_LIB @WITH_ANDROID_STATIC_LIB@)
-set(WITH_ANDROID_LITE_STATIC @WITH_ANDROID_LITE_STATIC@)
-set(WITH_ANDROID_OPENCV_STATIC @WITH_ANDROID_OPENCV_STATIC@)
-set(WITH_ANDROID_FLYCV_STATIC @WITH_ANDROID_FLYCV_STATIC@)
-set(WITH_ANDROID_OPENMP @WITH_ANDROID_OPENMP@)
-set(WITH_ANDROID_JAVA @WITH_ANDROID_JAVA@)
-set(WITH_ANDROID_TENSOR_FUNCS @WITH_ANDROID_TENSOR_FUNCS@)
-
-# encryption and auth
-set(PADDLEINFERENCE_WITH_ENCRYPT @PADDLEINFERENCE_WITH_ENCRYPT@)
-set(PADDLEINFERENCE_WITH_AUTH @PADDLEINFERENCE_WITH_AUTH@)
-
-set(FASTDEPLOY_LIBS "")
-set(FASTDEPLOY_INCS "")
-list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/include)
-
-# Note(zhoushunjie): include some useful utils function
-include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake)
-
-# Set C++11 as standard for the whole project
-if(NOT MSVC)
-  set(CMAKE_CXX_STANDARD 11)
-  set(CMAKE_CXX_FLAGS "-Wno-format")
-  if(NEED_ABI0)
-    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
-  else()
-    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
-  endif()
-endif(NOT MSVC)
-
-# Set FastDeploy static lib definitions
-if(WITH_ANDROID_LITE_STATIC)
-  add_definitions(-DWITH_LITE_STATIC)
-  add_definitions(-DWITH_ANDROID_LITE_STATIC)
-endif()
-
-if(WITH_ANDROID_STATIC_LIB)
-  add_definitions(-DWITH_STATIC_LIB)
-  add_definitions(-DWITH_ANDROID_STATIC_LIB)
-  # add_definitions(-DWITH_STATIC_WARNING)
-endif()
-
-# Still need omp while using FastDeploy static lib.
-# This is due to the use of openmp for Paddle Lite's
-# static library.
-if(ANDROID AND WITH_ANDROID_STATIC_LIB AND WITH_ANDROID_LITE_STATIC)
-  include(${CMAKE_CURRENT_LIST_DIR}/openmp.cmake)
-endif()
-
-if(ANDROID)
-  add_library(fastdeploy STATIC IMPORTED GLOBAL)
-  if(WITH_ANDROID_STATIC_LIB)
-    set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION
-                 ${CMAKE_CURRENT_LIST_DIR}/lib/${ANDROID_ABI}/lib${LIBRARY_NAME}_static.a)
-  else()
-    set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION
-                 ${CMAKE_CURRENT_LIST_DIR}/lib/${ANDROID_ABI}/lib${LIBRARY_NAME}.so)
-  endif()
-  list(APPEND FASTDEPLOY_LIBS fastdeploy)
-  if(WITH_ANDROID_OPENMP AND (NOT WITH_ANDROID_LITE_STATIC))
-    add_library(fastdeploy_omp STATIC IMPORTED GLOBAL)
-    set_property(TARGET fastdeploy_omp PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/lib/${ANDROID_ABI}/libomp.so)
-    list(APPEND FASTDEPLOY_LIBS fastdeploy_omp)
-  endif()
-else()
-  find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib NO_DEFAULT_PATH)
-  list(APPEND FASTDEPLOY_LIBS ${FDLIB})
-endif()
-
-if(ENABLE_ORT_BACKEND)
-  if (ORT_DIRECTORY)
-    set(ORT_LIB_PATH ${ORT_DIRECTORY}/lib)
-  else()
-    set(ORT_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib)
-  endif()
-  message(STATUS "The path of ONNXRuntime is ${ORT_LIB_PATH}.")
-  find_library(ORT_LIB onnxruntime ${ORT_LIB_PATH}  NO_DEFAULT_PATH)
-  list(APPEND FASTDEPLOY_LIBS ${ORT_LIB})
-endif()
-
-if(ENABLE_TVM_BACKEND)
-    if(APPLE)
-        set(TVM_RUNTIME_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tvm/lib/libtvm_runtime.dylib)
-    else()
-        set(TVM_RUNTIME_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tvm/lib/libtvm_runtime.so)
-    endif()
-    list(APPEND FASTDEPLOY_LIBS ${TVM_RUNTIME_LIB})
-endif()
-
-if(ENABLE_PADDLE_BACKEND)
-  find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib NO_DEFAULT_PATH)
-  if(WIN32)
-    set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/mkldnn.lib")
-    set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5md.lib")
-  elseif(APPLE)
-    message(STATUS "No third parties libs(mkldnn and omp) need to link into paddle_inference on MacOS OSX.")
-  else()
-    set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/libmkldnn.so.0")
-    set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5.so")
-    set(FDMODEL_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/fdmodel/lib/libfastdeploy_wenxin.so")
-    set(FDMODEL_MODEL_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/fdmodel/lib/libfastdeploy_model.so.2.0.0")
-    set(FDMODEL_AUTH_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/fdmodel/lib/libfastdeploy_auth.so")
-    if((EXISTS ${FDMODEL_LIB}) AND (EXISTS ${FDMODEL_MODEL_LIB}))
-      set(PADDLEINFERENCE_WITH_ENCRYPT ON CACHE BOOL "" FORCE)
-      list(APPEND FASTDEPLOY_LIBS ${FDMODEL_LIB} ${FDMODEL_MODEL_LIB})
-    endif()
-    if((EXISTS ${FDMODEL_LIB}) AND (EXISTS ${FDMODEL_AUTH_LIB}))
-      set(PADDLEINFERENCE_WITH_AUTH ON CACHE BOOL "" FORCE)
-      list(APPEND FASTDEPLOY_LIBS ${FDMODEL_AUTH_LIB})
-    endif()
-    if(PADDLEINFERENCE_WITH_ENCRYPT OR PADDLEINFERENCE_WITH_AUTH)
-      if(WITH_KUNLUNXIN)
-        list(APPEND FASTDEPLOY_LIBS -lssl -lcrypto)
-      endif()
-    endif()
-  endif()
-  list(APPEND FASTDEPLOY_LIBS ${PADDLE_LIB})
-  if(EXISTS "${DNNL_LIB}")
-    list(APPEND FASTDEPLOY_LIBS ${DNNL_LIB} ${IOMP_LIB})
-  endif()
-endif()
-
-if(ENABLE_OPENVINO_BACKEND)
-  if (OPENVINO_DIRECTORY)
-    set(OPENVINO_DIR ${OPENVINO_DIRECTORY})
-  else()
-    set(OPENVINO_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENVINO_FILENAME}/runtime)
-  endif()
-  get_openvino_libs(${OPENVINO_DIR})
-  message(STATUS "OPENVINO_LIBS = ${OPENVINO_LIBS}")
-  list(APPEND FASTDEPLOY_LIBS ${OPENVINO_LIBS})
-endif()
-
-if(ENABLE_RKNPU2_BACKEND)
-  if(RKNN2_TARGET_SOC STREQUAL "RK356X")
-    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/lib/librknnrt.so)
-  elseif (RKNN2_TARGET_SOC STREQUAL "RK3588")
-    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/lib/librknnrt.so)
-  else ()
-    message(FATAL_ERROR "RKNN2_TARGET_SOC is not set, ref value: RK356X or RK3588")
-  endif()
-  message(STATUS "The path of RKNPU2 is ${RKNPU2_LIB}.")
-  list(APPEND FASTDEPLOY_LIBS ${RKNPU2_LIB})
-endif()
-
-if(ENABLE_HORIZON_BACKEND)
-  set(DNN_PATH ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/dnn)
-  set(APPSDK_PATH ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/appsdk/appuser/)
-
-  set(DNN_LIB_PATH ${DNN_PATH}/lib)
-  set(APPSDK_LIB_PATH ${APPSDK_PATH}/lib/hbbpu)
-  set(BPU_libs dnn cnn_intf hbrt_bernoulli_aarch64)
-
-  link_directories(${DNN_LIB_PATH}
-                  ${APPSDK_PATH}/lib/hbbpu
-                  ${APPSDK_PATH}/lib)
-
-  list(APPEND FASTDEPLOY_LIBS ${BPU_libs})
-
-endif()
-if(ENABLE_LITE_BACKEND)
-  set(LITE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${PADDLELITE_FILENAME})
-  if(ANDROID)
-    if(WITH_ANDROID_LITE_STATIC)
-      if(WITH_ANDROID_STATIC_LIB)
-        list(APPEND FASTDEPLOY_INCS ${LITE_DIR}/include)
-      endif()
-    else()
-      add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL)
-      set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_full_api_shared.so)
-      list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared)
-    endif()
-  else()
-    # Linux/Mac/Win/...
-    find_library(LITE_LIB paddle_full_api_shared ${LITE_DIR}/lib NO_DEFAULT_PATH)
-    list(APPEND FASTDEPLOY_LIBS ${LITE_LIB})
-  endif()
-endif()
-
-if(ENABLE_POROS_BACKEND)
-  find_library(POROS_LIB poros ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/poros/lib NO_DEFAULT_PATH)
-  find_library(TORCH_LIB torch ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/torch/lib NO_DEFAULT_PATH)
-  set(TORCH_INCLUDE "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/torch/include")
-  list(APPEND FASTDEPLOY_LIBS ${POROS_LIB} ${TORCH_LIB})
-  list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/poros/include ${TORCH_INCLUDE})
-endif()
-
-if(WITH_GPU)
-  if(NOT CUDA_DIRECTORY)
-    set(CUDA_DIRECTORY "/usr/local/cuda")
-  endif()
-  if(WIN32)
-    find_library(CUDA_LIB cudart ${CUDA_DIRECTORY}/lib/x64)
-    find_library(NVJPEG_LIB nvjpeg ${CUDA_DIRECTORY}/lib/x64)
-  else()
-    find_library(CUDA_LIB cudart ${CUDA_DIRECTORY}/lib64)
-    if(NOT BUILD_ON_JETSON)
-      find_library(NVJPEG_LIB nvjpeg ${CUDA_DIRECTORY}/lib64)
-    endif()
-  endif()
-  if(NOT CUDA_LIB)
-    message(FATAL_ERROR "[FastDeploy] Cannot find library cudart in ${CUDA_DIRECTORY}, Please define CUDA_DIRECTORY, e.g -DCUDA_DIRECTORY=/path/to/cuda")
-  endif()
-  list(APPEND FASTDEPLOY_LIBS ${CUDA_LIB} ${NVJPEG_LIB})
-  list(APPEND FASTDEPLOY_INCS ${CUDA_DIRECTORY}/include)
-
-  if(ENABLE_TRT_BACKEND)
-    if(BUILD_ON_JETSON)
-      find_library(TRT_INFER_LIB nvinfer /usr/lib/aarch64-linux-gnu/)
-      find_library(TRT_ONNX_LIB nvonnxparser /usr/lib/aarch64-linux-gnu/)
-      find_library(TRT_PLUGIN_LIB nvinfer_plugin /usr/lib/aarch64-linux-gnu/)
-    else()
-      if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/)
-        find_library(TRT_INFER_LIB nvinfer ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-        find_library(TRT_ONNX_LIB nvonnxparser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-        find_library(TRT_PLUGIN_LIB nvinfer_plugin ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-      else()
-        find_library(TRT_INFER_LIB nvinfer /usr/lib/x86_64-linux-gnu/)
-        find_library(TRT_ONNX_LIB nvonnxparser /usr/lib/x86_64-linux-gnu/)
-        find_library(TRT_PLUGIN_LIB nvinfer_plugin /usr/lib/x86_64-linux-gnu/)
-      endif()
-    endif()
-    list(APPEND FASTDEPLOY_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_PLUGIN_LIB})
-  endif()
-endif()
-
-if(ENABLE_VISION)
-  if(OPENCV_DIRECTORY)
-    set(OpenCV_DIR ${OPENCV_DIRECTORY})
-  else()
-    if(ANDROID)
-      set(OpenCV_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENCV_FILENAME}/sdk/native/jni)
-      set(OpenCV_NATIVE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENCV_FILENAME}/sdk/native)
-    else()
-      set(OpenCV_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENCV_FILENAME})
-      if(WIN32)
-        set(OpenCV_DIR ${OpenCV_DIR}/build)
-      endif()
-    endif()
-  endif()
-  message(STATUS "The path of OpenCV is ${OpenCV_DIR}.")
-  if(ANDROID)
-    if(WITH_ANDROID_OPENCV_STATIC)
-      if(WITH_ANDROID_STATIC_LIB)
-        # Only need the headers of opencv while using FastDeploy static lib.
-        list(APPEND FASTDEPLOY_INCS ${OpenCV_DIR}/include)
-      else()
-        find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR})
-        list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS})
-        # For now, we still need to link OpenCV static libs.
-        # Users may use some of opencv's apis, but they may
-        # not have been compiled into fastdeploy.
-        # list(APPEND FASTDEPLOY_LIBS ${OpenCV_LIBS})
-        list(APPEND FASTDEPLOY_LIBS opencv_core opencv_video opencv_highgui opencv_imgproc opencv_imgcodecs)
-      endif()
-    else()
-      set(OpenCV_INCLUDE_DIRS ${OpenCV_DIR}/include)
-      get_filename_component(OpenCV_NATIVE_DIR ${OpenCV_DIR} DIRECTORY)
-      set(OpenCV_LIBS_DIR ${OpenCV_NATIVE_DIR}/libs)
-      if(ANDROID_TOOLCHAIN MATCHES "clang")  # use opencv 4.x
-        add_library(opencv_java4 STATIC IMPORTED GLOBAL)
-        set_property(TARGET opencv_java4 PROPERTY IMPORTED_LOCATION ${OpenCV_LIBS_DIR}/${ANDROID_ABI}/libopencv_java4.so)
-        list(APPEND FASTDEPLOY_LIBS opencv_java4)
-      elseif(ANDROID_TOOLCHAIN MATCHES "gcc")  # use opencv 3.x
-        add_library(opencv_java3 STATIC IMPORTED GLOBAL)
-        set_property(TARGET opencv_java3 PROPERTY IMPORTED_LOCATION ${OpenCV_LIBS_DIR}/${ANDROID_ABI}/opencv_java3.so)
-        list(APPEND FASTDEPLOY_LIBS opencv_java3)
-      else()
-        message(FATAL_ERROR "Only support clang/gcc toolchain, but found ${ANDROID_TOOLCHAIN}.")
-      endif()
-      list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS})
-      message(STATUS "FASTDEPLOY_INCS: ${FASTDEPLOY_INCS}")
-    endif()
-  # Win/Linux/Mac
-  else()
-    find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR} NO_DEFAULT_PATH)
-    list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS})
-    list(APPEND FASTDEPLOY_LIBS ${OpenCV_LIBS})
-  endif()
-
-  if(ENABLE_FLYCV)
-    include_directories(${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/include)
-    set(FLYCV_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/lib)
-    if(ANDROID)
-      if(NOT WITH_ANDROID_FLYCV_STATIC)
-        add_library(flycv_shared STATIC IMPORTED GLOBAL)
-        set_property(TARGET flycv_shared PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libflycv_shared.so)
-        list(APPEND FASTDEPLOY_LIBS flycv_shared)
-      else()
-        # This code may be needed later. Therefore, I choose to
-        # comment it rather than delete it. (TODO:qiuyanjun)
-        # add_library(flycv_static STATIC IMPORTED GLOBAL)
-        # add_library(flycv_png16 STATIC IMPORTED GLOBAL)
-        # add_library(flycv_turbojpeg STATIC IMPORTED GLOBAL)
-        # add_library(flycv_z STATIC IMPORTED GLOBAL)
-        # set_property(TARGET flycv_static PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libflycv_static.a)
-        # set_property(TARGET flycv_png16 PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libpng16.a)
-        # set_property(TARGET flycv_turbojpeg PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libturbojpeg.a)
-        # set_property(TARGET flycv_z PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libz.a)
-        # list(APPEND FASTDEPLOY_LIBS flycv_static)
-        # list(APPEND FASTDEPLOY_LIBS flycv_png16)
-        # list(APPEND FASTDEPLOY_LIBS flycv_turbojpeg)
-        # list(APPEND FASTDEPLOY_LIBS flycv_z)
-      endif()
-    else()
-      find_library(FLYCV_LIB flycv_shared ${FLYCV_LIB_DIR} NO_DEFAULT_PATH)
-      list(APPEND FASTDEPLOY_LIBS ${FLYCV_LIB})
-    endif()
-  endif()
-
-  if(ENABLE_CVCUDA)
-    find_library(CVCUDA_LIB cvcuda ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/cvcuda/lib NO_DEFAULT_PATH)
-    find_library(NVCV_TYPES_LIB nvcv_types ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/cvcuda/lib NO_DEFAULT_PATH)
-    list(APPEND FASTDEPLOY_LIBS ${CVCUDA_LIB} ${NVCV_TYPES_LIB})
-    list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/cvcuda/include NO_DEFAULT_PATH)
-    add_definitions(-DENABLE_CVCUDA)
-  endif()
-
-endif()
-
-if (ENABLE_TEXT)
-  if(ANDROID)
-    if(NOT ANDROID_TOOLCHAIN MATCHES "clang")
-      message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with FastTokenizer, but found ${ANDROID_TOOLCHAIN}.")
-    endif()
-    add_library(core_tokenizers STATIC IMPORTED GLOBAL)
-    set_property(TARGET core_tokenizers PROPERTY IMPORTED_LOCATION
-                ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/lib/${ANDROID_ABI}/libcore_tokenizers.so)
-    list(APPEND FASTDEPLOY_LIBS core_tokenizers)
-  else()
-    # Add dependency libs later: Linux/Mac/Win/...
-    find_library(FAST_TOKENIZER_LIB core_tokenizers ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/lib NO_DEFAULT_PATH)
-    list(APPEND FASTDEPLOY_LIBS ${FAST_TOKENIZER_LIB})
-  endif()
-  list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/include)
-  list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/third_party/include)
-endif()
-
-if(ENABLE_PADDLE2ONNX)
-  if(ANDROID)
-    message(FATAL_ERROR "Not support fastdeploy-paddle2onnx APIs with Android now!")
-  endif()
-  if(NOT BUILD_PADDLE2ONNX)
-    find_library(PADDLE2ONNX_LIB paddle2onnx  ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib NO_DEFAULT_PATH)
-    list(APPEND FASTDEPLOY_LIBS ${PADDLE2ONNX_LIB})
-  endif()
-endif()
-
-if(WITH_KUNLUNXIN)
-  list(APPEND FASTDEPLOY_LIBS -lpthread -lrt -ldl)
-endif()
-
-# log lib for Android
-if(ANDROID)
-  find_library(log-lib log)
-  list(APPEND FASTDEPLOY_LIBS ${log-lib})
-endif()
-
-# Update CXX LINKER's FLAGS, reference: https://zhuanlan.zhihu.com/p/595527528
-if(ANDROID AND (WITH_ANDROID_OPENCV_STATIC OR WITH_ANDROID_LITE_STATIC))
-  set(COMMON_LINK_FLAGS_REL "-Wl,-s,--gc-sections,-exclude-libs,ALL")
-  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${COMMON_LINK_FLAGS_REL} -Wl,-allow-multiple-definition" CACHE INTERNAL "" FORCE)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${COMMON_LINK_FLAGS_REL} -Wl,-allow-multiple-definition" CACHE INTERNAL "" FORCE)
-endif()
-
-remove_duplicate_libraries(FASTDEPLOY_LIBS)
-
-include(${CMAKE_CURRENT_LIST_DIR}/summary.cmake)
-fastdeploy_summary()
-message(STATUS "  DEPENDENCY_LIBS           : ${FASTDEPLOY_LIBS}")
-
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4.0")
-    string(STRIP "${CMAKE_CXX_COMPILER_VERSION}" CMAKE_CXX_COMPILER_VERSION)
-    message(FATAL_ERROR "[ERROR] FastDeploy require g++ version >= 5.4.0, but now your g++ version is ${CMAKE_CXX_COMPILER_VERSION}, this may cause failure! Use -DCMAKE_CXX_COMPILER to define path of your compiler.")
-  endif()
-endif()
-
-function(install_fastdeploy_libraries DESTINATION_DIR)
-  # No dynamic libs need to install while using
-  # FastDeploy static lib.
-  if(ANDROID AND WITH_ANDROID_STATIC_LIB)
-    return()
-  endif()
-  set(DYN_LIB_SUFFIX "*.so*")
-  if(WIN32)
-    set(DYN_LIB_SUFFIX "*.dll")
-  elseif(APPLE)
-    set(DYN_LIB_SUFFIX "*.dylib*")
-  endif()
-  if(FastDeploy_DIR)
-    set(DYN_SEARCH_DIR ${FastDeploy_DIR})
-  elseif(FASTDEPLOY_INSTALL_DIR)
-    set(DYN_SEARCH_DIR ${FASTDEPLOY_INSTALL_DIR})
-  else()
-    message(FATAL_ERROR "Please set FastDeploy_DIR/FASTDEPLOY_INSTALL_DIR before call install_fastdeploy_libraries.")
-  endif()
-  file(GLOB_RECURSE ALL_NEED_DYN_LIBS ${DYN_SEARCH_DIR}/lib/${DYN_LIB_SUFFIX})
-  file(GLOB_RECURSE ALL_DEPS_DYN_LIBS ${DYN_SEARCH_DIR}/third_libs/${DYN_LIB_SUFFIX})
-
-  if(ENABLE_VISION)
-    # OpenCV
-    if(ANDROID)
-      file(GLOB_RECURSE ALL_OPENCV_DYN_LIBS ${OpenCV_NATIVE_DIR}/libs/${DYN_LIB_SUFFIX})
-    else()
-      file(GLOB_RECURSE ALL_OPENCV_DYN_LIBS ${OpenCV_DIR}/${DYN_LIB_SUFFIX})
-    endif()
-    list(REMOVE_ITEM ALL_DEPS_DYN_LIBS ${ALL_OPENCV_DYN_LIBS})
-
-    if(WIN32)
-      file(GLOB OPENCV_DYN_LIBS ${OpenCV_DIR}/x64/vc15/bin/${DYN_LIB_SUFFIX})
-      file(INSTALL ${OPENCV_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-    elseif(ANDROID AND (NOT WITH_ANDROID_OPENCV_STATIC))
-      file(GLOB OPENCV_DYN_LIBS ${OpenCV_NATIVE_DIR}/libs/${ANDROID_ABI}/${DYN_LIB_SUFFIX})
-      file(INSTALL ${OPENCV_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-    else() # linux/mac
-      file(GLOB OPENCV_DYN_LIBS ${OpenCV_DIR}/lib/${DYN_LIB_SUFFIX})
-      file(INSTALL ${OPENCV_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-    endif()
-
-    # FlyCV
-    if(ENABLE_FLYCV)
-      file(GLOB_RECURSE ALL_FLYCV_DYN_LIBS ${FLYCV_LIB_DIR}/${DYN_LIB_SUFFIX})
-      list(REMOVE_ITEM ALL_DEPS_DYN_LIBS ${ALL_FLYCV_DYN_LIBS})
-      if(ANDROID AND (NOT WITH_ANDROID_FLYCV_STATIC))
-        file(INSTALL ${ALL_FLYCV_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-      endif()
-    endif()
-  endif()
-
-  if(ENABLE_OPENVINO_BACKEND)
-    # need plugins.xml for openvino backend
-    set(OPENVINO_RUNTIME_BIN_DIR ${OPENVINO_DIR}/bin)
-    file(GLOB OPENVINO_PLUGIN_XML ${OPENVINO_RUNTIME_BIN_DIR}/*.xml)
-    file(INSTALL ${OPENVINO_PLUGIN_XML} DESTINATION ${DESTINATION_DIR})
-  endif()
-
-  # Install other libraries
-  file(INSTALL ${ALL_NEED_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-  file(INSTALL ${ALL_DEPS_DYN_LIBS} DESTINATION ${DESTINATION_DIR})
-endfunction()

FastDeployCSharp.cmake.in

@@ -1,13 +0,0 @@
-list(APPEND FASTDEPLOY_DOTNET_REFERENCES
-    "Microsoft.CSharp"
-    "System"
-    "System.Core"
-    "System.Data"
-    "System.Deployment"
-    "System.Drawing"
-    "System.Net.Http"
-    "System.Xml"
-    "System.Reflection"
-    "${CMAKE_CURRENT_LIST_DIR}/csharp_lib/fastdeploy_csharp.dll")
-
-set(FASTDEPLOY_PACKAGE_REFERENCES "OpenCvSharp4_4.7.0.20230115;OpenCvSharp4.runtime.win_4.7.0.20230115")

LICENSE

@@ -1,3 +1,5 @@
+Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +188,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1 +0,0 @@
-README_EN.md

README.md

@@ -0,0 +1,91 @@
+English | [简体中文](README_CN.md)
+<p align="center">
+  <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><img src="https://github.com/user-attachments/assets/42b0039f-39e3-4279-afda-6d1865dfbffb" width="500"></a>
+</p>
+<p align="center">
+    <a href=""><img src="https://img.shields.io/badge/python-3.10-aff.svg"></a>
+    <a href=""><img src="https://img.shields.io/badge/os-linux-pink.svg"></a>
+    <a href="https://github.com/PaddlePaddle/FastDeploy/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/FastDeploy?color=9ea"></a>
+    <a href="https://github.com/PaddlePaddle/FastDeploy/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/FastDeploy?color=3af"></a>
+    <a href="https://github.com/PaddlePaddle/FastDeploy/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/FastDeploy?color=9cc"></a>
+    <a href="https://github.com/PaddlePaddle/FastDeploy/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/FastDeploy?color=ccf"></a>
+
+</p>
+
+<p align="center">
+     <a href="https://trendshift.io/repositories/4046" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4046" alt="PaddlePaddle%2FFastDeploy | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a></br>
+    <a href="https://paddlepaddle.github.io/FastDeploy/get_started/installation/nvidia_gpu/"><b> Installation </b></a>
+    |
+    <a href="https://paddlepaddle.github.io/FastDeploy/get_started/quick_start"><b> Quick Start </b></a>
+    |
+    <a href="https://paddlepaddle.github.io/FastDeploy/supported_models/"><b> Supported Models </b></a>
+
+</p>
+
+--------------------------------------------------------------------------------
+# FastDeploy : Inference and Deployment Toolkit for LLMs and VLMs based on PaddlePaddle
+
+## News
+**[2025-09] 🔥 FastDeploy v2.2 is newly released!** It now offers compatibility with models in the HuggingFace ecosystem, has further optimized performance, and newly adds support for [baidu/ERNIE-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking)!
+
+**[2025-08] 🔥 Released FastDeploy v2.1:** A brand-new KV Cache scheduling strategy has been introduced, and expanded support for PD separation and CUDA Graph across more models. Enhanced hardware support has been added for platforms like Kunlun and Hygon, along with comprehensive optimizations to improve the performance of both the service and inference engine.
+
+**[2025-07] The FastDeploy 2.0 Inference Deployment Challenge is now live!** Complete the inference deployment task for the ERNIE 4.5 series open-source models to win official FastDeploy 2.0 merch and generous prizes! 🎁 You're welcome to try it out and share your feedback! 📌[Sign up here](https://www.wjx.top/vm/meSsp3L.aspx#) 📌[Event details](https://github.com/PaddlePaddle/FastDeploy/discussions/2728)
+
+**[2025-06] 🔥 Released FastDeploy v2.0:** Supports inference and deployment for ERNIE 4.5. Furthermore, we open-source an industrial-grade PD disaggregation with context caching, dynamic role switching for effective resource utilization to further enhance inference performance for MoE models.
+
+## About
+
+**FastDeploy** is an inference and deployment toolkit for large language models and visual language models based on PaddlePaddle. It delivers **production-ready, out-of-the-box deployment solutions** with core acceleration technologies:
+
+- 🚀 **Load-Balanced PD Disaggregation**: Industrial-grade solution featuring context caching and dynamic instance role switching. Optimizes resource utilization while balancing SLO compliance and throughput.
+- 🔄 **Unified KV Cache Transmission**: Lightweight high-performance transport library with intelligent NVLink/RDMA selection.
+- 🤝 **OpenAI API Server and vLLM Compatible**: One-command deployment with [vLLM](https://github.com/vllm-project/vllm/) interface compatibility.
+- 🧮 **Comprehensive Quantization Format Support**: W8A16, W8A8, W4A16, W4A8, W2A16, FP8, and more.
+- ⏩ **Advanced Acceleration Techniques**: Speculative decoding, Multi-Token Prediction (MTP) and Chunked Prefill.
+- 🖥️ **Multi-Hardware Support**: NVIDIA GPU, Kunlunxin XPU, Hygon DCU, Ascend NPU, Iluvatar GPU, Enflame GCU, MetaX GPU etc.
+
+## Requirements
+
+- OS: Linux
+- Python: 3.10 ~ 3.12
+
+## Installation
+
+FastDeploy supports inference deployment on **NVIDIA GPUs**, **Kunlunxin XPUs**, **Iluvatar GPUs**, **Enflame GCUs**, **Hygon DCUs** and other hardware. For detailed installation instructions:
+
+- [NVIDIA GPU](./docs/get_started/installation/nvidia_gpu.md)
+- [Kunlunxin XPU](./docs/get_started/installation/kunlunxin_xpu.md)
+- [Iluvatar GPU](./docs/get_started/installation/iluvatar_gpu.md)
+- [Enflame GCU](./docs/get_started/installation/Enflame_gcu.md)
+- [Hygon DCU](./docs/get_started/installation/hygon_dcu.md)
+- [MetaX GPU](./docs/get_started/installation/metax_gpu.md.md)
+
+**Note:** We are actively working on expanding hardware support. Additional hardware platforms including Ascend NPU are currently under development and testing. Stay tuned for updates!
+
+## Get Started
+
+Learn how to use FastDeploy through our documentation:
+- [10-Minutes Quick Deployment](./docs/get_started/quick_start.md)
+- [ERNIE-4.5 Large Language Model Deployment](./docs/get_started/ernie-4.5.md)
+- [ERNIE-4.5-VL Multimodal Model Deployment](./docs/get_started/ernie-4.5-vl.md)
+- [Offline Inference Development](./docs/offline_inference.md)
+- [Online Service Deployment](./docs/online_serving/README.md)
+- [Best Practices](./docs/best_practices/README.md)
+
+## Supported Models
+
+Learn how to download models, enable using the torch format, and more:
+- [Full Supported Models List](./docs/supported_models.md)
+
+## Advanced Usage
+
+- [Quantization](./docs/quantization/README.md)
+- [PD Disaggregation Deployment](./docs/features/disaggregated.md)
+- [Speculative Decoding](./docs/features/speculative_decoding.md)
+- [Prefix Caching](./docs/features/prefix_caching.md)
+- [Chunked Prefill](./docs/features/chunked_prefill.md)
+
+## Acknowledgement
+
+FastDeploy is licensed under the [Apache-2.0 open-source license](./LICENSE). During development, portions of [vLLM](https://github.com/vllm-project/vllm) code were referenced and incorporated to maintain interface compatibility, for which we express our gratitude.

README_CN.md

@@ -1,416 +1,89 @@
-[English](README_EN.md) | 简体中文 | [हिन्दी](./docs/docs_i18n/README_हिन्दी.md) | [日本語](./docs/docs_i18n/README_日本語.md) | [한국인](./docs/docs_i18n/README_한국인.md) | [Pу́сский язы́к](./docs/docs_i18n/README_Pу́сский_язы́к.md)
-
-![⚡️FastDeploy](https://user-images.githubusercontent.com/31974251/185771818-5d4423cd-c94c-4a49-9894-bc7a8d1c29d0.png)
-
-</p>
-
+[English](README.md) | 简体中文
 <p align="center">
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/FastDeploy?color=ffa"></a>
-    <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
-    <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
+  <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><img src="https://github.com/user-attachments/assets/42b0039f-39e3-4279-afda-6d1865dfbffb" width="500"></a>
+</p>
+<p align="center">
+    <a href=""><img src="https://img.shields.io/badge/python-3.10-aff.svg"></a>
+    <a href=""><img src="https://img.shields.io/badge/os-linux-pink.svg"></a>
     <a href="https://github.com/PaddlePaddle/FastDeploy/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/FastDeploy?color=9ea"></a>
     <a href="https://github.com/PaddlePaddle/FastDeploy/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/FastDeploy?color=3af"></a>
     <a href="https://github.com/PaddlePaddle/FastDeploy/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/FastDeploy?color=9cc"></a>
     <a href="https://github.com/PaddlePaddle/FastDeploy/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/FastDeploy?color=ccf"></a>
+
 </p>
 
 <p align="center">
-    <a href="/docs/cn/build_and_install"><b> 安装 </b></a>
+     <a href="https://trendshift.io/repositories/4046" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4046" alt="PaddlePaddle%2FFastDeploy | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a></br>
+    <a href="https://paddlepaddle.github.io/FastDeploy/zh/get_started/installation/nvidia_gpu/"><b> 安装指导 </b></a>
     |
-    <a href="docs/README_CN.md"><b> 使用文档 </b></a>
+    <a href="https://paddlepaddle.github.io/FastDeploy/zh/get_started/quick_start"><b> 快速入门 </b></a>
     |
-    <a href="README_CN.md#fastdeploy-quick-start-python"><b> 快速开始 </b></a>
-    |
-    <a href="https://baidu-paddle.github.io/fastdeploy-api/"><b> API文档 </b></a>
-    |
-    <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><b> 更新日志 </b></a>
+    <a href="https://paddlepaddle.github.io/FastDeploy/zh/supported_models/"><b> 支持模型列表 </b></a>
+
 </p>
 
-<div align="center">
+--------------------------------------------------------------------------------
+# FastDeploy ：基于飞桨的大语言模型与视觉语言模型推理部署工具包
 
-[<img src='https://user-images.githubusercontent.com/54695910/200465949-da478e1b-21ce-43b8-9f3f-287460e786bd.png' height="80px" width="110px">](examples/vision/classification)
-[<img src='https://user-images.githubusercontent.com/54695910/188054680-2f8d1952-c120-4b67-88fc-7d2d7d2378b4.gif' height="80px" width="110px">](examples/vision/detection)
-[<img src='https://user-images.githubusercontent.com/54695910/188054711-6119f0e7-d741-43b1-b273-9493d103d49f.gif' height="80px" width="110px">](examples/vision/segmentation/paddleseg)
-[<img src='https://user-images.githubusercontent.com/54695910/188054718-6395321c-8937-4fa0-881c-5b20deb92aaa.gif' height="80px" width="110px">](examples/vision/segmentation/paddleseg)
-[<img src='https://user-images.githubusercontent.com/54695910/188058231-a5fe1ce1-0a38-460f-9582-e0b881514908.gif' height="80px" width="110px">](examples/vision/matting)
-[<img src='https://user-images.githubusercontent.com/54695910/188054691-e4cb1a70-09fe-4691-bc62-5552d50bd853.gif' height="80px" width="110px">](examples/vision/matting)
-[<img src='https://user-images.githubusercontent.com/54695910/188054669-a85996ba-f7f3-4646-ae1f-3b7e3e353e7d.gif' height="80px" width="110px">](examples/vision/ocr)<br>
-[<img src='https://user-images.githubusercontent.com/54695910/188059460-9845e717-c30a-4252-bd80-b7f6d4cf30cb.png' height="80px" width="110px">](examples/vision/facealign)
-[<img src='https://user-images.githubusercontent.com/54695910/188054671-394db8dd-537c-42b1-9d90-468d7ad1530e.gif' height="80px" width="110px">](examples/vision/keypointdetection)
-[<img src='https://user-images.githubusercontent.com/48054808/173034825-623e4f78-22a5-4f14-9b83-dc47aa868478.gif' height="80px" width="110px">](https://user-images.githubusercontent.com/54695910/200162475-f5d85d70-18fb-4930-8e7e-9ca065c1d618.gif)
-[<img src='https://user-images.githubusercontent.com/54695910/200162475-f5d85d70-18fb-4930-8e7e-9ca065c1d618.gif' height="80px" width="110px">](examples/text)
-[<img src='https://user-images.githubusercontent.com/54695910/212314909-77624bdd-1d12-4431-9cca-7a944ec705d3.png' height="80px" width="110px">](https://paddlespeech.bj.bcebos.com/Parakeet/docs/demos/parakeet_espnet_fs2_pwg_demo/tn_g2p/parakeet/001.wav)
-</div>
+## 最新活动
+**[2025-09] 🔥 FastDeploy v2.2 全新发布**: HuggingFace生态模型兼容，性能进一步优化，更新增对[baidu/ERNIE-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking)支持!
 
+**[2025-08] FastDeploy v2.1 发布**:全新的KV Cache调度策略，更多模型支持PD分离和CUDA Graph，昆仑、海光等更多硬件支持增强，全方面优化服务和推理引擎的性能。
 
- **⚡️FastDeploy**是一款**全场景**、**易用灵活**、**极致高效**的AI推理部署工具， 支持**云边端**部署。提供超过 🔥160+ **Text**，**Vision**， **Speech**和**跨模态**模型📦**开箱即用**的部署体验，并实现🔚**端到端**的推理性能优化。包括 [物体检测](./examples/vision/detection)、[字符识别（OCR）](./examples/vision/ocr)、[人脸](./examples/vision/facedet)、[人像扣图](./examples/vision/matting)、[多目标跟踪系统](./examples/vision/tracking/pptracking)、[NLP](./examples/text)、[Stable Diffusion文图生成](./examples/multimodal/stable_diffusion)、[TTS](./examples/audio/pp-tts) 等几十种任务场景，满足开发者**多场景、多硬件、多平台**的产业部署需求。
+**[2025-07] 《FastDeploy2.0推理部署实测》专题活动已上线!** 完成文心4.5系列开源模型的推理部署等任务，即可获得骨瓷马克杯等FastDeploy2.0官方周边及丰富奖金！🎁 欢迎大家体验反馈～ 📌[报名地址](https://www.wjx.top/vm/meSsp3L.aspx#)   📌[活动详情](https://github.com/PaddlePaddle/FastDeploy/discussions/2728)
 
-<div align="center">
+## 关于
 
-<img src="https://user-images.githubusercontent.com/54695910/213087733-7f2ea97b-baa4-4b0d-9b71-202ff6032a30.png" >
+**FastDeploy** 是基于飞桨（PaddlePaddle）的大语言模型（LLM）与视觉语言模型（VLM）推理部署工具包，提供**开箱即用的生产级部署方案**，核心技术特性包括：
 
-</div>
+- 🚀 **负载均衡式PD分解**：工业级解决方案，支持上下文缓存与动态实例角色切换，在保障SLO达标和吞吐量的同时优化资源利用率
+- 🔄 **统一KV缓存传输**：轻量级高性能传输库，支持智能NVLink/RDMA选择
+- 🤝 **OpenAI API服务与vLLM兼容**：单命令部署，兼容[vLLM](https://github.com/vllm-project/vllm/)接口
+- 🧮 **全量化格式支持**：W8A16、W8A8、W4A16、W4A8、W2A16、FP8等
+- ⏩ **高级加速技术**：推测解码、多令牌预测（MTP）及分块预填充
+- 🖥️ **多硬件支持**：NVIDIA GPU、昆仑芯XPU、海光DCU、昇腾NPU、天数智芯GPU、燧原GCU、沐曦GPU等
 
+## 要求
 
-## 🌠 近期更新
+- 操作系统: Linux
+- Python: 3.10 ~ 3.12
 
-- FastDeploy系列[**直播课程回放**](https://aistudio.baidu.com/aistudio/education/group/info/27800)
+## 安装
 
-- 服务化部署结合VisualDL新增支持可视化部署。在FastDeploy容器中启动VDL服务后，即可在VDL界面修改模型配置、启动/管理模型服务、查看性能数据、发送请求等，详细操作可参考相关文档
-    - [Serving可视化部署](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/zh_CN/vdl_management.md)
-    - [Serving可视化请求](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/zh_CN/client.md#%E4%BD%BF%E7%94%A8fastdeploy-client%E8%BF%9B%E8%A1%8C%E5%8F%AF%E8%A7%86%E5%8C%96%E8%AF%B7%E6%B1%82)
+FastDeploy 支持在**英伟达（NVIDIA）GPU**、**昆仑芯（Kunlunxin）XPU**、**天数（Iluvatar）GPU**、**燧原（Enflame）GCU**、**海光（Hygon）DCU** 以及其他硬件上进行推理部署。详细安装说明如下：
 
+- [英伟达 GPU](./docs/zh/get_started/installation/nvidia_gpu.md)
+- [昆仑芯 XPU](./docs/zh/get_started/installation/kunlunxin_xpu.md)
+- [天数 CoreX](./docs/zh/get_started/installation/iluvatar_gpu.md)
+- [燧原 S60](./docs/zh/get_started/installation/Enflame_gcu.md)
+- [海光 DCU](./docs/zh/get_started/installation/hygon_dcu.md)
+- [沐曦 GPU](./docs/zh/get_started/installation/metax_gpu.md.md)
 
-- **✨👥✨ 社区交流**
+**注意:** 我们正在积极拓展硬件支持范围。目前，包括昇腾（Ascend）NPU 等其他硬件平台正在开发测试中。敬请关注更新！
 
-    - **Slack**：Join our [Slack community](https://join.slack.com/t/fastdeployworkspace/shared_invite/zt-1o50e4voz-zbiIneCNRf_eH99eS2NVLg) and chat with other community members about ideas
+## 入门指南
 
-    - **微信**：扫描二维码，填写问卷加入技术社区，与社区开发者交流部署产业落地痛点问题
+通过我们的文档了解如何使用 FastDeploy：
+- [10分钟快速部署](./docs/zh/get_started/quick_start.md)
+- [ERNIE-4.5 部署](./docs/zh/get_started/ernie-4.5.md)
+- [ERNIE-4.5-VL 部署](./docs/zh/get_started/ernie-4.5-vl.md)
+- [离线推理](./docs/zh/offline_inference.md)
+- [在线服务](./docs/zh/online_serving/README.md)
+- [最佳实践](./docs/zh/best_practices/README.md)
 
-<div align="center">
-    <img src="https://user-images.githubusercontent.com/54695910/216615983-bbb78319-0231-4635-86d1-f2ebf9eac85d.jpg" width = "150" height = "150" />
-</div>
+## 支持模型列表
 
+通过我们的文档了解如何下载模型，如何支持torch格式等：
+- [模型支持列表](./docs/zh/supported_models.md)
 
-<div id="fastdeploy-acknowledge"></div>
+## 进阶用法
 
-## 🌌 推理后端及能力
+- [量化](./docs/zh/quantization/README.md)
+- [分离式部署](./docs/zh/features/disaggregated.md)
+- [投机解码](./docs/zh/features/speculative_decoding.md)
+- [前缀缓存](./docs/zh/features/prefix_caching.md)
+- [分块预填充](./docs/zh/features/chunked_prefill.md)
 
-<font size=0.5em>
+## 致谢
 
-|  | <img src="https://user-images.githubusercontent.com/54695910/212475832-f32502e2-4be2-42fc-a380-2ae265417938.png" height = "26" />  | <img src="https://user-images.githubusercontent.com/54695910/212475828-240036b0-f06c-4c44-830a-d8b136099b09.png" height = "27"  /> |<img src="https://user-images.githubusercontent.com/54695910/212475827-b73a1191-b3a8-4ad5-b6f6-855b3d1ffc09.png" height = "26" />| <img src="https://user-images.githubusercontent.com/54695910/212475826-f52b0ef3-e512-49fe-9b52-e1b9d1e8b6c2.png" height = "30" />  | <img src="https://user-images.githubusercontent.com/54695910/212475825-9686ae78-bad9-4be9-852e-6ad23be209da.png" height = "30" />  | <img src="https://user-images.githubusercontent.com/54695910/212475822-067349d2-8c4a-4431-bf02-05387e2962a8.png" height = "30" />  |<img src="https://user-images.githubusercontent.com/54695910/212475820-5210efe0-3e9a-429a-ad9d-48e8da2ffd0b.png" height = "30" /> |
-|:----------|:----------:|:----------:|:----------:|:----------:|:----------:|:----------:|:----------:|
-| X86_64&nbsp;CPU  |     |&nbsp;&nbsp;&nbsp;<img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>&nbsp;&nbsp;&nbsp;    |   <img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>  | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>   |   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>   |
-| NVDIA&nbsp;GPU    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212474106-a297aa0d-9225-458e-b5b7-e31aec7cfa79.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>  |     |    |
-|飞腾 CPU  |    |     |  <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>  | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>   |    |    |   |
-| 昆仑芯 XPU |    |    | <img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |     |   |    |
-| 华为昇腾 NPU |     |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>| <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |   |     |    |
-|Graphcore&nbsp;IPU   |    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    |    |  <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/>  |    |    |  |
-| 算能    |     |     |     | <img src="https://user-images.githubusercontent.com/54695910/212473382-e3e9063f-c298-4b61-ad35-a114aa6e6555.svg" height = "17"/>   |    |  |    |
-|Intel 显卡  |     |     |     | <img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/>   |    |   | |
-|Jetson  | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    |<img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474106-a297aa0d-9225-458e-b5b7-e31aec7cfa79.svg" height = "17"/>   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>  |     |    |
-|ARM&nbsp;CPU |    |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>| <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>   |    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>  |  <img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>  |
-|RK3588等 |   |    | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473387-2559cc2a-024b-4452-806c-6105d8eb2339.svg" height = "17"/>  |    |    |    |
-|RV1126等 |    |    | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |     |     |    |
-| 晶晨 |   |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>   |     |     |   |
-|  恩智浦 |   |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>   |     |    |    |
-
-</font>
-
-
-## 🔮 文档教程
-- [✴️ Python SDK快速开始](#fastdeploy-quick-start-python)  
-- [✴️ C++ SDK快速开始](#fastdeploy-quick-start-cpp)
-- **安装文档**
-    - [预编译库下载安装](docs/cn/build_and_install/download_prebuilt_libraries.md)
-    - [GPU部署环境编译安装](docs/cn/build_and_install/gpu.md)
-    - [CPU部署环境编译安装](docs/cn/build_and_install/cpu.md)
-    - [IPU部署环境编译安装](docs/cn/build_and_install/ipu.md)
-    - [昆仑芯XPU部署环境编译安装](docs/cn/build_and_install/kunlunxin.md)
-    - [瑞芯微RV1126部署环境编译安装](docs/cn/build_and_install/rv1126.md)
-    - [瑞芯微RK3588部署环境编译安装](docs/cn/build_and_install/rknpu2.md)
-    - [晶晨A311D部署环境编译安装](docs/cn/build_and_install/a311d.md)
-    - [华为昇腾部署环境编译安装](docs/cn/build_and_install/huawei_ascend.md)
-    - [Jetson部署环境编译安装](docs/cn/build_and_install/jetson.md)
-    - [Android平台部署环境编译安装](docs/cn/build_and_install/android.md)
-- **快速使用**
-    - [PP-YOLOE Python部署示例](docs/cn/quick_start/models/python.md)
-    - [PP-YOLOE C++部署示例](docs/cn/quick_start/models/cpp.md)
-- **不同后端使用**
-    - [Runtime Python使用示例](docs/cn/quick_start/runtime/python.md)
-    - [Runtime C++使用示例](docs/cn/quick_start/runtime/cpp.md)
-    - [如何配置模型部署的推理后端](docs/cn/faq/how_to_change_backend.md)
-- **服务化部署**
-    - [服务化部署镜像编译安装](serving/docs/zh_CN/compile.md)
-    - [服务化部署](serving)
-- **API文档**
-    - [Python API文档](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/python/html/)
-    - [C++ API文档](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/cpp/html/)
-    - [Android Java API文档](java/android)
-- **性能调优**
-    - [量化加速](docs/cn/quantize.md)
-    - [多线程多进程使用](/tutorials/multi_thread)
-- **常见问题**
-    - [1. Windows上C++ SDK如何使用](docs/cn/faq/use_sdk_on_windows.md)
-    - [2. Android上如何使用FastDeploy C++ SDK](docs/cn/faq/use_cpp_sdk_on_android.md)
-    - [3. TensorRT使用中的一些技巧](docs/cn/faq/tensorrt_tricks.md)
-- **更多FastDeploy部署模块**
-    - [Benchmark测试](benchmark)
-- **模型支持列表**
-    - [🖥️ 服务端 模型支持列表](#fastdeploy-server-models)
-    - [📳 移动端和端侧 模型支持列表](#fastdeploy-edge-models)
-    - [⚛️ Web和小程序 模型支持列表](#fastdeploy-web-models)
-- **💕开发者贡献**
-    - [增加新模型](docs/cn/faq/develop_a_new_model.md)
-
-
-
-<div id="fastdeploy-quick-start-python"></div>
-
-## 快速开始💨
-
-<details Open>
-
-<summary><b>Python SDK快速开始（点开收缩）</b></summary><div>
-
-### 🎆 快速安装
-
-#### 🔸 前置依赖
-
-- CUDA >= 11.2、cuDNN >= 8.0、Python >= 3.6
-- OS: Linux x86_64/macOS/Windows 10
-
-#### 🔸 安装GPU版本
-
-```bash
-pip install numpy opencv-python fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html
-```
-
-#### [🔸 Conda安装(推荐✨)](docs/cn/build_and_install/download_prebuilt_libraries.md)
-
-```bash
-conda config --add channels conda-forge && conda install cudatoolkit=11.2 cudnn=8.2
-```
-
-#### 🔸 安装CPU版本
-
-```bash
-pip install numpy opencv-python fastdeploy-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html
-```
-
-### 🎇 Python 推理示例
-
-* 准备模型和图片
-
-```bash
-wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
-tar xvf ppyoloe_crn_l_300e_coco.tgz
-wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
-```
-
-* 测试推理结果
-
-```python
-# GPU/TensorRT部署参考 examples/vision/detection/paddledetection/python
-import cv2
-import fastdeploy.vision as vision
-
-model = vision.detection.PPYOLOE("ppyoloe_crn_l_300e_coco/model.pdmodel",
-                                 "ppyoloe_crn_l_300e_coco/model.pdiparams",
-                                 "ppyoloe_crn_l_300e_coco/infer_cfg.yml")
-im = cv2.imread("000000014439.jpg")
-result = model.predict(im)
-print(result)
-
-vis_im = vision.vis_detection(im, result, score_threshold=0.5)
-cv2.imwrite("vis_image.jpg", vis_im)
-
-```
-
-</div></details>
-
-<div id="fastdeploy-quick-start-cpp"></div>
-
-<details close>
-
-<summary><b>C++ SDK快速开始（点开查看详情）</b></summary><div>
-
-
-### 🎆 安装
-
-- 参考[C++预编译库下载](docs/cn/build_and_install/download_prebuilt_libraries.md)文档  
-
-#### 🎇 C++ 推理示例
-
-* 准备模型和图片
-
-```bash
-wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
-tar xvf ppyoloe_crn_l_300e_coco.tgz
-wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
-```
-
-* 测试推理结果
-
-```C++
-// GPU/TensorRT部署参考 examples/vision/detection/paddledetection/cpp
-#include "fastdeploy/vision.h"
-
-int main(int argc, char* argv[]) {
-  namespace vision = fastdeploy::vision;
-  auto model = vision::detection::PPYOLOE("ppyoloe_crn_l_300e_coco/model.pdmodel",
-                                          "ppyoloe_crn_l_300e_coco/model.pdiparams",
-                                          "ppyoloe_crn_l_300e_coco/infer_cfg.yml");
-  auto im = cv::imread("000000014439.jpg");
-
-  vision::DetectionResult res;
-  model.Predict(im, &res);
-
-  auto vis_im = vision::VisDetection(im, res, 0.5);
-  cv::imwrite("vis_image.jpg", vis_im);
-  return 0;
-}
-```
-
-</div></details>
-
-更多部署案例请参考[模型部署示例](examples) .
-
-
-<div id="fastdeploy-server-models"></div>
-
-
-## ✴️ ✴️ 服务端模型支持列表 ✴️ ✴️
-
-符号说明: (1)  ✅ : 已经支持; (2) ❔: 正在进行中; (3) N/A : 暂不支持. <br>
-
-<details open><summary><b> 服务端模型支持列表（点击可收缩）</b></summary><div>
-
-<div align="center">
-  <img src="https://user-images.githubusercontent.com/115439700/212800663-894e9a7a-6d68-4b0b-bcd2-045732d08887.png" height ="40"/>
-</div>
-
-| 任务场景                   | 模型                                                                                           | Linux                                            | Linux      | Win     | Win        | Mac     | Mac     | Linux       | Linux           | Linux         | Linux         | Linux   | Linux   | Linux   |
-|:----------------------:|:--------------------------------------------------------------------------------------------:|:------------------------------------------------:|:----------:|:-------:|:----------:|:-------:|:-------:|:-----------:|:---------------:|:-------------:|:-------------:|:-------:|:-------:|:-------:|
-| ---                    | ---                                                                                          | X86 CPU                                          | NVIDIA GPU | X86 CPU | NVIDIA GPU | X86 CPU | Arm CPU | AArch64 CPU | 飞腾D2000 aarch64 | [NVIDIA Jetson](./docs/cn/build_and_install/jetson.md) | [Graphcore IPU](./docs/cn/build_and_install/ipu.md) | [昆仑芯 XPU](./docs/cn/build_and_install/kunlunxin.md) |[华为 昇腾](./docs/cn/build_and_install/huawei_ascend.md) |  [Serving](./serving) |
-| Classification         | [PaddleClas/ResNet50](./examples/vision/classification/paddleclas)                           | [✅](./examples/vision/classification/paddleclas) | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       | ✅       |✅       |
-| Classification         | [TorchVison/ResNet](examples/vision/classification/resnet)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Classification         | [ultralytics/YOLOv5Cls](examples/vision/classification/yolov5cls)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ❔       |❔       |
-| Classification         | [PaddleClas/PP-LCNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           |  ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/PP-LCNetv2](./examples/vision/classification/paddleclas)                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/EfficientNet](./examples/vision/classification/paddleclas)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/GhostNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV1](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV2](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV3](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/ShuffleNetV2](./examples/vision/classification/paddleclas)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/SqueeezeNetV1.1](./examples/vision/classification/paddleclas)                    | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/Inceptionv3](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/PP-HGNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Detection              | 🔥🔥[PaddleDetection/PP-YOLOE+](./examples/vision/detection/paddledetection)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ✅       |
-| Detection              | [🔥PaddleDetection/YOLOv8](./examples/vision/detection/paddledetection)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔      |
-| Detection              | [🔥ultralytics/YOLOv8](./examples/vision/detection/yolov8)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔               | ✅                           | ❔                           | ❔      |❔       | ❔      |
-| Detection              | [PaddleDetection/PicoDet](./examples/vision/detection/paddledetection)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ❔       | ✅       |
-| Detection              | [PaddleDetection/YOLOX](./examples/vision/detection/paddledetection)                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅               | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/YOLOv3](./examples/vision/detection/paddledetection)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/PP-YOLO](./examples/vision/detection/paddledetection)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/PP-YOLOv2](./examples/vision/detection/paddledetection)                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/Faster-RCNN](./examples/vision/detection/paddledetection)                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |❔        | ✅       |
-| Detection              | [PaddleDetection/Mask-RCNN](./examples/vision/detection/paddledetection)                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |❔        | ✅       |
-| Detection              | [Megvii-BaseDetection/YOLOX](./examples/vision/detection/yolox)                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7](./examples/vision/detection/yolov7)                                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7end2end_trt](./examples/vision/detection/yolov7end2end_trt)                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7end2end_ort](./examples/vision/detection/yolov7end2end_ort)               | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [meituan/YOLOv6](./examples/vision/detection/yolov6)                                         | ✅                                                | ✅                        | ✅                        |✅       |  ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅                          | ❔       |
-| Detection              | [ultralytics/YOLOv5](./examples/vision/detection/yolov5)                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       |✅       |
-| Detection              | [WongKinYiu/YOLOR](./examples/vision/detection/yolor)                                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔      | ✅       | ❔       |
-| Detection              | [WongKinYiu/ScaledYOLOv4](./examples/vision/detection/scaledyolov4)                          | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [ppogg/YOLOv5Lite](./examples/vision/detection/yolov5lite)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           |  ?       | ❔       |❔       |❔       |
-| Detection              | [RangiLyu/NanoDetPlus](./examples/vision/detection/nanodet_plus)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Perception              | [Paddle3D/Smoke](./examples/vision/perception/paddle3d/smoke)                             | ❔                                                 | ✅                         | ❔                         | ✅                         | ❔                       | ❔                      | ❔                           | ❔                 | ❔                            | ❔                           | ❔       |❔       | ✅       |
-| KeyPoint               | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose)                    | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅        |❔       | ❔       |
-| KeyPoint               | [PaddleDetection/PicoDet + TinyPose](./examples/vision/keypointdetection/det_keypoint_unite) | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅        | ❔       |❔       |
-| HeadPose               | [omasaht/headpose](examples/vision/headpose)                                                 | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔       | ❔       |❔       |
-| Tracking               | [PaddleDetection/PP-Tracking](examples/vision/tracking/pptracking)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       | ❔       |❔       |
-| OCR                    | [PaddleOCR/PP-OCRv2](./examples/vision/ocr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅              |✅       | ❔       |
-| OCR                    | [PaddleOCR/PP-OCRv3](./examples/vision/ocr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅      |✅        | ✅       |
-| Segmentation           | [PaddleSeg/PP-LiteSeg](./examples/vision/segmentation/paddleseg)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                 |❔         | ❔       |
-| Segmentation           | [PaddleSeg/PP-HumanSegLite](./examples/vision/segmentation/paddleseg)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                     |✅        | ❔       |
-| Segmentation           | [PaddleSeg/HRNet](./examples/vision/segmentation/paddleseg)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                | ✅        |❔       |
-| Segmentation           | [PaddleSeg/PP-HumanSegServer](./examples/vision/segmentation/paddleseg)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅        | ✅        |❔       |
-| Segmentation           | [PaddleSeg/Unet](./examples/vision/segmentation/paddleseg)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ✅               | ✅         | ✅        |❔       |
-| Segmentation           | [PaddleSeg/Deeplabv3](./examples/vision/segmentation/paddleseg)                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                     | ✅               | ✅        |❔       |
-| FaceDetection          | [biubug6/RetinaFace](./examples/vision/facedet/retinaface)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                  | ❔       |  ❔       | ❔       |
-| FaceDetection          | [Linzaer/UltraFace](./examples/vision/facedet/ultraface)                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceDetection          | [deepcam-cn/YOLOv5Face](./examples/vision/facedet/yolov5face)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceDetection          | [insightface/SCRFD](./examples/vision/facedet/scrfd)                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceAlign              | [Hsintao/PFLD](examples/vision/facealign/pfld)                                               | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    |  ❔       | ❔       |❔       |
-| FaceAlign              | [Single430/FaceLandmark1000](./examples/vision/facealign/face_landmark_1000)                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| FaceAlign              | [jhb86253817/PIPNet](./examples/vision/facealign)                                            | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| FaceRecognition        | [insightface/ArcFace](./examples/vision/faceid/insightface)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceRecognition        | [insightface/CosFace](./examples/vision/faceid/insightface)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| FaceRecognition        | [insightface/PartialFC](./examples/vision/faceid/insightface)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| FaceRecognition        | [insightface/VPL](./examples/vision/faceid/insightface)                                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| Matting                | [ZHKKKe/MODNet](./examples/vision/matting/modnet)                                            | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    |  ❔       | ❔       |❔       |
-| Matting                | [PeterL1n/RobustVideoMatting]()                                                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |  ❔       | ❔       |
-| Matting                | [PaddleSeg/PP-Matting](./examples/vision/matting/ppmatting)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ✅           |✅           | ❔       |
-| Matting                | [PaddleSeg/PP-HumanMatting](./examples/vision/matting/modnet)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                   | ✅       |✅           |❔       |
-| Matting                | [PaddleSeg/ModNet](./examples/vision/matting/modnet)                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔        |❔       |   ❔       |
-| Video Super-Resolution | [PaddleGAN/BasicVSR](./)                                                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| Video Super-Resolution | [PaddleGAN/EDVR](./examples/vision/sr/edvr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |❔       | ❔       |
-| Video Super-Resolution | [PaddleGAN/PP-MSVSR](./examples/vision/sr/ppmsvsr)                                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |❔       | ❔       |
-| Information Extraction | [PaddleNLP/UIE](./examples/text/uie)                                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔         |❔       |         |
-| NLP                    | [PaddleNLP/ERNIE-3.0](./examples/text/ernie-3.0)                                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ❔                           | ❔                    | ✅       |❔       | ✅       |
-| Speech                 | [PaddleSpeech/PP-TTS](./examples/audio/pp-tts)                                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ❔                           | --                          |❔       |❔       | ✅       |
-
-</div></details>
-
-<div id="fastdeploy-edge-models"></div>
-
-## 📳 移动端和端侧 模型支持列表
-
-<details open><summary><b> 端侧模型支持列表（点击可收缩）</b></summary><div>
-
-<div align="center">
-  <img src="https://user-images.githubusercontent.com/115439700/212800663-894e9a7a-6d68-4b0b-bcd2-045732d08887.png" height ="40"/>
-</div>
-
-| 任务场景               | 模型                                                                                        | 大小(MB)   | Linux   | Android | Linux     | Linux                   | Linux                          | Linux                       | Linux                            | 更新中...  |
-|:------------------:|:-----------------------------------------------------------------------------------------:|:--------:|:-------:|:-------:|:-------:|:-----------------------:|:------------------------------:|:---------------------------:|:--------------------------------:|:-------:|
-| ---                | ---                                                                                       | ---      | ARM CPU | [ARM CPU](./java/android) | [瑞芯微NPU<br>RK3588/RK3568/RK3566](./docs/cn/build_and_install/rknpu2.md) | [瑞芯微NPU<br>RV1109/RV1126/RK1808](./docs/cn/build_and_install/rv1126.md) | [晶晨NPU <br>A311D/S905D/C308X](./docs/cn/build_and_install/a311d.md) | 恩智浦NPU<br>i.MX&nbsp;8M&nbsp;Plus | 更新中... |
-| Classification     | [PaddleClas/ResNet50](examples/vision/classification/paddleclas)                         | 98        | ✅       | ✅       |  [✅](./examples/vision/classification/paddleclas/rknpu2)                             |      ✅                                |                                   |                                   |         |
-| Classification     | [PaddleClas/PP-LCNet](examples/vision/classification/paddleclas)                         | 11.9      | ✅       | ✅       | ❔                             | ✅                                 | --                                | --                                | --      |
-| Classification     | [PaddleClas/PP-LCNetv2](examples/vision/classification/paddleclas)                       | 26.6      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/EfficientNet](examples/vision/classification/paddleclas)                     | 31.4      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/GhostNet](examples/vision/classification/paddleclas)                         | 20.8      | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV1](examples/vision/classification/paddleclas)                      | 17        | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV2](examples/vision/classification/paddleclas)                      | 14.2      | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV3](examples/vision/classification/paddleclas)                      | 22        | ✅       | ✅       | ❔                             | ✅                                    | ❔                                 | ❔                                 | --      |
-| Classification     | [PaddleClas/ShuffleNetV2](examples/vision/classification/paddleclas)                     | 9.2       | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/SqueezeNetV1.1](examples/vision/classification/paddleclas)                   | 5         | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/Inceptionv3](examples/vision/classification/paddleclas)                      | 95.5      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/PP-HGNet](examples/vision/classification/paddleclas)                         | 59        | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Detection          | [PaddleDetection/PicoDet_s](examples/vision/detection/paddledetection) | 4.9       | ✅       | ✅       | [✅](./examples/vision/detection/paddledetection/rknpu2)                             | ✅                                    | ✅                                 | ✅                                 | --      |
-| Detection          | [YOLOv5](./examples/vision/detection/rkyolo) |        |  ❔     |  ❔      | [✅](./examples/vision/detection/rkyolo)                             | ❔                                   | ❔                                 | ❔                                 | --      |
-| Face Detection     | [deepinsight/SCRFD](./examples/vision/facedet/scrfd)                                     | 2.5       | ✅       | ✅       | [✅](./examples/vision/facedet/scrfd/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Keypoint Detection | [PaddleDetection/PP-TinyPose](examples/vision/keypointdetection/tiny_pose)               | 5.5       | ✅       | ✅       | ❔                             | ❔                                    | ❔                                 | ❔                                 | --      |
-| Segmentation       | [PaddleSeg/PP-LiteSeg(STDC1)](examples/vision/segmentation/paddleseg)                    | 32.2      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/PP-HumanSeg-Lite](examples/vision/segmentation/paddleseg)                     | 0.556     | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/HRNet-w18](examples/vision/segmentation/paddleseg)                            | 38.7      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/PP-HumanSeg](examples/vision/segmentation/paddleseg)                          | 107.2     | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/Unet](examples/vision/segmentation/paddleseg)                                 | 53.7      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/Deeplabv3](examples/vision/segmentation/paddleseg)                            | 150       | ❔       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             |                                      |                                   |                                   |         |
-| OCR                | [PaddleOCR/PP-OCRv2](examples/vision/ocr/PP-OCRv2)                                       | 2.3+4.4   | ✅       | ✅       | ❔                             | --                                   | --                                | --                                | --      |
-| OCR                | [PaddleOCR/PP-OCRv3](examples/vision/ocr/PP-OCRv3)                                       | 2.4+10.6  | ✅       | ❔       | ❔                             | ❔                                    | ❔                                 | ❔                                 | --      |
-
-</div></details>
-
-## ⚛️ Web和小程序 模型支持列表
-
-<div id="fastdeploy-web-models"></div>
-
-<details open><summary><b> Web和小程序部署支持列表（点击可收缩）</b></summary><div>
-
-| 任务场景               | 模型                                                                                          | [web_demo](examples/application/js/web_demo) |
-|:------------------:|:-------------------------------------------------------------------------------------------:|:--------------------------------------------:|
-| ---                | ---                                                                                         | [Paddle.js](examples/application/js)         |
-| Detection          | [FaceDetection](examples/application/js/web_demo/src/pages/cv/detection)                    | ✅                                            |
-| Detection          | [ScrewDetection](examples/application/js/web_demo/src/pages/cv/detection)                   | ✅                                            |
-| Segmentation       | [PaddleSeg/HumanSeg](./examples/application/js/web_demo/src/pages/cv/segmentation/HumanSeg) | ✅                                            |
-| Object Recognition | [GestureRecognition](examples/application/js/web_demo/src/pages/cv/recognition)             | ✅                                            |
-| Object Recognition | [ItemIdentification](examples/application/js/web_demo/src/pages/cv/recognition)             | ✅                                            |
-| OCR                | [PaddleOCR/PP-OCRv3](./examples/application/js/web_demo/src/pages/cv/ocr)                   | ✅                                            |
-
-</div></details>
-
-
-## 💐 Acknowledge
-
-本项目中SDK生成和下载使用了[EasyEdge](https://ai.baidu.com/easyedge/app/openSource)中的免费开放能力，在此表示感谢。
-
-## ©️ License
-
-<div id="fastdeploy-license"></div>
-
-FastDeploy遵循[Apache-2.0开源协议](./LICENSE)。
+FastDeploy 依据 [Apache-2.0 开源许可证](./LICENSE). 进行授权。在开发过程中，我们参考并借鉴了 [vLLM](https://github.com/vllm-project/vllm) 的部分代码，以保持接口兼容性，在此表示衷心感谢。

README_EN.md

@@ -1,406 +0,0 @@
-English | [简体中文](README_CN.md) | [हिन्दी](./docs/docs_i18n/README_हिन्दी.md) | [日本語](./docs/docs_i18n/README_日本語.md) | [한국인](./docs/docs_i18n/README_한국인.md) | [Pу́сский язы́к](./docs/docs_i18n/README_Pу́сский_язы́к.md)
-
-
-![⚡️FastDeploy](https://user-images.githubusercontent.com/31974251/185771818-5d4423cd-c94c-4a49-9894-bc7a8d1c29d0.png)
-
-</p>
-
-<p align="center">
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/FastDeploy?color=ffa"></a>
-    <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
-    <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/FastDeploy?color=9ea"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/FastDeploy?color=3af"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/FastDeploy?color=9cc"></a>
-    <a href="https://github.com/PaddlePaddle/FastDeploy/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/FastDeploy?color=ccf"></a>
-</p>
-<p align="center">
-    <a href="/docs/en/build_and_install"><b> Installation </b></a>
-    |
-    <a href="docs/README_EN.md"><b> Documents </b></a>
-    | <a href="/README_EN.md#Quick-Start"><b> Quick Start </b></a> |
-    <a href="https://baidu-paddle.github.io/fastdeploy-api/"><b> API Docs </b></a>
-    |
-    <a href="https://github.com/PaddlePaddle/FastDeploy/releases"><b> Release Notes </b></a>
-</p>
-<div align="center">
-
-[<img src='https://user-images.githubusercontent.com/54695910/200465949-da478e1b-21ce-43b8-9f3f-287460e786bd.png' height="80px" width="110px">](examples/vision/classification)
-[<img src='https://user-images.githubusercontent.com/54695910/188054680-2f8d1952-c120-4b67-88fc-7d2d7d2378b4.gif' height="80px" width="110px">](examples/vision/detection)
-[<img src='https://user-images.githubusercontent.com/54695910/188054711-6119f0e7-d741-43b1-b273-9493d103d49f.gif' height="80px" width="110px">](examples/vision/segmentation/paddleseg)
-[<img src='https://user-images.githubusercontent.com/54695910/188054718-6395321c-8937-4fa0-881c-5b20deb92aaa.gif' height="80px" width="110px">](examples/vision/segmentation/paddleseg)
-[<img src='https://user-images.githubusercontent.com/54695910/188058231-a5fe1ce1-0a38-460f-9582-e0b881514908.gif' height="80px" width="110px">](examples/vision/matting)
-[<img src='https://user-images.githubusercontent.com/54695910/188054691-e4cb1a70-09fe-4691-bc62-5552d50bd853.gif' height="80px" width="110px">](examples/vision/matting)
-[<img src='https://user-images.githubusercontent.com/54695910/188054669-a85996ba-f7f3-4646-ae1f-3b7e3e353e7d.gif' height="80px" width="110px">](examples/vision/ocr)<br>
-[<img src='https://user-images.githubusercontent.com/54695910/188059460-9845e717-c30a-4252-bd80-b7f6d4cf30cb.png' height="80px" width="110px">](examples/vision/facealign)
-[<img src='https://user-images.githubusercontent.com/54695910/188054671-394db8dd-537c-42b1-9d90-468d7ad1530e.gif' height="80px" width="110px">](examples/vision/keypointdetection)
-[<img src='https://user-images.githubusercontent.com/48054808/173034825-623e4f78-22a5-4f14-9b83-dc47aa868478.gif' height="80px" width="110px">](https://user-images.githubusercontent.com/54695910/200162475-f5d85d70-18fb-4930-8e7e-9ca065c1d618.gif)
-[<img src='https://user-images.githubusercontent.com/54695910/200162475-f5d85d70-18fb-4930-8e7e-9ca065c1d618.gif' height="80px" width="110px">](examples/text)
-[<img src='https://user-images.githubusercontent.com/54695910/212314909-77624bdd-1d12-4431-9cca-7a944ec705d3.png' height="80px" width="110px">](https://paddlespeech.bj.bcebos.com/Parakeet/docs/demos/parakeet_espnet_fs2_pwg_demo/tn_g2p/parakeet/001.wav)
-</div>
-
-**⚡️FastDeploy** is an **Easy-to-use** and **High Performance** AI model deployment toolkit for Cloud, Mobile and Edge with 📦**out-of-the-box and unified experience**, 🔚**end-to-end optimization** for over **🔥160+ Text, Vision, Speech and Cross-modal AI models**.
-Including [image classification](examples/vision/classification), [object detection](examples/vision/detection), [OCR](./examples/vision/ocr), [face detection](./examples/vision/facedet), [matting](./examples/vision/matting), [pp-tracking](./examples/vision/tracking/pptracking), [NLP](./examples/text), [stable diffusion](./examples/multimodal/stable_diffusion), [TTS](./examples/audio/pp-tts) and other tasks to meet developers' industrial deployment needs for **multi-scenario**, **multi-hardware** and **multi-platform**.
-
-<div align="center">
-
-<img src="https://user-images.githubusercontent.com/54695910/213087724-7175953a-0e07-4af8-a4a1-5304163da2e0.png" >
-
-</div>
-
-
-
-##  🌠 Recent updates
-- ✨✨✨ In **2023.01.17** we released [**YOLOv8**](./examples/vision/detection/paddledetection/) for deployment on FastDeploy series hardware, which includes [**Paddle YOLOv8**](https://github.com/PaddlePaddle/PaddleYOLO/tree/release/2.5/configs/yolov8) and [**ultralytics YOLOv8**](https://github.com/ultralytics/ultralytics)
-
-    - You can deploy [**Paddle YOLOv8**](https://github.com/PaddlePaddle/PaddleYOLO/tree/release/2.5/configs/yolov8) on [**Intel CPU**](./examples/vision/detection/paddledetection/python/infer_yolov8.py), [**NVIDIA GPU**](./examples/vision/detection/paddledetection/python/infer_yolov8.py), [**Jetson**](./examples/vision/detection/paddledetection/python/infer_yolov8.py), [**Phytium**](./examples/vision/detection/paddledetection/python/infer_yolov8.py), [**Kunlunxin**](./examples/vision/detection/paddledetection/python/infer_yolov8.py), [**HUAWEI Ascend**](./examples/vision/detection/paddledetection/python/infer_yolov8.py) ,[**ARM CPU**](./examples/vision/detection/paddledetection/cpp/infer_yolov8.cc)  [**RK3588**](./examples/vision/detection/paddledetection/rknpu2) and [**Sophgo TPU**](./examples/vision/detection/paddledetection/sophgo). Both **Python** deployments and **C++** deployments are included.
-    - You can deploy [**ultralytics YOLOv8**](https://github.com/ultralytics/ultralytics) on [**Intel CPU**](./examples/vision/detection/yolov8), [**NVIDIA GPU**](./examples/vision/detection/yolov8), [**Jetson**](./examples/vision/detection/yolov8). Both **Python** deployments and **C++** deployments are included
-    -  Fastdeploy supports quick deployment of multiple models, including **YOLOv8**, **PP-YOLOE+**, **YOLOv5** and other models
--  Serving deployment combined with VisualDL supports visual deployment. After the VDL service is started in the FastDeploy container, you can modify the model configuration, start/manage the model service, view performance data, and send requests on the VDL interface. For details, see related documents
-    - [Serving deployment visualization](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/EN/vdl_management-en.md)
-    - [Serving request visualization](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/EN/client-en.md#use-visualdl-as-fastdeploy-client-for-request-visualization)
-
-- **✨👥✨ Community**
-
-  - **Slack**：Join our [Slack community](https://join.slack.com/t/fastdeployworkspace/shared_invite/zt-1m88mytoi-mBdMYcnTF~9LCKSOKXd6Tg) and chat with other community members about ideas
-  - **Wechat**：Scan the QR code below using WeChat, follow the PaddlePaddle official account and fill out the questionnaire to join the WeChat group, and share the deployment industry implementation pain points with the community developers
-
-<div align="center">
-<img src="https://user-images.githubusercontent.com/54695910/216615983-bbb78319-0231-4635-86d1-f2ebf9eac85d.jpg"  width = "150" height = "150" />
-</div>
-
-## 🌌 Inference Backend and Abilities
-
-<font size=0.5em>
-
-|  | <img src="https://user-images.githubusercontent.com/54695910/213093175-052c3e47-75dc-4be8-9be9-6565532efa1c.png" width = "60" height = "40" />  | <img src="https://user-images.githubusercontent.com/54695910/213093173-27847120-bbb0-47b0-947f-8cf87142ed52.png" width = "75" height = "50"  /> |<img src="https://user-images.githubusercontent.com/54695910/213096791-8b47c875-6c89-4e1d-8c67-e226636844e1.png" width = "85" height = "60" />| <img src="https://user-images.githubusercontent.com/54695910/212475826-f52b0ef3-e512-49fe-9b52-e1b9d1e8b6c2.png" height = "30" />  | <img src="https://user-images.githubusercontent.com/54695910/212475825-9686ae78-bad9-4be9-852e-6ad23be209da.png" height = "30" />  | <img src="https://user-images.githubusercontent.com/54695910/212475822-067349d2-8c4a-4431-bf02-05387e2962a8.png" height = "30" />  |<img src="https://user-images.githubusercontent.com/54695910/212475820-5210efe0-3e9a-429a-ad9d-48e8da2ffd0b.png" height = "30" /> |
-|:----------|:----------:|:----------:|:----------:|:----------:|:----------:|:----------:|:----------:|
-| X86_64&nbsp;CPU  |     |&nbsp;&nbsp;&nbsp;<img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>&nbsp;&nbsp;&nbsp;    |   <img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>  | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>   |   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>   |
-| NVDIA&nbsp;GPU    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212474106-a297aa0d-9225-458e-b5b7-e31aec7cfa79.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>  |     |    |
-|Phytium CPU  |    |     |  <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>  | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>   |    |    |   |
-| KunlunXin XPU |    |    | <img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |     |   |    |
-| Huawei Ascend NPU |     |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>| <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |   |     |    |
-|Graphcore&nbsp;IPU   |    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    |    |  <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/>  |    |    |  |
-| Sophgo    |     |     |     | <img src="https://user-images.githubusercontent.com/54695910/212473382-e3e9063f-c298-4b61-ad35-a114aa6e6555.svg" height = "17"/>   |    |  |    |
-|Intel graphics card  |     |     |     | <img src="https://user-images.githubusercontent.com/54695910/212473392-9df374d4-5daa-4e2b-856b-6e50ff1e4282.svg" height = "17"/>   |    |   | |
-| Jetson    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    | <img src="https://user-images.githubusercontent.com/54695910/212545467-e64ee45d-bf12-492c-b263-b860cb1e172b.png" height = "25"/>    |<img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474106-a297aa0d-9225-458e-b5b7-e31aec7cfa79.svg" height = "17"/>   | <img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473391-92c9f289-a81a-4927-9f31-1ab3fa3c2971.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473556-d2ebb7cc-e72b-4b49-896b-83f95ae1fe63.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473190-fdf3cee2-5670-47b5-85e7-6853a8dd200a.svg" height = "17"/>  |     |    |
-|ARM&nbsp;CPU |    |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212474104-d82f3545-04d4-4ddd-b240-ffac34d8a920.svg" height = "17"/>| <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/><br><img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>   |    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>  |  <img src="https://user-images.githubusercontent.com/54695910/212473393-ae1958bd-ab7d-4863-94b9-32863e600ba1.svg" height = "17"/>  |
-|RK3588 etc. |   |    | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473387-2559cc2a-024b-4452-806c-6105d8eb2339.svg" height = "17"/>  |    |    |    |
-|RV1126 etc. |    |    | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>    |     |     |    |
-| Amlogic |   |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    | <img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>   |     |     |   |
-|  NXP |   |     | <img src="https://user-images.githubusercontent.com/54695910/212474105-38051192-9a1c-4b24-8ad1-f842fb0bf39d.svg" height = "17"/>    |<img src="https://user-images.githubusercontent.com/54695910/212473389-8c341bbe-30d4-4a28-b50a-074be4e98ce6.svg" height = "17"/>   |     |    |    |
-</font>
-
-## 🔮 Contents
--  [✴️ A Quick Start for Python SDK](#fastdeploy-quick-start-python)  
--  [✴️ A Quick Start for C++ SDK](#fastdeploy-quick-start-cpp)
-- **Installation**
-    - [How to Install Prebuilt Library](docs/en/build_and_install/download_prebuilt_libraries.md)
-    - [How to Build GPU Deployment Environment](docs/en/build_and_install/gpu.md)
-    - [How to Build CPU Deployment Environment](docs/en/build_and_install/cpu.md)
-    - [How to Build IPU Deployment Environment](docs/en/build_and_install/ipu.md)
-    - [How to Build KunlunXin XPU Deployment Environment](docs/en/build_and_install/kunlunxin.md)
-    - [How to Build RV1126 Deployment Environment](docs/en/build_and_install/rv1126.md)
-    - [How to Build RKNPU2 Deployment Environment](docs/en/build_and_install/rknpu2.md)
-    - [How to Build A311D Deployment Environment](docs/en/build_and_install/a311d.md)
-    - [How to build Huawei Ascend Deployment Environment](docs/en/build_and_install/huawei_ascend.md)
-    - [How to Build FastDeploy Library on Nvidia Jetson Platform](docs/en/build_and_install/jetson.md)
-    - [How to Build FastDeploy Android C++ SDK](docs/en/build_and_install/android.md)
-- **Quick Start**
-    - [PP-YOLOE Python Deployment Example](docs/en/quick_start/models/python.md)
-    - [PP-YOLOE C++ Deployment Example](docs/en/quick_start/models/cpp.md)
-- **Demos on Different Backends**
-    - [Runtime Python Inference](docs/en/quick_start/runtime/python.md)
-    - [Runtime C++ Inference](docs/en/quick_start/runtime/cpp.md)
-    - [How to Change Model Inference Backend](docs/en/faq/how_to_change_backend.md)
-- **Serving Deployment**
-    - [FastDeploy Serving Deployment Image Compilation](serving/docs/EN/compile-en.md)
-    - [Serving Deployment](serving)
-- **API Documents**
-    - [Python API](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/python/html/)
-    - [C++ API](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/cpp/html/)
-    - [Android Java API](java/android)
-- **Performance Tune-up**
-    - [Quantization Acceleration](docs/en/quantize.md)
-    - [Multi thread](/tutorials/multi_thread)
-- **FAQ**
-    - [1. Using the FastDeploy C++ SDK on Windows Platform](docs/en/faq/use_sdk_on_windows.md)
-    - [2. FastDeploy to deploy on Android Platform](docs/en/faq/use_cpp_sdk_on_android.md)
-    - [3. TensorRT Q&As](docs/en/faq/tensorrt_tricks.md)
-- **More FastDeploy Deploy Modules**
-    - [Benchmark Testing](benchmark)
-- **Model list**
-  - [🖥️ Supported Server-side and Cloud Model List](#fastdeploy-server-models)
-  - [📳 Supported Mobile and Edge Model List](#fastdeploy-edge-models)
-  - [⚛️ Supported Web and Mini Program Model List](#fastdeploy-web-models)
-- **💕 Developer Contributions**
-    - [Develop a new model](docs/en/faq/develop_a_new_model.md)
-
-
-## Quick Start💨
-
-<div id="fastdeploy-quick-start-python"></div>
-
-<details Open>
-<summary><b>A Quick Start for Python SDK(click to fold)</b></summary><div>
-
-#### 🎆 Installation
-
-##### 🔸 Prerequisites
-
-- CUDA >= 11.2 、cuDNN >= 8.0  、 Python >= 3.6
-- OS: Linux x86_64/macOS/Windows 10
-
-##### 🔸 Install FastDeploy SDK with both CPU and GPU support
-
-```bash
-pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html
-```
-
-##### [🔸 Conda Installation (Recommended✨)](docs/en/build_and_install/download_prebuilt_libraries.md)
-
-```bash
-conda config --add channels conda-forge && conda install cudatoolkit=11.2 cudnn=8.2
-```
-
-##### 🔸 Install FastDeploy SDK with only CPU support
-
-```bash
-pip install fastdeploy-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html
-```
-
-#### 🎇 Python Inference Example
-
-* Prepare model and picture
-
-```bash
-wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
-tar xvf ppyoloe_crn_l_300e_coco.tgz
-wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
-```
-
-* Test inference results
-
-```python
-# For deployment of GPU/TensorRT, please refer to examples/vision/detection/paddledetection/python
-import cv2
-import fastdeploy.vision as vision
-
-im = cv2.imread("000000014439.jpg")
-model = vision.detection.PPYOLOE("ppyoloe_crn_l_300e_coco/model.pdmodel",
-                                 "ppyoloe_crn_l_300e_coco/model.pdiparams",
-                                 "ppyoloe_crn_l_300e_coco/infer_cfg.yml")
-
-result = model.predict(im)
-print(result)
-
-vis_im = vision.vis_detection(im, result, score_threshold=0.5)
-cv2.imwrite("vis_image.jpg", vis_im)
-```
-
-</div></details>
-
-<div id="fastdeploy-quick-start-cpp"></div>
-
-<details>
-<summary><b>A Quick Start for C++ SDK(click to expand)</b></summary><div>
-
-#### 🎆 Installation
-
-- Please refer to [C++ Prebuilt Libraries Download](docs/en/build_and_install/download_prebuilt_libraries.md)
-
-#### 🎇 C++ Inference Example
-
-* Prepare models and pictures
-
-```bash
-wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
-tar xvf ppyoloe_crn_l_300e_coco.tgz
-wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
-```
-
-* Test inference results
-
-```C++
-// For GPU/TensorRT deployment, please refer to examples/vision/detection/paddledetection/cpp
-#include "fastdeploy/vision.h"
-
-int main(int argc, char* argv[]) {
-  namespace vision = fastdeploy::vision;
-  auto im = cv::imread("000000014439.jpg");
-  auto model = vision::detection::PPYOLOE("ppyoloe_crn_l_300e_coco/model.pdmodel",
-                                          "ppyoloe_crn_l_300e_coco/model.pdiparams",
-                                          "ppyoloe_crn_l_300e_coco/infer_cfg.yml");
-
-  vision::DetectionResult res;
-  model.Predict(&im, &res);
-
-  auto vis_im = vision::VisDetection(im, res, 0.5);
-  cv::imwrite("vis_image.jpg", vis_im);
-  return 0;
- }
-```
-
-</div></details>
-
-For more deployment models, please refer to [Vision Model Deployment Examples](examples/vision) .
-
-<div id="fastdeploy-server-models"></div>
-
-## ✴️ ✴️ Server-side and Cloud Model List ✴️ ✴️
-
-Notes: ✅: already supported; ❔: to be supported in the future;  N/A: Not Available;
-
-<details open><summary><b> Server-side and cloud model list(click to fold)</b></summary><div>
-
-<div align="center">
-  <img src="https://user-images.githubusercontent.com/115439700/212801271-5621419f-3997-4f00-94d5-63c8b6474aa8.png" height = "40"/>
-</div>
-
-| Task                   | Model                                                                                           | Linux                                            | Linux      | Win     | Win        | Mac     | Mac     | Linux       | Linux           | Linux         | Linux         | Linux   | Linux   | Linux   |
-|:----------------------:|:--------------------------------------------------------------------------------------------:|:------------------------------------------------:|:----------:|:-------:|:----------:|:-------:|:-------:|:-----------:|:---------------:|:-------------:|:-------------:|:-------:|:-------:|:-------:|
-| ---                    | ---                                                                                          | X86 CPU                                          | NVIDIA GPU | X86 CPU | NVIDIA GPU | X86 CPU | Arm CPU | AArch64 CPU | Phytium D2000 aarch64 | [NVIDIA Jetson](./docs/en/build_and_install/jetson.md) | [Graphcore IPU](./docs/en/build_and_install/ipu.md) | [kunlunxin XPU](./docs/en/build_and_install/kunlunxin.md) |[Huawei Ascend](./docs/en/build_and_install/huawei_ascend.md) |  [Serving](./serving) |
-| Classification         | [PaddleClas/ResNet50](./examples/vision/classification/paddleclas)                           | [✅](./examples/vision/classification/paddleclas) | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       | ✅       |✅       |
-| Classification         | [TorchVison/ResNet](examples/vision/classification/resnet)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Classification         | [ltralytics/YOLOv5Cls](examples/vision/classification/yolov5cls)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ❔       |❔       |
-| Classification         | [PaddleClas/PP-LCNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           |  ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/PP-LCNetv2](./examples/vision/classification/paddleclas)                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/EfficientNet](./examples/vision/classification/paddleclas)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/GhostNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV1](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV2](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/MobileNetV3](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/ShuffleNetV2](./examples/vision/classification/paddleclas)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/SqueeezeNetV1.1](./examples/vision/classification/paddleclas)                    | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/Inceptionv3](./examples/vision/classification/paddleclas)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ✅       |
-| Classification         | [PaddleClas/PP-HGNet](./examples/vision/classification/paddleclas)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ✅                           | ✅       |✅       | ✅       |
-| Detection              | 🔥🔥[PaddleDetection/PP-YOLOE+](./examples/vision/detection/paddledetection)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ✅       |
-| Detection              | [🔥PaddleDetection/YOLOv8](./examples/vision/detection/paddledetection)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔      |
-| Detection              | [🔥ultralytics/YOLOv8](./examples/vision/detection/yolov8)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔               | ✅                           | ❔                           | ❔      |❔       | ❔      |
-| Detection              | [PaddleDetection/PicoDet](./examples/vision/detection/paddledetection)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ❔       | ✅       |
-| Detection              | [PaddleDetection/YOLOX](./examples/vision/detection/paddledetection)                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅               | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/YOLOv3](./examples/vision/detection/paddledetection)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/PP-YOLO](./examples/vision/detection/paddledetection)                       | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/PP-YOLOv2](./examples/vision/detection/paddledetection)                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       | ✅       |
-| Detection              | [PaddleDetection/Faster-RCNN](./examples/vision/detection/paddledetection)                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |❔        | ✅       |
-| Detection              | [PaddleDetection/Mask-RCNN](./examples/vision/detection/paddledetection)                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |❔        | ✅       |
-| Detection              | [Megvii-BaseDetection/YOLOX](./examples/vision/detection/yolox)                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7](./examples/vision/detection/yolov7)                                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       |✅       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7end2end_trt](./examples/vision/detection/yolov7end2end_trt)                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [WongKinYiu/YOLOv7end2end_ort](./examples/vision/detection/yolov7end2end_ort)               | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [meituan/YOLOv6](./examples/vision/detection/yolov6)                                         | ✅                                                | ✅                        | ✅                        |✅       |  ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅                          | ❔       |
-| Detection              | [ultralytics/YOLOv5](./examples/vision/detection/yolov5)                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅       | ✅       |✅       |
-| Detection              | [WongKinYiu/YOLOR](./examples/vision/detection/yolor)                                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔      | ✅       | ❔       |
-| Detection              | [WongKinYiu/ScaledYOLOv4](./examples/vision/detection/scaledyolov4)                          | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Detection              | [ppogg/YOLOv5Lite](./examples/vision/detection/yolov5lite)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           |  ?       | ❔       |❔       |❔       |
-| Detection              | [RangiLyu/NanoDetPlus](./examples/vision/detection/nanodet_plus)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       |❔       | ❔       |
-| Perception              | [Paddle3D/Smoke](./examples/vision/perception/paddle3d/smoke)                             | ❔                                                 | ✅                         | ❔                         | ✅                         | ❔                       | ❔                      | ❔                           | ❔                 | ❔                            | ❔                           | ❔       |❔       | ✅       |
-| KeyPoint               | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose)                    | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅        |❔       | ❔       |
-| KeyPoint               | [PaddleDetection/PicoDet + TinyPose](./examples/vision/keypointdetection/det_keypoint_unite) | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅        | ❔       |❔       |
-| HeadPose               | [omasaht/headpose](examples/vision/headpose)                                                 | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ❔       | ❔       |❔       |
-| Tracking               | [PaddleDetection/PP-Tracking](examples/vision/tracking/pptracking)                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ❔       | ❔       |❔       |
-| OCR                    | [PaddleOCR/PP-OCRv2](./examples/vision/ocr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅              |✅       | ❔       |
-| OCR                    | [PaddleOCR/PP-OCRv3](./examples/vision/ocr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                           | ✅      |✅        | ✅       |
-| Segmentation           | [PaddleSeg/PP-LiteSeg](./examples/vision/segmentation/paddleseg)                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                 |❔         | ❔       |
-| Segmentation           | [PaddleSeg/PP-HumanSegLite](./examples/vision/segmentation/paddleseg)                        | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                     |✅        | ❔       |
-| Segmentation           | [PaddleSeg/HRNet](./examples/vision/segmentation/paddleseg)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅                | ✅        |❔       |
-| Segmentation           | [PaddleSeg/PP-HumanSegServer](./examples/vision/segmentation/paddleseg)                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                           | ✅        | ✅        |❔       |
-| Segmentation           | [PaddleSeg/Unet](./examples/vision/segmentation/paddleseg)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ✅               | ✅         | ✅        |❔       |
-| Segmentation           | [PaddleSeg/Deeplabv3](./examples/vision/segmentation/paddleseg)                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                     | ✅               | ✅        |❔       |
-| FaceDetection          | [biubug6/RetinaFace](./examples/vision/facedet/retinaface)                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                  | ❔       |  ❔       | ❔       |
-| FaceDetection          | [Linzaer/UltraFace](./examples/vision/facedet/ultraface)                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceDetection          | [deepcam-cn/YOLOv5Face](./examples/vision/facedet/yolov5face)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceDetection          | [insightface/SCRFD](./examples/vision/facedet/scrfd)                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceAlign              | [Hsintao/PFLD](examples/vision/facealign/pfld)                                               | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    |  ❔       | ❔       |❔       |
-| FaceAlign              | [Single430/FaceLandmark1000](./examples/vision/facealign/face_landmark_1000)                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| FaceAlign              | [jhb86253817/PIPNet](./examples/vision/facealign)                                            | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| FaceRecognition        | [insightface/ArcFace](./examples/vision/faceid/insightface)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       |  ❔       |❔       |
-| FaceRecognition        | [insightface/CosFace](./examples/vision/faceid/insightface)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| FaceRecognition        | [insightface/PartialFC](./examples/vision/faceid/insightface)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| FaceRecognition        | [insightface/VPL](./examples/vision/faceid/insightface)                                      | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔       | ❔       | ❔       |
-| Matting                | [ZHKKKe/MODNet](./examples/vision/matting/modnet)                                            | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    |  ❔       | ❔       |❔       |
-| Matting                | [PeterL1n/RobustVideoMatting]()                                                              | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |  ❔       | ❔       |
-| Matting                | [PaddleSeg/PP-Matting](./examples/vision/matting/ppmatting)                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ✅           |✅           | ❔       |
-| Matting                | [PaddleSeg/PP-HumanMatting](./examples/vision/matting/modnet)                                | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                   | ✅       |✅           |❔       |
-| Matting                | [PaddleSeg/ModNet](./examples/vision/matting/modnet)                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ✅                | ✅                           | ❔                    | ❔        |❔       |   ❔       |
-| Video Super-Resolution | [PaddleGAN/BasicVSR](./)                                                                     | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       | ❔       |❔       |
-| Video Super-Resolution | [PaddleGAN/EDVR](./examples/vision/sr/edvr)                                                  | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |❔       | ❔       |
-| Video Super-Resolution | [PaddleGAN/PP-MSVSR](./examples/vision/sr/ppmsvsr)                                           | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔       |❔       | ❔       |
-| Information Extraction | [PaddleNLP/UIE](./examples/text/uie)                                                         | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ✅                           | ❔                    | ❔         |❔       |         |
-| NLP                    | [PaddleNLP/ERNIE-3.0](./examples/text/ernie-3.0)                                             | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ❔                           | ❔                    | ✅       |❔       | ✅       |
-| Speech                 | [PaddleSpeech/PP-TTS](./examples/audio/pp-tts)                                                   | ✅                                                | ✅                        | ✅                        | ✅                        | ✅                       | ✅                     | ✅                          | ❔                | ❔                           | --                          |❔       |❔       | ✅       |
-
-</div></details>
-
-<div id="fastdeploy-edge-doc"></div>
-
-## 📳 Mobile and Edge Device Deployment
-
-<div id="fastdeploy-edge-models"></div>
-
-<details open><summary><b> Mobile and Edge Model List（click to fold）</b></summary><div>
-
-<div align="center">
-  <img src="https://user-images.githubusercontent.com/115439700/212801271-5621419f-3997-4f00-94d5-63c8b6474aa8.png" height = "40"/>
-</div>
-
-| Task               | Model                                                                                        | Size(MB)   | Linux   | Android | Linux     | Linux                   | Linux                          | Linux                       | Linux                            | TBD ...  |
-|:------------------:|:-----------------------------------------------------------------------------------------:|:--------:|:-------:|:-------:|:-------:|:-----------------------:|:------------------------------:|:---------------------------:|:--------------------------------:|:-------:|
-| ---                | ---                                                                                       | ---      | ARM CPU | [ARM CPU](./java/android) | [Rockchip NPU<br>RK3588/RK3568/RK3566](./docs/en/build_and_install/rknpu2.md) | [Rockchip NPU<br>RV1109/RV1126/RK1808](./docs/en/build_and_install/rv1126.md) | [Amlogic NPU <br>A311D/S905D/C308X](./docs/en/build_and_install/a311d.md) | NXP NPU<br>i.MX&nbsp;8M&nbsp;Plus | TBD... |
-| Classification     | [PaddleClas/ResNet50](examples/vision/classification/paddleclas)                         | 98        | ✅       | ✅       |  [✅](./examples/vision/classification/paddleclas/rknpu2)                             |      ✅                                |                                   |                                   |         |
-| Classification     | [PaddleClas/PP-LCNet](examples/vision/classification/paddleclas)                         | 11.9      | ✅       | ✅       | ❔                             | ✅                                 | --                                | --                                | --      |
-| Classification     | [PaddleClas/PP-LCNetv2](examples/vision/classification/paddleclas)                       | 26.6      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/EfficientNet](examples/vision/classification/paddleclas)                     | 31.4      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/GhostNet](examples/vision/classification/paddleclas)                         | 20.8      | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV1](examples/vision/classification/paddleclas)                      | 17        | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV2](examples/vision/classification/paddleclas)                      | 14.2      | ✅       | ✅       | ❔                             | ✅                                  | --                                | --                                | --      |
-| Classification     | [PaddleClas/MobileNetV3](examples/vision/classification/paddleclas)                      | 22        | ✅       | ✅       | ❔                             | ✅                                    | ❔                                 | ❔                                 | --      |
-| Classification     | [PaddleClas/ShuffleNetV2](examples/vision/classification/paddleclas)                     | 9.2       | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/SqueezeNetV1.1](examples/vision/classification/paddleclas)                   | 5         | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/Inceptionv3](examples/vision/classification/paddleclas)                      | 95.5      | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Classification     | [PaddleClas/PP-HGNet](examples/vision/classification/paddleclas)                         | 59        | ✅       | ✅       | ❔                             | ✅                                   | --                                | --                                | --      |
-| Detection          | [PaddleDetection/PicoDet_s](examples/vision/detection/paddledetection) | 4.9       | ✅       | ✅       | [✅](./examples/vision/detection/paddledetection/rknpu2)                             | ✅                                    | ✅                                 | ✅                                 | --      |
-| Detection          | [YOLOv5](./examples/vision/detection/rkyolo) |        |  ❔     |  ❔      | [✅](./examples/vision/detection/rkyolo)                             | ❔                                   | ❔                                 | ❔                                 | --      |
-| Face Detection     | [deepinsight/SCRFD](./examples/vision/facedet/scrfd)                                     | 2.5       | ✅       | ✅       | [✅](./examples/vision/facedet/scrfd/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Keypoint Detection | [PaddleDetection/PP-TinyPose](examples/vision/keypointdetection/tiny_pose)               | 5.5       | ✅       | ✅       | ❔                             | ❔                                    | ❔                                 | ❔                                 | --      |
-| Segmentation       | [PaddleSeg/PP-LiteSeg(STDC1)](examples/vision/segmentation/paddleseg)                    | 32.2      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/PP-HumanSeg-Lite](examples/vision/segmentation/paddleseg)                     | 0.556     | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/HRNet-w18](examples/vision/segmentation/paddleseg)                            | 38.7      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/PP-HumanSeg](examples/vision/segmentation/paddleseg)                          | 107.2     | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/Unet](examples/vision/segmentation/paddleseg)                                 | 53.7      | ✅       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             | --                                   | --                                | --                                | --      |
-| Segmentation       | [PaddleSeg/Deeplabv3](examples/vision/segmentation/paddleseg)                            | 150       | ❔       | ✅       | [✅](./examples/vision/segmentation/paddleseg/rknpu2)                             |                                      |                                   |                                   |         |
-| OCR                | [PaddleOCR/PP-OCRv2](examples/vision/ocr/PP-OCRv2)                                       | 2.3+4.4   | ✅       | ✅       | ❔                             | --                                   | --                                | --                                | --      |
-| OCR                | [PaddleOCR/PP-OCRv3](examples/vision/ocr/PP-OCRv3)                                       | 2.4+10.6  | ✅       | ❔       | ❔                             | ❔                                    | ❔                                 | ❔                                 | --      |
-
-</div></details>
-
-## ⚛️ Web and Mini Program Model List
-
-<div id="fastdeploy-web-models"></div>
-
-<details open><summary><b> Web and mini program model list(click to fold)</b></summary><div>
-
-| Task               | Model                                                                                          | [web_demo](examples/application/js/web_demo) |
-|:------------------:|:-------------------------------------------------------------------------------------------:|:--------------------------------------------:|
-| ---                | ---                                                                                         | [Paddle.js](examples/application/js)         |
-| Detection          | [FaceDetection](examples/application/js/web_demo/src/pages/cv/detection)                    | ✅                                            |
-| Detection          | [ScrewDetection](examples/application/js/web_demo/src/pages/cv/detection)                   | ✅                                            |
-| Segmentation       | [PaddleSeg/HumanSeg](./examples/application/js/web_demo/src/pages/cv/segmentation/HumanSeg) | ✅                                            |
-| Object Recognition | [GestureRecognition](examples/application/js/web_demo/src/pages/cv/recognition)             | ✅                                            |
-| Object Recognition | [ItemIdentification](examples/application/js/web_demo/src/pages/cv/recognition)             | ✅                                            |
-| OCR                | [PaddleOCR/PP-OCRv3](./examples/application/js/web_demo/src/pages/cv/ocr)                   | ✅                                            |
-
-</div></details>
-
-## 💐 Acknowledge
-
-<div id="fastdeploy-acknowledge"></div>
-
-We sincerely appreciate the open-sourced capabilities in [EasyEdge](https://ai.baidu.com/easyedge/app/openSource) as we adopt it for the SDK generation and download in this project.
-
-## ©️ License
-
-<div id="fastdeploy-license"></div>
-
-FastDeploy is provided under the [Apache-2.0](./LICENSE).

VERSION_NUMBER

@@ -1 +0,0 @@
-0.0.0

benchmark/cpp/CMakeLists.txt

@@ -1,121 +0,0 @@
-PROJECT(infer_demo C CXX)
-CMAKE_MINIMUM_REQUIRED (VERSION 3.10)
-# specify the decompress directory of FastDeploy SDK
-option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
-include(${FASTDEPLOY_INSTALL_DIR}/utils/gflags.cmake)
-include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
-include_directories(${FASTDEPLOY_INCS})
-
-add_executable(benchmark ${PROJECT_SOURCE_DIR}/benchmark.cc)
-add_executable(benchmark_yolov5 ${PROJECT_SOURCE_DIR}/benchmark_yolov5.cc)
-add_executable(benchmark_ppyolov5 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov5.cc)
-add_executable(benchmark_ppyolov6 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov6.cc)
-add_executable(benchmark_ppyolov7 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov7.cc)
-add_executable(benchmark_ppyolov8 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov8.cc)
-add_executable(benchmark_ppyolox ${PROJECT_SOURCE_DIR}/benchmark_ppyolox.cc)
-add_executable(benchmark_ppyoloe ${PROJECT_SOURCE_DIR}/benchmark_ppyoloe.cc)
-add_executable(benchmark_picodet ${PROJECT_SOURCE_DIR}/benchmark_picodet.cc)
-add_executable(benchmark_ppcls ${PROJECT_SOURCE_DIR}/benchmark_ppcls.cc)
-add_executable(benchmark_ppseg ${PROJECT_SOURCE_DIR}/benchmark_ppseg.cc)
-add_executable(benchmark_ppmatting ${PROJECT_SOURCE_DIR}/benchmark_ppmatting.cc)
-add_executable(benchmark_ppocr_det ${PROJECT_SOURCE_DIR}/benchmark_ppocr_det.cc)
-add_executable(benchmark_ppocr_cls ${PROJECT_SOURCE_DIR}/benchmark_ppocr_cls.cc)
-add_executable(benchmark_ppocr_rec ${PROJECT_SOURCE_DIR}/benchmark_ppocr_rec.cc)
-add_executable(benchmark_structurev2_table ${PROJECT_SOURCE_DIR}/benchmark_structurev2_table.cc)
-add_executable(benchmark_structurev2_layout ${PROJECT_SOURCE_DIR}/benchmark_structurev2_layout.cc)
-add_executable(benchmark_ppyoloe_r ${PROJECT_SOURCE_DIR}/benchmark_ppyoloe_r.cc)
-add_executable(benchmark_ppyoloe_r_sophgo ${PROJECT_SOURCE_DIR}/benchmark_ppyoloe_r_sophgo.cc)
-add_executable(benchmark_ppyolo ${PROJECT_SOURCE_DIR}/benchmark_ppyolo.cc)
-add_executable(benchmark_yolov3 ${PROJECT_SOURCE_DIR}/benchmark_yolov3.cc)
-add_executable(benchmark_fasterrcnn ${PROJECT_SOURCE_DIR}/benchmark_fasterrcnn.cc)
-add_executable(benchmark_maskrcnn ${PROJECT_SOURCE_DIR}/benchmark_maskrcnn.cc)
-add_executable(benchmark_ssd ${PROJECT_SOURCE_DIR}/benchmark_ssd.cc)
-add_executable(benchmark_rtmdet ${PROJECT_SOURCE_DIR}/benchmark_rtmdet.cc)
-add_executable(benchmark_cascadercnn ${PROJECT_SOURCE_DIR}/benchmark_cascadercnn.cc)
-add_executable(benchmark_fcos ${PROJECT_SOURCE_DIR}/benchmark_fcos.cc)
-add_executable(benchmark_gfl ${PROJECT_SOURCE_DIR}/benchmark_gfl.cc)
-add_executable(benchmark_retinanet ${PROJECT_SOURCE_DIR}/benchmark_retinanet.cc)
-add_executable(benchmark_tood ${PROJECT_SOURCE_DIR}/benchmark_tood.cc)
-add_executable(benchmark_ttfnet ${PROJECT_SOURCE_DIR}/benchmark_ttfnet.cc)
-add_executable(benchmark_ppdet ${PROJECT_SOURCE_DIR}/benchmark_ppdet.cc)
-add_executable(benchmark_dino ${PROJECT_SOURCE_DIR}/benchmark_dino.cc)
-add_executable(benchmark_ppshituv2_rec ${PROJECT_SOURCE_DIR}/benchmark_ppshituv2_rec.cc)
-add_executable(benchmark_ppshituv2_det ${PROJECT_SOURCE_DIR}/benchmark_ppshituv2_det.cc)
-
-if(UNIX AND (NOT APPLE) AND (NOT ANDROID))
-  target_link_libraries(benchmark ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolov5 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolov6 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolov7 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolox ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyoloe ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyoloe_r ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyoloe_r_sophgo ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_picodet ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppmatting ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppocr_det ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppocr_cls ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppocr_rec ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_structurev2_table ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_structurev2_layout ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppyolo ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_yolov3 ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_fasterrcnn ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_maskrcnn ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ssd ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_rtmdet ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_cascadercnn ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_fcos ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_gfl ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_retinanet ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_tood ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ttfnet ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppdet ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_dino ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppshituv2_rec ${FASTDEPLOY_LIBS} gflags pthread)
-  target_link_libraries(benchmark_ppshituv2_det ${FASTDEPLOY_LIBS} gflags pthread)
-else()
-  target_link_libraries(benchmark ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolov5 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolov6 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolov7 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolox ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyoloe ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyoloe_r ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyoloe_r_sophgo ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_picodet ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppmatting ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppocr_det ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppocr_cls ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppocr_rec ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_structurev2_table ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_structurev2_layout ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppyolo ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_yolov3 ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_fasterrcnn ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_maskrcnn ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ssd ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_rtmdet ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_cascadercnn ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_fcos ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_gfl ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_retinanet ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_tood ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ttfnet ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppdet ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_dino ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppshituv2_rec ${FASTDEPLOY_LIBS} gflags)
-  target_link_libraries(benchmark_ppshituv2_det ${FASTDEPLOY_LIBS} gflags)
-endif()
-# only for Android ADB test
-if(ANDROID)
-  install_fastdeploy_libraries(${CMAKE_CURRENT_BINARY_DIR})
-endif()

benchmark/cpp/README.md

@@ -1,217 +0,0 @@
-# FastDeploy C++ Benchmarks
-
-## 1. 编译选项  
-以下选项为benchmark相关的编译选项，在编译用来跑benchmark的sdk时，必须开启。  
-
-|选项|需要设置的值|说明|
-|---|---|---|  
-| ENABLE_BENCHMARK  | ON | 默认OFF, 是否打开BENCHMARK模式 |
-| ENABLE_VISION     | ON | 默认OFF，是否编译集成视觉模型的部署模块 |
-| ENABLE_TEXT       | ON | 默认OFF，是否编译集成文本NLP模型的部署模块 |  
-
-运行FastDeploy C++ Benchmark，需先准备好相应的环境，并在ENABLE_BENCHMARK=ON模式下从源码编译FastDeploy C++ SDK. 以下将按照硬件维度，来说明相应的系统环境要求。不同环境下的详细要求，请参考[FastDeploy环境要求](../../docs/cn/build_and_install)  
-
-## 2. Benchmark 设置说明  
-
-具体flags.h提供选项如下:
-
-<div id="选项设置说明"></div>  
-
-| 选项                 | 作用                                        |
-| -------------------- | ------------------------------------------ |
-| --model              | 模型路径                                     |
-| --image              | 图片路径    |
-| --config_path        | config.txt路径，包含具体设备、后端等信息  |
-
-具体config.txt包含信息含义如下:
-
-<div id="参数设置说明"></div>  
-
-| 参数                 | 作用                                        |
-| -------------------- | ------------------------------------------ |
-| device             | 选择 CPU/GPU/XPU，默认为 CPU  |
-| device_id          | GPU/XPU 卡号，默认为 0 |
-| cpu_thread_nums     | CPU 线程数，默认为 1      |
-| warmup           | 跑benchmark的warmup次数，默认为 200 |
-| repeat           | 跑benchmark的循环次数，默认为 1000 |
-| backend            | 指定后端类型，有default, ort, ov, trt, paddle, paddle_trt, lite 等，为default时，会自动选择最优后端，推荐设置为显式设置明确的backend。默认为 default   |
-| profile_mode      | 指定需要测试性能的模式，可选值为`[runtime, end2end]`，默认为 runtime |
-| include_h2d_d2h   | 是否把H2D+D2H的耗时统计在内，该参数只在profile_mode为runtime时有效，默认为 false |  
-| use_fp16    | 是否开启fp16，当前只对 trt, paddle-trt, lite后端有效，默认为 false |
-| collect_memory_info    | 是否记录 cpu/gpu memory信息，默认 false  |
-| sampling_interval    | 记录 cpu/gpu memory信息采样时间间隔，单位ms，默认为 50  |
-| precision_compare    | 是否进行精度比较，默认为 false  |  
-| result_path    | 记录 Benchmark 数据的 txt 文件路径  |  
-| xpu_l3_cache | 设置XPU L3 Cache大小，默认值为0。设置策略，对于 昆仑2 XPU R200，L3 Cache可用的最大值为 62914560，对于 昆仑1 XPU 则为 16776192 |
-
-## 3. X86_64 CPU 和 NVIDIA GPU 环境下运行 Benchmark
-
-### 3.1 环境准备  
-
-Linux上编译需满足:
-  - gcc/g++ >= 5.4(推荐8.2)
-  - cmake >= 3.18.0
-  - CUDA >= 11.2
-  - cuDNN >= 8.2
-  - TensorRT >= 8.5
-
-在GPU上编译FastDeploy需要准备好相应的CUDA环境以及TensorRT，详细文档请参考[GPU编译文档](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/cn/build_and_install/gpu.md)。  
-
-### 3.2 编译FastDeploy C++ SDK  
-```bash
-# 源码编译SDK
-git clone https://github.com/PaddlePaddle/FastDeploy.git -b develop
-cd FastDeploy
-mkdir build && cd build
-cmake .. -DWITH_GPU=ON \
-         -DENABLE_ORT_BACKEND=ON \
-         -DENABLE_PADDLE_BACKEND=ON \
-         -DENABLE_OPENVINO_BACKEND=ON \
-         -DENABLE_TRT_BACKEND=ON \
-         -DENABLE_VISION=ON \
-         -DENABLE_TEXT=ON \
-         -DENABLE_BENCHMARK=ON \  # 开启benchmark模式
-         -DTRT_DIRECTORY=/Paddle/TensorRT-8.5.2.2 \
-         -DCUDA_DIRECTORY=/usr/local/cuda \
-         -DCMAKE_INSTALL_PREFIX=${PWD}/compiled_fastdeploy_sdk
-
-make -j12
-make install  
-
-# 配置SDK路径
-cd ..  
-export FD_GPU_SDK=${PWD}/build/compiled_fastdeploy_sdk
-```  
-### 3.3 编译 Benchmark 示例  
-```bash  
-cd benchmark/cpp
-mkdir build && cd build  
-cmake .. -DFASTDEPLOY_INSTALL_DIR=${FD_GPU_SDK}  
-make -j4
-```
-
-### 3.4 运行 Benchmark 示例  
-
-在X86 CPU + NVIDIA GPU下，FastDeploy 目前支持多种推理后端，下面以 PaddleYOLOv8 为例，跑出多后端在 CPU/GPU 对应 benchmark 数据。
-
-- 下载模型文件和测试图片  
-```bash  
-wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov8_s_500e_coco.tgz  
-wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
-tar -zxvf yolov8_s_500e_coco.tgz
-```
-
-- 运行 yolov8 benchmark 示例  
-
-```bash  
-
-# 统计性能，用户根据需求修改config.txt文件，具体含义参考上表
-# eg：如果想测paddle gpu backend，将device改为gpu，backend修改为paddle即可
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path config.txt
-```
-注意，为避免对性能统计产生影响，测试性能时，最好不要开启内存显存统计的功能，当把collect_memory_info参数设置为true时，只有内存显存参数是稳定可靠的。更多参数设置，请参考[参数设置说明](#参数设置说明)
-
-## 4. 各个硬件上的一键运行脚本  
-
-在准备好相关的环境配置和SDK后，可以使用本目录提供的脚本一键运行后的benchmark数据。
-- 获取模型和资源文件  
-```bash
-./get_models.sh
-```  
-- 运行benchmark脚本  
-```bash
-# x86 CPU Paddle backend fp32
-./benchmark_x86.sh config/config.x86.paddle.fp32.txt
-# x86 CPU ONNXRuntime backend fp32
-./benchmark_x86.sh config/config.x86.ort.fp32.txt
-# x86 CPU OpenVIVO backend fp32
-./benchmark_x86.sh config/config.x86.ov.fp32.txt
-# NVIDIA GPU Paddle backend fp32
-./benchmark_gpu.sh config/config.gpu.paddle.fp32.txt
-# NVIDIA GPU ONNXRuntime backend fp32
-./benchmark_gpu.sh config/config.gpu.ort.fp32.txt
-# NVIDIA GPU Paddle-TRT backend fp32
-./benchmark_gpu_trt.sh config/config.gpu.paddle_trt.fp32.txt
-# NVIDIA GPU Paddle-TRT backend fp16
-./benchmark_gpu_trt.sh config/config.gpu.paddle_trt.fp16.txt
-# NVIDIA GPU TRT backend fp32
-./benchmark_gpu_trt.sh config/config.gpu.trt.fp32.txt
-# NVIDIA GPU TRT backend fp16
-./benchmark_gpu_trt.sh config/config.gpu.trt.fp16.txt
-
-# Arm CPU Paddle Lite backend fp32
-./benchmark_arm.sh config/config.arm.lite.fp32.txt
-# Arm CPU Paddle Lite backend fp16
-./benchmark_arm.sh config/config.arm.lite.fp16.txt
-# XPU Paddle Lite backend fp32
-./benchmark_xpu.sh config/config.xpu.lite.fp32.txt
-```
-
-## 5. Benchmark工具用法  
-FastDeploy除了提供包含模型前后处理在内的benchmark_xxx外，也提供常规的benchmark工具，以支持对任意模型进行benchmark。在编译benchmark目录的源码之后，会生成一个benchmark可执行文件，该工具支持[选项设置说明](#选项设置说明)中的所有参数，并且提供一些额外参数，便于使用，额外的参数说明如下。注意：该工具仅支持测试纯模型推理时间和推理+H2D+D2H耗时（当config.txt中include_h2d_d2h为true时），不支持测试包含前后处理在内的时间。
-
-| 参数                 | 作用                                        |
-| -------------------- | ------------------------------------------ |
-| shapes             | Set input shape for model, default "1,3,224,224" |
-| names          |  Set input names for model, default "DEFAULT" |  
-| dtypes          | Set input dtypes for model, default "FP32" |  
-| trt_shapes          | Set min/opt/max shape for trt/paddle_trt backend. eg:--trt_shape 1,3,224,224:1,3,224,224:1,3,224,224", default "1,3,224,224:1,3,224,224:1,3,224,224" |  
-| batch          | TensorRT max batch size, default=1 |  
-| dump          | Whether to dump output tensors, default false. |  
-| info          | Only check the input infos of model, default false. |  
-| diff          | Check the diff between two tensors, default false. |  
-| tensors          | The paths to dumped tensors, should look like "tensor_a.txt:tensor_b.txt"|  
-| mem          | Whether to force to collect memory info, default false.  |  
-| model_file          | Optional, set specific model file, eg, model.pdmodel, model.onnx, default "UNKNOWN" |  
-| params_file          | Optional, set specific params file, eg, model.pdiparams, default "" |  
-| model_format          | Optional, set specific model format, eg, PADDLE/ONNX/RKNN/TORCHSCRIPT/SOPHGO, default "PADDLE" |  
-| disable_mkldnn          | Whether to disable mkldnn for paddle backend, default false. |  
-
-### 5.1 benchmark工具使用示例  
-- 用法说明  
-```bash
-./benchmark --helpshort
-benchmark: ./benchmark -[info|diff|check|dump|mem] -model xxx -config_path xxx -[shapes|dtypes|names|tensors] -[model_file|params_file|model_format]
-...
-```
-- 单输入示例：--model，指定模型文件夹，其中包括*.pdmodel/pdiparams文件
-```bash
-./benchmark --model ResNet50_vd_infer --config_path config/config.x86.ov.fp32.txt --shapes 1,3,224,224 --names inputs --dtypes FP32
-```  
-- 单输入示例：--model_file, --params_file，指定具体的模型文件和参数文件
-```bash
-./benchmark --model_file MobileNetV1_ssld_infer/inference.pdmodel --params_file MobileNetV1_ssld_infer/inference.pdiparams --config_path config/config.x86.ov.fp32.txt --shapes 1,3,224,224 --names inputs --dtypes FP32
-```  
-- 多输入示例：  
-```bash
-./benchmark --model yolov5_s_300e_coco --config_path config/config.arm.lite.fp32.txt --shapes 1,3,640,640:1,2 --names image:scale_factor --dtypes FP32:FP32
-```
-- Paddle-TRT示例 ：
-```bash  
-./benchmark --model ResNet50_vd_infer --config_path config/config.gpu.paddle_trt.fp16.txt --trt_shapes 1,3,224,224:1,3,224,224:1,3,224,224 --names inputs --dtypes FP32
-```
-- TensorRT/Paddle-TRT多输入示例：  
-```bash
-./benchmark --model rtdetr_r50vd_6x_coco --trt_shapes 1,2:1,2:1,2:1,3,640,640:1,3,640,640:1,3,640,640:1,2:1,2:1,2 --names im_shape:image:scale_factor --shapes 1,2:1,3,640,640:1,2  --config_path config/config.gpu.paddle_trt.fp32.txt --dtypes FP32:FP32:FP32
-```
-- 支持FD全部后端和全部模型格式：--model_file, --params_file(optional), --model_format
-```bash
-# ONNX模型示例
-./benchmark --model ResNet50_vd_infer --model_file inference.onnx --model_format ONNX --config_path config/config.gpu.trt.fp16.txt --trt_shapes 1,3,224,224:1,3,224,224:1,3,224,224 --names inputs --dtypes FP32
-```  
-- 统计内显存占用：--mem 或 在config.txt中指定  
-```bash
-./benchmark --mem --model ResNet50_vd_infer --config_path config/config.x86.ov.fp32.txt --shapes 1,3,224,224 --names inputs --dtypes FP32
-```
-- 推理并dump 输出 tensor用作对比： --dump
-```bash
-./benchmark --dump --model ResNet50_vd_infer --config_path config/config.x86.ov.fp32.txt --shapes 1,3,224,224 --names inputs --dtypes FP32
-```
-- 对比两个 dumped 的tensor :  --diff
-```bash
-./benchmark --diff --tensors ov_linear_77.tmp_1.txt:lite_linear_77.tmp_1.txt
-```
-- 显示模型的输入信息： --info
-```bash
-./benchmark --info --model picodet_l_640_coco_lcnet --config_path config/config.arm.lite.fp32.txt
-```

benchmark/cpp/benchmark.cc

@@ -1,404 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#include "fastdeploy/function/functions.h"
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(shapes, "1,3,224,224",
-              "Required, set input shape for model."
-              "default 1,3,224,224");
-DEFINE_string(names, "DEFAULT", "Required, set input names for model.");
-DEFINE_string(dtypes, "FP32",
-              "Required, set input dtypes for model."
-              "default FP32.");
-DEFINE_string(trt_shapes, "1,3,224,224:1,3,224,224:1,3,224,224",
-              "Optional, set min/opt/max shape for trt/paddle_trt."
-              "default 1,3,224,224:1,3,224,224:1,3,224,224");
-DEFINE_int32(batch, 1,
-             "Optional, set trt max batch size, "
-             "default 1");
-DEFINE_bool(dump, false,
-            "Optional, whether to dump output tensors, "
-            "default false.");
-DEFINE_bool(info, false,
-            "Optional, only check the input infos of model."
-            "default false.");
-DEFINE_bool(diff, false,
-            "Optional, check the diff between two tensors."
-            "default false.");
-DEFINE_string(tensors, "tensor_a.txt:tensor_b.txt",
-              "Optional, the paths to dumped tensors, "
-              "default tensor_a.txt:tensor_b.txt");
-DEFINE_bool(mem, false,
-            "Optional, whether to force to collect memory info, "
-            "default false.");
-DEFINE_int32(interval, -1,
-             "Optional, sampling interval for collect memory info, "
-             "default false.");
-DEFINE_string(model_format, "PADDLE",
-              "Optional, set specific model format,"
-              "eg, PADDLE/ONNX/RKNN/TORCHSCRIPT/SOPHGO"
-              "default PADDLE.");
-DEFINE_bool(disable_mkldnn, false,
-            "Optional, disable mkldnn for paddle backend. "
-            "default false.");
-DEFINE_string(optimized_model_dir, "",
-              "Optional, set optimized model dir for lite."
-              "eg: model.opt.nb, "
-              "default ''");
-DEFINE_bool(collect_trt_shape_by_device, false,
-            "Optional, whether collect trt shape by device. "
-            "default false.");
-DEFINE_double(custom_tensor_value, 1.0,
-              "Optional, set the value for fd tensor, "
-              "default 1.0");
-DEFINE_bool(collect_trt_shape_by_custom_tensor_value, false,
-            "Optional, whether collect trt shape by custom tensor value. "
-            "default false.");
-
-#if defined(ENABLE_BENCHMARK)
-static std::vector<int64_t> GetInt64Shape(const std::vector<int>& shape) {
-  std::vector<int64_t> new_shape;
-  new_shape.resize(shape.size());
-  for (int i = 0; i < shape.size(); ++i) {
-    new_shape[i] = static_cast<int64_t>(shape[i]);
-  }
-  return new_shape;
-}
-
-static fastdeploy::ModelFormat GetModelFormat(const std::string& model_format) {
-  if (model_format == "PADDLE") {
-    return fastdeploy::ModelFormat::PADDLE;
-  } else if (model_format == "ONNX") {
-    return fastdeploy::ModelFormat::ONNX;
-  } else if (model_format == "RKNN") {
-    return fastdeploy::ModelFormat::RKNN;
-  } else if (model_format == "TORCHSCRIPT") {
-    return fastdeploy::ModelFormat::TORCHSCRIPT;
-  } else if (model_format == "SOPHGO") {
-    return fastdeploy::ModelFormat::SOPHGO;
-  } else {
-    return fastdeploy::ModelFormat::PADDLE;
-  }
-}
-
-static void CheckTensorDiff(int argc, char* argv[]) {
-  google::ParseCommandLineFlags(&argc, &argv, true);
-  std::cout << "Check tensor diff ..." << std::endl;
-  std::vector<std::string> tensor_paths =
-      benchmark::ResultManager::SplitStr(FLAGS_tensors);
-  assert(tensor_paths.size() == 2);
-  fastdeploy::FDTensor tensor_a, tensor_b;
-  benchmark::ResultManager::LoadFDTensor(&tensor_a, tensor_paths[0]);
-  benchmark::ResultManager::LoadFDTensor(&tensor_b, tensor_paths[1]);
-  auto tensor_diff =
-      benchmark::ResultManager::CalculateDiffStatis(tensor_a, tensor_b);
-  std::cout << "Tensor diff: mean=" << tensor_diff.data.mean
-            << ", max=" << tensor_diff.data.max
-            << ", min=" << tensor_diff.data.min << std::endl;
-}
-
-static void RuntimeProfiling(int argc, char* argv[]) {
-  // Init runtime option
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return;
-  }
-  if (FLAGS_disable_mkldnn) {
-    option.paddle_infer_option.enable_mkldnn = false;
-  }
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  UpdateBaseCustomFlags(config_info);  // see flags.h
-  // Init log recorder
-  std::stringstream ss;
-  ss.precision(6);
-
-  // Memory resource moniter
-  int sampling_interval = FLAGS_interval >= 1
-                              ? FLAGS_interval
-                              : std::stoi(config_info["sampling_interval"]);
-
-  benchmark::ResourceUsageMonitor resource_moniter(
-      sampling_interval, std::stoi(config_info["device_id"]));
-
-  // Check model path and model format
-  std::string model_name, params_name, config_name;
-  std::string model_file, params_file;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (FLAGS_model_file != "UNKNOWN") {
-    // Set model file/param/format via command line
-    if (FLAGS_model != "") {
-      model_file = FLAGS_model + sep + FLAGS_model_file;
-      params_file = FLAGS_model + sep + FLAGS_params_file;
-    } else {
-      model_file = FLAGS_model_file;
-      params_file = FLAGS_params_file;
-    }
-    model_format = GetModelFormat(FLAGS_model_format);
-    if (model_format == fastdeploy::ModelFormat::PADDLE &&
-        FLAGS_params_file == "") {
-      if (config_info["backend"] != "lite") {
-        std::cout << "[ERROR] params_file can not be empty for PADDLE"
-                  << " format, Please, set your custom params_file manually."
-                  << std::endl;
-        return;
-      } else {
-        std::cout << "[INFO] Will using the lite light api for: " << model_file
-                  << std::endl;
-      }
-    }
-  } else {
-    // Set model file/param/format via model dir (only support
-    // for Paddle model format now)
-    if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                                 &model_format, config_info, false)) {
-      return;
-    }
-    model_file = FLAGS_model + sep + model_name;
-    params_file = FLAGS_model + sep + params_name;
-  }
-
-  option.SetModelPath(model_file, params_file, model_format);
-
-  // Set opt model dir
-  if (config_info["backend"] == "lite") {
-    if (FLAGS_optimized_model_dir != "") {
-      option.paddle_lite_option.optimized_model_dir = FLAGS_optimized_model_dir;
-    } else {
-      option.paddle_lite_option.optimized_model_dir = FLAGS_model;
-    }
-  }
-
-  // Get input shapes/names/dtypes
-  std::vector<std::vector<int32_t>> input_shapes =
-      benchmark::ResultManager::GetInputShapes(FLAGS_shapes);
-  std::vector<std::string> input_names =
-      benchmark::ResultManager::GetInputNames(FLAGS_names);
-  std::vector<fastdeploy::FDDataType> input_dtypes =
-      benchmark::ResultManager::GetInputDtypes(FLAGS_dtypes);
-
-  // Set tensorrt shapes
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-    option.paddle_infer_option.collect_trt_shape_by_device = 
-      FLAGS_collect_trt_shape_by_device;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.max_batch_size = FLAGS_batch;
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shapes);
-    if (input_names[0] == "DEFAULT") {
-      std::cout << "Please set the input names for TRT/Paddle-TRT backend!"
-                << std::endl;
-      return;
-    }
-    assert(input_names.size() == (trt_shapes.size() / 3));
-    for (int i = 0; i < input_shapes.size(); ++i) {
-      option.trt_option.SetShape(input_names[i], trt_shapes[i * 3],
-                                 trt_shapes[i * 3 + 1], trt_shapes[i * 3 + 2]);
-      // Set custom input data for collect trt shapes
-      if (FLAGS_collect_trt_shape_by_custom_tensor_value) {
-        int min_shape_num = std::accumulate(trt_shapes[i * 3].begin(),
-                                            trt_shapes[i * 3].end(), 1,
-                                            std::multiplies<int>());
-        int opt_shape_num = std::accumulate(trt_shapes[i * 3 + 1].begin(),
-                                            trt_shapes[i * 3 + 1].end(), 1,
-                                            std::multiplies<int>());
-        int max_shape_num = std::accumulate(trt_shapes[i * 3 + 2].begin(),
-                                            trt_shapes[i * 3 + 2].end(), 1,
-                                            std::multiplies<int>());                                                                        
-        std::vector<float> min_input_data(min_shape_num, FLAGS_custom_tensor_value);                     
-        std::vector<float> opt_input_data(opt_shape_num, FLAGS_custom_tensor_value);                     
-        std::vector<float> max_input_data(max_shape_num, FLAGS_custom_tensor_value);                     
-        option.trt_option.SetInputData(input_names[i], min_input_data, 
-                                       opt_input_data, max_input_data);
-      }
-    }
-  }
-
-  // Init runtime
-  fastdeploy::Runtime runtime;
-  if (!runtime.Init(option)) {
-    std::cout << "Initial Runtime failed!" << std::endl;
-  }
-
-  // Check default input names
-  if (input_names[0] == "DEFAULT") {
-    input_names.clear();
-    for (int i = 0; i < runtime.NumInputs(); ++i) {
-      input_names.push_back(runtime.GetInputInfo(i).name);
-    }
-  }
-
-  assert(runtime.NumInputs() == input_shapes.size());
-  assert(runtime.NumInputs() == input_names.size());
-  assert(runtime.NumInputs() == input_dtypes.size());
-
-  // Feed inputs, all values set as 1.
-  std::vector<fastdeploy::FDTensor> inputs(runtime.NumInputs());
-  for (int i = 0; i < inputs.size(); ++i) {
-    fastdeploy::function::Full(
-      FLAGS_custom_tensor_value, GetInt64Shape(input_shapes[i]),
-      &inputs[i], input_dtypes[i]);
-    inputs[i].name = input_names[i];
-  }
-
-  // Start memory resource moniter
-  if (config_info["collect_memory_info"] == "true" || FLAGS_mem) {
-    resource_moniter.Start();
-  }
-
-  // Run runtime profiling
-  std::vector<fastdeploy::FDTensor> outputs;
-  if (!runtime.Infer(inputs, &outputs)) {
-    std::cerr << "Failed to predict." << std::endl;
-    ss << "Runtime(ms): Failed" << std::endl;
-    if (config_info["collect_memory_info"] == "true") {
-      ss << "cpu_rss_mb: Failed" << std::endl;
-      ss << "gpu_rss_mb: Failed" << std::endl;
-      ss << "gpu_util: Failed" << std::endl;
-      resource_moniter.Stop();
-    }
-    benchmark::ResultManager::SaveBenchmarkResult(ss.str(),
-                                                  config_info["result_path"]);
-    return;
-  }
-
-  double profile_time = runtime.GetProfileTime() * 1000.0;
-  std::cout << "Runtime(ms): " << profile_time << "ms." << std::endl;
-  ss << "Runtime(ms): " << profile_time << "ms." << std::endl;
-
-  // Collect memory info
-  if (config_info["collect_memory_info"] == "true" || FLAGS_mem) {
-    float cpu_mem = resource_moniter.GetMaxCpuMem();
-    float gpu_mem = resource_moniter.GetMaxGpuMem();
-    float gpu_util = resource_moniter.GetMaxGpuUtil();
-    std::cout << "cpu_rss_mb: " << cpu_mem << "MB." << std::endl;
-    ss << "cpu_rss_mb: " << cpu_mem << "MB." << std::endl;
-    std::cout << "gpu_rss_mb: " << gpu_mem << "MB." << std::endl;
-    ss << "gpu_rss_mb: " << gpu_mem << "MB." << std::endl;
-    std::cout << "gpu_util: " << gpu_util << std::endl;
-    ss << "gpu_util: " << gpu_util << "MB." << std::endl;
-    resource_moniter.Stop();
-  }
-  benchmark::ResultManager::SaveBenchmarkResult(ss.str(),
-                                                config_info["result_path"]);
-
-  // Dump output tensors
-  if (FLAGS_dump) {
-    for (int i = 0; i < outputs.size(); ++i) {
-      auto name_tokens =
-          benchmark::ResultManager::SplitStr(outputs[i].name, '/');
-      std::string out_name = name_tokens[0];
-      for (int j = 1; j < name_tokens.size(); ++j) {
-        out_name += "_";
-        out_name += name_tokens[j];
-      }
-      std::string out_file = config_info["backend"] + "_" + out_name + ".txt";
-      benchmark::ResultManager::SaveFDTensor(outputs[i], out_file);
-      outputs[i].PrintInfo();
-      std::cout << "Saved: " << out_file << std::endl;
-    }
-  }
-}
-
-static void showInputInfos(int argc, char* argv[]) {
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return;
-  }
-  if (FLAGS_disable_mkldnn) {
-    option.paddle_infer_option.enable_mkldnn = false;
-  }
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  std::string model_file, params_file;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (FLAGS_model_file != "UNKNOWN") {
-    // Set model file/param/format via command line
-    if (FLAGS_model != "") {
-      model_file = FLAGS_model + sep + FLAGS_model_file;
-      params_file = FLAGS_model + sep + FLAGS_params_file;
-    } else {
-      model_file = FLAGS_model_file;
-      params_file = FLAGS_params_file;
-    }
-    model_format = GetModelFormat(FLAGS_model_format);
-    if (model_format == fastdeploy::ModelFormat::PADDLE &&
-        FLAGS_params_file == "") {
-      if (config_info["backend"] != "lite") {
-        std::cout << "[ERROR] params_file can not be empty for PADDLE"
-                  << " format, Please, set your custom params_file manually."
-                  << std::endl;
-        return;
-      } else {
-        std::cout << "[INFO] Will using the lite light api for: " << model_file
-                  << std::endl;
-      }
-    }
-  } else {
-    // Set model file/param/format via model dir (only support
-    // for Paddle model format now)
-    if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                                 &model_format, config_info, false)) {
-      return;
-    }
-    model_file = FLAGS_model + sep + model_name;
-    params_file = FLAGS_model + sep + params_name;
-  }
-
-  option.SetModelPath(model_file, params_file, model_format);
-
-  // Init runtime
-  fastdeploy::Runtime runtime;
-  if (!runtime.Init(option)) {
-    std::cout << "Initial Runtime failed!" << std::endl;
-  }
-  // Show input tensor infos
-  auto input_infos = runtime.GetInputInfos();
-  for (int i = 0; i < input_infos.size(); ++i) {
-    std::cout << input_infos[i] << std::endl;
-  }
-}
-#endif
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK)
-  google::SetVersionString("0.0.0");
-  google::SetUsageMessage(
-      "./benchmark -[info|diff|check|dump|mem] -model xxx -config_path xxx "
-      "-[shapes|dtypes|names|tensors] -[model_file|params_file|model_format]");
-  google::ParseCommandLineFlags(&argc, &argv, true);
-  if (FLAGS_diff) {
-    CheckTensorDiff(argc, argv);
-    return 0;
-  } else if (FLAGS_info) {
-    showInputInfos(argc, argv);
-    return 0;
-  } else {
-    RuntimeProfiling(argc, argv);
-    return 0;
-  }
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_arm.sh

@@ -1,122 +0,0 @@
-# Run all models specify hardware and specify backend
-export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
-
-CONFIG_PATH="config.arm.lite.fp32.txt"
-if [ ! "$1" = "$CONFIG_PATH" ]; then
-  if [ -f "$1" ]; then
-    CONFIG_PATH="$1"
-  fi
-fi
-DEFAULT_INTERLVAL=30
-if [ ! "$2" = "" ]; then
-  DEFAULT_INTERLVAL=$2
-fi
-
-sleep_seconds() {
-  sleep_interval=$DEFAULT_INTERLVAL
-  if [ ! "$1" = "" ]; then
-    sleep_interval=$1
-  fi
-  echo "[INFO][SLEEP] --- Sleep $sleep_interval seconds for Arm CPU mobile to prevent the phone from overheating ..."
-  sleep $sleep_interval
-}
-
-# PaddleClas
-./benchmark_ppcls --model PPLCNet_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model PPLCNetV2_base_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model EfficientNetB7_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model EfficientNetB0_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model GhostNet_x0_5_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model GhostNet_x1_3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model GhostNet_x1_3_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV1_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV1_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV2_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV3_small_x0_35_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV3_large_x1_0_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ShuffleNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ShuffleNetV2_x2_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model SqueezeNet1_1_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model InceptionV3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ResNet50_vd_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model PPHGNet_tiny_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model PPHGNet_base_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ResNet50_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model EfficientNetB0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV2_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model MobileNetV3_small_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ViT_large_patch16_224_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model ResNeXt50_32x4d_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model DenseNet121_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model PPHGNet_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppcls --model person_exists_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH && sleep_seconds
-
-# PaddleOCR
-./benchmark_ppocr_det --model ch_PP-OCRv3_det_infer --image 12.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH && sleep_seconds
-
-# PaddleDetection
-./benchmark_ppyolov5 --model yolov5_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppyolov6 --model yolov6_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppyolox --model yolox_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_picodet --model picodet_l_640_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolox --model yolox_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_picodet --model picodet_l_640_coco_lcnet_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolo --model ppyolo_r50vd_dcn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_yolov3 --model yolov3_darknet53_270e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolov2_r101vd_dcn_365e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_320_coco_lcnet --image 000000014439.jpg $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_r50_vd_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_maskrcnn --model mask_rcnn_r50_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_fpn_1x_coco --image 000000014439.jpg $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_vd_fpn_ssld_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fcos --model fcos_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_gfl --model gfl_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r101_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_tood --model tood_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ttfnet --model ttfnet_darknet53_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_400e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_x_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_enhance_3x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_80e_sliced_visdrone_640_025 --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_mobilenet_v1_300_120e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_vgg16_300_240e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssdlite_mobilenet_v1_300_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_x_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_l_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_m_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_n_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-# PaddleSeg
-./benchmark_ppseg --model Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppseg --model PP_HumanSegV2_Lite_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppseg --model PP_HumanSegV1_Lite_infer --image portrait_heng.jpg --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppseg --model PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppseg --model FCN_HRNet_W18_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH && sleep_seconds
-./benchmark_ppseg --model SegFormer_B0-cityscapes-with-argmax --image cityscapes_demo.png --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppseg --model Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppseg --model Unet_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppseg --model PP_HumanSegV1_Server_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppmatting --model PP-Matting-512 --image matting_input.jpg --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppmatting --model PPHumanMatting --image matting_input.jpg --config_path $CONFIG_PATH && sleep_seconds 60
-./benchmark_ppmatting --model PPModnet_MobileNetV2 --image matting_input.jpg --config_path $CONFIG_PATH && sleep_seconds 60

benchmark/cpp/benchmark_cascadercnn.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + "model.pdmodel";
-  auto params_file = FLAGS_model + sep + "model.pdiparams";
-  auto config_file = FLAGS_model + sep + "infer_cfg.yml";
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_cascade_rcnn = vision::detection::CascadeRCNN(model_file, params_file,
-                                                        config_file, option,model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_cascade_rcnn.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "cascade_rcnn_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_cascade_rcnn.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_cascade_rcnn, model_cascade_rcnn.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_dino.cc

@@ -1,118 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("im_shape",{1,2},{1,2},{1,2});
-    option.trt_option.SetShape("image", {1, 3, 320,320},{1, 3, 640, 640},
-                               {1, 3, 1280, 1280});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppdet = vision::detection::PaddleDetectionModel(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppdet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppdet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-    // 2. Test tensor diff
-    std::cout << "=============== Test tensor diff =================\n";
-    std::vector<vision::DetectionResult> batch_res;
-    std::vector<fastdeploy::FDTensor> input_tensors, output_tensors;
-    std::vector<cv::Mat> imgs;
-    imgs.push_back(im);
-    std::vector<vision::FDMat> fd_images = vision::WrapMat(imgs);
-
-    model_ppdet.GetPreprocessor().Run(&fd_images, &input_tensors);
-    input_tensors[0].name = "image";
-    input_tensors[1].name = "scale_factor";
-    input_tensors[2].name = "im_shape";
-    input_tensors.pop_back();
-    model_ppdet.Infer(input_tensors, &output_tensors);
-    model_ppdet.GetPostprocessor().Run(output_tensors, &batch_res);
-    // Save tensor to -> disk.
-    auto& tensor_dump = output_tensors[0];
-    std::string det_tensor_path = "ppdet_tensor.txt";
-    benchmark::ResultManager::SaveFDTensor(tensor_dump, det_tensor_path);
-    // Load tensor from <- disk.
-    fastdeploy::FDTensor tensor_loaded;
-    benchmark::ResultManager::LoadFDTensor(&tensor_loaded, det_tensor_path);
-    // Calculate diff between two tensors.
-    auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis(
-        tensor_dump, tensor_loaded);
-    std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean
-              << ", max=" << det_tensor_diff.data.max
-              << ", min=" << det_tensor_diff.data.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppdet.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppdet, model_ppdet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res,0.3);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_fasterrcnn.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_fasterrcnn = vision::detection::FasterRCNN(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_fasterrcnn.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "fasterrcnn_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_fasterrcnn.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_fasterrcnn, model_fasterrcnn.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_fcos.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_fcos = vision::detection::FCOS(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_fcos.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "fcos_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_fcos.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_fcos, model_fcos.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_gfl.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_gfl = vision::detection::GFL(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_gfl.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "gfl_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_gfl.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_gfl, model_gfl.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_gpu.sh

@@ -1,111 +0,0 @@
-# Run all models specify hardware and specify backend
-
-CONFIG_PATH="config.gpu.paddle.fp32.txt"
-if [ ! "$1" = "$CONFIG_PATH" ]; then
-  if [ -f "$1" ]; then
-    CONFIG_PATH="$1"
-  fi
-fi
-
-# PaddleClas
-./benchmark_ppcls --model PPLCNet_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPLCNetV2_base_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB7_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x0_5_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x0_35_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_large_x1_0_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x2_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model SqueezeNet1_1_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model InceptionV3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_vd_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_tiny_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_base_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ViT_large_patch16_224_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNeXt50_32x4d_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model DenseNet121_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model person_exists_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-
-# PaddleOCR
-./benchmark_ppocr_det --model ch_PP-OCRv3_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-./benchmark_ppocr_table --model en_ppstructure_mobile_v2.0_SLANet_infer --image table.jpg --table_char_dict_path table_structure_dict.txt --config_path $CONFIG_PATH
-
-# PaddleDetection
-./benchmark_ppyolov5 --model yolov5_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolox --model yolox_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_640_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolox --model yolox_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_picodet --model picodet_l_640_coco_lcnet_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-
-./benchmark_ppyolo --model ppyolo_r50vd_dcn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_yolov3 --model yolov3_darknet53_270e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolov2_r101vd_dcn_365e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_320_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_r50_vd_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_maskrcnn --model mask_rcnn_r50_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_vd_fpn_ssld_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fcos --model fcos_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_gfl --model gfl_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r101_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_tood --model tood_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ttfnet --model ttfnet_darknet53_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_400e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_x_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_enhance_3x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_80e_sliced_visdrone_640_025 --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_mobilenet_v1_300_120e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_vgg16_300_240e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssdlite_mobilenet_v1_300_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_x_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_l_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_m_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_n_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-# PaddleSeg
-./benchmark_ppseg --model Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV2_Lite_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV1_Lite_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV2_Mobile_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model SegFormer_B0-cityscapes-with-argmax --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PP-Matting-512 --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPHumanMatting --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPModnet_MobileNetV2 --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model Unet_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV1_Server_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model FCN_HRNet_W18_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH

benchmark/cpp/benchmark_gpu_trt.sh

@@ -1,103 +0,0 @@
-# Run all models specify hardware and specify backend
-
-CONFIG_PATH="config.gpu.paddle_trt.fp32.txt"
-if [ ! "$1" = "$CONFIG_PATH" ]; then
-  if [ -f "$1" ]; then
-    CONFIG_PATH="$1"
-  fi
-fi
-
-# PaddleClas
-./benchmark_ppcls --model PPLCNet_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model PPLCNetV2_base_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model EfficientNetB7_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,600,600:1,3,600,600:1,3,600,600" --input_name "x"
-./benchmark_ppcls --model EfficientNetB0_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model GhostNet_x0_5_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model GhostNet_x1_3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model GhostNet_x1_3_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV1_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV1_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV2_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV3_small_x0_35_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model MobileNetV3_large_x1_0_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model ShuffleNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model ShuffleNetV2_x2_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model SqueezeNet1_1_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model InceptionV3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,299,299:1,3,299,299:1,3,299,299" --input_name "x"
-./benchmark_ppcls --model ResNet50_vd_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model PPHGNet_tiny_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model PPHGNet_base_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model ResNet50_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model EfficientNetB0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model MobileNetV2_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model MobileNetV3_small_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model ViT_large_patch16_224_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model ResNeXt50_32x4d_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model DenseNet121_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "inputs"
-./benchmark_ppcls --model PPHGNet_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-./benchmark_ppcls --model person_exists_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH --trt_shape "1,3,224,224:1,3,224,224:1,3,224,224" --input_name "x"
-
-# PaddleOCR
-./benchmark_ppocr_det --model ch_PP-OCRv3_det_infer --image 12.jpg --config_path $CONFIG_PATH --trt_shape 1,3,960,608:1,3,960,608:1,3,960,608
-./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH --trt_shape 1,3,48,192:1,3,48,192:1,3,48,192
-./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH --trt_shape 1,3,48,572:1,3,48,572:1,3,48,572
-./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH --trt_shape 1,3,960,608:1,3,960,608:1,3,960,608
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH --trt_shape 1,3,32,10:1,3,32,320:1,3,32,2304
-
-
-# PaddleDetection
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolox --model yolox_s_300e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_640_coco_lcnet_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_trt_nms --image 000000014439.jpg --config_path $CONFIG_PATH
-
-./benchmark_ppyolov5 --model yolov5_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolox --model yolox_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_640_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolox --model yolox_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_picodet --model picodet_l_640_coco_lcnet_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_yolov3 --model yolov3_darknet53_270e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_320_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_400e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_x_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_80e_sliced_visdrone_640_025 --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_x_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_l_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_m_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_n_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-# PaddleSeg
-./benchmark_ppseg --model Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --trt_shape 1,3,144,256:1,3,144,256:1,3,144,256
-./benchmark_ppseg --model PP_HumanSegV2_Lite_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --trt_shape 1,3,192,192:1,3,192,192:1,3,192,192
-./benchmark_ppseg --model PP_HumanSegV1_Lite_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --trt_shape 1,3,192,192:1,3,192,192:1,3,192,192
-./benchmark_ppseg --model PP_HumanSegV2_Mobile_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --trt_shape 1,3,192,192:1,3,192,192:1,3,192,192
-./benchmark_ppseg --model Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppseg --model PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppseg --model SegFormer_B0-cityscapes-with-argmax --image cityscapes_demo.png --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppmatting --model PP-Matting-512 --image matting_input.jpg --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppmatting --model PPHumanMatting --image matting_input.jpg --config_path $CONFIG_PATH --trt_shape 1,3,2048,2048:1,3,2048,2048:1,3,2048,2048
-./benchmark_ppmatting --model PPModnet_MobileNetV2 --image matting_input.jpg --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppseg --model Unet_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppseg --model PP_HumanSegV1_Server_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512
-./benchmark_ppseg --model FCN_HRNet_W18_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --trt_shape 1,3,512,512:1,3,512,512:1,3,512,512

benchmark/cpp/benchmark_maskrcnn.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_maskrcnn = vision::detection::MaskRCNN(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_maskrcnn.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "maskrcnn_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_maskrcnn.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_maskrcnn, model_maskrcnn.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_picodet.cc

@@ -1,88 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2}, {1, 2});
-  }
-  auto model_picodet = vision::detection::PicoDet(
-      model_file, params_file, config_file, option, model_format);
-  if (FLAGS_no_nms) {
-    model_picodet.GetPostprocessor().ApplyNMS();
-  }
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_picodet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "picodet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  BENCHMARK_MODEL(model_picodet, model_picodet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res, 0.5f);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppcls.cc

@@ -1,93 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
- 
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,224,224:1,3,224,224:1,3,224,224",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,224,224:1,3,224,224:1,3,224,224");
- 
-DEFINE_string(input_name, "x",
-              "Set input name for trt/paddle_trt backend."
-              "eg:--input_names x");
- 
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  // Set max_batch_size 1 for best performance
-  if (config_info["backend"] == "paddle_trt") {
-    option.trt_option.max_batch_size = 1;
-  }
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape(FLAGS_input_name, trt_shapes[0], 
-                               trt_shapes[1], trt_shapes[2]);
-  }
- 
-  auto model_ppcls = vision::classification::PaddleClasModel(
-      model_file, params_file, config_file, option, model_format);
-  vision::ClassifyResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppcls.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string cls_result_path = "ppcls_result.txt";
-    benchmark::ResultManager::SaveClassifyResult(res, cls_result_path);
-    // Load result from <- disk.
-    vision::ClassifyResult res_loaded;
-    benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path);
-    // Calculate diff between two results.
-    auto cls_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Labels diff: mean=" << cls_diff.labels.mean
-              << ", max=" << cls_diff.labels.max
-              << ", min=" << cls_diff.labels.min << std::endl;
-    std::cout << "Scores diff: mean=" << cls_diff.scores.mean
-              << ", max=" << cls_diff.scores.max
-              << ", min=" << cls_diff.scores.min << std::endl;
-  }
-  BENCHMARK_MODEL(model_ppcls, model_ppcls.Predict(im, &res))
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppdet.cc

@@ -1,117 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppdet = vision::detection::PaddleDetectionModel(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppdet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppdet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-    // 2. Test tensor diff
-    std::cout << "=============== Test tensor diff =================\n";
-    std::vector<vision::DetectionResult> batch_res;
-    std::vector<fastdeploy::FDTensor> input_tensors, output_tensors;
-    std::vector<cv::Mat> imgs;
-    imgs.push_back(im);
-    std::vector<vision::FDMat> fd_images = vision::WrapMat(imgs);
-
-    model_ppdet.GetPreprocessor().Run(&fd_images, &input_tensors);
-    input_tensors[0].name = "image";
-    input_tensors[1].name = "scale_factor";
-    input_tensors[2].name = "im_shape";
-    input_tensors.pop_back();
-    model_ppdet.Infer(input_tensors, &output_tensors);
-    model_ppdet.GetPostprocessor().Run(output_tensors, &batch_res);
-    // Save tensor to -> disk.
-    auto& tensor_dump = output_tensors[0];
-    std::string det_tensor_path = "ppdet_tensor.txt";
-    benchmark::ResultManager::SaveFDTensor(tensor_dump, det_tensor_path);
-    // Load tensor from <- disk.
-    fastdeploy::FDTensor tensor_loaded;
-    benchmark::ResultManager::LoadFDTensor(&tensor_loaded, det_tensor_path);
-    // Calculate diff between two tensors.
-    auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis(
-        tensor_dump, tensor_loaded);
-    std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean
-              << ", max=" << det_tensor_diff.data.max
-              << ", min=" << det_tensor_diff.data.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppdet.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppdet, model_ppdet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res,0.3);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppmatting.cc

@@ -1,90 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,512,512:1,3,512,512:1,3,512,512",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,512,512:1,3,512,512:1,3,512,512");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("img", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-  auto model_ppmatting = vision::matting::PPMatting(
-      model_file, params_file, config_file, option, model_format);
-  vision::MattingResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppmatting.Predict(&im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string matting_result_path = "ppmatting_result.txt";
-    benchmark::ResultManager::SaveMattingResult(res, matting_result_path);
-    // Load result from <- disk.
-    vision::MattingResult res_loaded;
-    benchmark::ResultManager::LoadMattingResult(&res_loaded,
-                                                matting_result_path);
-    // Calculate diff between two results.
-    auto matting_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Alpha diff: mean=" << matting_diff.alpha.mean
-              << ", max=" << matting_diff.alpha.max
-              << ", min=" << matting_diff.alpha.min << std::endl;
-    if (res_loaded.contain_foreground) {
-      std::cout << "Foreground diff: mean=" << matting_diff.foreground.mean
-                << ", max=" << matting_diff.foreground.max
-                << ", min=" << matting_diff.foreground.min << std::endl;
-    }
-  }
-  BENCHMARK_MODEL(model_ppmatting, model_ppmatting.Predict(&im, &res))
-  auto vis_im = vision::VisMatting(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppocr_cls.cc

@@ -1,78 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,48,10:4,3,48,320:8,3,48,1024",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,48,10:4,3,48,320:8,3,48,1024");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info, false)) {
-    return -1;
-  }
-  // Classification Model
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("x", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-  auto model_ppocr_cls =
-      vision::ocr::Classifier(model_file, params_file, option, model_format);
-  int32_t res_label;
-  float res_score;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppocr_cls.Predict(im, &res_label, &res_score);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    int32_t res_label_expect = 0;
-    float res_score_expect = 1.0;
-    // Calculate diff between two results.
-    auto ppocr_cls_label_diff = res_label - res_label_expect;
-    auto ppocr_cls_score_diff = res_score - res_score_expect;
-    std::cout << "PPOCR Cls label diff: " << ppocr_cls_label_diff << std::endl;
-    std::cout << "PPOCR Cls score diff: " << abs(ppocr_cls_score_diff)
-              << std::endl;
-  }
-  BENCHMARK_MODEL(model_ppocr_cls,
-                  model_ppocr_cls.Predict(im, &res_label, &res_score));
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppocr_det.cc

@@ -1,82 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,64,64:1,3,640,640:1,3,960,960",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,64,64:1,3,640,640:1,3,960,960");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  // Detection Model
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info, false)) {
-    return -1;
-  }
-  // Classification Model
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("x", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-  auto model_ppocr_det =
-      vision::ocr::DBDetector(model_file, params_file, option, model_format);
-  std::vector<std::array<int, 8>> res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppocr_det.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string ppocr_det_result_path = "ppocr_det_result.txt";
-    benchmark::ResultManager::SaveOCRDetResult(res, ppocr_det_result_path);
-    // Load result from <- disk.
-    std::vector<std::array<int, 8>> res_loaded;
-    benchmark::ResultManager::LoadOCRDetResult(&res_loaded,
-                                               ppocr_det_result_path);
-    // Calculate diff between two results.
-    auto ppocr_det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "PPOCR Boxes diff: mean=" << ppocr_det_diff.boxes.mean
-              << ", max=" << ppocr_det_diff.boxes.max
-              << ", min=" << ppocr_det_diff.boxes.min << std::endl;
-  }
-  BENCHMARK_MODEL(model_ppocr_det, model_ppocr_det.Predict(im, &res));
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppocr_rec.cc

@@ -1,83 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR.");
-DEFINE_string(trt_shape, "1,3,48,10:4,3,48,320:8,3,48,2304",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,48,10:4,3,48,320:8,3,48,2304");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info, false)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("x", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-  auto model_ppocr_rec = vision::ocr::Recognizer(
-      model_file, params_file, FLAGS_rec_label_file, option, model_format);
-  std::vector<std::string> model_names;
-  fastdeploy::benchmark::Split(FLAGS_model, model_names, sep);
-  if (model_names[model_names.size() - 1] == "ch_PP-OCRv2_rec_infer") {
-    model_ppocr_rec.GetPreprocessor().SetRecImageShape({3, 32, 320});
-  }
-  std::string text;
-  float rec_score;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppocr_rec.Predict(im, &text, &rec_score);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    std::string text_expect = "上海斯格威铂尔大酒店";
-    float res_score_expect = 0.993308;
-    // Calculate diff between two results.
-    auto ppocr_rec_text_diff = text.compare(text_expect);
-    auto ppocr_rec_score_diff = rec_score - res_score_expect;
-    std::cout << "PPOCR Rec text diff: " << ppocr_rec_text_diff << std::endl;
-    std::cout << "PPOCR Rec score diff: " << abs(ppocr_rec_score_diff)
-              << std::endl;
-  }
-  BENCHMARK_MODEL(model_ppocr_rec,
-                  model_ppocr_rec.Predict(im, &text, &rec_score));
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppseg.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,192,192:1,3,192,192:1,3,192,192",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,192,192:1,3,192,192:1,3,192,192");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("x", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-  auto model_ppseg = vision::segmentation::PaddleSegModel(
-      model_file, params_file, config_file, option, model_format);
-  vision::SegmentationResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppseg.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string seg_result_path = "ppseg_result.txt";
-    benchmark::ResultManager::SaveSegmentationResult(res, seg_result_path);
-    // Load result from <- disk.
-    vision::SegmentationResult res_loaded;
-    benchmark::ResultManager::LoadSegmentationResult(&res_loaded,
-                                                     seg_result_path);
-    // Calculate diff between two results.
-    auto seg_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Labels diff: mean=" << seg_diff.labels.mean
-              << ", max=" << seg_diff.labels.max
-              << ", min=" << seg_diff.labels.min << std::endl;
-    if (res_loaded.contain_score_map) {
-      std::cout << "Scores diff: mean=" << seg_diff.scores.mean
-                << ", max=" << seg_diff.scores.max
-                << ", min=" << seg_diff.scores.min << std::endl;
-    }
-  }
-  BENCHMARK_MODEL(model_ppseg, model_ppseg.Predict(im, &res))
-  auto vis_im = vision::VisSegmentation(im, res, 0.5);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppshituv2_det.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2}, {1, 2});
-    option.trt_option.SetShape("im_shape", {1, 2}, {1, 2}, {1, 2});
-  }
-  auto model = vision::classification::PPShiTuV2Detector(
-      model_file, params_file, config_file, option, model_format);
-  if (FLAGS_no_nms) {
-    model.GetPostprocessor().ApplyNMS();
-  }
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppshituv2_det_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  BENCHMARK_MODEL(model, model.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res, 0.5f);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppshituv2_rec.cc

@@ -1,93 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(trt_shape, "1,3,224,224:1,3,224,224:1,3,224,224",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,224,224:1,3,224,224:1,3,224,224");
-
-DEFINE_string(input_name, "x",
-              "Set input name for trt/paddle_trt backend."
-              "eg:--input_names x");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  // Set max_batch_size 1 for best performance
-  if (config_info["backend"] == "paddle_trt") {
-    option.trt_option.max_batch_size = 1;
-  }
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape(FLAGS_input_name, trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-
-  auto model = vision::classification::PPShiTuV2Recognizer(
-      model_file, params_file, config_file, option, model_format);
-  vision::ClassifyResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string cls_result_path = "ppcls_result.txt";
-    benchmark::ResultManager::SaveClassifyResult(res, cls_result_path);
-    // Load result from <- disk.
-    vision::ClassifyResult res_loaded;
-    benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path);
-    // Calculate diff between two results.
-    auto cls_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Labels diff: mean=" << cls_diff.labels.mean
-              << ", max=" << cls_diff.labels.max
-              << ", min=" << cls_diff.labels.min << std::endl;
-    std::cout << "Scores diff: mean=" << cls_diff.scores.mean
-              << ", max=" << cls_diff.scores.max
-              << ", min=" << cls_diff.scores.min << std::endl;
-  }
-  BENCHMARK_MODEL(model, model.Predict(im, &res))
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppyolo.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolo = vision::detection::PPYOLO(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolo.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolo_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolo.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolo, model_ppyolo.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppyoloe.cc

@@ -1,88 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyoloe = vision::detection::PPYOLOE(
-      model_file, params_file, config_file, option, model_format);
-  if (FLAGS_no_nms) {
-    model_ppyoloe.GetPostprocessor().ApplyNMS();
-  }
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyoloe.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyoloe_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  BENCHMARK_MODEL(model_ppyoloe, model_ppyoloe.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppyoloe_r.cc

@@ -1,60 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <fstream>
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-
-  auto model_ppyoloe_r = vision::detection::PPYOLOER(
-      model_file, params_file, config_file, option, model_format);
-
-  vision::DetectionResult res;
-
-  // Run profiling
-  BENCHMARK_MODEL(model_ppyoloe_r, model_ppyoloe_r.Predict(im, &res))
-
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppyoloe_r_sophgo.cc

@@ -1,61 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <fstream>
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::SOPHGO;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-
-  auto model_ppyoloe_r = vision::detection::PPYOLOER(
-      model_file, params_file, config_file, option, model_format);
-
-  vision::DetectionResult res;
-
-  // Run profiling
-  BENCHMARK_MODEL(model_ppyoloe_r, model_ppyoloe_r.Predict(im, &res))
-
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_ppyolov5.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolov5 = vision::detection::PaddleYOLOv5(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolov5.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolov5_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolov5.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolov5, model_ppyolov5.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppyolov6.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolov6 = vision::detection::PaddleYOLOv6(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolov6.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolov6_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolov6.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolov6, model_ppyolov6.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppyolov7.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolov7 = vision::detection::PaddleYOLOv7(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolov7.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolov7_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolov7.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolov7, model_ppyolov7.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppyolov8.cc

@@ -1,117 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolov8 = vision::detection::PaddleYOLOv8(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolov8.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolov8_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-    // 2. Test tensor diff
-    std::cout << "=============== Test tensor diff =================\n";
-    std::vector<vision::DetectionResult> batch_res;
-    std::vector<fastdeploy::FDTensor> input_tensors, output_tensors;
-    std::vector<cv::Mat> imgs;
-    imgs.push_back(im);
-    std::vector<vision::FDMat> fd_images = vision::WrapMat(imgs);
-
-    model_ppyolov8.GetPreprocessor().Run(&fd_images, &input_tensors);
-    input_tensors[0].name = "image";
-    input_tensors[1].name = "scale_factor";
-    input_tensors[2].name = "im_shape";
-    input_tensors.pop_back();
-    model_ppyolov8.Infer(input_tensors, &output_tensors);
-    model_ppyolov8.GetPostprocessor().Run(output_tensors, &batch_res);
-    // Save tensor to -> disk.
-    auto& tensor_dump = output_tensors[0];
-    std::string det_tensor_path = "ppyolov8_tensor.txt";
-    benchmark::ResultManager::SaveFDTensor(tensor_dump, det_tensor_path);
-    // Load tensor from <- disk.
-    fastdeploy::FDTensor tensor_loaded;
-    benchmark::ResultManager::LoadFDTensor(&tensor_loaded, det_tensor_path);
-    // Calculate diff between two tensors.
-    auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis(
-        tensor_dump, tensor_loaded);
-    std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean
-              << ", max=" << det_tensor_diff.data.max
-              << ", min=" << det_tensor_diff.data.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolov8.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolov8, model_ppyolov8.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ppyolox.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ppyolox = vision::detection::PaddleYOLOX(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ppyolox.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ppyolox_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ppyolox.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ppyolox, model_ppyolox.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_retinanet.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_retinanet = vision::detection::RetinaNet(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_retinanet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "retinanet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_retinanet.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_retinanet, model_retinanet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_rtmdet.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_rtmdet = vision::detection::RTMDet(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_rtmdet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "rtmdet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_rtmdet.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_rtmdet, model_rtmdet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ssd.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ssd = vision::detection::SSD(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ssd.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ssd_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ssd.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ssd, model_ssd.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_structurev2_layout.cc

@@ -1,93 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info, false)) {
-    return -1;
-  }
-
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 800, 608}, {1, 3, 800, 608},
-                               {1, 3, 800, 608});
-  }
-  auto layout_model = vision::ocr::StructureV2Layout(model_file, params_file,
-                                                     option, model_format);
-  // 5 for publaynet, 10 for cdla
-  layout_model.GetPostprocessor().SetNumClass(5);
-
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    layout_model.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string layout_result_path = "layout_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, layout_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded,
-                                                  layout_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  BENCHMARK_MODEL(layout_model, layout_model.Predict(im, &res))
-  std::vector<std::string> labels = {"text", "title", "list", "table",
-                                     "figure"};
-  if (layout_model.GetPostprocessor().GetNumClass() == 10) {
-    labels = {"text",      "title",         "figure", "figure_caption",
-              "table",     "table_caption", "header", "footer",
-              "reference", "equation"};
-  }
-  auto vis_im =
-      vision::VisDetection(im, res, labels, 0.3, 2, .5f, {255, 0, 0}, 2);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_structurev2_table.cc

@@ -1,161 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_string(table_char_dict_path, "",
-              "Path of table character dict of PPOCR.");
-DEFINE_string(trt_shape, "1,3,48,10:4,3,48,320:8,3,48,2304",
-              "Set min/opt/max shape for trt/paddle_trt backend."
-              "eg:--trt_shape 1,3,48,10:4,3,48,320:8,3,48,2304");
-
-int main(int argc, char *argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info, false)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    std::vector<std::vector<int32_t>> trt_shapes =
-        benchmark::ResultManager::GetInputShapes(FLAGS_trt_shape);
-    option.trt_option.SetShape("x", trt_shapes[0], trt_shapes[1],
-                               trt_shapes[2]);
-  }
-
-  auto model_ppocr_table = vision::ocr::StructureV2Table(
-      model_file, params_file, FLAGS_table_char_dict_path, option,
-      model_format);
-  fastdeploy::vision::OCRResult result;
-
-  if (config_info["precision_compare"] == "true") {
-    std::string expect_structure_html =
-        "<html><body><table><thead><tr><td></td><td></td><td></td><td></"
-        "td><td></td></tr></thead><tbody><tr><td></td><td></td><td></td><td></"
-        "td><td></td></tr><tr><td></td><td></td><td></td><td></td><td></td></"
-        "tr><tr><td></td><td></td><td></td><td></td><td></td></tr><tr><td></"
-        "td><td></td><td></td><td></td><td></td></tr><tr><td></td><td></"
-        "td><td></td><td></td><td></td></tr><tr><td></td><td></td><td></"
-        "td><td></td><td></td></tr><tr><td></td><td></td><td></td><td></"
-        "td><td></td></tr><tr><td></td><td></td><td></td><td></td><td></td></"
-        "tr><tr><td></td><td></td><td></td><td></td><td></td></tr><tr><td></"
-        "td><td></td><td></td><td></td><td></td></tr><tr><td></td><td></"
-        "td><td></td><td></td><td></td></tr><tr><td></td><td></td><td></"
-        "td><td></td><td></td></tr><tr><td></td><td></td><td></td><td></"
-        "td><td></td></tr><tr><td></td><td></td><td></td><td></td><td></td></"
-        "tr><tr><td></td><td></td><td></td><td></td><td></td></tr></tbody></"
-        "table></body></html>";
-    std::vector<int> expect_box_coord{
-        41,  4,   97,  18,  161, 4,   173, 18,  216, 4,   225, 17,  272, 4,
-        283, 17,  321, 4,   348, 18,  33,  20,  106, 38,  150, 22,  180, 38,
-        202, 22,  235, 38,  262, 21,  293, 38,  326, 23,  343, 37,  27,  38,
-        109, 56,  150, 39,  179, 56,  204, 39,  236, 56,  263, 39,  292, 55,
-        329, 40,  343, 54,  22,  57,  118, 74,  152, 58,  176, 74,  204, 58,
-        236, 75,  262, 58,  291, 74,  326, 58,  344, 74,  27,  75,  119, 92,
-        150, 75,  177, 92,  204, 75,  235, 92,  260, 75,  292, 92,  326, 75,
-        346, 92,  44,  92,  102, 110, 150, 92,  177, 110, 205, 92,  236, 110,
-        262, 92,  290, 110, 329, 93,  339, 110, 41,  109, 102, 128, 151, 110,
-        175, 128, 205, 110, 236, 128, 262, 110, 291, 127, 329, 110, 338, 127,
-        42,  128, 102, 146, 149, 128, 177, 146, 205, 128, 237, 146, 262, 128,
-        291, 146, 329, 128, 339, 145, 31,  145, 110, 163, 150, 145, 178, 163,
-        206, 145, 237, 164, 262, 145, 292, 163, 324, 145, 342, 162, 40,  162,
-        108, 180, 154, 162, 175, 180, 209, 162, 231, 180, 266, 162, 286, 180,
-        325, 162, 341, 179, 38,  180, 105, 197, 152, 180, 177, 197, 207, 180,
-        236, 197, 262, 180, 291, 197, 329, 181, 339, 196, 42,  196, 102, 214,
-        151, 197, 179, 214, 205, 197, 236, 214, 263, 197, 291, 214, 320, 197,
-        349, 214, 46,  215, 100, 233, 149, 216, 179, 233, 204, 216, 238, 233,
-        262, 216, 291, 233, 321, 216, 345, 232, 42,  233, 104, 251, 147, 234,
-        179, 251, 203, 233, 237, 251, 260, 233, 294, 251, 326, 234, 341, 250,
-        19,  251, 120, 269, 148, 253, 180, 270, 202, 252, 240, 270, 259, 252,
-        294, 270, 324, 252, 347, 268, 16,  270, 123, 286, 146, 270, 182, 287,
-        200, 270, 238, 287, 256, 270, 294, 286, 319, 270, 353, 286};
-
-    // Run once at least
-    if (!model_ppocr_table.Predict(im, &result)) {
-      std::cerr << "Failed to predict." << std::endl;
-      return -1;
-    }
-
-    // 1. Test result diff
-    std::cout << "=============== Test Table Result diff =================\n";
-    // Calculate diff between two results.
-    std::string result_table_structure;
-    for (auto &structure : result.table_structure) {
-      result_table_structure += structure;
-    }
-    if (expect_structure_html == result_table_structure) {
-      std::cout << "PPOCR Table structure has no diff" << std::endl;
-    } else {
-      std::cout << "PPOCR Table structure has diff" << std::endl;
-      std::cout << "expected: " << expect_structure_html << std::endl;
-      std::cout << "result: " << result_table_structure << std::endl;
-    }
-
-    std::vector<int> table_box_coord;
-    for (auto &box : result.table_boxes) {
-      // x1 y1 x2 y1 x2 y2 x1 y2 => x1 y1 x2 y2
-      table_box_coord.push_back(box[0]);
-      table_box_coord.push_back(box[1]);
-      table_box_coord.push_back(box[2]);
-      table_box_coord.push_back(box[5]);
-    }
-
-    if (expect_box_coord.size() == table_box_coord.size()) {
-      std::cout << "table boxes num matched with expected: "
-                << table_box_coord.size() << std::endl;
-      int max_diff = 0;
-      int total_diff = 0;
-      for (int i = 0; i < table_box_coord.size(); i++) {
-        int diff = std::abs(table_box_coord[i] - expect_box_coord[i]);
-        if (diff > max_diff) {
-          max_diff = diff;
-        }
-        total_diff += diff;
-      }
-      std::cout << "box coords, max_diff: " << max_diff << ", "
-                << ", total diff: " << total_diff << ", average diff: "
-                << total_diff / float(table_box_coord.size()) << std::endl;
-    } else {
-      std::cout << "boxes num has diff, expect box num: "
-                << expect_box_coord.size() / 4
-                << ", result box num:" << table_box_coord.size() / 4
-                << std::endl;
-    }
-  }
-
-  BENCHMARK_MODEL(model_ppocr_table, model_ppocr_table.Predict(im, &result));
-#endif
-  return 0;
-}

benchmark/cpp/benchmark_tood.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_tood = vision::detection::TOOD(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_tood.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "tood_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_tood.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_tood, model_tood.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_ttfnet.cc

@@ -1,89 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  if (config_info["backend"] == "paddle_trt") {
-    option.paddle_infer_option.collect_trt_shape = true;
-  }
-  if (config_info["backend"] == "paddle_trt" ||
-      config_info["backend"] == "trt") {
-    option.trt_option.SetShape("image", {1, 3, 640, 640}, {1, 3, 640, 640},
-                               {1, 3, 640, 640});
-    option.trt_option.SetShape("scale_factor", {1, 2}, {1, 2},
-                               {1, 2});
-  }
-  auto model_ttfnet = vision::detection::TTFNet(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_ttfnet.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "ttfnet_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_ttfnet.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_ttfnet, model_ttfnet.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_x86.sh

@@ -1,109 +0,0 @@
-# Run all models specify hardware and specify backend
-
-CONFIG_PATH="config.x86.paddle.fp32.txt"
-if [ ! "$1" = "$CONFIG_PATH" ]; then
-  if [ -f "$1" ]; then
-    CONFIG_PATH="$1"
-  fi
-fi
-
-# PaddleClas
-./benchmark_ppcls --model PPLCNet_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPLCNetV2_base_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB7_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x0_5_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x0_35_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_large_x1_0_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x2_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model SqueezeNet1_1_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model InceptionV3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_vd_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_tiny_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_base_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ViT_large_patch16_224_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNeXt50_32x4d_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model DenseNet121_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model person_exists_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-
-# PaddleOCR
-./benchmark_ppocr_det --model ch_PP-OCRv3_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-
-# PaddleDetection
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolox --model yolox_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_picodet --model picodet_l_640_coco_lcnet_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-
-./benchmark_ppyolov6 --model yolov6_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolox --model yolox_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_640_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolo_r50vd_dcn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_yolov3 --model yolov3_darknet53_270e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolov2_r101vd_dcn_365e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_320_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_r50_vd_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_maskrcnn --model mask_rcnn_r50_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_vd_fpn_ssld_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fcos --model fcos_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_gfl --model gfl_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r101_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_tood --model tood_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ttfnet --model ttfnet_darknet53_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_400e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_x_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_enhance_3x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_80e_sliced_visdrone_640_025 --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_mobilenet_v1_300_120e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_vgg16_300_240e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssdlite_mobilenet_v1_300_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_x_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_l_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_m_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_n_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-# PaddleSeg
-./benchmark_ppseg --model Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV2_Lite_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV1_Lite_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV2_Mobile_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model SegFormer_B0-cityscapes-with-argmax --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer --image cityscapes_demo.png --warmup 10 --repeat 50 --config_path $CONFIG_PATH
-./benchmark_ppseg --model Unet_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH
-./benchmark_ppseg --model PP_HumanSegV1_Server_with_argmax_infer --image portrait_heng.jpg --warmup 10 --repeat 50 --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PP-Matting-512 --image matting_input.jpg --warmup 10 --repeat 50 --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPHumanMatting --image matting_input.jpg --warmup 10 --repeat 50 --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPModnet_MobileNetV2 --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppseg --model FCN_HRNet_W18_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH

benchmark/cpp/benchmark_xpu.sh

@@ -1,109 +0,0 @@
-# Run all models specify hardware and specify backend
-
-CONFIG_PATH="config.xpu.lite.fp32.txt"
-if [ ! "$1" = "$CONFIG_PATH" ]; then
-  if [ -f "$1" ]; then
-    CONFIG_PATH="$1"
-  fi
-fi
-
-# PaddleClas
-./benchmark_ppcls --model PPLCNet_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPLCNetV2_base_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB7_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x0_5_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model GhostNet_x1_3_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV1_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x0_35_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_large_x1_0_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x0_25_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ShuffleNetV2_x2_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model SqueezeNet1_1_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model InceptionV3_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_vd_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_tiny_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_base_ssld_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNet50_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model EfficientNetB0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV2_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model MobileNetV3_small_x1_0_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ViT_large_patch16_224_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model ResNeXt50_32x4d_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model DenseNet121_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model PPHGNet_small_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-./benchmark_ppcls --model person_exists_infer --image ILSVRC2012_val_00000010.jpeg --config_path $CONFIG_PATH
-
-# PaddleOCR
-./benchmark_ppocr_det --model ch_PP-OCRv3_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH
-
-# PaddleDetection
-./benchmark_ppyolov5 --model yolov5_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov6 --model yolov6_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov7 --model yolov7_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolov8 --model yolov8_s_500e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyolox --model yolox_s_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-./benchmark_picodet --model picodet_l_640_coco_lcnet_no_nms --image 000000014439.jpg --config_path $CONFIG_PATH --no_nms
-
-./benchmark_ppyolov5 --model yolov5_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolox --model yolox_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_plus_crn_m_80e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_640_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolo_r50vd_dcn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_yolov3 --model yolov3_darknet53_270e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolo --model ppyolov2_r101vd_dcn_365e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_picodet --model picodet_l_320_coco_lcnet --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_r50_vd_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_maskrcnn --model mask_rcnn_r50_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_rtmdet --model rtmdet_s_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_cascadercnn --model cascade_rcnn_r50_vd_fpn_ssld_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fcos --model fcos_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_gfl --model gfl_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r101_fpn_2x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_retinanet --model retinanet_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_tood --model tood_r50_fpn_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ttfnet --model ttfnet_darknet53_1x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov5 --model yolov5_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov6 --model yolov6_s_400e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_l_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov7 --model yolov7_x_300e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_fasterrcnn --model faster_rcnn_enhance_3x_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyoloe --model ppyoloe_crn_l_80e_sliced_visdrone_640_025 --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_mobilenet_v1_300_120e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssd_vgg16_300_240e_voc --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ssd --model ssdlite_mobilenet_v1_300_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8 --model yolov8_x_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8  --model yolov8_l_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8  --model yolov8_m_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-./benchmark_ppyolov8  --model yolov8_n_500e_coco --image 000000014439.jpg --config_path $CONFIG_PATH
-
-# PaddleSeg
-./benchmark_ppseg --model Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model PP_HumanSegV2_Lite_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model PP_HumanSegV1_Lite_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model PP_HumanSegV2_Mobile_192x192_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model FCN_HRNet_W18_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model SegFormer_B0-cityscapes-with-argmax --image cityscapes_demo.png --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model Unet_cityscapes_with_argmax_infer --image cityscapes_demo.png --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppseg --model PP_HumanSegV1_Server_with_argmax_infer --image portrait_heng.jpg --config_path $CONFIG_PATH --xpu_l3_cache 0
-./benchmark_ppmatting --model PP-Matting-512 --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPHumanMatting --image matting_input.jpg --config_path $CONFIG_PATH
-./benchmark_ppmatting --model PPModnet_MobileNetV2 --image matting_input.jpg --config_path $CONFIG_PATH

benchmark/cpp/benchmark_yolov3.cc

@@ -1,79 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-DEFINE_bool(no_nms, false, "Whether the model contains nms.");
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  std::string model_name, params_name, config_name;
-  auto model_format = fastdeploy::ModelFormat::PADDLE;
-  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
-                               &model_format, config_info)) {
-    return -1;
-  }
-  auto model_file = FLAGS_model + sep + model_name;
-  auto params_file = FLAGS_model + sep + params_name;
-  auto config_file = FLAGS_model + sep + config_name;
-  auto model_yolov3 = vision::detection::YOLOv3(
-      model_file, params_file, config_file, option, model_format);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_yolov3.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "yolov3_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  // Run profiling
-  if (FLAGS_no_nms) {
-    model_yolov3.GetPostprocessor().ApplyNMS();
-  }
-  BENCHMARK_MODEL(model_yolov3, model_yolov3.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-
-  return 0;
-}

benchmark/cpp/benchmark_yolov5.cc

@@ -1,62 +0,0 @@
-// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "flags.h"
-#include "macros.h"
-#include "option.h"
-
-namespace vision = fastdeploy::vision;
-namespace benchmark = fastdeploy::benchmark;
-
-int main(int argc, char* argv[]) {
-#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
-  // Initialization
-  auto option = fastdeploy::RuntimeOption();
-  if (!CreateRuntimeOption(&option, argc, argv, true)) {
-    return -1;
-  }
-  auto im = cv::imread(FLAGS_image);
-  std::unordered_map<std::string, std::string> config_info;
-  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
-                                                &config_info);
-  auto model_yolov5 = vision::detection::YOLOv5(FLAGS_model, "", option);
-  vision::DetectionResult res;
-  if (config_info["precision_compare"] == "true") {
-    // Run once at least
-    model_yolov5.Predict(im, &res);
-    // 1. Test result diff
-    std::cout << "=============== Test result diff =================\n";
-    // Save result to -> disk.
-    std::string det_result_path = "yolov5_result.txt";
-    benchmark::ResultManager::SaveDetectionResult(res, det_result_path);
-    // Load result from <- disk.
-    vision::DetectionResult res_loaded;
-    benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
-    // Calculate diff between two results.
-    auto det_diff =
-        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
-    std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-              << ", max=" << det_diff.boxes.max
-              << ", min=" << det_diff.boxes.min << std::endl;
-    std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
-              << ", max=" << det_diff.labels.max
-              << ", min=" << det_diff.labels.min << std::endl;
-  }
-  BENCHMARK_MODEL(model_yolov5, model_yolov5.Predict(im, &res))
-  auto vis_im = vision::VisDetection(im, res);
-  cv::imwrite("vis_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-#endif
-  return 0;
-}

benchmark/cpp/config/config.arm.lite.fp16.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 10
-repeat: 20
-backend: lite
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp16_e2e_mem.txt

benchmark/cpp/config/config.arm.lite.fp16.e2e.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: lite
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp16_e2e.txt

benchmark/cpp/config/config.arm.lite.fp16.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: lite
-profile_mode: runtime
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp16.txt

benchmark/cpp/config/config.arm.lite.fp32.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 10
-repeat: 20
-backend: lite
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp32_e2e_mem.txt

benchmark/cpp/config/config.arm.lite.fp32.e2e.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: lite
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp32_e2e.txt

benchmark/cpp/config/config.arm.lite.fp32.txt

@@ -1,14 +0,0 @@
-device: cpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: lite
-profile_mode: runtime
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_arm_lite_fp32.txt

benchmark/cpp/config/config.gpu.ort.fp32.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: ort
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_ort_fp32_e2e_mem.txt

benchmark/cpp/config/config.gpu.ort.fp32.e2e.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: ort
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_ort_fp32_e2e.txt

benchmark/cpp/config/config.gpu.ort.fp32.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: ort
-profile_mode: runtime
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_ort_fp32.txt

benchmark/cpp/config/config.gpu.paddle.fp32.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: paddle
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_fp32_e2e_mem.txt

benchmark/cpp/config/config.gpu.paddle.fp32.e2e.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: paddle
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_fp32_e2e.txt

benchmark/cpp/config/config.gpu.paddle.fp32.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: paddle
-profile_mode: runtime
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_fp32.txt

benchmark/cpp/config/config.gpu.paddle_trt.fp16.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: paddle_trt
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_trt_fp16_e2e_mem.txt

benchmark/cpp/config/config.gpu.paddle_trt.fp16.e2e.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: paddle_trt
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_trt_fp16_e2e.txt

benchmark/cpp/config/config.gpu.paddle_trt.fp16.h2d.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: paddle_trt
-profile_mode: runtime
-include_h2d_d2h: true
-use_fp16: true
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_trt_fp16_h2d.txt

benchmark/cpp/config/config.gpu.paddle_trt.fp16.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 200
-repeat: 1000
-backend: paddle_trt
-profile_mode: runtime
-include_h2d_d2h: false
-use_fp16: true
-collect_memory_info: false
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_trt_fp16.txt

benchmark/cpp/config/config.gpu.paddle_trt.fp32.e2e.mem.txt

@@ -1,14 +0,0 @@
-device: gpu
-device_id: 0
-cpu_thread_nums: 1
-warmup: 20
-repeat: 100
-backend: paddle_trt
-profile_mode: end2end
-include_h2d_d2h: false
-use_fp16: false
-collect_memory_info: true
-sampling_interval: 1
-precision_compare: false
-xpu_l3_cache: 0
-result_path: benchmark_gpu_paddle_trt_fp32_e2e_mem.txt