apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

[LLM] First commit the llm deployment code

Browse Source
This commit is contained in:
jiangjiajun
2025-06-09 19:20:15 +08:00
committed by XieYunshen
parent 8513414112
commit 149c79699d
11814 changed files with 127294 additions and 1293102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
scripts/__init__.py
View File

94
scripts/android/build_android_aar.sh
View File

@@ -1,94 +0,0 @@
#!/bin/bash
set -e
set +x
FASTDEPLOY_DIR=$(pwd)
BUILT_PACKAGE_DIR=build/Android
CXX_PACKAGE_PREFIX=fastdeploy-android-latest-shared-dev
CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/${CXX_PACKAGE_PREFIX}
ARMV8_CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/arm64-v8a-api-21/install
ARMV7_CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/armeabi-v7a-api-21/install
# check package name
echo "[INFO] --- FASTDEPLOY_DIR: ${FASTDEPLOY_DIR}"
# check arm v7 & v8 c++ sdk
if [ ! -d "${BUILT_PACKAGE_DIR}" ]; then
echo "[ERROR] --- ${BUILT_PACKAGE_DIR} not exist, please build c++ sdk first!"
exit 0
fi
if [ ! -d "${ARMV8_CXX_PACKAGE_NAME}" ]; then
echo "[ERROR] --- ${ARMV8_CXX_PACKAGE_NAME} not exist, please build c++ sdk first!"
exit 0
fi
if [ ! -d "${ARMV7_CXX_PACKAGE_NAME}" ]; then
echo "[ERROR] --- ${ARMV7_CXX_PACKAGE_NAME} not exist, please build c++ sdk first!"
exit 0
fi
# remove old package
echo "[INFO] --- Packing ${CXX_PACKAGE_NAME} package ..."
if [ -d "${CXX_PACKAGE_NAME}" ]; then
rm -rf ${CXX_PACKAGE_NAME}
echo "[INFO] --- Removed old ${CXX_PACKAGE_NAME} done !"
if [ -f "${CXX_PACKAGE_NAME}.tgz" ]; then
rm ${CXX_PACKAGE_NAME}.tgz
echo "[INFO] --- Removed old ${CXX_PACKAGE_NAME} done !"
fi
fi
# package latest c++ sdk
mkdir ${CXX_PACKAGE_NAME}
echo "[INFO] --- Collecting package contents ..."
cp -r ${ARMV7_CXX_PACKAGE_NAME}/* ${CXX_PACKAGE_NAME}/
cp -r ${ARMV8_CXX_PACKAGE_NAME}/* ${CXX_PACKAGE_NAME}/
# Fix Paddle Lite headers to support FastDeploy static lib
SDK_LITE_INCLUDE_DIR=${CXX_PACKAGE_NAME}/third_libs/install/paddlelite/include
ARMV7_LITE_INCLUDE_DIR=${ARMV7_CXX_PACKAGE_NAME}/third_libs/install/paddlelite/include
ARMV8_LITE_INCLUDE_DIR=${ARMV8_CXX_PACKAGE_NAME}/third_libs/install/paddlelite/include
rm ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
cat ${ARMV8_LITE_INCLUDE_DIR}/paddle_use_kernels.h | head -n 3 >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
echo "#if (defined(__aarch64__) || defined(_M_ARM64))" >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
cat ${ARMV8_LITE_INCLUDE_DIR}/paddle_use_kernels.h | grep -v "#" | grep "USE" >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
echo "#else" >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
cat ${ARMV7_LITE_INCLUDE_DIR}/paddle_use_kernels.h | grep -v "#" | grep "USE" >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
echo "#endif" >> ${SDK_LITE_INCLUDE_DIR}/paddle_use_kernels.h
echo "[INFO] --- Fixed Paddle Lite paddle_use_kernels.h to support FastDeploy static lib."
if [ -d "${CXX_PACKAGE_NAME}/examples" ]; then
rm -rf ${CXX_PACKAGE_NAME}/examples
fi
echo "[INFO] --- Removed examples files ..."
echo "[INFO] --- Removing static .a files: "
static_files=$(find ${CXX_PACKAGE_NAME}/third_libs/install/ -name "*.a")
if [ ${#static_files[@]} -gt 10 ]; then
echo "${#static_files[@]}: ${static_files}"
rm $(find ${CXX_PACKAGE_NAME}/third_libs/install/ -name "*.a")
fi
echo "[INFO] --- Taring ${CXX_PACKAGE_NAME}.tgz package ..."
tar -zcvf ${CXX_PACKAGE_NAME}.tgz ${CXX_PACKAGE_NAME}/* >> ${BUILT_PACKAGE_DIR}/pkg.log 2>&1
echo "[INFO] --- Package ${CXX_PACKAGE_NAME}.tgz done ! Package size info: "
du -sh ${BUILT_PACKAGE_DIR}/* | grep ${CXX_PACKAGE_PREFIX}
# update c++ sdk to jni lib
echo "[INFO] --- Update c++ sdk for jni lib ..."
JAVA_ANDROID_DIR=${FASTDEPLOY_DIR}/java/android
JNI_LIB_DIR=${JAVA_ANDROID_DIR}/fastdeploy
CXX_LIB_FOR_JNI_DIR=${JNI_LIB_DIR}/libs/${CXX_PACKAGE_PREFIX}
if [ -d "${CXX_LIB_FOR_JNI_DIR}" ]; then
rm -rf ${CXX_LIB_FOR_JNI_DIR}
echo "[INFO] --- Remove old ${CXX_LIB_FOR_JNI_DIR} done!"
fi
cp -r ${CXX_PACKAGE_NAME} ${JNI_LIB_DIR}/libs
echo "[INFO] --- Update ${CXX_LIB_FOR_JNI_DIR} done!"
# build java aar package
cd ${JAVA_ANDROID_DIR}
echo "[INFO] --- JAVA_ANDROID_DIR: ${JAVA_ANDROID_DIR}"
echo "[INFO] --- Building java aar package ... "
chmod +x gradlew
./gradlew fastdeploy:assembleDebug
echo "[INFO] --- Built java aar package!"
ls -lh ${JNI_LIB_DIR}/build/outputs/aar/
# Usage:
# ./scripts/android/build_android_aar.sh

37
scripts/android/build_android_app.sh
View File

@@ -1,37 +0,0 @@
# build FastDeploy app
FASTDEPLOY_DIR=$(pwd)
JAVA_ANDROID_DIR=${FASTDEPLOY_DIR}/java/android
JNI_LIB_DIR=${JAVA_ANDROID_DIR}/fastdeploy
AAR_DEBUG_PACKAGE=${JNI_LIB_DIR}/build/outputs/aar/fastdeploy-debug.aar
APP_DIR=${JAVA_ANDROID_DIR}/app
APP_LIBS_DIR=${APP_DIR}/libs
cd ${JAVA_ANDROID_DIR}
# check aar package first!
echo "[INFO] --- JAVA_ANDROID_DIR: ${JAVA_ANDROID_DIR}"
if [ ! -d "${JNI_LIB_DIR}/build/outputs/aar" ]; then
echo "-- [ERROR] ${JNI_LIB_DIR} not exists, please build aar package first!"
exit 0
fi
ls -lh ${JNI_LIB_DIR}/build/outputs/aar/
if [ ! -d "${APP_LIBS_DIR}" ]; then
mkdir -p "${APP_LIBS_DIR}" && echo "-- [INFO] Created ${APP_LIBS_DIR} !"
fi
# update aar package
echo "[INFO] --- Update aar package ..."
if [ -f "${APP_LIBS_DIR}/fastdeploy-android-sdk-latest-dev.aar" ]; then
rm -f "${APP_LIBS_DIR}/fastdeploy-android-sdk-latest-dev.aar"
echo "[INFO] --- Removed old aar package: ${APP_LIBS_DIR}/fastdeploy-android-sdk-latest-dev.aar"
fi
cp ${AAR_DEBUG_PACKAGE} ${APP_LIBS_DIR}/fastdeploy-android-sdk-latest-dev.aar
if [ -f "${APP_LIBS_DIR}/fastdeploy-android-sdk-latest-dev.aar" ]; then
echo "[INFO] --- Update aar package done!"
fi
# build android app
echo "[INFO] --- Building FastDeploy Android App ..."
chmod +x gradlew
./gradlew app:assembleDebug
echo "[INFO] --- Built FastDeploy Android app."
# Usage:
# ./scripts/android/build_android_app.sh

117
scripts/android/build_android_cpp.sh
View File

@@ -1,117 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=OFF \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=ON \
-DWITH_ANDROID_OPENCV_STATIC=OFF \
-DWITH_ANDROID_LITE_STATIC=OFF \
-DWITH_ANDROID_OPENMP=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp.sh arm64-v8a 21
# ./scripts/android/build_android_cpp.sh armeabi-v7a 21

120
scripts/android/build_android_cpp_opencl_with_benchmark.sh
View File

@@ -1,120 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=OFF \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=OFF \
-DENABLE_BENCHMARK=ON \
-DWITH_OPENCL=ON \
-DWITH_ANDROID_OPENCV_STATIC=OFF \
-DWITH_ANDROID_LITE_STATIC=OFF \
-DWITH_ANDROID_OPENMP=OFF \
-DWITH_TESTING=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_opencl_with_benchmark.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_opencl_with_benchmark.sh armeabi-v7a 21

119
scripts/android/build_android_cpp_with_benchmark.sh
View File

@@ -1,119 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=OFF \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=OFF \
-DENABLE_BENCHMARK=ON \
-DWITH_ANDROID_OPENCV_STATIC=OFF \
-DWITH_ANDROID_LITE_STATIC=OFF \
-DWITH_ANDROID_OPENMP=OFF \
-DWITH_TESTING=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_benchmark.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_benchmark.sh armeabi-v7a 21

114
scripts/android/build_android_cpp_with_runtime_only.sh
View File

@@ -1,114 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_FLYCV=OFF \
-DENABLE_TEXT=OFF \
-DENABLE_VISION=OFF \
-DWITH_ANDROID_LITE_STATIC=ON \
-DWITH_ANDROID_TENSOR_FUNCS=OFF \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_runtime_only.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_runtime_only.sh armeabi-v7a 21

119
scripts/android/build_android_cpp_with_static_deps.sh
View File

@@ -1,119 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=OFF \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=OFF \
-DWITH_ANDROID_OPENCV_STATIC=ON \
-DWITH_ANDROID_FLYCV_STATIC=ON \
-DWITH_ANDROID_LITE_STATIC=ON \
-DWITH_ANDROID_OPENMP=ON \
-DWITH_ANDROID_JAVA=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_static_deps.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_static_deps.sh armeabi-v7a 21

120
scripts/android/build_android_cpp_with_static_deps_with_text_api.sh
View File

@@ -1,120 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=ON \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=OFF \
-DWITH_ANDROID_OPENCV_STATIC=ON \
-DWITH_ANDROID_FLYCV_STATIC=ON \
-DWITH_ANDROID_LITE_STATIC=ON \
-DWITH_ANDROID_STATIC_LIB=OFF \
-DWITH_ANDROID_OPENMP=ON \
-DWITH_ANDROID_JAVA=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_static_deps_with_text_api.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_static_deps_with_text_api.sh armeabi-v7a 21

118
scripts/android/build_android_cpp_with_text_api.sh
View File

@@ -1,118 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=ON \
-DENABLE_TEXT=ON \
-DENABLE_VISION=ON \
-DBUILD_EXAMPLES=ON \
-DWITH_ANDROID_OPENCV_STATIC=OFF \
-DWITH_ANDROID_FLYCV_STATIC=OFF \
-DWITH_ANDROID_LITE_STATIC=OFF \
-DWITH_ANDROID_OPENMP=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_text_api.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_text_api.sh armeabi-v7a 21

116
scripts/android/build_android_cpp_with_text_api_only.sh
View File

@@ -1,116 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# mutable global variables
# -------------------------------------------------------------------------------
TOOLCHAIN=clang # gcc/clang toolchain
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly ANDROID_ABI=$1
readonly ANDROID_PLATFORM="android-$2"
readonly BUILD_ROOT=build/Android
readonly BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-$2
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${ANDROID_ABI}"
unset C_INCLUDE_PATH
fi
}
__set_android_ndk() {
if [ -z $ANDROID_NDK ]; then
echo "-- [INFO] ANDROID_NDK not exists, please setup manually ..."
exit 0
else
echo "-- [INFO] Found ANDROID_NDK: ${ANDROID_NDK}"
fi
if [ "$ANDROID_NDK" ]; then
NDK_VERSION=$(echo $ANDROID_NDK | egrep -o "[0-9]{2}" | head -n 1)
if [ "$NDK_VERSION" -gt 17 ]; then
TOOLCHAIN=clang
fi
echo "-- [INFO] Checked ndk version: ${NDK_VERSION}"
echo "-- [INFO] Selected toolchain: ${TOOLCHAIN}"
fi
}
__build_fastdeploy_android_shared() {
local ANDROID_STL=c++_shared # c++_static
local ANDROID_TOOLCHAIN=${TOOLCHAIN}
local TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DANDROID_ABI=${ANDROID_ABI} \
-DANDROID_NDK=${ANDROID_NDK} \
-DANDROID_PLATFORM=${ANDROID_PLATFORM} \
-DANDROID_STL=${ANDROID_STL} \
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_LITE_BACKEND=ON \
-DENABLE_PADDLE2ONNX=OFF \
-DENABLE_FLYCV=OFF \
-DENABLE_TEXT=ON \
-DENABLE_VISION=OFF \
-DBUILD_EXAMPLES=OFF \
-DWITH_ANDROID_LITE_STATIC=ON \
-DWITH_ANDROID_OPENMP=ON \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${ANDROID_ABI}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__set_android_ndk
__build_fastdeploy_android_shared
exit 0
}
main
# Usage:
# ./scripts/android/build_android_cpp_with_text_api_only.sh arm64-v8a 21
# ./scripts/android/build_android_cpp_with_text_api_only.sh armeabi-v7a 21

61
scripts/android/bundle_android_cpp_with_text_api_only.sh
View File

@@ -1,61 +0,0 @@
#!/bin/bash
set -e
set +x
FASTDEPLOY_DIR=$(pwd)
BUILT_PACKAGE_DIR=build/Android
CXX_PACKAGE_PREFIX=fastdeploy-android-latest-shared-dev
CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/${CXX_PACKAGE_PREFIX}
ARMV8_CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/arm64-v8a-api-21/install
ARMV7_CXX_PACKAGE_NAME=${BUILT_PACKAGE_DIR}/armeabi-v7a-api-21/install
# check package name
echo "[INFO] --- FASTDEPLOY_DIR: ${FASTDEPLOY_DIR}"
# check arm v7 & v8 c++ sdk
if [ ! -d "${BUILT_PACKAGE_DIR}" ]; then
echo "[ERROR] --- ${BUILT_PACKAGE_DIR} not exist, please build c++ sdk first!"
exit 0
fi
if [ ! -d "${ARMV8_CXX_PACKAGE_NAME}" ]; then
echo "[ERROR] --- ${ARMV8_CXX_PACKAGE_NAME} not exist, please build c++ sdk first!"
exit 0
fi
if [ ! -d "${ARMV7_CXX_PACKAGE_NAME}" ]; then
echo "[ERROR] --- ${ARMV7_CXX_PACKAGE_NAME} not exist, please build c++ sdk first!"
exit 0
fi
# remove old package
echo "[INFO] --- Packing ${CXX_PACKAGE_NAME} package ..."
if [ -d "${CXX_PACKAGE_NAME}" ]; then
rm -rf ${CXX_PACKAGE_NAME}
echo "[INFO] --- Removed old ${CXX_PACKAGE_NAME} done !"
if [ -f "${CXX_PACKAGE_NAME}.tgz" ]; then
rm ${CXX_PACKAGE_NAME}.tgz
echo "[INFO] --- Removed old ${CXX_PACKAGE_NAME} done !"
fi
fi
# package latest c++ sdk
mkdir ${CXX_PACKAGE_NAME}
echo "[INFO] --- Collecting package contents ..."
cp -r ${ARMV7_CXX_PACKAGE_NAME}/* ${CXX_PACKAGE_NAME}/
cp -r ${ARMV8_CXX_PACKAGE_NAME}/* ${CXX_PACKAGE_NAME}/
if [ -d "${CXX_PACKAGE_NAME}/examples" ]; then
rm -rf ${CXX_PACKAGE_NAME}/examples
fi
echo "[INFO] --- Removed examples files ..."
echo "[INFO] --- Removing static .a files: "
static_files=$(find ${CXX_PACKAGE_NAME}/third_libs/install/ -name "*.a")
if [ ${#static_files[@]} -gt 10 ]; then
echo "${#static_files[@]}: ${static_files}"
rm $(find ${CXX_PACKAGE_NAME}/third_libs/install/ -name "*.a")
fi
echo "[INFO] --- Taring ${CXX_PACKAGE_NAME}.tgz package ..."
tar -zcvf ${CXX_PACKAGE_NAME}.tgz ${CXX_PACKAGE_NAME}/* >> ${BUILT_PACKAGE_DIR}/pkg.log 2>&1
echo "[INFO] --- Package ${CXX_PACKAGE_NAME}.tgz done ! Package size info: "
du -sh ${BUILT_PACKAGE_DIR}/* | grep ${CXX_PACKAGE_PREFIX}
# Usage:
# ./scripts/android/bundle_android_cpp_with_text_api_only.sh

13
scripts/ascend_init.sh
View File

@@ -1,13 +0,0 @@
# Set huawei ascend toolkit correctly.
HUAWEI_ASCEND_TOOLKIT_HOME="/usr/local/Ascend/ascend-toolkit/latest"
HUAWEI_ASCEND_DRIVER_PATH="/usr/local/Ascend/driver"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HUAWEI_ASCEND_DRIVER_PATH/lib64/driver:$HUAWEI_ASCEND_DRIVER_PATH/lib64:$HUAWEI_ASCEND_DRIVER_PATH/lib64/stub:$HUAWEI_ASCEND_TOOLKIT_HOME/acllib/lib64:$HUAWEI_ASCEND_TOOLKIT_HOME/atc/lib64:$HUAWEI_ASCEND_TOOLKIT_HOME/opp/op_proto/built-in
export PYTHONPATH=$PYTHONPATH:$HUAWEI_ASCEND_TOOLKIT_HOME/fwkacllib/python/site-packages:$HUAWEI_ASCEND_TOOLKIT_HOME/acllib/python/site-packages:$HUAWEI_ASCEND_TOOLKIT_HOME/toolkit/python/site-packages:$HUAWEI_ASCEND_TOOLKIT_HOME/atc/python/site-packages:$HUAWEI_ASCEND_TOOLKIT_HOME/pyACL/python/site-packages/acl
export PATH=$PATH:$HUAWEI_ASCEND_TOOLKIT_HOME/atc/ccec_compiler/bin:${HUAWEI_ASCEND_TOOLKIT_HOME}/acllib/bin:$HUAWEI_ASCEND_TOOLKIT_HOME/atc/bin
export ASCEND_AICPU_PATH=$HUAWEI_ASCEND_TOOLKIT_HOME
export ASCEND_OPP_PATH=$HUAWEI_ASCEND_TOOLKIT_HOME/opp
export TOOLCHAIN_HOME=$HUAWEI_ASCEND_TOOLKIT_HOME/toolkit
export ASCEND_SLOG_PRINT_TO_STDOUT=0
export ASCEND_GLOBAL_LOG_LEVEL=3
echo "===== Finish Initializing Environment for Ascend Deployment ====="

25
scripts/build_bcloud_lib.py
View File

@@ -1,25 +0,0 @@
import sys
import os
import shutil
dirname = sys.argv[1]
bc_dirname = sys.argv[2]
if os.path.exists(bc_dirname):
raise Exception("Path {} is already exists.".format(bc_dirname))
os.makedirs(bc_dirname)
# copy include files
shutil.copytree(os.path.join(dirname, "include"), os.path.join(bc_dirname, "include"))
# copy libraries
shutil.copytree(os.path.join(dirname, "lib"), os.path.join(bc_dirname, "lib"))
third_libs = os.path.join(dirname, "third_libs")
for root, dirs, files in os.walk(third_libs):
for f in files:
if f.strip().count(".so") > 0 or f.strip() == "plugins.xml":
full_path = os.path.join(root, f)
shutil.copy(full_path, os.path.join(bc_dirname, "lib"), follow_symlinks=False)

206
scripts/build_wheel_pipeline_cu123.sh Normal file
View File

@@ -0,0 +1,206 @@
#!/usr/bin/env bash
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
PYTHON_VERSION=python
PYTHON_VERSION=${1:-$PYTHON_VERSION}
export python=$PYTHON_VERSION
CPU_USE_BF16="false"
CPU_USE_BF16=${2:-$CPU_USE_BF16}
WITH_CPU="false"
# paddle distributed use to set archs
unset PADDLE_CUDA_ARCH_LIST
# directory config
DIST_DIR="dist"
BUILD_DIR="build"
EGG_DIR="fastdeploy.egg-info"
# custom_ops directory config
OPS_SRC_DIR="custom_ops"
OPS_BUILD_DIR="build"
OPS_EGG_DIR="efficitentllm_ops.egg-info"
OPS_TMP_DIR_BASE="tmp_base"
OPS_TMP_DIR="tmp"
OPS_TMP_DIR_CPU="tmp_cpu"
TEST_DIR="tests"
# command line log config
RED='\033[0;31m'
BLUE='\033[0;34m'
GREEN='\033[1;32m'
BOLD='\033[1m'
NONE='\033[0m'
function python_version_check() {
PY_MAIN_VERSION=`${python} -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1}'`
PY_SUB_VERSION=`${python} -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $2}'`
echo -e "find python version ${PY_MAIN_VERSION}.${PY_SUB_VERSION}"
if [ $PY_MAIN_VERSION -ne "3" -o $PY_SUB_VERSION -lt "8" ]; then
echo -e "${RED}FAIL:${NONE} please use Python >= 3.8 !"
exit 1
fi
}
function init() {
echo -e "${BLUE}[init]${NONE} removing building directory..."
rm -rf $DIST_DIR $BUILD_DIR $EGG_DIR
if [ `${python} -m pip list | grep fastdeploy | wc -l` -gt 0 ]; then
echo -e "${BLUE}[init]${NONE} uninstalling fastdeploy..."
${python} -m pip uninstall -y fastdeploy
fi
${python} -m pip install setuptools_scm
echo -e "${BLUE}[init]${NONE} installing requirements..."
${python} -m pip install --force-reinstall --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu123/
${python} -m pip install --upgrade --force-reinstall -r requirements.txt --ignore-installed PyYAML
echo -e "${BLUE}[init]${NONE} ${GREEN}init success\n"
}
function copy_ops(){
OPS_VERSION="0.0.0"
PY_MAIN_VERSION=`${python} -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1}'`
PY_SUB_VERSION=`${python} -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $2}'`
PY_VERSION="py${PY_MAIN_VERSION}.${PY_SUB_VERSION}"
SYSTEM_VERSION=`${python} -c "import platform; print(platform.system().lower())"`
PROCESSOR_VERSION=`${python} -c "import platform; print(platform.processor())"`
WHEEL_BASE_NAME="fastdeploy_base_ops-${OPS_VERSION}-${PY_VERSION}-${SYSTEM_VERSION}-${PROCESSOR_VERSION}.egg"
WHEEL_NAME="fastdeploy_ops-${OPS_VERSION}-${PY_VERSION}-${SYSTEM_VERSION}-${PROCESSOR_VERSION}.egg"
echo -e "OPS are for BASE"
mkdir -p ../fastdeploy/model_executor/ops/base && cp -r ./${OPS_TMP_DIR_BASE}/${WHEEL_BASE_NAME}/* ../fastdeploy/model_executor/ops/base
echo -e "OPS are for CUDA"
cp -r ./${OPS_TMP_DIR}/${WHEEL_NAME}/* ../fastdeploy/model_executor/ops/gpu
if [ "$WITH_CPU" == "true" ]; then
WHEEL_CPU_NAME="fastdeploy_cpu_ops-${OPS_VERSION}-${PY_VERSION}-${SYSTEM_VERSION}-${PROCESSOR_VERSION}.egg"
echo -e "OPS are for CPU"
cd ${OPS_TMP_DIR_CPU}/${WHEEL_CPU_NAME}/xFasterTransformer/build/
for file in *_pd_.so; do
mv "$file" "${file/_pd_/}"
done
cd ../../x86-simd-sort/builddir/
for file in *_pd_.so; do
mv "$file" "${file/_pd_/}"
done
cd ../../../../
cp -r ${OPS_TMP_DIR_CPU}/${WHEEL_CPU_NAME}/* ../fastdeploy/model_executor/ops/cpu
fi
return
}
function build_and_install_ops() {
cd $OPS_SRC_DIR
export no_proxy=bcebos.com,paddlepaddle.org.cn,${no_proxy}
echo -e "${BLUE}[build]${NONE} build and install fastdeploy_custom_ops..."
echo -e "${BLUE}[build]${NONE} build and install fastdeploy_base_ops..."
${python} setup_ops_base.py install --install-lib ${OPS_TMP_DIR_BASE}
echo -e "${BLUE}[build]${NONE} build and install fastdeploy_custom_ops gpu ops..."
BUILDING_ARCS="[80, 90]" ${python} setup_ops.py install --install-lib ${OPS_TMP_DIR}
if [ "$WITH_CPU" == "true" ]; then
echo -e "${BLUE}[build]${NONE} build and install fastdeploy_custom_ops cpu ops..."
if [ "$CPU_USE_BF16" == "true" ]; then
CPU_USE_BF16=True ${python} setup_ops_cpu.py install --install-lib ${OPS_TMP_DIR_CPU}
:
elif [ "$CPU_USE_BF16" == "false" ]; then
${python} setup_ops_cpu.py install --install-lib ${OPS_TMP_DIR_CPU}
:
else
echo "Error: Invalid parameter '$CPU_USE_BF16'. Please use true or false."
exit 1
fi
fi
if [ $? -ne 0 ]; then
echo -e "${RED}[FAIL]${NONE} build fastdeploy_custom_ops wheel failed !"
exit 1
fi
echo -e "${BLUE}[build]${NONE} ${GREEN}build fastdeploy_custom_ops wheel success\n"
copy_ops
cd ..
}
function build_and_install() {
echo -e "${BLUE}[build]${NONE} building fastdeploy wheel..."
${python} setup.py bdist_wheel --python-tag py3
if [ $? -ne 0 ]; then
echo -e "${RED}[FAIL]${NONE} build fastdeploy wheel failed !"
exit 1
fi
echo -e "${BLUE}[build]${NONE} ${GREEN}build fastdeploy wheel success\n"
echo -e "${BLUE}[install]${NONE} installing fastdeploy..."
cd $DIST_DIR
find . -name "fastdeploy*.whl" | xargs ${python} -m pip install
if [ $? -ne 0 ]; then
cd ..
echo -e "${RED}[FAIL]${NONE} install fastdeploy wheel failed !"
exit 1
fi
echo -e "${BLUE}[install]${NONE} ${GREEN}fastdeploy install success\n"
cd ..
}
function cleanup() {
rm -rf $BUILD_DIR $EGG_DIR
${python} -m pip uninstall -y fastdeploy
rm -rf $OPS_SRC_DIR/$BUILD_DIR $OPS_SRC_DIR/$EGG_DIR
}
function abort() {
echo -e "${RED}[FAIL]${NONE} build wheel failed !
please check your code" 1>&2
cur_dir=`basename "$pwd"`
rm -rf $BUILD_DIR $EGG_DIR $DIST_DIR
${python} -m pip uninstall -y fastdeploy
rm -rf $OPS_SRC_DIR/$BUILD_DIR $OPS_SRC_DIR/$EGG_DIR
}
python_version_check
trap 'abort' 0
set -e
init
build_and_install_ops
build_and_install
cleanup
# get Paddle version
PADDLE_VERSION=`${python} -c "import paddle; print(paddle.version.full_version)"`
PADDLE_COMMIT=`${python} -c "import paddle; print(paddle.version.commit)"`
# get fastdeploy version
FASTDEPLOY_BRANCH=`git rev-parse --abbrev-ref HEAD`
FASTDEPLOY_COMMIT=`git rev-parse --short HEAD`
# get Python version
PYTHON_VERSION=`${python} -c "import platform; print(platform.python_version())"`
echo -e "\n${GREEN}fastdeploy wheel compiled and checked success !${NONE}
${BLUE}Python version:${NONE} $PYTHON_VERSION
${BLUE}Paddle version:${NONE} $PADDLE_VERSION ($PADDLE_COMMIT)
${BLUE}fastdeploy branch:${NONE} $FASTDEPLOY_BRANCH ($FASTDEPLOY_COMMIT)\n"
echo -e "${GREEN}wheel saved under${NONE} ${RED}${BOLD}./dist${NONE}"
trap : 0

7
scripts/clean_sdk.sh
View File

@@ -1,7 +0,0 @@
origin_install_dir=$1
rm -rf $origin_install_dir/include/onnx $origin_install_dir/include/paddle2onnx
mv $origin_install_dir/lib $origin_install_dir/lib_bak
mkdir $origin_install_dir/lib
cp $origin_install_dir/lib_bak/*fastdeploy* $origin_install_dir/lib
rm -rf $origin_install_dir/lib_bak

497
scripts/codestyle/clang-tidy.py Normal file
View File

@@ -0,0 +1,497 @@
#!/usr/bin/env python3
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===------------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py
"""
Parallel clang-tidy runner
==========================
Runs clang-tidy over all files in a compilation database. Requires clang-tidy
and clang-apply-replacements in $PATH.
Example invocations.
- Run clang-tidy on all files in the current working directory with a default
set of checks and show warnings in the cpp files and all project headers.
run-clang-tidy.py $PWD
- Fix all header guards.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard
- Fix all header guards included from clang-tidy and header guards
for clang-tidy headers.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
-header-filter=extra/clang-tidy
Compilation database setup:
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
"""
import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
try:
import yaml
except ImportError:
yaml = None
is_py2 = sys.version[0] == "2"
if is_py2:
import Queue as queue
else:
import queue
def find_compilation_database(path, result="./"):
"""Adjusts the directory until a compilation database is found."""
result = "./"
while not os.path.isfile(os.path.join(result, path)):
if os.path.realpath(result) == "/":
print("Warning: could not find compilation database.")
return None
result += "../"
return os.path.realpath(result)
def make_absolute(f, directory):
"""Convert a relative file path to an absolute file path."""
if os.path.isabs(f):
return f
return os.path.normpath(os.path.join(directory, f))
def analysis_gitignore(path, filename=".gitignore"):
"""Analysis gitignore file and return ignore file list"""
with open(path + "/" + filename, "r") as f:
lines = f.readlines()
ignore_file_list = []
for line in lines:
# Blank row
if line == "\n" or line == "\r\n":
continue
# explanatory note
line = line.replace("\n", "").strip()
if "#" in line:
if not line.startswith("#"):
ignore_file_list.append(line[: line.index("#")].replace(" ", ""))
continue
# TODO(gouzil): support more gitignore rules
if "*" in line:
continue
ignore_file_list.append(line.replace(" ", ""))
return ignore_file_list
def skip_check_file(database, build_path):
"""Skip checking some files"""
skip_file_list = []
skip_file_list.append(".cu")
skip_file_list.append(os.path.join(os.getcwd(), build_path))
skip_file_list += analysis_gitignore(os.getcwd())
res_list = []
for entry in database:
write_in = True
for ignore_file in skip_file_list:
if ignore_file in entry["file"]:
write_in = False
break
if write_in:
res_list.append(entry)
return res_list
def get_tidy_invocation(
f,
clang_tidy_binary,
checks,
tmpdir,
build_path,
header_filter,
extra_arg,
extra_arg_before,
quiet,
config,
):
"""Gets a command line for clang-tidy."""
start = [clang_tidy_binary]
if header_filter is not None:
start.append("-header-filter=" + header_filter)
if checks:
start.append("-checks=" + checks)
if tmpdir is not None:
start.append("-export-fixes")
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
os.close(handle)
start.append(name)
for arg in extra_arg:
start.append(f"-extra-arg={arg}")
for arg in extra_arg_before:
start.append(f"-extra-arg-before={arg}")
start.append("-p=" + build_path)
if quiet:
start.append("-quiet")
if config:
start.append("-config=" + config)
start.append(f)
return start
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
content = yaml.safe_load(open(replacefile, "r"))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {"MainSourceFile": "", mergekey: merged}
with open(mergefile, "w") as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, "w").close()
def check_clang_apply_replacements_binary(args):
"""Checks if invoking supplied clang-apply-replacements binary works."""
try:
subprocess.check_call([args.clang_apply_replacements_binary, "--version"])
except:
print(
"Unable to run clang-apply-replacements. Is clang-apply-replacements "
"binary correctly specified?",
file=sys.stderr,
)
traceback.print_exc()
sys.exit(1)
def apply_fixes(args, tmpdir):
"""Calls clang-apply-fixes on a given directory."""
invocation = [args.clang_apply_replacements_binary]
if args.format:
invocation.append("-format")
if args.style:
invocation.append("-style=" + args.style)
invocation.append(tmpdir)
subprocess.call(invocation)
def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
name = queue.get()
invocation = get_tidy_invocation(
name,
args.clang_tidy_binary,
args.checks,
tmpdir,
build_path,
args.header_filter,
args.extra_arg,
args.extra_arg_before,
args.quiet,
args.config,
)
proc = subprocess.Popen(
invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
output, err = proc.communicate()
if proc.returncode != 0:
failed_files.append(name)
with lock:
sys.stdout.write(" ".join(invocation) + "\n" + output.decode("utf-8"))
if len(err) > 0:
sys.stdout.flush()
sys.stderr.write(err.decode("utf-8"))
queue.task_done()
def main():
"""Runs clang-tidy over all files in a compilation database."""
parser = argparse.ArgumentParser(
description="Runs clang-tidy over all files "
"in a compilation database. Requires "
"clang-tidy and clang-apply-replacements in "
"$PATH."
)
parser.add_argument(
"-clang-tidy-binary",
metavar="PATH",
default="clang-tidy-15",
help="path to clang-tidy binary",
)
parser.add_argument(
"-clang-apply-replacements-binary",
metavar="PATH",
default="clang-apply-replacements-15",
help="path to clang-apply-replacements binary",
)
parser.add_argument(
"-checks",
default=None,
help="checks filter, when not specified, use clang-tidy " "default",
)
parser.add_argument(
"-config",
default=None,
help="Specifies a configuration in YAML/JSON format: "
" -config=\"{Checks: '*', "
" CheckOptions: [{key: x, "
' value: y}]}" '
"When the value is empty, clang-tidy will "
"attempt to find a file named .clang-tidy for "
"each source file in its parent directories.",
)
parser.add_argument(
"-header-filter",
default=None,
help="regular expression matching the names of the "
"headers to output diagnostics from. Diagnostics from "
"the main file of each translation unit are always "
"displayed.",
)
if yaml:
parser.add_argument(
"-export-fixes",
metavar="filename",
dest="export_fixes",
help="Create a yaml file to store suggested fixes in, "
"which can be applied with clang-apply-replacements.",
)
parser.add_argument(
"-j",
type=int,
default=0,
help="number of tidy instances to be run in parallel.",
)
parser.add_argument(
"files",
nargs="*",
default=[".*"],
help="files to be processed (regex on path)",
)
parser.add_argument("-fix", action="store_true", help="apply fix-its")
parser.add_argument(
"-format",
action="store_true",
help="Reformat code " "after applying fixes",
)
parser.add_argument(
"-style",
default="file",
help="The style of reformat " "code after applying fixes",
)
parser.add_argument(
"-p",
dest="build_path",
help="Path used to read a compile command database.",
)
parser.add_argument(
"-extra-arg",
dest="extra_arg",
action="append",
default=[],
help="Additional argument to append to the compiler " "command line.",
)
parser.add_argument(
"-extra-arg-before",
dest="extra_arg_before",
action="append",
default=[],
help="Additional argument to prepend to the compiler " "command line.",
)
parser.add_argument(
"-quiet", action="store_true", help="Run clang-tidy in quiet mode"
)
args = parser.parse_args()
db_path = "compile_commands.json"
if args.build_path is not None:
build_path = args.build_path
if not os.path.isfile(os.path.join(build_path, db_path)):
print(
f"Warning: could not find compilation database in {build_path}, skip clang-tidy check."
)
build_path = None
else:
# Find our database
build_path = find_compilation_database(db_path)
if build_path is None:
sys.exit(0)
try:
invocation = [args.clang_tidy_binary, "-list-checks"]
invocation.append("-p=" + build_path)
if args.checks:
invocation.append("-checks=" + args.checks)
invocation.append("-")
if args.quiet:
# Even with -quiet we still want to check if we can call clang-tidy.
with open(os.devnull, "w") as dev_null:
subprocess.check_call(invocation, stdout=dev_null)
else:
subprocess.check_call(invocation)
except:
print("Unable to run clang-tidy.", file=sys.stderr)
sys.exit(0)
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
database = skip_check_file(database, build_path)
files = {make_absolute(entry["file"], entry["directory"]) for entry in database}
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
tmpdir = None
if args.fix or (yaml and args.export_fixes):
check_clang_apply_replacements_binary(args)
tmpdir = tempfile.mkdtemp()
# Build up a big regexy filter from all command line arguments.
file_name_re = re.compile("|".join(args.files))
return_code = 0
try:
# Spin up a bunch of tidy-launching threads.
task_queue = queue.Queue(max_task)
# List of files with a non-zero return code.
failed_files = []
lock = threading.Lock()
for _ in range(max_task):
t = threading.Thread(
target=run_tidy,
args=(args, tmpdir, build_path, task_queue, lock, failed_files),
)
t.daemon = True
t.start()
# Fill the queue with files.
for name in files:
if file_name_re.search(name):
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
if len(failed_files):
return_code = 1
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print("\nCtrl-C detected, goodbye.")
if tmpdir:
shutil.rmtree(tmpdir)
os.kill(0, 9)
if yaml and args.export_fixes:
print("Writing fixes to " + args.export_fixes + " ...")
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
print("Error exporting fixes.\n", file=sys.stderr)
traceback.print_exc()
return_code = 1
if args.fix:
print("Applying fixes ...")
try:
apply_fixes(args, tmpdir)
except:
print("Error applying fixes.\n", file=sys.stderr)
traceback.print_exc()
return_code = 1
if tmpdir:
shutil.rmtree(tmpdir)
sys.exit(return_code)
if __name__ == "__main__":
if os.getenv("SKIP_CLANG_TIDY_CHECK", "").lower() in [
"y",
"yes",
"t",
"true",
"on",
"1",
]:
print(
"SKIP_CLANG_TIDY_CHECK is set, skip clang-tidy check.",
file=sys.stderr,
)
sys.exit(0)
target_version = "15.0.2"
try:
out = subprocess.check_output(["clang-tidy", "--version"], shell=True)
version = out.decode("utf-8")
if version.find(target_version) == -1:
print(
f"clang-tidy version == {target_version} not found, attempting auto-install...",
file=sys.stderr,
)
subprocess.check_output(
'pip install --no-cache clang-tidy=="15.0.2.1"',
shell=True,
)
except:
print("clang-tidy not found, attempting auto-install...", file=sys.stderr)
subprocess.check_output(
'pip install --no-cache clang-tidy=="15.0.2.1"',
shell=True,
)
main()

35
scripts/codestyle/clang_format.sh Normal file
View File

@@ -0,0 +1,35 @@
#!/bin/bash
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
readonly VERSION="13.0.0"
version=$(clang-format -version)
if ! [[ $(python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1$2}') -ge 36 ]]; then
echo "clang-format installation by pip need python version great equal 3.6,
please change the default python to higher version."
exit 1
fi
if ! [[ $version == *"$VERSION"* ]]; then
# low version of pip may not have the source of clang-format whl
pip install --upgrade pip
pip install clang-format==13.0.0
fi
clang-format $@

132
scripts/codestyle/copyright.py Normal file
View File

@@ -0,0 +1,132 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import datetime
import os
import re
import sys
COPYRIGHT = """Copyright (c) {year} PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""
def _generate_copyright(comment_mark):
year = datetime.datetime.now().year
copyright = COPYRIGHT.format(year=year)
return [
(
f"{comment_mark} {line}{os.linesep}"
if line
else f"{comment_mark}{os.linesep}"
)
for line in copyright.splitlines()
]
def _get_comment_mark(path):
lang_type = re.compile(r"\.(py|pyi|sh)$")
if lang_type.search(path) is not None:
return "#"
lang_type = re.compile(r"\.(h|c|hpp|cc|cpp|cu|go|cuh|proto)$")
if lang_type.search(path) is not None:
return "//"
return None
RE_ENCODE = re.compile(r"^[ \t\v]*#.*?coding[:=]", re.IGNORECASE)
RE_COPYRIGHT = re.compile(r".*Copyright \(c\) \d{4}", re.IGNORECASE)
RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
def _check_copyright(path):
head = []
try:
with open(path, "r", encoding="utf-8") as f:
head = [next(f) for x in range(4)]
except StopIteration:
pass
for idx, line in enumerate(head):
if RE_COPYRIGHT.search(line) is not None:
return True
return False
def generate_copyright(path, comment_mark):
original_contents = open(path, "r", encoding="utf-8").readlines()
head = original_contents[0:4]
insert_line_no = 0
for i, line in enumerate(head):
if RE_ENCODE.search(line) or RE_SHEBANG.search(line):
insert_line_no = i + 1
copyright = _generate_copyright(comment_mark)
if insert_line_no == 0:
new_contents = copyright
if len(original_contents) > 0 and len(original_contents[0].strip()) != 0:
new_contents.append(os.linesep)
new_contents.extend(original_contents)
else:
new_contents = original_contents[0:insert_line_no]
new_contents.append(os.linesep)
new_contents.extend(copyright)
if (
len(original_contents) > insert_line_no
and len(original_contents[insert_line_no].strip()) != 0
):
new_contents.append(os.linesep)
new_contents.extend(original_contents[insert_line_no:])
new_contents = "".join(new_contents)
with open(path, "w", encoding="utf-8") as output_file:
output_file.write(new_contents)
def main(argv=None):
parser = argparse.ArgumentParser(description="Checker for copyright declaration.")
parser.add_argument("filenames", nargs="*", help="Filenames to check")
args = parser.parse_args(argv)
retv = 0
for path in args.filenames:
comment_mark = _get_comment_mark(path)
if comment_mark is None:
print("warning:Unsupported file", path, file=sys.stderr)
continue
if _check_copyright(path):
continue
generate_copyright(path, comment_mark)
if __name__ == "__main__":
sys.exit(main())

14
scripts/patch_lib.sh → scripts/codestyle/cpplint_pre_commit.sh Executable file → Normal file
View File

@@ -1,4 +1,6 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#!/bin/bash
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,4 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
patchelf --set-rpath '${ORIGIN}' $1/*.so*
readonly VERSION="1.6.0"
version=$(cpplint --version)
if ! [[ $version == *"$VERSION"* ]]; then
pip install cpplint==1.6.0
fi
cpplint $@

68
scripts/codestyle/pre_commit.sh Executable file
View File

@@ -0,0 +1,68 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set +x
# use pre-commit 2.17
if ! [[ $(pre-commit --version) == *"2.17.0"* ]]; then
pip install pre-commit==2.17.0 1>nul
fi
# Install clang-format before git commit to avoid repeat installation due to
# pre-commit multi-thread running.
readonly VERSION="13.0.0"
version=$(clang-format -version)
if ! [[ $(python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1$2}') -ge 36 ]]; then
echo "clang-format installation by pip need python version great equal 3.6,
please change the default python to higher version."
exit 1
fi
diff_files=$(git diff --name-only --diff-filter=ACMR ${BRANCH})
num_diff_files=$(echo "$diff_files" | wc -l)
echo -e "diff files between pr and ${BRANCH}:\n${diff_files}"
echo "Checking code style by pre-commit ..."
pre-commit run --files ${diff_files};check_error=$?
if test ! -z "$(git diff)"; then
echo -e '\n************************************************************************************'
echo -e "These files have been formatted by code format hook. You should use pre-commit to \
format them before git push."
echo -e '************************************************************************************\n'
git diff 2>&1
fi
echo -e '\n************************************************************************************'
if [ ${check_error} != 0 ];then
echo "Your PR code style check failed."
echo "Please install pre-commit locally and set up git hook scripts:"
echo ""
echo " pip install pre-commit==2.17.0"
echo " pre-commit install"
echo ""
if [[ $num_diff_files -le 100 ]];then
echo "Then, run pre-commit to check codestyle issues in your PR:"
echo ""
echo " pre-commit run --files" $(echo ${diff_files} | tr "\n" " ")
echo ""
fi
echo "For more information, please refer to our codestyle check guide:"
echo "https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/dev_guides/git_guides/codestyle_check_guide_cn.html"
else
echo "Your PR code style check passed."
fi
echo -e '************************************************************************************\n'
exit ${check_error}

23
scripts/copy_directory.py → scripts/codestyle/sort_txt_file.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,21 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import shutil
import os
import sys
def copy_directory(src, dst):
if os.path.exists(dst):
raise Exception("Destination {} is already exist.".format(dst))
if not os.path.exists(src):
raise Exception("Source {} is not exist.".format(src))
try:
shutil.copytree(src, dst, symlinks=True)
except:
raise Exception("Copy {} to {} failed.".format(src, dst))
def sort_by_dict_order(file_path):
with open(file_path, "r") as f:
lines = f.readlines()
sorted_lines = sorted(lines)
with open(file_path, "w") as f:
f.writelines(sorted_lines)
if __name__ == "__main__":
copy_directory(sys.argv[1], sys.argv[2])
file_paths = sys.argv[1:]
for file_path in file_paths:
file_path = os.path.normpath(file_path)
sort_by_dict_order(file_path)

59
scripts/convert_ep_state_from_ep8.py Normal file
View File

@@ -0,0 +1,59 @@
"""
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
import paddle
import paddle.distributed as dist
from glob import glob
import os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--model_dir", type=str, required=True)
args = parser.parse_args()
rank = dist.get_rank()
ep_num = dist.get_world_size()
print("rank: ", rank)
# merge tpn -> tp1
model_dir = args.model_dir
save_merged_pp_dir = os.path.join(model_dir, "merged_tp1_state_split")
os.makedirs(save_merged_pp_dir, exist_ok=True)
model_path_pp = glob(os.path.join(model_dir, "shangxianv1_ep_hadamard_quantmodel_to_eval_pp*"))
for p in model_path_pp:
model_path_ep = os.path.join(p, f"model_state.ep0{rank}.pdparams")
print(p, model_path_ep)
state_dicts = paddle.load(model_path_ep, return_numpy=True)
print("merge ep")
print("p: ", p)
for k, v in state_dicts.items():
v = paddle.to_tensor(v)
if "mlp.experts" in k:
k_list = k.split(".")
export_id = rank * ep_num + int(k_list[5])
k_list[5] = str(export_id)
k = ".".join(k_list)
print(f"key: {k}")
save_split_path = os.path.join(save_merged_pp_dir, k)
paddle.save(v, save_split_path)
elif rank == 0:
save_split_path = os.path.join(save_merged_pp_dir, k)
paddle.save(paddle.to_tensor(v), save_split_path)
print(f"merge {p} end")
print("merge end")

6
scripts/convert_ep_state_from_ep8.sh Normal file
View File

@@ -0,0 +1,6 @@
export devices=0,1,2,3,4,5,6,7
python -m paddle.distributed.launch --gpus ${devices} convert_ep_state_from_ep8.py --model_dir /path/to/model_dir

86
scripts/convert_ep_state_from_tp8.py Normal file
View File

@@ -0,0 +1,86 @@
"""
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
import paddle
import paddle.distributed as dist
import pdb
from glob import glob
import os
import numpy as np
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--model_dir", type=str, required=True)
args = parser.parse_args()
rank = dist.get_rank()
print("rank: ", rank)
# merge tpn -> tp1
model_dir = args.model_dir
model_path_pp = glob(os.path.join(model_dir, f"pp{rank}"))
model_path_pp_tp = []
for p in model_path_pp:
model_path_tp = glob(os.path.join(p, "model_state*"))
model_path_tp = sorted(model_path_tp)
save_merged_pp_path = os.path.join(p, "merged_tp1_state.pdparams")
save_merged_pp_dir = os.path.join(model_dir, "merged_tp1_state_split")
os.makedirs(save_merged_pp_dir, exist_ok=True)
print(p, model_path_tp)
state_dicts = [paddle.load(path, return_numpy=True) for path in model_path_tp]
state = state_dicts[0]
print("merge tp")
print("p: ", p)
for k, v in state.items():
save_split_path = os.path.join(save_merged_pp_dir, k)
state_now = []
for i in range(len(state_dicts)):
state_now.append(state_dicts[i][k])
print("k: ", k, ", v.shape: ", v.shape)
if "qkv_proj" in k:
"""not need prmt"""
# qkv not prmt
ori_q = [s[:, :1024] for s in state_now]
ori_k = [s[:, 1024:1152] for s in state_now]
ori_v = [s[:, 1152:] for s in state_now]
new_q = np.concatenate(ori_q, axis=1)
new_k = np.concatenate(ori_k, axis=1)
new_v = np.concatenate(ori_v, axis=1)
print(new_q.shape)
print(new_k.shape)
print(new_v.shape)
new_w = np.concatenate([new_q, new_k, new_v], axis=1)
# new_w = np.concatenate(state_now, axis=1)
elif "o_proj" in k or "down_proj" in k:
new_w = np.concatenate(state_now, axis=0)
elif "embed_tokens" in k:
new_w = np.concatenate(state_now, axis=0)
elif "up_gate_proj" in k:
dim = state_now[0].shape[1]
half_ffn1_1 = [s[:, :(dim // 2)] for s in state_now]
half_ffn1_2 = [s[:, (dim // 2):] for s in state_now]
new_ffn1_1 = np.concatenate(half_ffn1_1, axis=1)
new_ffn1_2 = np.concatenate(half_ffn1_2, axis=1)
new_w = np.concatenate([new_ffn1_1, new_ffn1_2], axis=1)
elif "lm_head" in k or "mtp_linear_proj" in k:
new_w = np.concatenate(state_now, axis=1)
else:
new_w = v
print("merged_shape: ", new_w.shape)
paddle.save(paddle.to_tensor(new_w), save_split_path)
print("merge end")

3
scripts/convert_ep_state_from_tp8.sh Normal file
View File

@@ -0,0 +1,3 @@
export devices=0,1,2,3,4,5,6,7
python -m paddle.distributed.launch --gpus ${devices} convert_ep_state_from_tp8.py --model_dir /path/to/model_dir

167
scripts/fastdeploy_init.bat
View File

@@ -1,167 +0,0 @@
@echo off
set __script_action_type=%1
set __fastdeploy_sdk_dir=%2
set __another_target_dir=%3
set __install_infos_flag=%4
@rem help
if "%__script_action_type%" == "help" (
call:__print_long_line
echo [1] [help] print help information: fastdeploy_init.bat help
echo [2] [show] show all dlls/libs/include paths: fastdeploy_init.bat show fastdeploy-sdk-dir
echo [3] [init] init all dlls paths for current terminal: fastdeploy_init.bat init fastdeploy-sdk-dir [WARNING: need copy onnxruntime.dll manually]
echo [4] [setup] setup path env for current terminal: fastdeploy_init.bat setup fastdeploy-sdk-dir [WARNING: need copy onnxruntime.dll manually]
echo [5] [install] install all dlls to a specific dir: fastdeploy_init.bat install fastdeploy-sdk-dir another-dir-to-install-dlls **[RECOMMEND]**
echo [6] [install] install all dlls with logging infos: fastdeploy_init.bat install fastdeploy-sdk-dir another-dir-to-install-dlls info
call:__print_long_line
goto:eof
)
@rem show dlls and libs
if "%__script_action_type%" == "show" (
call:__print_long_line
echo [SDK] %__fastdeploy_sdk_dir%
call:__print_long_line
set __have_openvino_flag=false
set __fastdeploy_lib_dir=%__fastdeploy_sdk_dir%\lib
@setlocal enabledelayedexpansion
echo [DLL] !__fastdeploy_lib_dir!\fastdeploy.dll **[NEEDED]**
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir%\third_libs ^| findstr /e \.dll ^| findstr /v "vc14\\bin\\opencv"') do (
set __3rd_dll_file=%%a && set __3rd_needed_flag=true
echo !__3rd_dll_file! | findstr "openvino">nul && set __have_openvino_flag=true
echo !__3rd_dll_file! | findstr d\.dll>nul && set __3rd_needed_flag=false
if "!__3rd_needed_flag!"=="false" (echo !__3rd_dll_file! | findstr /v opencv>nul && set __3rd_needed_flag=true)
echo !__3rd_dll_file! | findstr debug\.dll>nul && set __3rd_needed_flag=false
if "!__3rd_needed_flag!"=="true" (echo [DLL] !__3rd_dll_file! **[NEEDED]**) else (echo [DLL] !__3rd_dll_file!)
)
call:__print_long_line
echo [Lib] !__fastdeploy_lib_dir!\fastdeploy.lib **[NEEDED][fastdeploy]**
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir%\third_libs ^| findstr /e \.lib ^| findstr /v "vc14\\lib\\opencv"') do (
set __3rd_lib_file=%%a && set __3rd_needed_flag=false && set __api_tag=fastdeploy
echo !__3rd_lib_file! | findstr "opencv">nul && set __3rd_needed_flag=true
echo !__3rd_lib_file! | findstr "opencv">nul && set __api_tag=!__api_tag!::vision
if "!__3rd_needed_flag!"=="true" (echo !__3rd_lib_file! | findstr d\.lib>nul && set __3rd_needed_flag=false)
echo !__3rd_lib_file! | findstr "fast_tokenizer">nul && set __3rd_needed_flag=true
echo !__3rd_lib_file! | findstr "fast_tokenizer">nul && set __api_tag=!__api_tag!::text
if "!__3rd_needed_flag!"=="true" (echo [Lib] !__3rd_lib_file! **[NEEDED][!__api_tag!]**) else (echo [Lib] !__3rd_lib_file!)
)
call:__print_long_line
set __fastdeploy_include_dir=%__fastdeploy_sdk_dir%\include
echo [Include] !__fastdeploy_include_dir! **[NEEDED][fastdeploy]**
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir%\third_libs ^| findstr /e include ^| findstr /v "vc14\\bin\\opencv"') do (
set __3rd_include_dir=%%a && set __3rd_needed_flag=false && set __api_tag=fastdeploy
echo !__3rd_include_dir! | findstr "opencv">nul && set __3rd_needed_flag=true
echo !__3rd_include_dir! | findstr "opencv">nul && set __api_tag=!__api_tag!::vision
echo !__3rd_include_dir! | findstr "fast_tokenizer">nul && set __3rd_needed_flag=true
echo !__3rd_include_dir! | findstr "fast_tokenizer">nul && set __api_tag=!__api_tag!::text
if "!__3rd_needed_flag!"=="true" (echo [Include] !__3rd_include_dir! **[NEEDED][!__api_tag!]**) else (echo [Include] !__3rd_include_dir!)
)
call:__print_long_line
if "!__have_openvino_flag!"=="true" (
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir%\third_libs ^| findstr /e \.xml ^| findstr "openvino"') do (
set __openvino_plugin_xml=%%a
echo [XML] !__openvino_plugin_xml! **[NEEDED]**
)
call:__print_long_line
)
@setlocal disabledelayedexpansion
goto:eof
)
@rem init all paths for dlls
if "%__script_action_type%" == "init" (
@setlocal enabledelayedexpansion
set /p yes_or_no=Init dll paths for FastDeploy in current terminal: [y/n]
if "!yes_or_no!"=="y" (echo YES.) else (echo NO. && pause && goto:eof)
@setlocal disabledelayedexpansion
if exist bin.txt (del /Q bin.txt)
if exist lib.txt (del /Q lib.txt)
for /f "delims= " %%a in ('dir /s /b /A:D %__fastdeploy_sdk_dir% ^| findstr /v include ^| findstr /e bin ^| findstr /v "vc14\\bin"') do (>>bin.txt set /p=%%a;<nul)
for /f "delims= " %%a in ('dir /s /b /A:D %__fastdeploy_sdk_dir% ^| findstr /v include ^| findstr /e lib ^| findstr /v "vc14\\lib"') do (>>lib.txt set /p=%%a;<nul)
for /f %%i in (bin.txt) do (endlocal & set __fastdeploy_bin_paths=%%i)
for /f %%j in (lib.txt) do (endlocal & set __fastdeploy_lib_paths=%%j)
pause
call:__print_long_line
echo [INFO] FastDeploy dlls paths init done! Please run: [fastdeploy_init.bat setup fastdeploy-sdk-dir]
echo [INFO] command to push these dlls paths into PATH ENV in current terminal.
call:__print_long_line
goto:eof
)
@rem setup PATH ENV for all dlls
if "%__script_action_type%" == "setup" (
@setlocal enabledelayedexpansion
set /p yes_or_no=Setup PATH ENV for FastDeploy in current terminal: [y/n]
if "!yes_or_no!"=="y" (echo YES.) else (echo NO. && pause && goto:eof)
@setlocal disabledelayedexpansion
if not exist bin.txt (echo Can not found bin.txt, Please run init before setup && goto:eof)
if not exist lib.txt (echo Can not found lib.txt, Please run init before setup && goto:eof)
for /f %%i in (bin.txt) do (endlocal & set __fastdeploy_bin_paths=%%i)
for /f %%j in (lib.txt) do (endlocal & set __fastdeploy_lib_paths=%%j)
set "PATH=%__fastdeploy_bin_paths%%__fastdeploy_lib_paths%%PATH%"
pause
call:__print_long_line
echo [INFO] FastDeploy PATH ENV setup done! Please use [set PATH] to check PATH ENV in current terminal.
echo [INFO] Just setup once again if the paths of FastDeploy can not be found in your PATH ENV.
call:__print_long_line
echo [WARN] Please copy all onnxruntime dlls manually to your-exe-or-custom-dll-dir if ENABLE_ORT_BACKEND=ON.
echo [WARN] Use [fastdeploy_init.bat show fastdeploy-sdk-dir] to find the dll's location of onnxruntime.
call:__print_long_line
goto:eof
)
@rem copy all dlls to a specific location
if "%__script_action_type%" == "install" (
@setlocal enabledelayedexpansion
if "!__install_infos_flag!"=="noconfirm" (
echo [INFO] Installing all FastDeploy dlls ...
) else (
echo [INFO] Do you want to install all FastDeploy dlls ?
echo [INFO] From: !__fastdeploy_sdk_dir!
echo [INFO] To: !__another_target_dir!
set /p yes_or_no=Choose y means YES, n means NO: [y/n]
if "!yes_or_no!"=="y" (echo YES.) else (echo NO. && pause && goto:eof)
pause
)
@setlocal disabledelayedexpansion
if not exist %__fastdeploy_sdk_dir% ( echo [ERROR] %__fastdeploy_sdk_dir% is not exist ! && goto:eof )
if not exist %__another_target_dir% ( mkdir %__another_target_dir% && echo [INFO] Created %__another_target_dir% done!)
set __have_openvino_flag=false
@setlocal enabledelayedexpansion
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir% ^| findstr /e \.dll ^| findstr /v "vc14\\bin\\opencv"') do (
set __3rd_or_fd_dll_file=%%a && set __3rd_or_fd_needed_flag=true
echo !__3rd_or_fd_dll_file! | findstr "openvino">nul && set __have_openvino_flag=true
echo !__3rd_or_fd_dll_file! | findstr d\.dll>nul && set __3rd_or_fd_needed_flag=false
if "!__3rd_or_fd_needed_flag!"=="false" ( echo !__3rd_or_fd_dll_file! | findstr /v opencv>nul && set __3rd_or_fd_needed_flag=true)
echo !__3rd_or_fd_dll_file! | findstr debug\.dll>nul && set __3rd_or_fd_needed_flag=false
if "!__3rd_or_fd_needed_flag!"=="true" (
copy /Y !__3rd_or_fd_dll_file! %__another_target_dir%
if "!__install_infos_flag!"=="info" ( echo [Installed][DLL] !__3rd_or_fd_dll_file! "--->" %__another_target_dir%)
)
)
if "!__have_openvino_flag!"=="true" (
for /f "delims= " %%a in ('dir /s /b %__fastdeploy_sdk_dir% ^| findstr /e \.xml ^| findstr "openvino"') do (
set __openvino_plugin_xml=%%a
copy /Y !__openvino_plugin_xml! %__another_target_dir%
if "!__install_infos_flag!"=="info" ( echo [Installed][XML] !__openvino_plugin_xml! "--->" %__another_target_dir% )
)
)
@setlocal disabledelayedexpansion
goto:eof
)
goto:eof
@rem helpers
:__print_long_line
echo ------------------------------------------------------------------------------------------------------------------------------------------------------------
goto:eof
@rem end
@echo on

61
scripts/fastdeploy_init.sh
View File

@@ -1,61 +0,0 @@
# source this file to import libraries
PLATFORM=`uname`
FASTDEPLOY_LIBRARY_PATH=${BASH_SOURCE:-$0}
if [[ "$PLATFORM" = "Linux" ]];then
FASTDEPLOY_LIBRARY_PATH=`readlink -f ${FASTDEPLOY_LIBRARY_PATH}`
fi
FASTDEPLOY_LIBRARY_PATH=$(cd `dirname ${FASTDEPLOY_LIBRARY_PATH}`; pwd)
echo "=============== Information ======================"
echo "FastDeploy Library Path: $FASTDEPLOY_LIBRARY_PATH"
echo "Platform: $PLATFORM"
echo "=================================================="
# Find all the .so files' path
if [[ "$(ps -a $$)" =~ "zsh" ]]; then
ALL_SO_FILES=(`find $FASTDEPLOY_LIBRARY_PATH -name "*.so*"`)
ALL_DYLIB_FILES=(`find $FASTDEPLOY_LIBRARY_PATH -name "*.dylib*"`)
else
ALL_SO_FILES=`find $FASTDEPLOY_LIBRARY_PATH -name "*.so*"`
ALL_DYLIB_FILES=`find $FASTDEPLOY_LIBRARY_PATH -name "*.dylib*"`
fi
for SO_FILE in $ALL_SO_FILES;do
LIBS_DIRECTORIES+=(${SO_FILE%/*})
done
# Find all the .dylib files' path
# ALL_DYLIB_FILES=(`find $FASTDEPLOY_LIBRARY_PATH -name "*.dylib*"`)
for DYLIB_FILE in $ALL_DYLIB_FILES;do
LIBS_DIRECTORIES+=(${DYLIB_FILE%/*})
done
# Remove the dumplicate directories
LIBS_DIRECTORIES=($(awk -v RS=' ' '!a[$1]++' <<< ${LIBS_DIRECTORIES[@]}))
# Print the dynamic library location and output the configuration file
IMPORT_PATH=""
output_file=${FASTDEPLOY_LIBRARY_PATH}/fastdeploy_libs.conf
rm -rf $output_file
for LIB_DIR in ${LIBS_DIRECTORIES[@]};do
echo "Find Library Directory: $LIB_DIR"
echo "$LIB_DIR" >> $output_file
IMPORT_PATH=${LIB_DIR}":"$IMPORT_PATH
done
if [ -f "ascend_init.sh" ]
then
source ascend_init.sh
fi
echo "[Execute] Will try to export all the library directories to environments, if not work, please try to export these path by your self."
PLATFORM=`uname`
if [[ "$PLATFORM" = "Linux" ]];then
NEW_LIB_PATH=$(tr ":" "\n" <<< "${IMPORT_PATH}:$LD_LIBRARY_PATH" | sort | uniq | tr "\n" ":")
export LD_LIBRARY_PATH=$NEW_LIB_PATH
fi
if [[ "$PLATFORM" = "Darwin" ]];then
NEW_LIB_PATH=$(tr ":" "\n" <<< "${IMPORT_PATH}:$DYLD_LIBRARY_PATH" | sort | uniq | tr "\n" ":")
export DYLD_LIBRARY_PATH=$NEW_LIB_PATH
fi

225
scripts/get_rdma_nics.sh Normal file
View File

@@ -0,0 +1,225 @@
#!/bin/bash
Cur_Dir=$(cd `dirname $0`; pwd)
NICNAME_TYPE=xgbe # 默认检测类型
type=$1
if [ "$ENABLE_EP_DP" == "1" ]; then
gpu_root_port_filename="${Cur_Dir}/gpu_rootport_${DP_RANK}.txt"
else
gpu_root_port_filename="${Cur_Dir}/gpu_rootport.txt"
fi
function __NEW_GPU_ROOTPORT_FILE__() {
touch ${gpu_root_port_filename} 2>/dev/null
echo "" > ${gpu_root_port_filename} 2>/dev/null
for gpu_bus in $(lspci 2>/dev/null | grep -iE "Communication controller: | controller: NVIDIA" | awk '{print $1}')
do
readlink "/sys/bus/pci/devices/0000:${gpu_bus}" 2>/dev/null | awk -F [/] '{print $6}' >> ${gpu_root_port_filename}
done
}
function __RM_GPU_ROOTPORT_FILE__() {
rm -rf ${gpu_root_port_filename} 2>/dev/null
}
function __JUDGE_NIC_TYPE__() {
XGBE_NUM=$(ip a 2>/dev/null | grep -c ": ${NICNAME_TYPE}")
gpu_first=true
xpu_first=true
cpu_first=true
for (( xgbe_no=0; xgbe_no < XGBE_NUM; xgbe_no++ ))
do
[ ! -d "/sys/class/net/${NICNAME_TYPE}${xgbe_no}" ] && continue
PCI_ADDRESS=$(ethtool -i "${NICNAME_TYPE}${xgbe_no}" 2>/dev/null | awk -F '0000:' '/bus-info/{print $2}')
[ -z "$PCI_ADDRESS" ] && continue
NIC_ROOTPORT=$(readlink "/sys/bus/pci/devices/0000:${PCI_ADDRESS}" 2>/dev/null | awk -F '/' '{print $6}')
NIC_TYPE="CPU_NIC"
grep -qxF "$NIC_ROOTPORT" ${gpu_root_port_filename} 2>/dev/null && NIC_TYPE="GPU_NIC"
if [[ "$type" == "gpu" && "$NIC_TYPE" == "GPU_NIC" ]]; then
ibdev=$(ibdev2netdev 2>/dev/null | awk -v nic="${NICNAME_TYPE}${xgbe_no}" '$5 == nic {print $1}')
if [ -n "$ibdev" ] && ip link show "${NICNAME_TYPE}${xgbe_no}" | grep -q "state UP"; then
if $gpu_first; then
printf "KVCACHE_RDMA_NICS=%s" "$ibdev"
gpu_first=false
else
printf ",%s" "$ibdev"
fi
fi
fi
if [[ "$type" == "xpu" && "$NIC_TYPE" == "GPU_NIC" ]]; then
ibdev=$(ibdev2netdev 2>/dev/null | awk -v nic="${NICNAME_TYPE}${xgbe_no}" '$5 == nic {print $1}')
if [ -n "$ibdev" ] && ip link show "${NICNAME_TYPE}${xgbe_no}" | grep -q "state UP"; then
if $xpu_first; then
printf "KVCACHE_RDMA_NICS=%s,%s" "$ibdev" "$ibdev"
xpu_first=false
else
printf ",%s,%s" "$ibdev" "$ibdev"
fi
fi
fi
if [[ "$type" == "cpu" ]]; then
for (( xgbe_no=0; xgbe_no < XGBE_NUM; xgbe_no++ ))
do
[ ! -d "/sys/class/net/${NICNAME_TYPE}${xgbe_no}" ] && continue
PCI_ADDRESS=$(ethtool -i "${NICNAME_TYPE}${xgbe_no}" 2>/dev/null | awk -F '0000:' '/bus-info/{print $2}')
[ -z "$PCI_ADDRESS" ] && continue
NIC_ROOTPORT=$(readlink "/sys/bus/pci/devices/0000:${PCI_ADDRESS}" 2>/dev/null | awk -F '/' '{print $6}')
grep -qxF "$NIC_ROOTPORT" ${gpu_root_port_filename} 2>/dev/null && continue
if ip link show "${NICNAME_TYPE}${xgbe_no}" | grep -q "state UP" && \
ip a show "${NICNAME_TYPE}${xgbe_no}" | grep -q "inet"; then
printf "KV_CACHE_SOCKET_IFNAME=%s\n" "${NICNAME_TYPE}${xgbe_no}"
return 0
fi
done
echo "ERROR: No active CPU NIC with IP found!" >&2
return 1
fi
if [[ "$type" == "cpu_ib" && "$NIC_TYPE" == "CPU_NIC" ]]; then
ibdev=$(ibdev2netdev 2>/dev/null | awk -v nic="${NICNAME_TYPE}${xgbe_no}" '$5 == nic {print $1}')
if [ -n "$ibdev" ] && ip link show "${NICNAME_TYPE}${xgbe_no}" | grep -q "state UP" && \
ip a show "${NICNAME_TYPE}${xgbe_no}" | grep -q "inet "; then
if $cpu_ib_first; then
printf "KVCACHE_RDMA_NICS=%s" "$ibdev"
cpu_ib_first=false
else
printf ",%s" "$ibdev"
fi
fi
fi
done
case "$type" in
gpu) ! $gpu_first && printf "\n" ;;
xpu) ! $xpu_first && printf "\n" ;;
cpu) ! $cpu_first && printf "\n" ;;
cpu_ib) ! $cpu_ib_first && printf "\n" ;;
esac
}
function get_vxpu_nics() {
local topo_output=$(xpu-smi topo -m)
local xpu_info=$(echo "$topo_output" | grep -E '^XPU[0-9]+')
local nic_mapping=()
while IFS= read -r line; do
if [[ $line =~ NIC([0-9]+):\ +(mlx[0-9_]+) ]]; then
local nic_idx=${BASH_REMATCH[1]}
local nic_name=${BASH_REMATCH[2]}
nic_mapping[$nic_idx]=$nic_name
fi
done < <(echo "$topo_output" | grep -E '^\s*NIC[0-9]+:')
local nic_count=${#nic_mapping[@]}
declare -A priority_map=([PIX]=2 [NODE]=1 [SYS]=0)
local optimal_nics=()
while IFS= read -r line; do
local fields=($line)
local nic_start_index=5
local max_nics=$(( ${#fields[@]} - nic_start_index ))
local actual_nic_count=$(( max_nics < nic_count ? max_nics : nic_count ))
local best_priority=-1
local best_nic=""
for ((nic_idx=0; nic_idx<actual_nic_count; nic_idx++)); do
local conn_type=${fields[nic_idx+nic_start_index]}
local current_priority=${priority_map[$conn_type]:--1}
if (( current_priority > best_priority )); then
best_priority=$current_priority
best_nic="${nic_mapping[$nic_idx]}"
fi
done
if [[ -n "$best_nic" ]]; then
optimal_nics+=("$best_nic")
fi
done <<< "$xpu_info"
local IFS=,
export KVCACHE_RDMA_NICS="${optimal_nics[*]}"
echo "KVCACHE_RDMA_NICS=${optimal_nics[*]}"
}
function get_vcpu_nics() {
ip -o addr show | awk '$3 == "inet" && $4 ~ /^10\./ {print "KV_CACHE_SOCKET_IFNAME="$2; exit}'
}
function __main__() {
if [[ "$type" == "vxpu" ]]; then
get_vxpu_nics
return 0
fi
if [[ "$type" == "vcpu" ]]; then
get_vcpu_nics
return 0
fi
# 处理 bond 情况
if [[ "$type" == "cpu" ]]; then
for bond in $(ls -d /sys/class/net/bond* 2>/dev/null); do
bond_if=$(basename "$bond")
if ip link show "$bond_if" | grep -q "state UP" && \
ip a show "$bond_if" | grep -q "inet "; then
printf "KV_CACHE_SOCKET_IFNAME=%s\n" "$bond_if"
return 0
fi
done
fi
if [[ "$type" == "cpu_ib" ]]; then
first=true
for bond in $(ls -d /sys/class/net/bond* 2>/dev/null); do
bond_if=$(basename "$bond")
__NEW_GPU_ROOTPORT_FILE__
ibdev=$(ibdev2netdev 2>/dev/null | grep -w "$bond_if" | awk '{print $1}')
if [ -n "$ibdev" ] && ip link show "$bond_if" | grep -q "state UP" && \
ip a show "$bond_if" | grep -q "inet "; then
if $first; then
printf "KVCACHE_RDMA_NICS=%s" "$ibdev"
first=false
else
printf ",%s" "$ibdev"
fi
fi
bondib=$(show_gids 2>/dev/null | grep -w "$bond_if" | awk '{print $1}' | grep "mlx.*bond" | head -1)
if [ -n "$bondib" ] && ip link show "$bond_if" | grep -q "state UP" && \
ip a show "$bond_if" | grep -q "inet " && $first; then
printf "KVCACHE_RDMA_NICS=%s" "$bondib"
first=false
fi
__RM_GPU_ROOTPORT_FILE__
done
! $first && printf "\n"
[ ! $first ] && return 0
fi
local nic_types=("eth" "ib" "xgbe")
for nt in "${nic_types[@]}"; do
if ip a | grep -iq "$nt"; then
__NEW_GPU_ROOTPORT_FILE__
NICNAME_TYPE=$nt
__JUDGE_NIC_TYPE__
__RM_GPU_ROOTPORT_FILE__
fi
done
}
__main__

119
scripts/linux/build_linux_x86_64_cpp_cpu.sh
View File

@@ -1,119 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly BUILD_ROOT=build/Linux
readonly BUILD_DIR=${BUILD_ROOT}/x86_64
readonly PADDLEINFERENCE_DIRECTORY=$1
readonly PADDLEINFERENCE_VERSION=$2
BUILD_WITH_CUSTOM_PADDLE='OFF'
if [[ "$PADDLEINFERENCE_DIRECTORY" != "" ]]; then
BUILD_WITH_CUSTOM_PADDLE='ON'
fi
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=OFF \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=OFF \
-DENABLE_OPENVINO_BACKEND=OFF \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=OFF \
-DENABLE_BENCHMARK=OFF \
-DENABLE_FLYCV=OFF \
-DWITH_FLYCV_STATIC=OFF \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64]][${BUILD_DIR}/install]"
}
__build_fastdeploy_linux_x86_64_shared_custom_paddle() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=OFF \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=OFF \
-DENABLE_OPENVINO_BACKEND=OFF \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=OFF \
-DENABLE_BENCHMARK=OFF \
-DPADDLEINFERENCE_DIRECTORY=${PADDLEINFERENCE_DIRECTORY} \
-DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
-DENABLE_FLYCV=OFF \
-DWITH_FLYCV_STATIC=OFF \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64]][${BUILD_DIR}/install]"
echo "-- [INFO][${PADDLEINFERENCE_VERSION}][${PADDLEINFERENCE_DIRECTORY}]"
}
main() {
__make_build_dir
__check_cxx_envs
if [ "${BUILD_WITH_CUSTOM_PADDLE}" == "ON" ]; then
__build_fastdeploy_linux_x86_64_shared_custom_paddle
else
__build_fastdeploy_linux_x86_64_shared
fi
exit 0
}
main
# Usage:
# ./scripts/linux/build_linux_x86_64_cpp_cpu.sh
# ./scripts/linux/build_linux_x86_64_cpp_cpu.sh paddle_inference-linux-x64-mkl-avx-2.4.2 paddle2.4.2

125
scripts/linux/build_linux_x86_64_cpp_cpu_with_benchmark.sh
View File

@@ -1,125 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
ROOT_PATH=$(pwd)
BUILD_ROOT=build/Linux
BUILD_DIR="${BUILD_ROOT}/x86_64_gpu"
PADDLEINFERENCE_DIRECTORY=$1
PADDLEINFERENCE_VERSION=$2
PADDLEINFERENCE_API_CUSTOM_OP=$3
BUILD_WITH_CUSTOM_PADDLE='OFF'
if [[ -d "$1" ]]; then
BUILD_WITH_CUSTOM_PADDLE='ON'
else
if [[ "$1" == "ON" ]]; then
PADDLEINFERENCE_API_CUSTOM_OP='ON'
fi
fi
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_gpu_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=OFF \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=OFF \
-DENABLE_PADDLE_BACKEND=ON \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=OFF \
-DPADDLEINFERENCE_API_CUSTOM_OP=${PADDLEINFERENCE_API_CUSTOM_OP:-"OFF"} \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
}
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=OFF \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=OFF \
-DENABLE_PADDLE_BACKEND=ON \
-DPADDLEINFERENCE_DIRECTORY=${PADDLEINFERENCE_DIRECTORY} \
-DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
-DPADDLEINFERENCE_API_CUSTOM_OP=${PADDLEINFERENCE_API_CUSTOM_OP:-"OFF"} \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
echo "-- [INFO][${PADDLEINFERENCE_VERSION}][${PADDLEINFERENCE_DIRECTORY}]"
}
main() {
__make_build_dir
__check_cxx_envs
if [ "${BUILD_WITH_CUSTOM_PADDLE}" == "ON" ]; then
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle
else
__build_fastdeploy_linux_x86_64_gpu_shared
fi
exit 0
}
main
# Usage:
# ./scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh
# ./scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh $PADDLEINFERENCE_DIRECTORY $PADDLEINFERENCE_VERSION

121
scripts/linux/build_linux_x86_64_cpp_gpu.sh
View File

@@ -1,121 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly BUILD_ROOT=build/Linux
readonly BUILD_DIR="${BUILD_ROOT}/x86_64_gpu"
readonly PADDLEINFERENCE_DIRECTORY=$1
readonly PADDLEINFERENCE_VERSION=$2
BUILD_WITH_CUSTOM_PADDLE='OFF'
if [[ "$PADDLEINFERENCE_DIRECTORY" != "" ]]; then
BUILD_WITH_CUSTOM_PADDLE='ON'
fi
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_gpu_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=ON \
-DTRT_DIRECTORY=${TRT_DIRECTORY} \
-DCUDA_DIRECTORY=${CUDA_DIRECTORY} \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=OFF \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
}
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=ON \
-DTRT_DIRECTORY=${TRT_DIRECTORY} \
-DCUDA_DIRECTORY=${CUDA_DIRECTORY} \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DPADDLEINFERENCE_DIRECTORY=${PADDLEINFERENCE_DIRECTORY} \
-DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=OFF \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
echo "-- [INFO][${PADDLEINFERENCE_VERSION}][${PADDLEINFERENCE_DIRECTORY}]"
}
main() {
__make_build_dir
__check_cxx_envs
if [ "${BUILD_WITH_CUSTOM_PADDLE}" == "ON" ]; then
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle
else
__build_fastdeploy_linux_x86_64_gpu_shared
fi
exit 0
}
main
# Usage:
# ./scripts/linux/build_linux_x86_64_cpp_gpu.sh
# ./scripts/linux/build_linux_x86_64_cpp_gpu.sh paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-2.4.2 paddle2.4.2

106
scripts/linux/build_linux_x86_64_cpp_gpu_encrypt_runtime.sh
View File

@@ -1,106 +0,0 @@
#!/usr/bin/env bash
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly BUILD_ROOT=build/Linux
readonly BUILD_DIR="${BUILD_ROOT}/x86_64_gpu"
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/fastdeploy_install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=ON \
-DTRT_DIRECTORY=${TRT_DIRECTORY} \
-DCUDA_DIRECTORY=${CUDA_DIRECTORY} \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DPADDLEINFERENCE_DIRECTORY=${PADDLEINFERENCE_DIRECTORY} \
-DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=OFF \
-DENABLE_BENCHMARK=OFF \
-DBUILD_EXAMPLES=OFF \
-DPython_EXECUTABLE=/usr/bin/python3 \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-DLIBRARY_NAME=fastdeploy_runtime \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${FASDEPLOY_INSTALL_DIR}]"
echo "-- [INFO][${PADDLEINFERENCE_DIRECTORY}][${PADDLEINFERENCE_VERSION}]"
}
main() {
__make_build_dir
__check_cxx_envs
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle
exit 0
}
main
# Usage:
# export PADDLEINFERENCE_DIRECTORY=xxx
# export PADDLEINFERENCE_VERSION=xxx
# export CUDA_DIRECTOY=/usr/local/cuda
# export TRT_DIRECTORY=/home/qiuyanjun/TensorRT-8.5.2.2
# ./scripts/linux/build_linux_x86_64_cpp_gpu_encrypt_runtime.sh

129
scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh
View File

@@ -1,129 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
ROOT_PATH=$(pwd)
BUILD_ROOT=build/Linux
BUILD_DIR="${BUILD_ROOT}/x86_64_gpu"
PADDLEINFERENCE_DIRECTORY=$1
PADDLEINFERENCE_VERSION=$2
PADDLEINFERENCE_API_CUSTOM_OP=$3
BUILD_WITH_CUSTOM_PADDLE='OFF'
if [[ -d "$1" ]]; then
BUILD_WITH_CUSTOM_PADDLE='ON'
else
if [[ "$1" == "ON" ]]; then
PADDLEINFERENCE_API_CUSTOM_OP='ON'
fi
fi
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_gpu_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=ON \
-DTRT_DIRECTORY=${TRT_DIRECTORY} \
-DCUDA_DIRECTORY=${CUDA_DIRECTORY} \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=OFF \
-DPADDLEINFERENCE_API_CUSTOM_OP=${PADDLEINFERENCE_API_CUSTOM_OP:-"OFF"} \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
}
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_GPU=ON \
-DTRT_DIRECTORY=${TRT_DIRECTORY} \
-DCUDA_DIRECTORY=${CUDA_DIRECTORY} \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_TRT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DPADDLEINFERENCE_DIRECTORY=${PADDLEINFERENCE_DIRECTORY} \
-DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
-DPADDLEINFERENCE_API_CUSTOM_OP=${PADDLEINFERENCE_API_CUSTOM_OP:-"OFF"} \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_gpu}][${BUILD_DIR}/install]"
echo "-- [INFO][${PADDLEINFERENCE_VERSION}][${PADDLEINFERENCE_DIRECTORY}]"
}
main() {
__make_build_dir
__check_cxx_envs
if [ "${BUILD_WITH_CUSTOM_PADDLE}" == "ON" ]; then
__build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle
else
__build_fastdeploy_linux_x86_64_gpu_shared
fi
exit 0
}
main
# Usage:
# ./scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh
# ./scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh $PADDLEINFERENCE_DIRECTORY $PADDLEINFERENCE_VERSION

79
scripts/linux/build_linux_x86_64_cpp_xpu_with_benchmark.sh
View File

@@ -1,79 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly BUILD_ROOT=build/Linux
readonly BUILD_DIR="${BUILD_ROOT}/x86_64_xpu"
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_linux_x86_64_xpu_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=Release \
-DWITH_KUNLUNXIN=ON \
-DENABLE_ORT_BACKEND=OFF \
-DENABLE_PADDLE_BACKEND=ON \
-DENABLE_LITE_BACKEND=OFF \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][x86_64_xpu}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
__build_fastdeploy_linux_x86_64_xpu_shared
exit 0
}
main
# Usage:
# ./scripts/linux/build_linux_x86_64_cpp_gpu.sh

102
scripts/macosx/build_macosx_cpp.sh
View File

@@ -1,102 +0,0 @@
#!/bin/bash
set -e
set +x
# -------------------------------------------------------------------------------
# readonly global variables
# -------------------------------------------------------------------------------
readonly ROOT_PATH=$(pwd)
readonly BUILD_ROOT=build/MacOSX
readonly OSX_ARCH=$1 # arm64, x86_64
readonly BUILD_DIR=${BUILD_ROOT}/${OSX_ARCH}
# -------------------------------------------------------------------------------
# tasks
# -------------------------------------------------------------------------------
__make_build_dir() {
if [ ! -d "${BUILD_DIR}" ]; then
echo "-- [INFO] BUILD_DIR: ${BUILD_DIR} not exists, setup manually ..."
if [ ! -d "${BUILD_ROOT}" ]; then
mkdir -p "${BUILD_ROOT}" && echo "-- [INFO] Created ${BUILD_ROOT} !"
fi
mkdir -p "${BUILD_DIR}" && echo "-- [INFO] Created ${BUILD_DIR} !"
else
echo "-- [INFO] Found BUILD_DIR: ${BUILD_DIR}"
fi
}
__check_cxx_envs() {
if [ $LDFLAGS ]; then
echo "-- [INFO] Found LDFLAGS: ${LDFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset LDFLAGS
fi
if [ $CPPFLAGS ]; then
echo "-- [INFO] Found CPPFLAGS: ${CPPFLAGS}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPPFLAGS
fi
if [ $CPLUS_INCLUDE_PATH ]; then
echo "-- [INFO] Found CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset CPLUS_INCLUDE_PATH
fi
if [ $C_INCLUDE_PATH ]; then
echo "-- [INFO] Found C_INCLUDE_PATH: ${C_INCLUDE_PATH}, \c"
echo "unset it before crossing compiling ${BUILD_DIR}"
unset C_INCLUDE_PATH
fi
}
__build_fastdeploy_osx_arm64_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=MinSizeRel \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=ON \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${OSX_ARCH}][${BUILD_DIR}/install]"
}
__build_fastdeploy_osx_x86_64_shared() {
local FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install"
cd "${BUILD_DIR}" && echo "-- [INFO] Working Dir: ${PWD}"
cmake -DCMAKE_BUILD_TYPE=MinSizeRel \
-DENABLE_ORT_BACKEND=ON \
-DENABLE_PADDLE_BACKEND=ON \
-DENABLE_OPENVINO_BACKEND=ON \
-DENABLE_PADDLE2ONNX=ON \
-DENABLE_VISION=ON \
-DENABLE_BENCHMARK=ON \
-DBUILD_EXAMPLES=ON \
-DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
-Wno-dev ../../.. && make -j8 && make install
echo "-- [INFO][built][${OSX_ARCH}][${BUILD_DIR}/install]"
}
main() {
__make_build_dir
__check_cxx_envs
if [ "$OSX_ARCH" = "arm64" ]; then
__build_fastdeploy_osx_arm64_shared
else
__build_fastdeploy_osx_x86_64_shared
fi
exit 0
}
main
# Usage:
# ./scripts/macosx/build_macosx_cpp.sh arm64
# ./scripts/macosx/build_macosx_cpp.sh x86_64

51
scripts/merge_cache_scale.py Normal file
View File

@@ -0,0 +1,51 @@
"""
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
import os
import json
import argparse
import numpy as np
parser = argparse.ArgumentParser()
parser.add_argument("--model_dir", type=str, required=True)
args = parser.parse_args()
ep_num = 4
model_dir = args.model_dir
scale_dicts = []
for i in range(ep_num):
scale_dict = {}
path = os.path.join(model_dir, f"cachekv_scales_{i}.json")
with open(path, "r") as file:
scale_c = json.load(file)
for k, v in scale_c.items():
scale_dict[k] = v
scale_dicts.append(scale_dict)
new_dict = {}
for k in scale_dicts[0].keys():
v_list = [scale_dicts[i][k] for i in range(ep_num)]
v = np.concatenate(v_list, axis=1)
v = v.tolist()
new_dict[k] = v
res_file = os.path.join(model_dir, "cachekv_scales.json")
b = json.dumps(new_dict)
f = open(res_file, "w")
f.write(b)
f.close()

44
scripts/patch_paddle_inference.py
View File

@@ -1,44 +0,0 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import shutil
import subprocess
import platform
import sys
def process_paddle_inference(paddle_inference_so_file):
if platform.system().lower() != "linux":
return
rpaths = [
"$ORIGIN", "$ORIGIN/../../third_party/install/mkldnn/lib/",
"$ORIGIN/../../third_party/install/mklml/lib/",
"$ORIGIN/../../third_party/install/xpu/lib/",
"$ORIGIN/../../third_party/install/fdmodel/lib/",
"$ORIGIN/../../../tensorrt/lib/"
]
patchelf_exe = os.getenv("PATCHELF_EXE", "patchelf")
command = "{} --force-rpath --set-rpath '{}' {}".format(
patchelf_exe, ":".join(rpaths), paddle_inference_so_file)
if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
assert os.system(
command) == 0, "patchelf {} failed, the command: {}".format(
paddle_inference_so_file, command)
if __name__ == "__main__":
process_paddle_inference(sys.argv[1])

43
scripts/patch_paddle_lite.py
View File

@@ -1,43 +0,0 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import shutil
import subprocess
import platform
import sys
def process_paddle_lite(paddle_lite_so_path):
if platform.system().lower() != "linux":
return
rpaths = ["$ORIGIN", "$ORIGIN/mklml/lib/"]
patchelf_exe = os.getenv("PATCHELF_EXE", "patchelf")
for root, dirs, files in os.walk(paddle_lite_so_path):
for lib in files:
if ".so" in lib:
paddle_lite_so_file = os.path.join(root, lib)
command = "{} --set-rpath '{}' {}".format(
patchelf_exe, ":".join(rpaths), paddle_lite_so_file)
if platform.machine() != 'sw_64' and platform.machine(
) != 'mips64':
assert os.system(
command
) == 0, "patchelf {} failed, the command: {}".format(
paddle_lite_so_file, command)
if __name__ == "__main__":
process_paddle_lite(sys.argv[1])

78
scripts/prefill_fake_server.sh Normal file
View File

@@ -0,0 +1,78 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cd ../custom_ops/gpu_ops/fp8_deep_gemm
rm -rf build dist deep_gemm.egg-info
pip uninstall deep_gemm
python setup.py develop
cd ../../../scripts
rm -rf log
rm -f core*
export NVIDIA_TF32_OVERRIDE=0
export NCCL_ALGO=Tree
export FLAGS_allocator_strategy=auto_growth
export FLAGS_fraction_of_gpu_memory_to_use=0.98
export FLAGS_gemm_use_half_precision_compute_type=False
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
export PYTHONPATH=$(dirname $(pwd)):$PYTHONPATH
export FLAGS_enable_pir_api=0
export FLAGS_use_append_attn=1
export FLAGS_use_fa3=1
export devices=0,1,2,3,4,5,6,7
export CUDA_VISIBLE_DEVICES=${devices}
# export FLAGS_enable_blaslt_global_search=1
# export FLAGS_cublaslt_device_best_config=/path/to/cublaslt_device_best_config.csv
# export FLAGS_use_cutlass_device_best_config_path=/path/to/cutlass_device_best_config.json
model_path=${1:-"/path/to/model"}
for name in `env | grep -E 'PADDLE|ENDPOINT' | awk -F'=' '{print $1}'`; do
unset ${name}
done
export PADDLE_TRAINER_ID=0
export PADDLE_TRAINERS_NUM=1
export TRAINER_INSTANCES_NUM=1
export TRAINER_INSTANCES=`hostname -i`
self_ip=`hostname -i`
python -m paddle.distributed.launch \
--gpus ${devices} \
fake_p_server.py \
--model_name_or_path ${model_path} \
--input_file "../data/qf_turbopro_5k_pd5_eb" \
--output_file ./predict_out.json \
--predict_model_type "WINT8" \
--dtype bfloat16 \
--data_format "pt" \
--append_bos_token "False" \
--max_dec_len 1 \
--top_p 0 \
--batch_size 8 \
--moe_quant_type "weight_only_int4" \
--use_ep "True" \
--generation_phase 1 \
--benchmark "False" \
--fake_server_p "True" \
--use_cache_kv_int8 "False" \
--scale_dir "None"

38
scripts/run_offline_quantization.sh Normal file
View File

@@ -0,0 +1,38 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
rm -rf log
rm -f core*
export devices=0
export CUDA_VISIBLE_DEVICES=${devices}
#*/merged_tp1_state_split/safetensors
model_path=${1:-"/path/to/model"}
output_path=${2:-"/path/to/quanted"}
for name in `env | grep -E 'PADDLE|ENDPOINT' | awk -F'=' '{print $1}'`; do
unset ${name}
done
export PADDLE_TRAINER_ID=0
export PADDLE_TRAINERS_NUM=1
export TRAINER_INSTANCES_NUM=1
export TRAINER_INSTANCES=`hostname -i`
self_ip=`hostname -i`
python offline_quantization.py \
--model_name_or_path ${model_path} \
--predict_model_type "weight_only_int8" \
--moe_quant_type "fp8" \
--output_dir ${output_path} \
--safe_serialization "True" \
--dtype "bfloat16"

63
scripts/run_unittest.sh Normal file
View File

@@ -0,0 +1,63 @@
#!/bin/bash
set -x
set -e
cd FastDeploy
export no_proxy=agent.baidu.com:8118,localhost,127.0.0.1,localaddress,.localdomain.com,.cdn.bcebos.com,.baidu.com,bcebos.com
export https_proxy=
export http_proxy=
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export LD_LIBRARY_PATH=/usr/local/cuda/compat/:/usr/lib64/:$LD_LIBRARY_PATH
ldconfig
python -V
pwd
git config --global --add safe.directory /workspace1/FastDeploy
python -m pip install --force-reinstall --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install --upgrade --force-reinstall -r requirements/unittest/requirements.txt
bash tools/build_wheel.sh
# 获取显存占用最小的gpu
gpu_info=$(nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits 2>/dev/null)
if [ -z "$gpu_info" ]; then
echo "错误: 未找到 GPU 或 nvidia-smi 不可用。"
exit 1
fi
min_vram=999999
min_gpu=-1
while read -r line; do
gpu_id=$(echo "$line" | awk -F', ' '{print $1}' | tr -d ' ')
vram=$(echo "$line" | awk -F', ' '{print $2}' | tr -d ' ')
if [ "$vram" -lt "$min_vram" ]; then
min_vram=$vram
min_gpu=$gpu_id
fi
done <<< "$gpu_info"
export CUDA_VISIBLE_DEVICES=${min_gpu}
# 使用 find 命令查找 test 目录下的 .py 文件
test_files=$(find test -type f -name "test*.py")
# 遍历每个找到的测试文件
for test_file in $test_files; do
python $test_file
# 检查前一个命令的退出状态码
if [ $? -ne 0 ]; then
echo $test_file
exit 1
fi
done
echo "All tests passed."
exit 0

45
scripts/tune_cublaslt_int8_gemm.py Normal file
View File

@@ -0,0 +1,45 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""tune_cublaslt_gemm"""
import paddle
from fastdeploy.utils import llm_logger as logger
def tune_cublaslt_int8_gemm(
ns: list,
ks: list,
m_min: int = 32,
m_max: int = 32768,
dtype="int8",
is_test=True,
is_read_from_file=False,
path="./cublaslt_gemm_search.csv",
):
"""
tune cublaslt int8 gemm performance
"""
K_tensor = paddle.to_tensor(ks)
N_tensor = paddle.to_tensor(ns)
try:
from fastdeploy.model_executor.ops.gpu import tune_cublaslt_gemm
except ImportError:
logger.warning(
"From fastdeploy.model_executor.ops.gpu import tune_cublaslt_gemm Failed!"
)
return
tune_cublaslt_gemm(K_tensor, N_tensor, m_min, m_max, dtype, is_test,
is_read_from_file, path)

198
scripts/tune_cutlass_fp8_gemm.py Normal file
View File

@@ -0,0 +1,198 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" UT for cutlass_fp8_fp8_half_gemm_fused """
import paddle
from fastdeploy.utils import llm_logger as logger
def tune_cutlass_fp8_fp8_half_gemm_fused(
ns: list,
ks: list,
m_min: int = 32,
m_max: int = 32768,
):
"""
Tune fp8 gemm.
"""
assert len(ns) == len(
ks), "The length of `ns` must be equal to that of `ks`"
try:
from fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_half_gemm_fused
except ImportError:
logger.warning(
"From fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_half_gemm_fused failed, \
fp8 is only support cuda arch 89+.")
return
paddle.seed(2003)
for m in range(m_min, m_max + 32, 32):
if m > m_max:
break
for idx in range(len(ns)):
n = ns[idx]
k = ks[idx]
A = paddle.rand(shape=[m, k],
dtype="bfloat16").astype("float8_e4m3fn")
B = paddle.rand(shape=[n, k],
dtype="bfloat16").astype("float8_e4m3fn")
cutlass_fp8_fp8_half_gemm_fused(
A,
B,
bias=None,
transpose_x=False,
transpose_y=True,
output_dtype="bfloat16",
scale=0.5,
activation_type="identity",
)
paddle.device.cuda.empty_cache()
def tune_cutlass_fp8_fp8_fp8_dual_gemm_fused(
ns: list,
ks: list,
m_min: int = 32,
m_max: int = 32768,
):
"""
Tune fp8 dual-gemm.
"""
assert len(ns) == len(
ks), "The length of `ns` must be equal to that of `ks`"
try:
from fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_fp8_dual_gemm_fused
except ImportError:
logger.warning(
"From fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_fp8_dual_gemm_fused failed, \
fp8 is only support cuda arch 89+.")
return
paddle.seed(2003)
for m in range(m_min, m_max + 32, 32):
if m > m_max:
break
for idx in range(len(ns)):
n = ns[idx]
k = ks[idx]
A = paddle.rand(shape=[m, k],
dtype="bfloat16").astype("float8_e4m3fn")
B0 = paddle.rand(shape=[n, k],
dtype="bfloat16").astype("float8_e4m3fn")
B1 = paddle.rand(shape=[n, k],
dtype="bfloat16").astype("float8_e4m3fn")
cutlass_fp8_fp8_fp8_dual_gemm_fused(
A,
B0,
B1,
bias0=None,
bias1=None,
transpose_x=False,
transpose_y=True,
scale0=0.1,
scale1=0.1,
scale_out=0.5,
activation_type="swiglu",
)
paddle.device.cuda.empty_cache()
def tune_per_channel_fp8_gemm_fused(
ns: list,
ks: list,
m_min: int = 32,
m_max: int = 32768,
):
"""
Tune per-channel quant gemm.
"""
assert len(ns) == len(
ks), "The length of `ns` must be equal to that of `ks`"
try:
from fastdeploy.model_executor.ops.gpu import per_channel_fp8_fp8_half_gemm_fused
except ImportError:
logger.warning(
"From fastdeploy.model_executor.ops.gpu import per_channel_fp8_fp8_half_gemm_fused failed, \
fp8 is only support cuda arch 89+.")
return
paddle.seed(2003)
for m in range(m_min, m_max + 32, 32):
if m > m_max:
break
for idx in range(len(ns)):
n = ns[idx]
k = ks[idx]
A = paddle.rand(shape=[m, k],
dtype="bfloat16").astype("float8_e4m3fn")
B = paddle.rand(shape=[n, k],
dtype="bfloat16").astype("float8_e4m3fn")
scalar_scale = paddle.full([1], 0.168, dtype="float32")
channel_scale = paddle.rand(shape=[n], dtype="float32")
per_channel_fp8_fp8_half_gemm_fused(
A,
B,
bias=None,
scalar_scale=scalar_scale,
channel_scale=channel_scale,
transpose_x=False,
transpose_y=True,
output_dtype="bfloat16",
)
paddle.device.cuda.empty_cache()
def tune_blockwise_fp8_gemm_fused(
ns: list,
ks: list,
m_min: int = 32,
m_max: int = 32768,
):
"""
Tune per-channel quant gemm.
"""
assert len(ns) == len(
ks), "The length of `ns` must be equal to that of `ks`"
try:
from fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_half_block_gemm_fused
except ImportError:
logger.warning(
"From fastdeploy.model_executor.ops.gpu import cutlass_fp8_fp8_half_block_gemm_fused failed, \
fp8 is only support cuda arch 90+.")
return
paddle.seed(2003)
for m in range(m_min, m_max + 32, 32):
if m > m_max:
break
for idx in range(len(ns)):
n = ns[idx]
k = ks[idx]
scale_n = (n + 128 - 1) // 128
scale_k = (k + 128 - 1) // 128
A = paddle.rand(shape=[m, k],
dtype="bfloat16").astype("float8_e4m3fn")
B = paddle.rand(shape=[n, k],
dtype="bfloat16").astype("float8_e4m3fn")
a_scale = paddle.randn([scale_k, m], dtype="float32")
b_scale = paddle.randn([scale_n, scale_k], dtype="float32")
cutlass_fp8_fp8_half_block_gemm_fused(
A,
B,
x_sacle=a_scale,
y_sacle=b_scale,
bias=None,
transpose_x=False,
transpose_y=True,
output_dtype="bfloat16",
)
paddle.device.cuda.empty_cache()

74
scripts/tune_scaled_gemm_f8_i4_f16.py Normal file
View File

@@ -0,0 +1,74 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""tune_cutlass_fp8int4_gemm"""
import os
import paddle
from fastdeploy.model_executor.ops.gpu import scaled_gemm_f8_i4_f16
from tqdm import tqdm
def tune_scaled_gemm_f8_i4_f16(
ns: list, ks: list, dtype="int8", is_test=True, is_read_from_file=False
):
"""
Tune fp8 int4 gemm.
"""
assert len(ns) == len(ks), "list[n] and list[k] should have the same length!"
os.environ["FLAGS_fastdeploy_op_configs"] = "tune"
mm_tmp = []
for m in range(1, 4, 1):
mm_tmp.append(m)
for m in range(4, 16, 4):
mm_tmp.append(m)
for m in range(16, 64, 16):
mm_tmp.append(m)
for m in range(64, 256, 32):
mm_tmp.append(m)
for m in range(256, 512, 64):
mm_tmp.append(m)
for m in range(512, 1024, 128):
mm_tmp.append(m)
for m in range(1024, 8192, 1024):
mm_tmp.append(m)
# Note the end value is 32769 to include 32768
for m in range(8192, 32769, 4096):
mm_tmp.append(m)
for m in tqdm(mm_tmp):
for idx in range(0, len(ns)):
n = ns[idx]
k = ks[idx]
A = paddle.cast(paddle.ones((m, k)), "float8_e4m3fn")
B = paddle.cast(paddle.ones((n // 2, k)), "int8")
w_scale = paddle.ones(n)
scaled_gemm_f8_i4_f16(
x=A.cuda(),
y=B.cuda(),
scale=paddle.cast(w_scale, dtype).cuda(),
zero_points=None,
bias=None,
out_scale=1.0,
groupsize=-1,
out_dtype=dtype,
)