Optimize compilation process (#164)

* refine build scripts

* fix copy for trt

* fix library import

* fix lib import

* fix lib import

* Update CMakeLists.txt

* Update process_libraries.py

* polish cmake

Co-authored-by: dev5 <paddle_dev5@DESKTOP-O9GS2N7>

CMakeLists.txt

@@ -215,14 +215,14 @@ if(ENABLE_TRT_BACKEND)
   endif()
   find_package(Python COMPONENTS Interpreter Development REQUIRED)
   message(STATUS "Copying ${TRT_DIRECTORY}/lib to ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib ...")
-  execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib)
+  execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/build_scripts/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib)
   if(UNIX)
-  execute_process(COMMAND sh -c "ls *.so*" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
+    execute_process(COMMAND sh -c "ls *.so*" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
-	          COMMAND sh -c "xargs patchelf --set-rpath '$ORIGIN'" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
+  	          COMMAND sh -c "xargs patchelf --set-rpath '$ORIGIN'" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib
-	          RESULT_VARIABLE result
+  	          RESULT_VARIABLE result
-                  OUTPUT_VARIABLE curr_out
+                    OUTPUT_VARIABLE curr_out
-                  ERROR_VARIABLE  curr_out)
+                    ERROR_VARIABLE  curr_out)
-  message(STATUS "result:${result} out:${curr_out}")
+    message(STATUS "result:${result} out:${curr_out}")
   endif()
 
 endif()
@@ -295,6 +295,11 @@ if(MSVC)
 endif()
 target_link_libraries(${LIBRARY_NAME} ${DEPEND_LIBS})
 
+if(WIN32)
+  add_custom_target(copy_yaml_library ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/third_party/yaml-cpp/Release  ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/yaml-cpp/lib DEPENDS ${LIBRARY_NAME})
+  add_custom_target(copy_yaml_include ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include  ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/yaml-cpp/include DEPENDS ${LIBRARY_NAME})
+endif()
+
 # add examples after prepare include paths for third-parties
 if(BUILD_EXAMPLES AND EXISTS ${PROJECT_SOURCE_DIR}/examples)
   add_definitions(-DBUILD_EXAMPLES)
@@ -412,6 +417,17 @@ if(BUILD_FASTDEPLOY_PYTHON)
     target_compile_options(${PY_LIBRARY_NAME} PRIVATE $<$<NOT:$<CONFIG:Debug>>:/MT> $<$<CONFIG:Debug>:/MTd>)
   endif()
 
+  file(REMOVE_RECURSE ${PROJECT_SOURCE_DIR}/fastdeploy/libs)
+  file(MAKE_DIRECTORY ${PROJECT_SOURCE_DIR}/fastdeploy/libs)
+
+  if(WIN32)
+    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy_directory   ${CMAKE_CURRENT_BINARY_DIR}/Release ${PROJECT_SOURCE_DIR}/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
+  elseif(APPLE)
+    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/*.so** ${CMAKE_CURRENT_BINARY_DIR}/*.dylib** ${PROJECT_SOURCE_DIR}/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
+  else()
+    add_custom_target(copy_fd_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/*.so* ${PROJECT_SOURCE_DIR}/fastdeploy/libs/ DEPENDS ${PY_LIBRARY_NAME})
+  endif()
+  add_custom_target(copy_third_libraries ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install ${PROJECT_SOURCE_DIR}/fastdeploy/libs/third_libs DEPENDS ${PY_LIBRARY_NAME})
 endif(BUILD_FASTDEPLOY_PYTHON)
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")

build_scripts/process_libraries.py

@@ -0,0 +1,128 @@
+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import shutil
+import subprocess
+import platform
+
+
+def process_on_linux(current_dir):
+    rpaths = ["$ORIGIN:$ORIGIN/libs"]
+    fd_libs = list()
+    libs_path = os.path.join(current_dir, "fastdeploy", "libs")
+    for f in os.listdir(libs_path):
+        filename = os.path.join(libs_path, f)
+        if not os.path.isfile(filename):
+            continue
+        if f.count("fastdeploy") and f.count(".so") > 0:
+            fd_libs.append(filename)
+
+    third_libs_path = os.path.join(libs_path, "third_libs")
+    for root, dirs, files in os.walk(third_libs_path):
+        for d in dirs:
+            if d != "lib":
+                continue
+            rel_path = os.path.relpath(os.path.join(root, d), libs_path)
+            rpath = "$ORIGIN/" + rel_path
+            rpaths.append(rpath)
+
+    for lib in fd_libs:
+        command = "patchelf --set-rpath '{}' {}".format(":".join(rpaths), lib)
+        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
+            assert subprocess.Popen(
+                command,
+                shell=True) != 0, "patchelf {} failed, the command: {}".format(
+                    command, lib)
+
+
+def process_on_mac(current_dir):
+    fd_libs = list()
+    libs_path = os.path.join(current_dir, "fastdeploy", "libs")
+    for f in os.listdir(libs_path):
+        filename = os.path.join(libs_path, f)
+        if not os.path.isfile(filename):
+            continue
+        if f.count("fastdeploy") and (f.count(".dylib") > 0 or f.count(".so") > 0):
+            fd_libs.append(filename)
+
+    pre_commands = list()
+    for lib in fd_libs:
+        pre_commands.append(
+            "install_name_tool -delete_rpath '@loader_path/libs' " + lib)
+
+    commands = list()
+    third_libs_path = os.path.join(libs_path, "third_libs")
+    for root, dirs, files in os.walk(third_libs_path):
+        for d in dirs:
+            if d != "lib":
+                continue
+        rel_path = rel_path = os.path.relpath(os.path.join(root, d), libs_path)
+        rpath = "$loader_path/" + rel_path
+        for lib in fd_libs:
+            pre_commands.append(
+                "install_name_tool -delete_rpath '@loader_path/{}' {}".format(
+                    rpath, lib))
+            commands.append("install_name_tool -add_rpath 'loader_path/{}' {}".
+                            format(rpath, lib))
+
+    for cmd in pre_commands:
+        try:
+            subprocess.Popen(cmd, shell=True)
+        except:
+            print("Skip execute command:", cmd)
+
+    for cmd in commands:
+        assert subprocess.Popen(
+            cmd, shell=True) != 0, "Execute command failed: {}".format(cmd)
+
+def process_on_windows(current_dir):
+    libs_path = os.path.join(current_dir, "fastdeploy", "libs")
+    third_libs_path = os.path.join(libs_path, "third_libs")
+    for root, dirs, files in os.walk(third_libs_path):
+        for f in files:
+            file_path = os.path.join(root, f)
+            if f.count('onnxruntime') > 0 and f.endswith('.dll'):
+                shutil.copy(file_path, libs_path)
+
+def get_all_files(dirname):
+    files = list()
+    for root, dirs, filenames in os.walk(dirname):
+        for f in filenames:
+            fullname = os.path.join(root, f)
+            files.append(fullname)
+    return files
+
+
+def process_libraries(current_dir):
+    if platform.system().lower() == "linux":
+        process_on_linux(current_dir)
+    elif platform.system().lower() == "darwin":
+        process_on_mac(current_dir)
+    elif platform.system().lower() == "windows":
+        process_on_windows(current_dir)
+
+    all_files = get_all_files(os.path.join(current_dir, "fastdeploy", "libs"))
+    package_data = list()
+    filters = [".vcxproj", ".png", ".java", ".h", ".cc", ".cpp", ".hpp"]
+    for f in all_files:
+        remain = True
+        for flt in filters:
+            if f.count(flt) > 0:
+                remain = False
+        if remain:
+            package_data.append(
+                os.path.relpath(f, os.path.join(current_dir, "fastdeploy")))
+    return package_data

external/summary.cmake

@@ -52,4 +52,8 @@ function(fastdeploy_summary)
   message(STATUS "  ENABLE_TEXT               : ${ENABLE_TEXT}")
   message(STATUS "  ENABLE_DEBUG              : ${ENABLE_DEBUG}")
   message(STATUS "  ENABLE_VISION_VISUALIZE   : ${ENABLE_VISION_VISUALIZE}")
+  if (${BUILD_FASTDEPLOY_PYTHON})
+    message(STATUS "  Python executable         : ${PYTHON_EXECUTABLE}")
+    message(STATUS "  Python includes           : ${PYTHON_INCLUDE_DIR}")
+  endif()
 endfunction()

fastdeploy/c_lib_wrap.py.in

@@ -119,7 +119,7 @@ if os.name == "nt":
 
 
 try:
-    from .@PY_LIBRARY_NAME@ import *
+    from .libs.@PY_LIBRARY_NAME@ import *
 except:
     raise RuntimeError("FastDeploy initalized failed!")
 

setup.py

@@ -20,6 +20,7 @@ from __future__ import unicode_literals
 import shutil
 import os
 
+TOP_DIR = os.path.realpath(os.path.dirname(__file__))
 PACKAGE_NAME = os.getenv("PACKAGE_NAME", "fastdeploy")
 wheel_name = "fastdeploy-python"
 
@@ -40,14 +41,15 @@ import platform
 from textwrap import dedent
 import multiprocessing
 
-with open("requirements.txt") as fin:
+with open(os.path.join(TOP_DIR, "requirements.txt")) as fin:
     REQUIRED_PACKAGES = fin.read()
 
 setup_configs = dict()
 setup_configs["ENABLE_PADDLE_FRONTEND"] = os.getenv("ENABLE_PADDLE_FRONTEND",
                                                     "ON")
 setup_configs["ENABLE_ORT_BACKEND"] = os.getenv("ENABLE_ORT_BACKEND", "ON")
-setup_configs["ENABLE_OPENVINO_BACKEND"] = os.getenv("ENABLE_OPENVINO_BACKEND", "OFF")
+setup_configs["ENABLE_OPENVINO_BACKEND"] = os.getenv("ENABLE_OPENVINO_BACKEND",
+                                                     "OFF")
 setup_configs["ENABLE_PADDLE_BACKEND"] = os.getenv("ENABLE_PADDLE_BACKEND",
                                                    "OFF")
 setup_configs["ENABLE_VISION"] = os.getenv("ENABLE_VISION", "ON")
@@ -65,7 +67,6 @@ if setup_configs["WITH_GPU"] == "ON":
 if os.getenv("CMAKE_CXX_COMPILER", None) is not None:
     setup_configs["CMAKE_CXX_COMPILER"] = os.getenv("CMAKE_CXX_COMPILER")
 
-TOP_DIR = os.path.realpath(os.path.dirname(__file__))
 SRC_DIR = os.path.join(TOP_DIR, PACKAGE_NAME)
 CMAKE_BUILD_DIR = os.path.join(TOP_DIR, '.setuptools-cmake-build')
 
@@ -324,9 +325,9 @@ ext_modules = [
 
 # no need to do fancy stuff so far
 if PACKAGE_NAME != "fastdeploy":
-    packages = setuptools.find_packages(exclude=['fastdeploy*'])
+    packages = setuptools.find_packages(exclude=['fastdeploy*', 'build_scripts'])
 else:
-    packages = setuptools.find_packages()
+    packages = setuptools.find_packages(exclude=['build_scripts'])
 
 ################################################################################
 # Test
@@ -343,155 +344,55 @@ if sys.version_info[0] == 3:
 package_data = {PACKAGE_NAME: ["LICENSE", "ThirdPartyNotices.txt"]}
 
 if sys.argv[1] == "install" or sys.argv[1] == "bdist_wheel":
-    if not os.path.exists(".setuptools-cmake-build"):
+    shutil.copy(os.path.join(TOP_DIR, "ThirdPartyNotices.txt"), os.path.join(TOP_DIR, PACKAGE_NAME))
-        print("Please execute `python setup.py build` first.")
+    shutil.copy(os.path.join(TOP_DIR, "LICENSE"), os.path.join(TOP_DIR, PACKAGE_NAME))
+    if not os.path.exists(os.path.join(TOP_DIR, "fastdeploy", "libs", "third_libs")):
+        print("Didn't detect path: fastdeploy/libs/third_libs exist, please execute `python setup.py build` first")
         sys.exit(0)
-    import shutil
+    sys.path.append(os.path.split(os.path.abspath(__file__))[0])
-
+    from build_scripts.process_libraries import process_libraries
-    shutil.copy("ThirdPartyNotices.txt", PACKAGE_NAME)
+    all_lib_data = process_libraries(
-    shutil.copy("LICENSE", PACKAGE_NAME)
+        os.path.split(os.path.abspath(__file__))[0])
-    depend_libs = list()
+    package_data[PACKAGE_NAME].extend(all_lib_data)
-
+    setuptools.setup(
-    # copy fastdeploy library
+        name=wheel_name,
-    pybind_so_file = None
+        version=VersionInfo.version,
-    for f in os.listdir(".setuptools-cmake-build"):
+        ext_modules=ext_modules,
-        if not os.path.isfile(os.path.join(".setuptools-cmake-build", f)):
+        description="Deploy Kit Tool For Deeplearning models.",
-            continue
+        packages=packages,
-        if f.count(PACKAGE_NAME) > 0:
+        package_data=package_data,
-            shutil.copy(
+        include_package_data=True,
-                os.path.join(".setuptools-cmake-build", f),
+        setup_requires=setup_requires,
-                os.path.join(PACKAGE_NAME, "libs"))
+        extras_require=extras_require,
-        if f.count(".cpython-") > 0:
+        author='fastdeploy',
-            pybind_so_file = os.path.join(".setuptools-cmake-build", f)
+        author_email='fastdeploy@baidu.com',
-
+        url='https://github.com/PaddlePaddle/FastDeploy.git',
-    if not os.path.exists(".setuptools-cmake-build/third_libs/install"):
+        install_requires=REQUIRED_PACKAGES,
-        raise Exception(
+        classifiers=[
-            "Cannot find directory third_libs/install in .setuptools-cmake-build."
+            "Programming Language :: Python :: 3",
-        )
+            "License :: OSI Approved :: Apache Software License",
-
+            "Operating System :: OS Independent",
-    if os.path.exists(os.path.join(PACKAGE_NAME, "libs/third_libs")):
+        ],
-        shutil.rmtree(os.path.join(PACKAGE_NAME, "libs/third_libs"))
+        license='Apache 2.0')
-    shutil.copytree(
+else:
-        ".setuptools-cmake-build/third_libs/install",
+    setuptools.setup(
-        os.path.join(PACKAGE_NAME, "libs/third_libs"),
+        name=wheel_name,
-        symlinks=True)
+        version=VersionInfo.version,
-
+        description="Deploy Kit Tool For Deeplearning models.",
-    third_party_path = os.path.join(".setuptools-cmake-build", "third_party")
+        ext_modules=ext_modules,
-    if os.path.exists(third_party_path):
+        cmdclass=cmdclass,
-        for f in os.listdir(third_party_path):
+        packages=packages,
-            lib_dir_name = os.path.join(third_party_path, f)
+        package_data=package_data,
-            if os.path.isfile(lib_dir_name):
+        include_package_data=True,
-                continue
+        setup_requires=setup_requires,
-            for f1 in os.listdir(lib_dir_name):
+        extras_require=extras_require,
-                release_dir = os.path.join(lib_dir_name, f1)
+        author='fastdeploy',
-                if f1 == "Release" and not os.path.isfile(release_dir):
+        author_email='fastdeploy@baidu.com',
-                    if os.path.exists(
+        url='https://github.com/PaddlePaddle/FastDeploy.git',
-                            os.path.join(PACKAGE_NAME, "libs/third_libs", f)):
+        install_requires=REQUIRED_PACKAGES,
-                        shutil.rmtree(
+        classifiers=[
-                            os.path.join(PACKAGE_NAME, "libs/third_libs", f))
+            "Programming Language :: Python :: 3",
-                    shutil.copytree(release_dir,
+            "License :: OSI Approved :: Apache Software License",
-                                    os.path.join(PACKAGE_NAME,
+            "Operating System :: OS Independent",
-                                                 "libs/third_libs", f, "lib"))
+        ],
-
+        license='Apache 2.0')
-    if platform.system().lower() == "windows":
-        release_dir = os.path.join(".setuptools-cmake-build", "Release")
-        for f in os.listdir(release_dir):
-            filename = os.path.join(release_dir, f)
-            if not os.path.isfile(filename):
-                continue
-            if filename.endswith(".pyd"):
-                continue
-            shutil.copy(filename, os.path.join(PACKAGE_NAME, "libs"))
-
-    if platform.system().lower() == "linux":
-        rpaths = ["$ORIGIN:$ORIGIN/libs"]
-        for root, dirs, files in os.walk(
-                ".setuptools-cmake-build/third_libs/install"):
-            for d in dirs:
-                if d == "lib":
-                    path = os.path.relpath(
-                        os.path.join(root, d),
-                        ".setuptools-cmake-build/third_libs/install")
-                    rpaths.append("$ORIGIN/" + os.path.join("libs/third_libs",
-                                                            path))
-        rpaths = ":".join(rpaths)
-        command = "patchelf --force-rpath --set-rpath '{}' ".format(rpaths) + pybind_so_file
-        print(
-            "=========================Set rpath for library===================")
-        print(command)
-        # The sw_64 not suppot patchelf, so we just disable that.
-        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
-            assert subprocess.Popen(
-                command,
-                shell=True) != 0, "patchelf {} failed, the command: {}".format(
-                    command, pybind_so_file)
-    elif platform.system().lower() == "darwin":
-        pre_commands = [
-            "install_name_tool -delete_rpath '@loader_path/libs' " +
-            pybind_so_file
-        ]
-        commands = [
-            "install_name_tool -id '@loader_path/libs' " + pybind_so_file
-        ]
-        commands.append("install_name_tool -add_rpath '@loader_path/libs' " +
-                        pybind_so_file)
-        for root, dirs, files in os.walk(
-                ".setuptools-cmake-build/third_libs/install"):
-            for d in dirs:
-                if d == "lib":
-                    path = os.path.relpath(
-                        os.path.join(root, d),
-                        ".setuptools-cmake-build/third_libs/install")
-                    pre_commands.append(
-                        "install_name_tool -delete_rpath '@loader_path/{}' ".
-                        format(os.path.join("libs/third_libs",
-                                            path)) + pybind_so_file)
-                    commands.append(
-                        "install_name_tool -add_rpath '@loader_path/{}' ".
-                        format(os.path.join("libs/third_libs",
-                                            path)) + pybind_so_file)
-        for command in pre_commands:
-            try:
-                subprocess.Popen(command, shell=True)
-            except:
-                print("Skip execute command: " + command)
-        for command in commands:
-            assert subprocess.Popen(
-                command,
-                shell=True) != 0, "command execute failed! command: {}".format(
-                    command)
-
-    all_files = get_all_files(os.path.join(PACKAGE_NAME, "libs"))
-    for f in all_files:
-        # remove un-need ocv samples files to avoid too long file path
-        # in windows which can make building process failed.
-        if platform.system().lower() == "windows":
-            if f.find(".vcxproj.") > 0:
-                continue
-            if f.find("opencv") > 0 and any(
-                (f.find("samples") > 0, f.find("java") > 0, f.find(".png") > 0,
-                 f.find(".jpg") > 0)):
-                continue
-        package_data[PACKAGE_NAME].append(os.path.relpath(f, PACKAGE_NAME))
-
-setuptools.setup(
-    name=wheel_name,
-    version=VersionInfo.version,
-    description="Deploy Kit Tool For Deeplearning models.",
-    ext_modules=ext_modules,
-    cmdclass=cmdclass,
-    packages=packages,
-    package_data=package_data,
-    include_package_data=True,
-    setup_requires=setup_requires,
-    extras_require=extras_require,
-    author='fastdeploy',
-    author_email='fastdeploy@baidu.com',
-    url='https://github.com/PaddlePaddle/FastDeploy.git',
-    install_requires=REQUIRED_PACKAGES,
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: Apache Software License",
-        "Operating System :: OS Independent",
-    ],
-    license='Apache 2.0')