mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-24 17:10:35 +08:00
[Build] Support build with source code of Paddle2ONNX (#1559)
* Add notes for tensors * Optimize some apis * move some warnings * Support build with Paddle2ONNX * Add protobuf support * Fix compile on mac * add clearn package script * Add paddle2onnx code * remove submodule * Add onnx ocde * remove softlink * add onnx code * fix error * Add cmake file * fix patchelf * update paddle2onnx * Delete .gitmodules --------- Co-authored-by: PaddleCI <paddle_ci@example.com> Co-authored-by: pangyoki <pangyoki@126.com> Co-authored-by: jiangjiajun <jiangjiajun@baidu.lcom>
This commit is contained in:
Jason
357
third_party/onnx/setup.py
vendored
Normal file
357
third_party/onnx/setup.py
vendored
Normal file
@@ -0,0 +1,357 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from distutils.spawn import find_executable
|
||||
from distutils import sysconfig, log
|
||||
import setuptools
|
||||
import setuptools.command.build_py
|
||||
import setuptools.command.develop
|
||||
import setuptools.command.build_ext
|
||||
|
||||
from collections import namedtuple
|
||||
from contextlib import contextmanager
|
||||
from datetime import date
|
||||
import glob
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import platform
|
||||
from textwrap import dedent
|
||||
import multiprocessing
|
||||
|
||||
|
||||
TOP_DIR = os.path.realpath(os.path.dirname(__file__))
|
||||
SRC_DIR = os.path.join(TOP_DIR, "onnx")
|
||||
TP_DIR = os.path.join(TOP_DIR, "third_party")
|
||||
CMAKE_BUILD_DIR = os.path.join(TOP_DIR, ".setuptools-cmake-build")
|
||||
PACKAGE_NAME = "onnx"
|
||||
|
||||
WINDOWS = (os.name == "nt")
|
||||
|
||||
CMAKE = find_executable("cmake3") or find_executable("cmake")
|
||||
MAKE = find_executable("make")
|
||||
|
||||
install_requires = []
|
||||
setup_requires = []
|
||||
tests_require = []
|
||||
extras_require = {}
|
||||
|
||||
################################################################################
|
||||
# Global variables for controlling the build variant
|
||||
################################################################################
|
||||
|
||||
# Default value is set to TRUE\1 to keep the settings same as the current ones.
|
||||
# However going forward the recommended way to is to set this to False\0
|
||||
ONNX_ML = not bool(os.getenv("ONNX_ML") == "0")
|
||||
ONNX_VERIFY_PROTO3 = bool(os.getenv("ONNX_VERIFY_PROTO3") == "1")
|
||||
ONNX_NAMESPACE = os.getenv("ONNX_NAMESPACE", "onnx")
|
||||
ONNX_BUILD_TESTS = bool(os.getenv("ONNX_BUILD_TESTS") == "1")
|
||||
ONNX_DISABLE_EXCEPTIONS = bool(os.getenv("ONNX_DISABLE_EXCEPTIONS") == "1")
|
||||
|
||||
USE_MSVC_STATIC_RUNTIME = bool(os.getenv("USE_MSVC_STATIC_RUNTIME", "0") == "1")
|
||||
DEBUG = bool(os.getenv("DEBUG", "0") == "1")
|
||||
COVERAGE = bool(os.getenv("COVERAGE", "0") == "1")
|
||||
|
||||
################################################################################
|
||||
# Version
|
||||
################################################################################
|
||||
|
||||
try:
|
||||
git_version = subprocess.check_output(["git", "rev-parse", "HEAD"],
|
||||
cwd=TOP_DIR).decode("ascii").strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
git_version = None
|
||||
|
||||
with open(os.path.join(TOP_DIR, "VERSION_NUMBER")) as version_file:
|
||||
VERSION_NUMBER = version_file.read().strip()
|
||||
if "--weekly_build" in sys.argv:
|
||||
today_number = date.today().strftime("%Y%m%d")
|
||||
VERSION_NUMBER += ".dev" + today_number
|
||||
PACKAGE_NAME = "onnx-weekly"
|
||||
sys.argv.remove("--weekly_build")
|
||||
VersionInfo = namedtuple("VersionInfo", ["version", "git_version"])(
|
||||
version=VERSION_NUMBER,
|
||||
git_version=git_version
|
||||
)
|
||||
|
||||
################################################################################
|
||||
# Pre Check
|
||||
################################################################################
|
||||
|
||||
assert CMAKE, "Could not find cmake executable!"
|
||||
|
||||
################################################################################
|
||||
# Utilities
|
||||
################################################################################
|
||||
|
||||
|
||||
@contextmanager
|
||||
def cd(path):
|
||||
if not os.path.isabs(path):
|
||||
raise RuntimeError("Can only cd to absolute path, got: {}".format(path))
|
||||
orig_path = os.getcwd()
|
||||
os.chdir(path)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
os.chdir(orig_path)
|
||||
|
||||
|
||||
################################################################################
|
||||
# Customized commands
|
||||
################################################################################
|
||||
|
||||
|
||||
class ONNXCommand(setuptools.Command):
|
||||
user_options = []
|
||||
|
||||
def initialize_options(self):
|
||||
pass
|
||||
|
||||
def finalize_options(self):
|
||||
pass
|
||||
|
||||
|
||||
class create_version(ONNXCommand):
|
||||
def run(self):
|
||||
with open(os.path.join(SRC_DIR, "version.py"), "w") as f:
|
||||
f.write(dedent("""\
|
||||
# This file is generated by setup.py. DO NOT EDIT!
|
||||
|
||||
|
||||
version = "{version}"
|
||||
git_version = "{git_version}"
|
||||
""".format(**dict(VersionInfo._asdict()))))
|
||||
|
||||
|
||||
class cmake_build(setuptools.Command):
|
||||
"""
|
||||
Compiles everything when `python setup.py build` is run using cmake.
|
||||
|
||||
Custom args can be passed to cmake by specifying the `CMAKE_ARGS`
|
||||
environment variable.
|
||||
|
||||
The number of CPUs used by `make` can be specified by passing `-j<ncpus>`
|
||||
to `setup.py build`. By default all CPUs are used.
|
||||
"""
|
||||
user_options = [
|
||||
(str("jobs="), str("j"), str("Specifies the number of jobs to use with make"))
|
||||
]
|
||||
|
||||
built = False
|
||||
|
||||
def initialize_options(self):
|
||||
self.jobs = None
|
||||
|
||||
def finalize_options(self):
|
||||
self.set_undefined_options("build", ("parallel", "jobs"))
|
||||
if self.jobs is None and os.getenv("MAX_JOBS") is not None:
|
||||
self.jobs = os.getenv("MAX_JOBS")
|
||||
self.jobs = multiprocessing.cpu_count() if self.jobs is None else int(self.jobs)
|
||||
|
||||
def run(self):
|
||||
if cmake_build.built:
|
||||
return
|
||||
cmake_build.built = True
|
||||
if not os.path.exists(CMAKE_BUILD_DIR):
|
||||
os.makedirs(CMAKE_BUILD_DIR)
|
||||
|
||||
with cd(CMAKE_BUILD_DIR):
|
||||
build_type = "Release"
|
||||
# configure
|
||||
cmake_args = [
|
||||
CMAKE,
|
||||
"-DPYTHON_INCLUDE_DIR={}".format(sysconfig.get_python_inc()),
|
||||
"-DPYTHON_EXECUTABLE={}".format(sys.executable),
|
||||
"-DBUILD_ONNX_PYTHON=ON",
|
||||
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
|
||||
"-DONNX_NAMESPACE={}".format(ONNX_NAMESPACE),
|
||||
"-DPY_EXT_SUFFIX={}".format(sysconfig.get_config_var("EXT_SUFFIX") or ''),
|
||||
]
|
||||
if COVERAGE:
|
||||
cmake_args.append("-DONNX_COVERAGE=ON")
|
||||
if COVERAGE or DEBUG:
|
||||
# in order to get accurate coverage information, the
|
||||
# build needs to turn off optimizations
|
||||
build_type = "Debug"
|
||||
cmake_args.append("-DCMAKE_BUILD_TYPE=%s" % build_type)
|
||||
if WINDOWS:
|
||||
cmake_args.extend([
|
||||
# we need to link with libpython on windows, so
|
||||
# passing python version to window in order to
|
||||
# find python in cmake
|
||||
"-DPY_VERSION={}".format("{0}.{1}".format(*sys.version_info[:2])),
|
||||
])
|
||||
if USE_MSVC_STATIC_RUNTIME:
|
||||
cmake_args.append("-DONNX_USE_MSVC_STATIC_RUNTIME=ON")
|
||||
if platform.architecture()[0] == "64bit":
|
||||
cmake_args.extend(["-A", "x64", "-T", "host=x64"])
|
||||
else:
|
||||
cmake_args.extend(["-A", "Win32", "-T", "host=x86"])
|
||||
if ONNX_ML:
|
||||
cmake_args.append("-DONNX_ML=1")
|
||||
if ONNX_VERIFY_PROTO3:
|
||||
cmake_args.append("-DONNX_VERIFY_PROTO3=1")
|
||||
if ONNX_BUILD_TESTS:
|
||||
cmake_args.append("-DONNX_BUILD_TESTS=ON")
|
||||
if ONNX_DISABLE_EXCEPTIONS:
|
||||
cmake_args.append("-DONNX_DISABLE_EXCEPTIONS=ON")
|
||||
if "CMAKE_ARGS" in os.environ:
|
||||
extra_cmake_args = shlex.split(os.environ["CMAKE_ARGS"])
|
||||
# prevent crossfire with downstream scripts
|
||||
del os.environ["CMAKE_ARGS"]
|
||||
log.info("Extra cmake args: {}".format(extra_cmake_args))
|
||||
cmake_args.extend(extra_cmake_args)
|
||||
cmake_args.append(TOP_DIR)
|
||||
log.info("Using cmake args: {}".format(cmake_args))
|
||||
if "-DONNX_DISABLE_EXCEPTIONS=ON" in cmake_args:
|
||||
raise RuntimeError("-DONNX_DISABLE_EXCEPTIONS=ON option is only available for c++ builds. Python binding require exceptions to be enabled.")
|
||||
subprocess.check_call(cmake_args)
|
||||
|
||||
build_args = [CMAKE, "--build", os.curdir]
|
||||
if WINDOWS:
|
||||
build_args.extend(["--config", build_type])
|
||||
build_args.extend(["--", "/maxcpucount:{}".format(self.jobs)])
|
||||
else:
|
||||
build_args.extend(["--", "-j", str(self.jobs)])
|
||||
subprocess.check_call(build_args)
|
||||
|
||||
|
||||
class build_py(setuptools.command.build_py.build_py):
|
||||
def run(self):
|
||||
self.run_command("create_version")
|
||||
self.run_command("cmake_build")
|
||||
|
||||
generated_python_files = \
|
||||
glob.glob(os.path.join(CMAKE_BUILD_DIR, "onnx", "*.py")) + \
|
||||
glob.glob(os.path.join(CMAKE_BUILD_DIR, "onnx", "*.pyi"))
|
||||
|
||||
for src in generated_python_files:
|
||||
dst = os.path.join(
|
||||
TOP_DIR, os.path.relpath(src, CMAKE_BUILD_DIR))
|
||||
self.copy_file(src, dst)
|
||||
|
||||
return setuptools.command.build_py.build_py.run(self)
|
||||
|
||||
|
||||
class develop(setuptools.command.develop.develop):
|
||||
def run(self):
|
||||
self.run_command("build_py")
|
||||
setuptools.command.develop.develop.run(self)
|
||||
|
||||
|
||||
class build_ext(setuptools.command.build_ext.build_ext):
|
||||
def run(self):
|
||||
self.run_command("cmake_build")
|
||||
setuptools.command.build_ext.build_ext.run(self)
|
||||
|
||||
def build_extensions(self):
|
||||
for ext in self.extensions:
|
||||
fullname = self.get_ext_fullname(ext.name)
|
||||
filename = os.path.basename(self.get_ext_filename(fullname))
|
||||
|
||||
lib_path = CMAKE_BUILD_DIR
|
||||
if os.name == "nt":
|
||||
debug_lib_dir = os.path.join(lib_path, "Debug")
|
||||
release_lib_dir = os.path.join(lib_path, "Release")
|
||||
if os.path.exists(debug_lib_dir):
|
||||
lib_path = debug_lib_dir
|
||||
elif os.path.exists(release_lib_dir):
|
||||
lib_path = release_lib_dir
|
||||
src = os.path.join(lib_path, filename)
|
||||
dst = os.path.join(os.path.realpath(self.build_lib), "onnx", filename)
|
||||
self.copy_file(src, dst)
|
||||
|
||||
|
||||
class mypy_type_check(ONNXCommand):
|
||||
description = "Run MyPy type checker"
|
||||
|
||||
def run(self):
|
||||
"""Run command."""
|
||||
onnx_script = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "tools/mypy-onnx.py"))
|
||||
returncode = subprocess.call([sys.executable, onnx_script])
|
||||
sys.exit(returncode)
|
||||
|
||||
|
||||
cmdclass = {
|
||||
"create_version": create_version,
|
||||
"cmake_build": cmake_build,
|
||||
"build_py": build_py,
|
||||
"develop": develop,
|
||||
"build_ext": build_ext,
|
||||
"typecheck": mypy_type_check,
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Extensions
|
||||
################################################################################
|
||||
|
||||
ext_modules = [
|
||||
setuptools.Extension(
|
||||
name=str("onnx.onnx_cpp2py_export"),
|
||||
sources=[])
|
||||
]
|
||||
|
||||
################################################################################
|
||||
# Packages
|
||||
################################################################################
|
||||
|
||||
# no need to do fancy stuff so far
|
||||
packages = setuptools.find_packages()
|
||||
|
||||
requirements_file = "requirements.txt"
|
||||
requirements_path = os.path.join(os.getcwd(), requirements_file)
|
||||
if not os.path.exists(requirements_path):
|
||||
this = os.path.dirname(__file__)
|
||||
requirements_path = os.path.join(this, requirements_file)
|
||||
if not os.path.exists(requirements_path):
|
||||
raise FileNotFoundError("Unable to find " + requirements_file)
|
||||
with open(requirements_path) as f:
|
||||
install_requires = f.read().splitlines()
|
||||
|
||||
################################################################################
|
||||
# Test
|
||||
################################################################################
|
||||
|
||||
setup_requires.append("pytest-runner")
|
||||
tests_require.append("pytest")
|
||||
tests_require.append("nbval")
|
||||
tests_require.append("tabulate")
|
||||
|
||||
extras_require["lint"] = [
|
||||
"clang-format==13.0.0",
|
||||
"flake8",
|
||||
"mypy==0.782",
|
||||
"types-protobuf==3.18.4",
|
||||
]
|
||||
|
||||
################################################################################
|
||||
# Final
|
||||
################################################################################
|
||||
|
||||
setuptools.setup(
|
||||
name=PACKAGE_NAME,
|
||||
version=VersionInfo.version,
|
||||
description="Open Neural Network Exchange",
|
||||
long_description=open("README.md").read(),
|
||||
long_description_content_type="text/markdown",
|
||||
ext_modules=ext_modules,
|
||||
cmdclass=cmdclass,
|
||||
packages=packages,
|
||||
license="Apache License v2.0",
|
||||
include_package_data=True,
|
||||
install_requires=install_requires,
|
||||
setup_requires=setup_requires,
|
||||
tests_require=tests_require,
|
||||
extras_require=extras_require,
|
||||
author="ONNX",
|
||||
author_email="onnx-technical-discuss@lists.lfaidata.foundation",
|
||||
url="https://github.com/onnx/onnx",
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"check-model = onnx.bin.checker:check_model",
|
||||
"check-node = onnx.bin.checker:check_node",
|
||||
"backend-test-tools = onnx.backend.test.cmd_tools:main",
|
||||
]
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user