diff --git a/cmake/paddle_inference.cmake b/cmake/paddle_inference.cmake index dce29cddd..6e3d4d689 100755 --- a/cmake/paddle_inference.cmake +++ b/cmake/paddle_inference.cmake @@ -13,6 +13,9 @@ # limitations under the License. include(ExternalProject) +# The priority strategy for Paddle inference is as follows: +# PADDLEINFERENCE_DIRECTORY > custom PADDLEINFERENCE_URL > default PADDLEINFERENCE_URL. + if(WITH_GPU AND WITH_IPU) message(FATAL_ERROR "Cannot build with WITH_GPU=ON and WITH_IPU=ON on the same time.") endif() @@ -86,55 +89,61 @@ if(PADDLEINFERENCE_DIRECTORY) execute_process(COMMAND rm -rf ${THIRD_PARTY_PATH}/install/paddle_inference/paddle/lib/*.a) endif() else() - # Use default Paddle Inference libs. - set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/") - if(WIN32) - if (WITH_GPU) - set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.575cafb44b.zip") - set(PADDLEINFERENCE_VERSION "0.0.0.575cafb44b") - else() - set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-mkl-avx-0.0.0.cbdba50933.zip") - set(PADDLEINFERENCE_VERSION "0.0.0.cbdba50933") - endif() - elseif(APPLE) - if(CURRENT_OSX_ARCH MATCHES "arm64") - message(FATAL_ERROR "Paddle Backend doesn't support Mac OSX with Arm64 now.") - set(PADDLEINFERENCE_FILE "paddle_inference-osx-arm64-openblas-0.0.0.660f781b77.tgz") - else() - # TODO(qiuyanjun): Should remove this old paddle inference lib - # set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-2.4-dev3.tgz") - set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-openblas-0.0.0.660f781b77.tgz") - endif() - set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") - else() - # Linux with x86 CPU/Arm CPU/GPU/IPU ... - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") - message(FATAL_ERROR "Paddle Backend doesn't support linux aarch64 now.") - else() - if(WITH_GPU) - set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.660f781b77.tgz") - set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") + + # Custom Paddle Inference URL + if (NOT PADDLEINFERENCE_URL) + + # Use default Paddle Inference libs. + set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/") + if(WIN32) + if (WITH_GPU) + set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.575cafb44b.zip") + set(PADDLEINFERENCE_VERSION "0.0.0.575cafb44b") else() - set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-mkl-avx-0.0.0.660f781b77.tgz") - set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") + set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-mkl-avx-0.0.0.cbdba50933.zip") + set(PADDLEINFERENCE_VERSION "0.0.0.cbdba50933") endif() - if (WITH_IPU) - set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-ipu-2.4-dev1.tgz") - # TODO(qiuyanjun): Should use the commit id to tag the version - set(PADDLEINFERENCE_VERSION "2.4-dev1") + elseif(APPLE) + if(CURRENT_OSX_ARCH MATCHES "arm64") + message(FATAL_ERROR "Paddle Backend doesn't support Mac OSX with Arm64 now.") + set(PADDLEINFERENCE_FILE "paddle_inference-osx-arm64-openblas-0.0.0.660f781b77.tgz") + else() + # TODO(qiuyanjun): Should remove this old paddle inference lib + # set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-2.4-dev3.tgz") + set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-openblas-0.0.0.660f781b77.tgz") endif() - - if(NEED_ABI0) - if(WITH_GPU OR WITH_PU) - message(WARNING "While NEED_ABI0=ON, only support CPU now, will fallback to CPU.") + set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") + else() + # Linux with x86 CPU/Arm CPU/GPU/IPU ... + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + message(FATAL_ERROR "Paddle Backend doesn't support linux aarch64 now.") + else() + if(WITH_GPU) + set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.660f781b77.tgz") + set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") + else() + set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-mkl-avx-0.0.0.660f781b77.tgz") + set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77") + endif() + if (WITH_IPU) + set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-ipu-2.4-dev1.tgz") + # TODO(qiuyanjun): Should use the commit id to tag the version + set(PADDLEINFERENCE_VERSION "2.4-dev1") + endif() + + if(NEED_ABI0) + if(WITH_GPU OR WITH_PU) + message(WARNING "While NEED_ABI0=ON, only support CPU now, will fallback to CPU.") + endif() + set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-2.4.0-abi0.tgz") + set(PADDLEINFERENCE_VERSION "2.4.0-abi0") endif() - set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-2.4.0-abi0.tgz") - set(PADDLEINFERENCE_VERSION "2.4.0-abi0") endif() endif() - endif() - set(PADDLEINFERENCE_URL "${PADDLEINFERENCE_URL_BASE}${PADDLEINFERENCE_FILE}") + set(PADDLEINFERENCE_URL "${PADDLEINFERENCE_URL_BASE}${PADDLEINFERENCE_FILE}") + endif(PADDLEINFERENCE_URL) + ExternalProject_Add( ${PADDLEINFERENCE_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index 8c615ca0e..e40286961 100644 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -672,8 +672,8 @@ std::string OCRResult::Str() { out = out + "]"; if (rec_scores.size() > 0) { - out = out + "rec text: " + text[n] + - " rec score:" + std::to_string(rec_scores[n]) + " "; + out = out + "rec text: " + text[n] + " rec score:" + + std::to_string(rec_scores[n]) + " "; } if (cls_labels.size() > 0) { out = out + "cls label: " + std::to_string(cls_labels[n]) + @@ -689,20 +689,20 @@ std::string OCRResult::Str() { out = out + "[" + std::to_string(table_boxes[n][i * 2]) + "," + std::to_string(table_boxes[n][i * 2 + 1]) + "]"; - if (i != 1) { + if (i != 3) { out = out + ","; } } - out = out + "]"; + out = out + "]\n"; } - out = out + "\ntable structure: "; + out = out + "\ntable structure: \n"; for (int m = 0; m < table_structure.size(); m++) { out += table_structure[m]; } if (!table_html.empty()) { - out = out + "\n" + "table html: " + table_html; + out = out + "\n" + "table html: \n" + table_html; } } std::vector> table_boxes; @@ -713,8 +713,8 @@ std::string OCRResult::Str() { cls_scores.size() > 0) { std::string out; for (int i = 0; i < rec_scores.size(); i++) { - out = out + "rec text: " + text[i] + - " rec score:" + std::to_string(rec_scores[i]) + " "; + out = out + "rec text: " + text[i] + " rec score:" + + std::to_string(rec_scores[i]) + " "; out = out + "cls label: " + std::to_string(cls_labels[i]) + " cls score: " + std::to_string(cls_scores[i]); out = out + "\n"; @@ -733,8 +733,8 @@ std::string OCRResult::Str() { cls_scores.size() == 0) { std::string out; for (int i = 0; i < rec_scores.size(); i++) { - out = out + "rec text: " + text[i] + - " rec score:" + std::to_string(rec_scores[i]) + " "; + out = out + "rec text: " + text[i] + " rec score:" + + std::to_string(rec_scores[i]) + " "; out = out + "\n"; } return out; @@ -742,25 +742,25 @@ std::string OCRResult::Str() { table_structure.size() > 0) { std::string out; for (int n = 0; n < table_boxes.size(); n++) { - out = out + ", table boxes: ["; - for (int i = 0; i < 2; i++) { + out = out + "table boxes: ["; + for (int i = 0; i < 4; i++) { out = out + "[" + std::to_string(table_boxes[n][i * 2]) + "," + std::to_string(table_boxes[n][i * 2 + 1]) + "]"; - if (i != 1) { + if (i != 3) { out = out + ","; } } - out = out + "]"; + out = out + "]\n"; } - out = out + "\ntable structure: "; + out = out + "\ntable structure: \n"; for (int m = 0; m < table_structure.size(); m++) { out += table_structure[m]; } if (!table_html.empty()) { - out = out + "\n" + "table html: " + table_html; + out = out + "\n" + "table html: \n" + table_html; } return out; } @@ -781,9 +781,9 @@ std::string HeadPoseResult::Str() { std::string out; out = "HeadPoseResult: [yaw, pitch, roll]\n"; - out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + - "pitch: " + std::to_string(euler_angles[1]) + "\n" + - "roll: " + std::to_string(euler_angles[2]) + "\n"; + out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + "pitch: " + + std::to_string(euler_angles[1]) + "\n" + "roll: " + + std::to_string(euler_angles[2]) + "\n"; return out; } diff --git a/python/setup.py b/python/setup.py index dcf3d873f..12db68e60 100755 --- a/python/setup.py +++ b/python/setup.py @@ -90,6 +90,7 @@ setup_configs["PADDLEINFERENCE_DIRECTORY"] = os.getenv( "PADDLEINFERENCE_DIRECTORY", "") setup_configs["PADDLEINFERENCE_VERSION"] = os.getenv("PADDLEINFERENCE_VERSION", "") +setup_configs["PADDLEINFERENCE_URL"] = os.getenv("PADDLEINFERENCE_URL", "") setup_configs["RKNN2_TARGET_SOC"] = os.getenv("RKNN2_TARGET_SOC", "") if setup_configs["RKNN2_TARGET_SOC"] != "" or setup_configs[ diff --git a/scripts/linux/build_linux_x86_64_cpp_cpu.sh b/scripts/linux/build_linux_x86_64_cpp_cpu.sh index 9bd69c08e..83a255324 100755 --- a/scripts/linux/build_linux_x86_64_cpp_cpu.sh +++ b/scripts/linux/build_linux_x86_64_cpp_cpu.sh @@ -69,7 +69,7 @@ __build_fastdeploy_linux_x86_64_shared() { -DENABLE_BENCHMARK=ON \ -DENABLE_FLYCV=OFF \ -DWITH_FLYCV_STATIC=OFF \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install @@ -93,7 +93,7 @@ __build_fastdeploy_linux_x86_64_shared_custom_paddle() { -DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \ -DENABLE_FLYCV=OFF \ -DWITH_FLYCV_STATIC=OFF \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install diff --git a/scripts/linux/build_linux_x86_64_cpp_gpu.sh b/scripts/linux/build_linux_x86_64_cpp_gpu.sh index 57f66f5b8..849b742ec 100755 --- a/scripts/linux/build_linux_x86_64_cpp_gpu.sh +++ b/scripts/linux/build_linux_x86_64_cpp_gpu.sh @@ -70,7 +70,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared() { -DENABLE_PADDLE2ONNX=ON \ -DENABLE_VISION=ON \ -DENABLE_BENCHMARK=OFF \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install @@ -95,7 +95,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() { -DENABLE_PADDLE2ONNX=ON \ -DENABLE_VISION=ON \ -DENABLE_BENCHMARK=OFF \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install diff --git a/scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh b/scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh index b2e0cd8cf..c37d87e03 100755 --- a/scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh +++ b/scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh @@ -70,7 +70,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared() { -DENABLE_PADDLE2ONNX=ON \ -DENABLE_VISION=ON \ -DENABLE_BENCHMARK=ON \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install @@ -95,7 +95,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() { -DENABLE_PADDLE2ONNX=ON \ -DENABLE_VISION=ON \ -DENABLE_BENCHMARK=ON \ - -DBUILD_EXAMPLES=ON \ + -DBUILD_EXAMPLES=OFF \ -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ -Wno-dev ../../.. && make -j8 && make install diff --git a/serving/Dockerfile_CUDA_11_2_TRT_8_5_PADDLE_2_4_2 b/serving/Dockerfile_CUDA_11_2_TRT_8_5_PADDLE_2_4_2 new file mode 100644 index 000000000..e5d509fe7 --- /dev/null +++ b/serving/Dockerfile_CUDA_11_2_TRT_8_5_PADDLE_2_4_2 @@ -0,0 +1,62 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG http_proxy +ARG https_proxy + +FROM nvcr.io/nvidia/tritonserver:21.10-py3 as full +FROM nvcr.io/nvidia/tritonserver:21.10-py3-min + +COPY --from=full /opt/tritonserver/bin/tritonserver /opt/tritonserver/bin/fastdeployserver +COPY --from=full /opt/tritonserver/lib /opt/tritonserver/lib +COPY --from=full /opt/tritonserver/include /opt/tritonserver/include +COPY --from=full /opt/tritonserver/backends/python /opt/tritonserver/backends/python + +COPY serving/TensorRT-8.5.2.2 /opt/TensorRT-8.5.2.2 + +ENV TZ=Asia/Shanghai \ + DEBIAN_FRONTEND=noninteractive \ + DCGM_VERSION=2.2.9 \ + http_proxy=$http_proxy \ + https_proxy=$http_proxy + +RUN apt-get update \ + && apt-key del 7fa2af80 \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \ + && dpkg -i cuda-keyring_1.0-1_all.deb \ + && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub \ + && apt-get update && apt-get install -y --no-install-recommends datacenter-gpu-manager=1:2.2.9 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libre2-5 libb64-0d python3 python3-pip libarchive-dev ffmpeg libsm6 libxext6 \ + && python3 -m pip install -U pip \ + && python3 -m pip install paddlenlp fast-tokenizer-python + +COPY python/dist/*.whl /opt/fastdeploy/ +RUN python3 -m pip install /opt/fastdeploy/*.whl \ + && rm -rf /opt/fastdeploy/*.whl + +# unset proxy +# ENV http_proxy='' +# ENV https_proxy='' +# RUN python3 -m pip install paddlepaddle-gpu==2.4.2.post112 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html +RUN unset http_proxy +RUN unset https_proxy +RUN python3 -m pip install https://paddle-wheel.bj.bcebos.com/2.4.2/linux/linux-gpu-cuda11.2-cudnn8-mkl-gcc8.2-avx/paddlepaddle_gpu-2.4.2.post112-cp38-cp38-linux_x86_64.whl + +COPY serving/build/libtriton_fastdeploy.so /opt/tritonserver/backends/fastdeploy/ +COPY build/fastdeploy_install /opt/fastdeploy/ + +ENV LD_LIBRARY_PATH="/opt/TensorRT-8.5.2.2/lib/:/opt/fastdeploy/lib:/opt/fastdeploy/third_libs/install/onnxruntime/lib:/opt/fastdeploy/third_libs/install/paddle2onnx/lib:/opt/fastdeploy/third_libs/install/tensorrt/lib:/opt/fastdeploy/third_libs/install/paddle_inference/paddle/lib:/opt/fastdeploy/third_libs/install/paddle_inference/third_party/install/mkldnn/lib:/opt/fastdeploy/third_libs/install/paddle_inference/third_party/install/mklml/lib:/opt/fastdeploy/third_libs/install/openvino/runtime/lib:$LD_LIBRARY_PATH" +ENV PATH="/opt/tritonserver/bin:$PATH" diff --git a/serving/scripts/build_fd_cuda_11_2_trt_8_5_paddle_2_4_2.sh b/serving/scripts/build_fd_cuda_11_2_trt_8_5_paddle_2_4_2.sh new file mode 100755 index 000000000..0c98972b9 --- /dev/null +++ b/serving/scripts/build_fd_cuda_11_2_trt_8_5_paddle_2_4_2.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -d "./cmake-3.18.6-Linux-x86_64/" ]; then + wget https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz + tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz + rm -rf cmake-3.18.6-Linux-x86_64.tar.gz +fi + +if [ ! -d "./TensorRT-8.5.2.2/" ]; then + wget https://fastdeploy.bj.bcebos.com/resource/TensorRT/TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz + tar -zxvf TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz + rm -rf TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz +fi + +# build vision、runtime、backend +docker run -it --rm --name build_fd_libs \ + -v`pwd`/..:/workspace/fastdeploy \ + -e "http_proxy=${http_proxy}" \ + -e "https_proxy=${https_proxy}" \ + nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \ + bash -c \ + 'cd /workspace/fastdeploy/python; + rm -rf .setuptools-cmake-build dist build fastdeploy/libs/third_libs; + apt-get update; + apt-get install -y --no-install-recommends patchelf python3-dev python3-pip rapidjson-dev git; + ln -s /usr/bin/python3 /usr/bin/python; + export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH; + export WITH_GPU=ON; + export ENABLE_TRT_BACKEND=OFF; + export TRT_DIRECTORY=/workspace/fastdeploy/serving/TensorRT-8.5.2.2/; + export ENABLE_ORT_BACKEND=OFF; + export ENABLE_PADDLE_BACKEND=OFF; + export ENABLE_OPENVINO_BACKEND=OFF; + export ENABLE_VISION=ON; + export ENABLE_TEXT=ON; + export PADDLEINFERENCE_URL=https://bj.bcebos.com/fastdeploy/third_libs/paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-2.4.2.tgz; + export PADDLEINFERENCE_VERSION=paddle2.4.2; + python setup.py build; + python setup.py bdist_wheel; + cd /workspace/fastdeploy; + rm -rf build; mkdir -p build;cd build; + cmake .. -DENABLE_TRT_BACKEND=ON -DPADDLEINFERENCE_URL=${PADDLEINFERENCE_URL} -DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy_install -DWITH_GPU=ON -DTRT_DIRECTORY=/workspace/fastdeploy/serving/TensorRT-8.5.2.2/ -DENABLE_PADDLE_BACKEND=ON -DENABLE_ORT_BACKEND=ON -DENABLE_OPENVINO_BACKEND=ON -DENABLE_VISION=ON -DBUILD_FASTDEPLOY_PYTHON=OFF -DENABLE_PADDLE2ONNX=ON -DENABLE_TEXT=OFF -DLIBRARY_NAME=fastdeploy_runtime; + make -j`nproc`; + make install; + cd /workspace/fastdeploy/serving; + rm -rf build; mkdir build; cd build; + cmake .. -DFASTDEPLOY_DIR=/workspace/fastdeploy/build/fastdeploy_install -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 -DTRITON_BACKEND_REPO_TAG=r21.10; + make -j`nproc`'