[cmake] Support custom paddle inference url (#1939)

* [cmake] Support custom paddle inference url

* [Python] Add custom Paddle Inference URL support for python

* [Docker] Add fd serving Dockerfile for paddle2.4.2

* [Docker] Add fd serving Dockerfile for paddle2.4.2

* [Docker] Add fd serving Dockerfile for paddle2.4.2

* [Docker] Add fd serving Dockerfile for paddle2.4.2

* [Bug Fix] fixed result format string error

* rerunning the re-touch CIs

* rerunning CIs

cmake/paddle_inference.cmake

@@ -13,6 +13,9 @@
 # limitations under the License.
 include(ExternalProject)
 
+# The priority strategy for Paddle inference is as follows:
+# PADDLEINFERENCE_DIRECTORY > custom PADDLEINFERENCE_URL > default PADDLEINFERENCE_URL.
+
 if(WITH_GPU AND WITH_IPU)
   message(FATAL_ERROR "Cannot build with WITH_GPU=ON and WITH_IPU=ON on the same time.")
 endif()
@@ -86,55 +89,61 @@ if(PADDLEINFERENCE_DIRECTORY)
     execute_process(COMMAND rm -rf ${THIRD_PARTY_PATH}/install/paddle_inference/paddle/lib/*.a)
   endif()
 else()
-  # Use default Paddle Inference libs.
+
-  set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/")
+  # Custom Paddle Inference URL
-  if(WIN32)
+  if (NOT PADDLEINFERENCE_URL)
-    if (WITH_GPU)
+
-      set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.575cafb44b.zip")
+    # Use default Paddle Inference libs.
-      set(PADDLEINFERENCE_VERSION "0.0.0.575cafb44b")
+    set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/")
-    else()
+    if(WIN32)
-      set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-mkl-avx-0.0.0.cbdba50933.zip")
+      if (WITH_GPU)
-      set(PADDLEINFERENCE_VERSION "0.0.0.cbdba50933")
+        set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.575cafb44b.zip")
-    endif()
+        set(PADDLEINFERENCE_VERSION "0.0.0.575cafb44b")
-  elseif(APPLE)
-    if(CURRENT_OSX_ARCH MATCHES "arm64")
-      message(FATAL_ERROR "Paddle Backend doesn't support Mac OSX with Arm64 now.")
-      set(PADDLEINFERENCE_FILE "paddle_inference-osx-arm64-openblas-0.0.0.660f781b77.tgz")
-    else()
-      # TODO(qiuyanjun): Should remove this old paddle inference lib
-      # set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-2.4-dev3.tgz")
-      set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-openblas-0.0.0.660f781b77.tgz")
-    endif()
-    set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
-  else()
-    # Linux with x86 CPU/Arm CPU/GPU/IPU ...
-    if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
-      message(FATAL_ERROR "Paddle Backend doesn't support linux aarch64 now.")
-    else()
-      if(WITH_GPU)
-        set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.660f781b77.tgz")
-        set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
       else()
-        set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-mkl-avx-0.0.0.660f781b77.tgz")
+        set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-mkl-avx-0.0.0.cbdba50933.zip")
-        set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
+        set(PADDLEINFERENCE_VERSION "0.0.0.cbdba50933")
       endif()
-      if (WITH_IPU)
+    elseif(APPLE)
-        set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-ipu-2.4-dev1.tgz")
+      if(CURRENT_OSX_ARCH MATCHES "arm64")
-        # TODO(qiuyanjun): Should use the commit id to tag the version
+        message(FATAL_ERROR "Paddle Backend doesn't support Mac OSX with Arm64 now.")
-        set(PADDLEINFERENCE_VERSION "2.4-dev1")
+        set(PADDLEINFERENCE_FILE "paddle_inference-osx-arm64-openblas-0.0.0.660f781b77.tgz")
+      else()
+        # TODO(qiuyanjun): Should remove this old paddle inference lib
+        # set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-2.4-dev3.tgz")
+        set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-openblas-0.0.0.660f781b77.tgz")
       endif()
-
+      set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
-      if(NEED_ABI0)
+    else()
-        if(WITH_GPU OR WITH_PU)
+      # Linux with x86 CPU/Arm CPU/GPU/IPU ...
-          message(WARNING "While NEED_ABI0=ON, only support CPU now, will fallback to CPU.")
+      if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
+        message(FATAL_ERROR "Paddle Backend doesn't support linux aarch64 now.")
+      else()
+        if(WITH_GPU)
+          set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-0.0.0.660f781b77.tgz")
+          set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
+        else()
+          set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-mkl-avx-0.0.0.660f781b77.tgz")
+          set(PADDLEINFERENCE_VERSION "0.0.0.660f781b77")
+        endif()
+        if (WITH_IPU)
+          set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-ipu-2.4-dev1.tgz")
+          # TODO(qiuyanjun): Should use the commit id to tag the version
+          set(PADDLEINFERENCE_VERSION "2.4-dev1")
+        endif()
+
+        if(NEED_ABI0)
+          if(WITH_GPU OR WITH_PU)
+            message(WARNING "While NEED_ABI0=ON, only support CPU now, will fallback to CPU.")
+          endif()
+          set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-2.4.0-abi0.tgz")
+          set(PADDLEINFERENCE_VERSION "2.4.0-abi0")
         endif()
-        set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-2.4.0-abi0.tgz")
-        set(PADDLEINFERENCE_VERSION "2.4.0-abi0")
       endif()
     endif()
-  endif()
+    set(PADDLEINFERENCE_URL "${PADDLEINFERENCE_URL_BASE}${PADDLEINFERENCE_FILE}")
-  set(PADDLEINFERENCE_URL "${PADDLEINFERENCE_URL_BASE}${PADDLEINFERENCE_FILE}")
 
+  endif(PADDLEINFERENCE_URL)
+  
   ExternalProject_Add(
     ${PADDLEINFERENCE_PROJECT}
     ${EXTERNAL_PROJECT_LOG_ARGS}

fastdeploy/vision/common/result.cc

@@ -672,8 +672,8 @@ std::string OCRResult::Str() {
       out = out + "]";
 
       if (rec_scores.size() > 0) {
-        out = out + "rec text: " + text[n] +
+        out = out + "rec text: " + text[n] + " rec score:" +
-              " rec score:" + std::to_string(rec_scores[n]) + " ";
+              std::to_string(rec_scores[n]) + " ";
       }
       if (cls_labels.size() > 0) {
         out = out + "cls label: " + std::to_string(cls_labels[n]) +
@@ -689,20 +689,20 @@ std::string OCRResult::Str() {
           out = out + "[" + std::to_string(table_boxes[n][i * 2]) + "," +
                 std::to_string(table_boxes[n][i * 2 + 1]) + "]";
 
-          if (i != 1) {
+          if (i != 3) {
             out = out + ",";
           }
         }
-        out = out + "]";
+        out = out + "]\n";
       }
 
-      out = out + "\ntable structure: ";
+      out = out + "\ntable structure: \n";
       for (int m = 0; m < table_structure.size(); m++) {
         out += table_structure[m];
       }
 
       if (!table_html.empty()) {
-        out = out + "\n" + "table html: " + table_html;
+        out = out + "\n" + "table html: \n" + table_html;
       }
     }
     std::vector<std::array<int, 8>> table_boxes;
@@ -713,8 +713,8 @@ std::string OCRResult::Str() {
              cls_scores.size() > 0) {
     std::string out;
     for (int i = 0; i < rec_scores.size(); i++) {
-      out = out + "rec text: " + text[i] +
+      out = out + "rec text: " + text[i] + " rec score:" +
-            " rec score:" + std::to_string(rec_scores[i]) + " ";
+            std::to_string(rec_scores[i]) + " ";
       out = out + "cls label: " + std::to_string(cls_labels[i]) +
             " cls score: " + std::to_string(cls_scores[i]);
       out = out + "\n";
@@ -733,8 +733,8 @@ std::string OCRResult::Str() {
              cls_scores.size() == 0) {
     std::string out;
     for (int i = 0; i < rec_scores.size(); i++) {
-      out = out + "rec text: " + text[i] +
+      out = out + "rec text: " + text[i] + " rec score:" +
-            " rec score:" + std::to_string(rec_scores[i]) + " ";
+            std::to_string(rec_scores[i]) + " ";
       out = out + "\n";
     }
     return out;
@@ -742,25 +742,25 @@ std::string OCRResult::Str() {
              table_structure.size() > 0) {
     std::string out;
     for (int n = 0; n < table_boxes.size(); n++) {
-      out = out + ", table boxes: [";
+      out = out + "table boxes: [";
-      for (int i = 0; i < 2; i++) {
+      for (int i = 0; i < 4; i++) {
         out = out + "[" + std::to_string(table_boxes[n][i * 2]) + "," +
               std::to_string(table_boxes[n][i * 2 + 1]) + "]";
 
-        if (i != 1) {
+        if (i != 3) {
           out = out + ",";
         }
       }
-      out = out + "]";
+      out = out + "]\n";
     }
 
-    out = out + "\ntable structure: ";
+    out = out + "\ntable structure: \n";
     for (int m = 0; m < table_structure.size(); m++) {
       out += table_structure[m];
     }
 
     if (!table_html.empty()) {
-      out = out + "\n" + "table html: " + table_html;
+      out = out + "\n" + "table html: \n" + table_html;
     }
     return out;
   }
@@ -781,9 +781,9 @@ std::string HeadPoseResult::Str() {
   std::string out;
 
   out = "HeadPoseResult: [yaw, pitch, roll]\n";
-  out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" +
+  out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + "pitch: " +
-        "pitch: " + std::to_string(euler_angles[1]) + "\n" +
+        std::to_string(euler_angles[1]) + "\n" + "roll: " +
-        "roll: " + std::to_string(euler_angles[2]) + "\n";
+        std::to_string(euler_angles[2]) + "\n";
   return out;
 }
 

python/setup.py

@@ -90,6 +90,7 @@ setup_configs["PADDLEINFERENCE_DIRECTORY"] = os.getenv(
     "PADDLEINFERENCE_DIRECTORY", "")
 setup_configs["PADDLEINFERENCE_VERSION"] = os.getenv("PADDLEINFERENCE_VERSION",
                                                      "")
+setup_configs["PADDLEINFERENCE_URL"] = os.getenv("PADDLEINFERENCE_URL", "")
 
 setup_configs["RKNN2_TARGET_SOC"] = os.getenv("RKNN2_TARGET_SOC", "")
 if setup_configs["RKNN2_TARGET_SOC"] != "" or setup_configs[

scripts/linux/build_linux_x86_64_cpp_cpu.sh

@@ -69,7 +69,7 @@ __build_fastdeploy_linux_x86_64_shared() {
         -DENABLE_BENCHMARK=ON \
         -DENABLE_FLYCV=OFF \
         -DWITH_FLYCV_STATIC=OFF \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 
@@ -93,7 +93,7 @@ __build_fastdeploy_linux_x86_64_shared_custom_paddle() {
         -DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} \
         -DENABLE_FLYCV=OFF \
         -DWITH_FLYCV_STATIC=OFF \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 

scripts/linux/build_linux_x86_64_cpp_gpu.sh

@@ -70,7 +70,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared() {
         -DENABLE_PADDLE2ONNX=ON \
         -DENABLE_VISION=ON \
         -DENABLE_BENCHMARK=OFF \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 
@@ -95,7 +95,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
         -DENABLE_PADDLE2ONNX=ON \
         -DENABLE_VISION=ON \
         -DENABLE_BENCHMARK=OFF \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 

scripts/linux/build_linux_x86_64_cpp_gpu_with_benchmark.sh

@@ -70,7 +70,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared() {
         -DENABLE_PADDLE2ONNX=ON \
         -DENABLE_VISION=ON \
         -DENABLE_BENCHMARK=ON \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 
@@ -95,7 +95,7 @@ __build_fastdeploy_linux_x86_64_gpu_shared_custom_paddle() {
         -DENABLE_PADDLE2ONNX=ON \
         -DENABLE_VISION=ON \
         -DENABLE_BENCHMARK=ON \
-        -DBUILD_EXAMPLES=ON \
+        -DBUILD_EXAMPLES=OFF \
         -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \
         -Wno-dev ../../.. && make -j8 && make install
 

serving/Dockerfile_CUDA_11_2_TRT_8_5_PADDLE_2_4_2

@@ -0,0 +1,62 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG http_proxy
+ARG https_proxy
+
+FROM nvcr.io/nvidia/tritonserver:21.10-py3 as full
+FROM nvcr.io/nvidia/tritonserver:21.10-py3-min
+
+COPY --from=full /opt/tritonserver/bin/tritonserver /opt/tritonserver/bin/fastdeployserver
+COPY --from=full /opt/tritonserver/lib /opt/tritonserver/lib
+COPY --from=full /opt/tritonserver/include /opt/tritonserver/include
+COPY --from=full /opt/tritonserver/backends/python /opt/tritonserver/backends/python
+
+COPY serving/TensorRT-8.5.2.2 /opt/TensorRT-8.5.2.2
+
+ENV TZ=Asia/Shanghai \
+    DEBIAN_FRONTEND=noninteractive \
+    DCGM_VERSION=2.2.9 \
+    http_proxy=$http_proxy \
+    https_proxy=$http_proxy
+
+RUN apt-get update \
+    && apt-key del 7fa2af80 \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \
+    && dpkg -i cuda-keyring_1.0-1_all.deb \
+    && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub \
+    && apt-get update && apt-get install -y --no-install-recommends datacenter-gpu-manager=1:2.2.9
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libre2-5 libb64-0d python3 python3-pip libarchive-dev ffmpeg libsm6 libxext6 \
+    && python3 -m pip install -U pip \
+    && python3 -m pip install paddlenlp fast-tokenizer-python
+
+COPY python/dist/*.whl /opt/fastdeploy/
+RUN python3 -m pip install  /opt/fastdeploy/*.whl \
+    && rm -rf /opt/fastdeploy/*.whl
+
+# unset proxy
+# ENV http_proxy=''
+# ENV https_proxy=''
+# RUN python3 -m pip install paddlepaddle-gpu==2.4.2.post112 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html
+RUN unset http_proxy
+RUN unset https_proxy
+RUN python3 -m pip install https://paddle-wheel.bj.bcebos.com/2.4.2/linux/linux-gpu-cuda11.2-cudnn8-mkl-gcc8.2-avx/paddlepaddle_gpu-2.4.2.post112-cp38-cp38-linux_x86_64.whl
+
+COPY serving/build/libtriton_fastdeploy.so /opt/tritonserver/backends/fastdeploy/
+COPY build/fastdeploy_install /opt/fastdeploy/
+
+ENV LD_LIBRARY_PATH="/opt/TensorRT-8.5.2.2/lib/:/opt/fastdeploy/lib:/opt/fastdeploy/third_libs/install/onnxruntime/lib:/opt/fastdeploy/third_libs/install/paddle2onnx/lib:/opt/fastdeploy/third_libs/install/tensorrt/lib:/opt/fastdeploy/third_libs/install/paddle_inference/paddle/lib:/opt/fastdeploy/third_libs/install/paddle_inference/third_party/install/mkldnn/lib:/opt/fastdeploy/third_libs/install/paddle_inference/third_party/install/mklml/lib:/opt/fastdeploy/third_libs/install/openvino/runtime/lib:$LD_LIBRARY_PATH"
+ENV PATH="/opt/tritonserver/bin:$PATH"

serving/scripts/build_fd_cuda_11_2_trt_8_5_paddle_2_4_2.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ! -d "./cmake-3.18.6-Linux-x86_64/" ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz
+    tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz
+    rm -rf cmake-3.18.6-Linux-x86_64.tar.gz
+fi
+
+if [ ! -d "./TensorRT-8.5.2.2/" ]; then
+    wget https://fastdeploy.bj.bcebos.com/resource/TensorRT/TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz
+    tar -zxvf TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz
+    rm -rf TensorRT-8.5.2.2.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz
+fi
+
+# build vision、runtime、backend
+docker run -it --rm --name build_fd_libs \
+           -v`pwd`/..:/workspace/fastdeploy \
+           -e "http_proxy=${http_proxy}" \
+           -e "https_proxy=${https_proxy}" \
+           nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
+           bash -c \
+           'cd /workspace/fastdeploy/python;
+            rm -rf .setuptools-cmake-build dist build fastdeploy/libs/third_libs;
+            apt-get update;
+            apt-get install -y --no-install-recommends patchelf python3-dev python3-pip rapidjson-dev git;
+            ln -s /usr/bin/python3 /usr/bin/python;
+            export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
+            export WITH_GPU=ON;
+            export ENABLE_TRT_BACKEND=OFF;
+            export TRT_DIRECTORY=/workspace/fastdeploy/serving/TensorRT-8.5.2.2/;
+            export ENABLE_ORT_BACKEND=OFF;
+            export ENABLE_PADDLE_BACKEND=OFF;
+            export ENABLE_OPENVINO_BACKEND=OFF;
+            export ENABLE_VISION=ON;
+            export ENABLE_TEXT=ON;
+            export PADDLEINFERENCE_URL=https://bj.bcebos.com/fastdeploy/third_libs/paddle_inference-linux-x64-gpu-trt8.5.2.2-mkl-avx-2.4.2.tgz;
+            export PADDLEINFERENCE_VERSION=paddle2.4.2;
+            python setup.py build;
+            python setup.py bdist_wheel;
+            cd /workspace/fastdeploy;
+            rm -rf build; mkdir -p build;cd build;
+            cmake .. -DENABLE_TRT_BACKEND=ON -DPADDLEINFERENCE_URL=${PADDLEINFERENCE_URL} -DPADDLEINFERENCE_VERSION=${PADDLEINFERENCE_VERSION} -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy_install -DWITH_GPU=ON -DTRT_DIRECTORY=/workspace/fastdeploy/serving/TensorRT-8.5.2.2/ -DENABLE_PADDLE_BACKEND=ON -DENABLE_ORT_BACKEND=ON -DENABLE_OPENVINO_BACKEND=ON -DENABLE_VISION=ON -DBUILD_FASTDEPLOY_PYTHON=OFF -DENABLE_PADDLE2ONNX=ON -DENABLE_TEXT=OFF -DLIBRARY_NAME=fastdeploy_runtime;
+            make -j`nproc`;
+            make install;
+            cd /workspace/fastdeploy/serving;
+            rm -rf build; mkdir build; cd build;
+            cmake .. -DFASTDEPLOY_DIR=/workspace/fastdeploy/build/fastdeploy_install -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 -DTRITON_BACKEND_REPO_TAG=r21.10;
+            make -j`nproc`'