[Serving]: add ipu support for serving. (#10) (#470)

* feat(ipu): add ipu docker for serving. (#10)

* feat(ipu): add ipu docker for serving.

* feat(ipu): enable ipu docker in serving.

* fix(): fix typo and issues in IPU.

* remove unused env path.

* doc(ipu): add ipu docker build doc and fix typo.

* fix(): clean apt cache in docker.

* fix(ipu): fix typo.

serving/Dockerfile_ipu

@@ -0,0 +1,64 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM graphcore/poplar:3.0.0
+
+#Install the build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends curl wget vim git patchelf python3-dev python3-pip \
+    python3-setuptools build-essential libgl1-mesa-glx libglib2.0-dev ca-certificates \
+    libssl-dev zlib1g-dev rapidjson-dev libboost-dev libre2-dev librdmacm-dev libnuma-dev libarchive-dev unzip && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/bin/python3 /usr/bin/python;
+RUN pip3 install --upgrade pip
+
+# install cmake
+WORKDIR /home
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.6-Linux-x86_64/bin:$PATH
+
+
+#install triton
+ENV TAG=r21.10
+RUN git clone https://github.com/triton-inference-server/server.git -b $TAG && \
+    cd server && \
+    mkdir -p build/tritonserver/install && \
+    python3 build.py \
+      --build-dir `pwd`/build \
+      --no-container-build \
+      --endpoint=grpc \
+      --enable-logging \
+      --enable-stats \
+      --cmake-dir `pwd`/build \
+      --repo-tag=common:$TAG \
+      --repo-tag=core:$TAG \
+      --repo-tag=backend:$TAG \
+      --repo-tag=thirdparty:$TAG \
+      --backend=python:$TAG
+
+COPY python/dist/*.whl /opt/fastdeploy/
+RUN python3 -m pip install  /opt/fastdeploy/*.whl \
+    && rm -rf /opt/fastdeploy/*.whl
+
+# triton server
+RUN mkdir -p /opt/tritonserver && cp -r /home/server/build/tritonserver/install/* /opt/tritonserver
+# python backend
+RUN mkdir -p /opt/tritonserver/backends/python && cp -r /home/server/build/python/install/backends/python /opt/tritonserver/backends/
+# fd_backend
+COPY serving/build/libtriton_fastdeploy.so /opt/tritonserver/backends/fastdeploy/
+
+COPY build/fastdeploy-0.0.3 /opt/fastdeploy/
+RUN mv /opt/tritonserver/bin/tritonserver /opt/tritonserver/bin/fastdeployserver
+ENV LD_LIBRARY_PATH="/opt/fastdeploy/lib:/opt/fastdeploy/third_libs/install/onnxruntime/lib:/opt/fastdeploy/third_libs/install/paddle2onnx/lib:/opt/fastdeploy/third_libs/install/paddle_inference/paddle/lib:$LD_LIBRARY_PATH"
+ENV PATH="/opt/tritonserver/bin:$PATH"

serving/docs/zh_CN/compile.md

@@ -7,7 +7,7 @@
 FastDploy发布的GPU镜像基于[Triton Inference Server](https://github.com/triton-inference-server/server)的21.10版本进行制作，如果有其他CUDA版本需求，可以参照[NVIDIA 官网](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html)中展示的版本信息修改Dockerfile和scripts中的脚本.
 
 ```
-# 进入serving目录执行脚本编译fastdeply和服务化的backend
+# 进入serving目录执行脚本编译fastdeploy和服务化的backend
 cd serving
 bash scripts/build.sh
 
@@ -19,7 +19,7 @@ docker build -t paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 -f servi
 ## 制作CPU镜像
 
 ```
-# 进入serving目录执行脚本编译fastdeply和服务化的backend
+# 进入serving目录执行脚本编译fastdeploy和服务化的backend
 cd serving
 bash scripts/build.sh OFF
 
@@ -27,3 +27,15 @@ bash scripts/build.sh OFF
 cd ../
 docker build -t paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 -f serving/Dockerfile_cpu .
 ```
+
+## 制作IPU镜像
+
+```
+# 进入serving目录执行脚本编译fastdeploy和服务化的backend
+cd serving
+bash scripts/build_fd_ipu.sh
+
+# 退出到FastDeploy主目录，制作镜像
+cd ../
+docker build -t paddlepaddle/fastdeploy:0.3.0-ipu-only-21.10 -f serving/Dockerfile_ipu .
+```

serving/scripts/build_fd_ipu.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ! -d "./cmake-3.18.6-Linux-x86_64/" ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz
+    tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz
+    rm -rf cmake-3.18.6-Linux-x86_64.tar.gz
+fi
+
+# build vision
+docker run -it --rm --name build_fd_vison \
+           -v`pwd`/..:/workspace/fastdeploy \
+           graphcore/poplar:3.0.0 \
+           bash -c \
+           'cd /workspace/fastdeploy/python;
+            rm -rf .setuptools-cmake-build dist;
+            apt-get update;
+            apt-get install -y --no-install-recommends patchelf python3-dev python3-pip python3-setuptools build-essential;
+            ln -s /usr/bin/python3 /usr/bin/python;
+            pip3 install wheel;
+            export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
+            export WITH_GPU=OFF;
+            export WITH_IPU=ON;
+            export ENABLE_PADDLE_BACKEND=ON;
+            export ENABLE_VISION=ON;
+            python setup.py build;
+            python setup.py bdist_wheel'
+
+# build runtime
+docker run -it --rm --name build_fd_runtime \
+           -v`pwd`/..:/workspace/fastdeploy \
+           graphcore/poplar:3.0.0 \
+           bash -c \
+           'cd /workspace/fastdeploy;
+            rm -rf build; mkdir build; cd build;
+            apt-get update;
+            apt-get install -y --no-install-recommends python3-dev python3-pip build-essential;
+            ln -s /usr/bin/python3 /usr/bin/python;
+            export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
+            cmake .. -DENABLE_ORT_BACKEND=OFF -DENABLE_TEXT=OFF -DENABLE_VISION=OFF -DBUILD_FASTDEPLOY_PYTHON=OFF -DENABLE_PADDLE_BACKEND=ON -DWITH_IPU=ON -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-0.0.3 -DLIBRARY_NAME=fastdeploy_runtime;
+            make -j`nproc`;
+            make install'
+
+# build backend
+docker run -it --rm --name build_fd_backend \
+           -v`pwd`/..:/workspace/fastdeploy \
+           graphcore/poplar:3.0.0 \
+           bash -c \
+           'cd /workspace/fastdeploy/serving;
+            rm -rf build; mkdir build; cd build;
+            apt-get update; apt-get install -y --no-install-recommends rapidjson-dev build-essential git ca-certificates;
+            export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
+            cmake .. -DTRITON_ENABLE_GPU=OFF -DFASTDEPLOY_DIR=/workspace/fastdeploy/build/fastdeploy-0.0.3 -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 -DTRITON_BACKEND_REPO_TAG=r21.10; make -j`nproc`'

serving/src/fastdeploy_runtime.cc

@@ -238,6 +238,8 @@ ModelState::ModelState(TRITONBACKEND_Model* triton_model)
               runtime_options_->SetPaddleMKLDNN(pd_enable_mkldnn);
             } else if (param_key == "use_paddle_log") {
                 runtime_options_->EnablePaddleLogInfo();
+            } else if (param_key == "use_ipu") {
+              runtime_options_->UseIpu();
             }
           }
         }
@@ -384,7 +386,10 @@ TRITONSERVER_Error* ModelState::LoadModel(
     runtime_options_->UseCpu();
   }
 #else
-  runtime_options_->UseCpu();
+  if (runtime_options_->device != fastdeploy::Device::IPU) {
+    // If Device is set to IPU, just skip CPU setting.
+    runtime_options_->UseCpu();
+  }
 #endif  // TRITON_ENABLE_GPU
 
   *runtime = new fastdeploy::Runtime();