[Model] Add Paddle3D CenterPoint model (#2078)

* add centerpoint

* update for review comments

examples/vision/perception/paddle3d/centerpoint/cpp/CMakeLists.txt

@@ -0,0 +1,14 @@
+PROJECT(infer_demo C CXX)
+CMAKE_MINIMUM_REQUIRED (VERSION 3.10)
+
+# 指定下载解压后的fastdeploy库路径
+option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
+
+include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
+
+# 添加FastDeploy依赖头文件
+include_directories(${FASTDEPLOY_INCS})
+
+add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc)
+# 添加FastDeploy库依赖
+target_link_libraries(infer_demo ${FASTDEPLOY_LIBS})

examples/vision/perception/paddle3d/centerpoint/cpp/infer.cc

@@ -0,0 +1,38 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/vision.h"
+
+int main(int argc, char* argv[]) {
+  fastdeploy::RuntimeOption option;
+  option.UseGpu();
+  option.UsePaddleBackend();
+
+  std::string model_file = argv[1];
+  std::string params_file = argv[2];
+  std::string test_point = argv[3];
+
+  auto model = fastdeploy::vision::perception::Centerpoint(
+      model_file, params_file, "test", option, fastdeploy::ModelFormat::PADDLE);
+  assert(model.Initialized());
+
+  fastdeploy::vision::PerceptionResult res;
+  if (!model.Predict(test_point, &res)) {
+    std::cerr << "Failed to predict." << std::endl;
+    return 1;
+  }
+  std::cout << "predict result:" << res.Str() << std::endl;
+
+  return 0;
+}

fastdeploy/vision.h

@@ -36,6 +36,7 @@
 #include "fastdeploy/vision/detection/contrib/rknpu2/model.h"
 #include "fastdeploy/vision/perception/paddle3d/smoke/smoke.h"
 #include "fastdeploy/vision/perception/paddle3d/petr/petr.h"
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/centerpoint.h"
 #include "fastdeploy/vision/detection/ppdet/model.h"
 #include "fastdeploy/vision/facealign/contrib/face_landmark_1000.h"
 #include "fastdeploy/vision/facealign/contrib/pfld.h"

fastdeploy/vision/common/result.cc

@@ -227,6 +227,7 @@ PerceptionResult::PerceptionResult(const PerceptionResult& res) {
                            res.observation_angle.end());
   yaw_angle.assign(res.yaw_angle.begin(), res.yaw_angle.end());
   velocity.assign(res.velocity.begin(), res.velocity.end());
+  valid.assign(res.valid.begin(), res.valid.end());
 }
 
 PerceptionResult& PerceptionResult::operator=(PerceptionResult&& other) {
@@ -238,6 +239,7 @@ PerceptionResult& PerceptionResult::operator=(PerceptionResult&& other) {
     observation_angle = std::move(other.observation_angle);
     yaw_angle = std::move(other.yaw_angle);
     velocity = std::move(other.velocity);
+    valid = std::move(other.valid);
   }
   return *this;
 }
@@ -250,6 +252,7 @@ void PerceptionResult::Free() {
   std::vector<float>().swap(observation_angle);
   std::vector<float>().swap(yaw_angle);
   std::vector<std::array<float, 3>>().swap(velocity);
+  std::vector<bool>().swap(valid);
 }
 
 void PerceptionResult::Clear() {
@@ -260,6 +263,7 @@ void PerceptionResult::Clear() {
   observation_angle.clear();
   yaw_angle.clear();
   velocity.clear();
+  valid.clear();
 }
 
 void PerceptionResult::Reserve(int size) {
@@ -284,22 +288,52 @@ void PerceptionResult::Resize(int size) {
 
 std::string PerceptionResult::Str() {
   std::string out;
-  out =
+  out = "PerceptionResult: [";
-      "PerceptionResult: [xmin, ymin, xmax, ymax, w, h, l, cx, cy, cz, "
+  if (valid[2]) {
-      "yaw_angle, "
+    out += "xmin, ymin, xmax, ymax, w, h, l,";
-      "ob_angle, score, label_id]\n";
+  }
+  if (valid[3]) {
+    out += " cx, cy, cz,";
+  }
+  if (valid[5]) {
+    out += " yaw_angle,";
+  }
+  if (valid[4]) {
+    out += " ob_angle,";
+  }
+  if (valid[0]) {
+    out += " score,";
+  }
+  if (valid[1]) {
+    out += " label_id,";
+  }
+  out += "]\n";
+
   for (size_t i = 0; i < boxes.size(); ++i) {
+    if (valid[2]) {
       out = out + std::to_string(boxes[i][0]) + "," +
             std::to_string(boxes[i][1]) + ", " + std::to_string(boxes[i][2]) +
             ", " + std::to_string(boxes[i][3]) + ", " +
             std::to_string(boxes[i][4]) + ", " + std::to_string(boxes[i][5]) +
-          ", " + std::to_string(boxes[i][6]) + ", " +
+            ", " + std::to_string(boxes[i][6]) + ", ";
-          std::to_string(center[i][0]) + ", " + std::to_string(center[i][1]) +
+    }
-          ", " + std::to_string(center[i][2]) + ", " +
+    if (valid[3]) {
-          std::to_string(yaw_angle[i]) + ", " +
+      out = out + std::to_string(center[i][0]) + ", " +
-          std::to_string(observation_angle[i]) + ", " +
+            std::to_string(center[i][1]) + ", " + std::to_string(center[i][2]) +
-          std::to_string(scores[i]) + ", " + std::to_string(label_ids[i]);
+            ", ";
-
+    }
+    if (valid[5]) {
+      out = out + std::to_string(yaw_angle[i]) + ", ";
+    }
+    if (valid[4]) {
+      out = out + std::to_string(observation_angle[i]) + ", ";
+    }
+    if (valid[0]) {
+      out = out + std::to_string(scores[i]) + ", ";
+    }
+    if (valid[1]) {
+      out = out + std::to_string(label_ids[i]);
+    }
     out += "\n";
   }
   return out;
@@ -672,8 +706,8 @@ std::string OCRResult::Str() {
       out = out + "]";
 
       if (rec_scores.size() > 0) {
-        out = out + "rec text: " + text[n] + " rec score:" +
+        out = out + "rec text: " + text[n] +
-              std::to_string(rec_scores[n]) + " ";
+              " rec score:" + std::to_string(rec_scores[n]) + " ";
       }
       if (cls_labels.size() > 0) {
         out = out + "cls label: " + std::to_string(cls_labels[n]) +
@@ -713,8 +747,8 @@ std::string OCRResult::Str() {
              cls_scores.size() > 0) {
     std::string out;
     for (int i = 0; i < rec_scores.size(); i++) {
-      out = out + "rec text: " + text[i] + " rec score:" +
+      out = out + "rec text: " + text[i] +
-            std::to_string(rec_scores[i]) + " ";
+            " rec score:" + std::to_string(rec_scores[i]) + " ";
       out = out + "cls label: " + std::to_string(cls_labels[i]) +
             " cls score: " + std::to_string(cls_scores[i]);
       out = out + "\n";
@@ -733,8 +767,8 @@ std::string OCRResult::Str() {
              cls_scores.size() == 0) {
     std::string out;
     for (int i = 0; i < rec_scores.size(); i++) {
-      out = out + "rec text: " + text[i] + " rec score:" +
+      out = out + "rec text: " + text[i] +
-            std::to_string(rec_scores[i]) + " ";
+            " rec score:" + std::to_string(rec_scores[i]) + " ";
       out = out + "\n";
     }
     return out;
@@ -781,9 +815,9 @@ std::string HeadPoseResult::Str() {
   std::string out;
 
   out = "HeadPoseResult: [yaw, pitch, roll]\n";
-  out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + "pitch: " +
+  out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" +
-        std::to_string(euler_angles[1]) + "\n" + "roll: " +
+        "pitch: " + std::to_string(euler_angles[1]) + "\n" +
-        std::to_string(euler_angles[2]) + "\n";
+        "roll: " + std::to_string(euler_angles[2]) + "\n";
   return out;
 }
 

fastdeploy/vision/common/result.h

@@ -35,8 +35,10 @@ enum FASTDEPLOY_DECL ResultType {
   MASK,
   KEYPOINT_DETECTION,
   HEADPOSE,
+  PERCEPTION,
 };
 
+
 struct FASTDEPLOY_DECL BaseResult {
   ResultType type = ResultType::UNKNOWN_RESULT;
 };
@@ -164,6 +166,17 @@ struct FASTDEPLOY_DECL PerceptionResult : public BaseResult {
   // vx, vy, vz
   std::vector<std::array<float, 3>> velocity;
 
+  // valid results for func Str(): True for printing
+  // 0 scores
+  // 1 label_ids
+  // 2 boxes
+  // 3 center
+  // 4 observation_angle
+  // 5 yaw_angle
+  // 6 velocity
+  std::vector<bool> valid;
+
+
   /// Copy constructor
   PerceptionResult(const PerceptionResult& res);
   /// Move assignment

fastdeploy/vision/perception/paddle3d/centerpoint/centerpoint.cc

@@ -0,0 +1,92 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/centerpoint.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace perception {
+
+Centerpoint::Centerpoint(const std::string& model_file,
+                         const std::string& params_file,
+                         const std::string& config_file,
+                         const RuntimeOption& custom_option,
+                         const ModelFormat& model_format)
+    : preprocessor_(config_file) {
+  valid_gpu_backends = {Backend::PDINFER};
+
+  runtime_option = custom_option;
+  runtime_option.model_format = model_format;
+  runtime_option.model_file = model_file;
+  runtime_option.params_file = params_file;
+  initialized = Initialize();
+}
+
+bool Centerpoint::Initialize() {
+  if (!InitRuntime()) {
+    FDERROR << "Failed to initialize fastdeploy backend." << std::endl;
+    return false;
+  }
+  return true;
+}
+
+bool Centerpoint::Predict(const std::string point_dir,
+                          PerceptionResult* result) {
+  std::vector<PerceptionResult> results;
+  if (!BatchPredict({point_dir}, &results)) {
+    return false;
+  }
+
+  if (results.size()) {
+    *result = std::move(results[0]);
+  }
+  return true;
+}
+
+bool Centerpoint::BatchPredict(std::vector<std::string> points_dir,
+                               std::vector<PerceptionResult>* results) {
+  int64_t num_point_dim = 5;
+  int with_timelag = 0;
+  if (!preprocessor_.Run(points_dir, num_point_dim, with_timelag,
+                         reused_input_tensors_)) {
+    FDERROR << "Failed to preprocess the input image." << std::endl;
+    return false;
+  }
+
+  results->resize(reused_input_tensors_.size());
+  for (int index = 0; index < reused_input_tensors_.size(); ++index) {
+    std::vector<FDTensor> input_tensor;
+    input_tensor.push_back(reused_input_tensors_[index]);
+
+    input_tensor[0].name = InputInfoOfRuntime(0).name;
+
+    if (!Infer(input_tensor, &reused_output_tensors_)) {
+      FDERROR << "Failed to inference by runtime." << std::endl;
+      return false;
+    }
+
+    (*results)[index].Clear();
+    (*results)[index].Reserve(reused_output_tensors_[0].shape[0]);
+    if (!postprocessor_.Run(reused_output_tensors_, &((*results)[index]))) {
+      FDERROR << "Failed to postprocess the inference results by runtime."
+              << std::endl;
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/centerpoint.h

@@ -0,0 +1,78 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/preprocessor.h"
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/postprocessor.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace perception {
+/*! @brief Centerpoint model object used when to load a Centerpoint model exported by Centerpoint.
+ */
+class FASTDEPLOY_DECL Centerpoint : public FastDeployModel {
+ public:
+  /** \brief  Set path of model file and the configuration of runtime.
+   *
+   * \param[in] model_file Path of model file, e.g Centerpoint/model.pdiparams
+   * \param[in] params_file Path of parameter file, e.g Centerpoint/model.pdiparams, if the model format is ONNX, this parameter will be ignored
+   * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends"
+   * \param[in] model_format Model format of the loaded model, default is Paddle format
+   */
+  Centerpoint(const std::string& model_file, const std::string& params_file,
+         const std::string& config_file,
+         const RuntimeOption& custom_option = RuntimeOption(),
+         const ModelFormat& model_format = ModelFormat::PADDLE);
+
+  std::string ModelName() const { return "Paddle3D/Centerpoint"; }
+
+  /** \brief Predict the perception result for an input image
+   *
+   * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format
+   * \param[in] result The output perception result will be writen to this structure
+   * \return true if the prediction successed, otherwise false
+   */
+  virtual bool Predict(std::string point_dir, PerceptionResult* result);
+
+  /** \brief Predict the perception results for a batch of input images
+   *
+   * \param[in] imgs, The input image list, each element comes from cv::imread()
+   * \param[in] results The output perception result list
+   * \return true if the prediction successed, otherwise false
+   */
+  virtual bool BatchPredict(std::vector<std::string> points_dir,
+                            std::vector<PerceptionResult>* results);
+
+  /// Get preprocessor reference of Centerpoint
+  virtual CenterpointPreprocessor& GetPreprocessor() {
+    return preprocessor_;
+  }
+
+  /// Get postprocessor reference of Centerpoint
+  virtual CenterpointPostprocessor& GetPostprocessor() {
+    return postprocessor_;
+  }
+
+ protected:
+  bool Initialize();
+  CenterpointPreprocessor preprocessor_;
+  CenterpointPostprocessor postprocessor_;
+  bool initialized_ = false;
+  std::vector<std::vector<FDTensor>> ouput_tensors;
+};
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/centerpoint_pybind.cc

@@ -0,0 +1,57 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindCenterpoint(pybind11::module& m) {
+  pybind11::class_<vision::perception::CenterpointPreprocessor,
+                   vision::ProcessorManager>(m, "CenterpointPreprocessor")
+      .def(pybind11::init<std::string>())
+      .def("run", [](vision::perception::CenterpointPreprocessor& self,
+                     std::vector<std::string> points_dir,
+                     const int64_t num_point_dim, const int with_timelag) {
+        std::vector<FDTensor> outputs;
+        if (!self.Run(points_dir, num_point_dim, with_timelag, outputs)) {
+          throw std::runtime_error(
+              "Failed to preprocess the input data in "
+              "CenterpointPreprocessor.");
+        }
+
+        return outputs;
+      });
+
+  pybind11::class_<vision::perception::Centerpoint, FastDeployModel>(
+      m, "Centerpoint")
+      .def(pybind11::init<std::string, std::string, std::string, RuntimeOption,
+                          ModelFormat>())
+      .def("predict",
+           [](vision::perception::Centerpoint& self, std::string point_dir) {
+             vision::PerceptionResult result;
+             self.Predict(point_dir, &result);
+             return result;
+           })
+      .def("batch_predict",
+           [](vision::perception::Centerpoint& self,
+              std::vector<std::string>& points_dir) {
+             std::vector<vision::PerceptionResult> results;
+             self.BatchPredict(points_dir, &results);
+             return results;
+           })
+      .def_property_readonly("preprocessor",
+                             &vision::perception::Centerpoint::GetPreprocessor)
+      .def_property_readonly(
+          "postprocessor", &vision::perception::Centerpoint::GetPostprocessor);
+}
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/postprocessor.cc

@@ -0,0 +1,71 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/postprocessor.h"
+
+#include "fastdeploy/vision/utils/utils.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace perception {
+
+CenterpointPostprocessor::CenterpointPostprocessor() {}
+
+bool CenterpointPostprocessor::Run(const std::vector<FDTensor>& tensors,
+                                   PerceptionResult* result) {
+  if (tensors[0].dtype != FDDataType::FP32) {
+    FDERROR << "Only support post process with float32 data." << std::endl;
+    return false;
+  }
+
+  const float* data_0 = reinterpret_cast<const float*>(tensors[0].Data());
+  for (int i = 0; i < tensors[0].shape[0] * tensors[0].shape[1]; i += 9) {
+    // item 1 ~ 3   :  box3d bottom center x, y, z
+    // item 4 ~ 6   :  box3d w, l, h
+    // item 7 ~ 8   :  speed x,y
+    // item 9   :  box3d yaw angle
+    std::vector<float> vec(data_0 + i, data_0 + i + 9);
+    result->boxes.emplace_back(
+        std::array<float, 7>{0, 0, 0, 0, vec[3], vec[4], vec[5]});
+    result->center.emplace_back(std::array<float, 3>{vec[0], vec[1], vec[2]});
+    result->yaw_angle.push_back(vec[8]);
+    result->velocity.push_back(std::array<float, 3>{vec[6], vec[7]});
+  }
+
+  const float* data_1 = reinterpret_cast<const float*>(tensors[2].Data());
+  for (int i = 0; i < tensors[1].shape[0]; i += 1) {
+    std::vector<float> vec(data_1 + i, data_1 + i + 1);
+    result->scores.push_back(vec[0]);
+  }
+
+  const long long* data_2 =
+      reinterpret_cast<const long long*>(tensors[1].Data());
+  for (int i = 0; i < tensors[2].shape[0]; i++) {
+    std::vector<long long> vec(data_2 + i, data_2 + i + 1);
+    result->label_ids.push_back(vec[0]);
+  }
+  result->valid.push_back(true);   // 0 scores
+  result->valid.push_back(true);   // 1 label_ids
+  result->valid.push_back(true);   // 2 boxes
+  result->valid.push_back(true);   // 3 center
+  result->valid.push_back(false);  // 4 observation_angle
+  result->valid.push_back(true);   // 5 yaw_angle
+  result->valid.push_back(true);   // 6 velocity
+
+  return true;
+}
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/postprocessor.h

@@ -0,0 +1,48 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+
+namespace fastdeploy {
+namespace vision {
+
+namespace perception {
+/*! @brief Postprocessor object for Centerpoint serials model.
+ */
+class FASTDEPLOY_DECL CenterpointPostprocessor {
+ public:
+  /** \brief Create a postprocessor instance for Centerpoint serials model
+   */
+  CenterpointPostprocessor();
+
+  /** \brief Process the result of runtime and fill to PerceptionResult structure
+   *
+   * \param[in] tensors The inference result from runtime
+   * \param[in] result The output result of detection
+   * \param[in] ims_info The shape info list, record input_shape and output_shape
+   * \return true if the postprocess successed, otherwise false
+   */
+  bool Run(const std::vector<FDTensor>& tensors,
+      PerceptionResult* results);
+
+
+ protected:
+  float conf_threshold_;
+};
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/preprocessor.cc

@@ -0,0 +1,108 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/vision/perception/paddle3d/centerpoint/preprocessor.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace perception {
+
+CenterpointPreprocessor::CenterpointPreprocessor(
+    const std::string &config_file) {
+  initialized_ = true;
+}
+
+bool CenterpointPreprocessor::ReadPoint(const std::string &file_path,
+                                        const int64_t num_point_dim,
+                                        void **buffer, int64_t *num_points) {
+  std::ifstream file_in(file_path, std::ios::in | std::ios::binary);
+  if (num_point_dim < 4) {
+    FDERROR << "Point dimension must not be less than 4, but received "
+            << "num_point_dim is " << num_point_dim << std::endl;
+  }
+
+  if (!file_in) {
+    FDERROR << "Failed to read file: " << file_path << std::endl;
+    return false;
+  }
+
+  std::streampos file_size;
+  file_in.seekg(0, std::ios::end);
+  file_size = file_in.tellg();
+  file_in.seekg(0, std::ios::beg);
+
+  *buffer = malloc(file_size);
+  if (*buffer == nullptr) {
+    FDERROR << "Failed to malloc memory of size: " << file_size << std::endl;
+    return false;
+  }
+  file_in.read(reinterpret_cast<char *>(*buffer), file_size);
+  file_in.close();
+
+  if (file_size / sizeof(float) % num_point_dim != 0) {
+    FDERROR << "Loaded file size (" << file_size
+            << ") is not evenly divisible by num_point_dim (" << num_point_dim
+            << ")\n";
+    return false;
+  }
+  *num_points = file_size / sizeof(float) / num_point_dim;
+  return true;
+}
+
+bool CenterpointPreprocessor::InsertTimeToPoints(const int64_t num_points,
+                                                 const int64_t num_point_dim,
+                                                 float *points) {
+  for (int64_t i = 0; i < num_points; ++i) {
+    *(points + i * num_point_dim + 4) = 0.;
+  }
+  return true;
+}
+
+bool CenterpointPreprocessor::Apply(std::vector<std::string> &points_dir,
+                                    const int64_t num_point_dim,
+                                    const int with_timelag,
+                                    std::vector<FDTensor> &outputs) {
+  for (int index = 0; index < points_dir.size(); ++index) {
+    std::string file_path = points_dir[index];
+    std::vector<int64_t> points_shape;
+    void *buffer = nullptr;
+    int64_t num_points;
+    if (!ReadPoint(file_path, num_point_dim, &buffer, &num_points)) {
+      return false;
+    }
+    float *points = static_cast<float *>(buffer);
+
+    if (!with_timelag && num_point_dim == 5 || num_point_dim > 5) {
+      InsertTimeToPoints(num_points, num_point_dim, points);
+    }
+    points_shape.push_back(num_points);
+    points_shape.push_back(num_point_dim);
+
+    FDTensor tensor;
+    tensor.SetData(points_shape, FDDataType::FP32, points, true);
+    outputs.push_back(tensor);
+  }
+  return true;
+}
+
+bool CenterpointPreprocessor::Run(std::vector<std::string> &points_dir,
+                                  const int64_t num_point_dim,
+                                  const int with_timelag,
+                                  std::vector<FDTensor> &outputs) {
+  bool ret = Apply(points_dir, num_point_dim, with_timelag, outputs);
+  return ret;
+}
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/centerpoint/preprocessor.h

@@ -0,0 +1,63 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/vision/common/processors/manager.h"
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+
+namespace fastdeploy {
+namespace vision {
+
+namespace perception {
+/*! @brief Preprocessor object for Centerpoint model.
+ */
+class FASTDEPLOY_DECL CenterpointPreprocessor : public ProcessorManager  {
+ public:
+  CenterpointPreprocessor() = default;
+  /** \brief Create a preprocessor instance for Centerpoint model
+   *
+   * \param[in] config_file Path of configuration file for deployment, e.g Centerpoint/infer_cfg.yml
+   */
+  explicit CenterpointPreprocessor(const std::string& config_file);
+
+  bool Apply(FDMatBatch* image_batch, std::vector<FDTensor>* outputs) {
+    return false;
+  }
+
+  bool Apply(std::vector<std::string>& points_dir,
+            const int64_t num_point_dim,
+            const int with_timelag,
+            std::vector<FDTensor>& outputs);
+
+  bool Run(std::vector<std::string>& points_dir,
+            const int64_t num_point_dim,
+            const int with_timelag,
+            std::vector<FDTensor>& outputs);
+
+
+ protected:
+  std::vector<std::shared_ptr<Processor>> processors_;
+  bool ReadPoint(const std::string &file_path,
+                const int64_t num_point_dim,
+                void **buffer, int64_t *num_points);
+  bool InsertTimeToPoints(const int64_t num_points,
+                             const int64_t num_point_dim,
+                             float *points);
+  bool initialized_ = false;
+};
+
+}  // namespace perception
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/perception/paddle3d/petr/postprocessor.cc

@@ -39,8 +39,8 @@ bool PetrPostprocessor::Run(const std::vector<FDTensor>& tensors,
     // item 7       :  box3d yaw angle
     // item 8 ~ 9   :  speed x,y
     std::vector<float> vec(data_0 + i, data_0 + i + 9);
-    result->boxes.emplace_back(std::array<float, 7>{
+    result->boxes.emplace_back(
-        0, 0, 0, 0, vec[0], vec[1], vec[2]});
+        std::array<float, 7>{0, 0, 0, 0, vec[0], vec[1], vec[2]});
     result->center.emplace_back(std::array<float, 3>{vec[3], vec[4], vec[5]});
     result->yaw_angle.push_back(vec[6]);
     result->velocity.push_back(std::array<float, 3>{vec[7], vec[8]});
@@ -50,11 +50,21 @@ bool PetrPostprocessor::Run(const std::vector<FDTensor>& tensors,
     std::vector<float> vec(data_1 + i, data_1 + i + 1);
     result->scores.push_back(vec[0]);
   }
-  const long long* data_2 = reinterpret_cast<const long long*>(tensors[2].Data());
+  const long long* data_2 =
+      reinterpret_cast<const long long*>(tensors[2].Data());
   for (int i = 0; i < tensors[2].shape[0]; i++) {
     std::vector<long long> vec(data_2 + i, data_2 + i + 1);
     result->label_ids.push_back(vec[0]);
   }
+
+  result->valid.push_back(true);   // 0 scores
+  result->valid.push_back(true);   // 1 label_ids
+  result->valid.push_back(true);   // 2 boxes
+  result->valid.push_back(true);   // 3 center
+  result->valid.push_back(false);  // 4 observation_angle
+  result->valid.push_back(true);   // 5 yaw_angle
+  result->valid.push_back(true);   // 6 velocity
+
   return true;
 }
 

fastdeploy/vision/perception/paddle3d/smoke/postprocessor.cc

@@ -50,6 +50,15 @@ bool SmokePostprocessor::Run(const std::vector<FDTensor>& tensors,
     result->observation_angle.push_back(vec[1]);
     result->yaw_angle.push_back(vec[12]);
   }
+
+  result->valid.push_back(true);   // 0 scores
+  result->valid.push_back(true);   // 1 label_ids
+  result->valid.push_back(true);   // 2 boxes
+  result->valid.push_back(true);   // 3 center
+  result->valid.push_back(true);   // 4 observation_angle
+  result->valid.push_back(true);   // 5 yaw_angle
+  result->valid.push_back(false);  // 6 velocity
+
   return true;
 }
 

fastdeploy/vision/perception/perception_pybind.cc

@@ -18,11 +18,13 @@ namespace fastdeploy {
 
 void BindSmoke(pybind11::module& m);
 void BindPetr(pybind11::module& m);
+void BindCenterpoint(pybind11::module& m);
 
 void BindPerception(pybind11::module& m) {
   auto perception_module =
       m.def_submodule("perception", "3D object perception models.");
   BindSmoke(perception_module);
   BindPetr(perception_module);
+  BindCenterpoint(perception_module);
 }
 }  // namespace fastdeploy

fastdeploy/vision/vision_pybind.cc

@@ -121,6 +121,7 @@ void BindVision(pybind11::module& m) {
 
   pybind11::class_<vision::PerceptionResult>(m, "PerceptionResult")
       .def(pybind11::init())
+      .def_readwrite("valid", &vision::PerceptionResult::valid)
       .def_readwrite("scores", &vision::PerceptionResult::scores)
       .def_readwrite("label_ids", &vision::PerceptionResult::label_ids)
       .def_readwrite("boxes", &vision::PerceptionResult::boxes)

python/fastdeploy/vision/perception/__init__.py

@@ -15,3 +15,4 @@
 from __future__ import absolute_import
 from .paddle3d.smoke import *
 from .paddle3d.petr import *
+from .paddle3d.centerpoint import *

python/fastdeploy/vision/perception/paddle3d/centerpoint.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import logging
+from .... import FastDeployModel, ModelFormat
+from .... import c_lib_wrap as C
+
+
+class CenterpointPreprocessor:
+    def __init__(self, config_file):
+        """Create a preprocessor for Centerpoint
+        """
+        self._preprocessor = C.vision.perception.CenterpointPreprocessor(
+            config_file)
+
+    def run(self, point_dirs, num_point_dim, with_timelag):
+        """Preprocess input images for Centerpoint
+
+        :param: input_ims: (list of numpy.ndarray)The input image
+        :return: list of FDTensor
+        """
+        return self._preprocessor.run(point_dirs, num_point_dim, with_timelag)
+
+
+class Centerpoint(FastDeployModel):
+    def __init__(self,
+                 model_file,
+                 params_file,
+                 config_file,
+                 runtime_option=None,
+                 model_format=ModelFormat.PADDLE):
+        """Load a Centerpoint model exported by Centerpoint.
+
+        :param model_file: (str)Path of model file, e.g ./Centerpoint.pdmodel
+        :param params_file: (str)Path of parameters file, e.g ./Centerpoint.pdiparams
+        :param config_file: (str)Path of config file, e.g ./infer_cfg.yaml
+        :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU
+        :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model
+        """
+        super(Centerpoint, self).__init__(runtime_option)
+
+        self._model = C.vision.perception.Centerpoint(
+            model_file, params_file, config_file, self._runtime_option,
+            model_format)
+        assert self.initialized, "Centerpoint initialize failed."
+
+    def predict(self, point_dir):
+        """Detect an input image
+
+        :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format
+        :param conf_threshold: confidence threshold for postprocessing, default is 0.25
+        :param nms_iou_threshold: iou threshold for NMS, default is 0.5
+        :return: PerceptionResult
+        """
+        return self._model.predict(point_dir)
+
+    def batch_predict(self, points_dir):
+        """Classify a batch of input image
+
+        :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format
+        :return list of PerceptionResult
+        """
+
+        return self._model.batch_predict(points_dir)
+
+    @property
+    def preprocessor(self):
+        """Get CenterpointPreprocessor object of the loaded model
+
+        :return CenterpointPreprocessor
+        """
+        return self._model.preprocessor
+
+    @property
+    def postprocessor(self):
+        """Get CenterpointPostprocessor object of the loaded model
+
+        :return CenterpointPostprocessor
+        """
+        return self._model.postprocessor