diff --git a/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc b/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc index 2e0f8dc77..455917938 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc @@ -34,7 +34,6 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { } auto im = cv::imread(image_file); - auto im_bak = im.clone(); fastdeploy::vision::DetectionResult res; if (!model.Predict(&im, &res)) { @@ -43,7 +42,7 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::Visualize::VisDetection(im_bak, res, 0.5); + auto vis_im = fastdeploy::vision::VisDetection(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } @@ -63,7 +62,6 @@ void GpuInfer(const std::string& model_dir, const std::string& image_file) { } auto im = cv::imread(image_file); - auto im_bak = im.clone(); fastdeploy::vision::DetectionResult res; if (!model.Predict(&im, &res)) { @@ -72,7 +70,7 @@ void GpuInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::Visualize::VisDetection(im_bak, res, 0.5); + auto vis_im = fastdeploy::vision::VisDetection(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } @@ -93,7 +91,6 @@ void TrtInfer(const std::string& model_dir, const std::string& image_file) { } auto im = cv::imread(image_file); - auto im_bak = im.clone(); fastdeploy::vision::DetectionResult res; if (!model.Predict(&im, &res)) { @@ -102,7 +99,7 @@ void TrtInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::Visualize::VisDetection(im_bak, res, 0.5); + auto vis_im = fastdeploy::vision::VisDetection(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } diff --git a/fastdeploy/fastdeploy_model.cc b/fastdeploy/fastdeploy_model.cc index ad4a61c06..ad8c1329d 100755 --- a/fastdeploy/fastdeploy_model.cc +++ b/fastdeploy/fastdeploy_model.cc @@ -16,97 +16,78 @@ namespace fastdeploy { -bool FastDeployModel::InitRuntime() { - FDASSERT( - CheckModelFormat(runtime_option.model_file, runtime_option.model_format), - "ModelFormatCheck Failed."); - if (runtime_initialized_) { - FDERROR << "The model is already initialized, cannot be initliazed again." +std::string Str(const std::vector& backends) { + std::ostringstream oss; + if (backends.size() == 0) { + oss << "[]"; + return oss.str(); + } + oss << "[ " << backends[0]; + for (int i = 1; i < backends.size(); ++i) { + oss << " ," << backends[i]; + } + oss << " ]"; + return oss.str(); +} + +bool IsSupported(const std::vector& backends, Backend backend) { + for (size_t i = 0; i < backends.size(); ++i) { + if (backends[i] == backend) { + return true; + } + } + return false; +} + +bool FastDeployModel::InitRuntimeWithSpecifiedBackend() { + if (!IsBackendAvailable(runtime_option.backend)) { + FDERROR << runtime_option.backend + << " is not compiled with current FastDeploy library." << std::endl; return false; } - if (runtime_option.backend != Backend::UNKNOWN) { - if (!IsBackendAvailable(runtime_option.backend)) { - FDERROR << Str(runtime_option.backend) - << " is not compiled with current FastDeploy library." - << std::endl; + + bool use_gpu = (runtime_option.device == Device::GPU); + bool use_ipu = (runtime_option.device == Device::IPU); + bool use_rknpu = (runtime_option.device == Device::RKNPU); + bool use_timvx = (runtime_option.device == Device::TIMVX); + + if (use_gpu) { + if (!IsSupported(valid_gpu_backends, runtime_option.backend)) { + FDERROR << "The valid gpu backends of model " << ModelName() << " are " << Str(valid_gpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; return false; } - - bool use_gpu = (runtime_option.device == Device::GPU); - bool use_ipu = (runtime_option.device == Device::IPU); -#ifndef WITH_GPU - use_gpu = false; -#endif -#ifndef WITH_IPU - use_ipu = false; -#endif - bool use_rknpu = (runtime_option.device == Device::RKNPU); - bool use_timvx = (runtime_option.device == Device::TIMVX); - - // whether the model is supported by the setted backend - bool is_supported = false; - if (use_gpu) { - for (auto& item : valid_gpu_backends) { - if (item == runtime_option.backend) { - is_supported = true; - break; - } - } - } else if (use_rknpu) { - for (auto& item : valid_rknpu_backends) { - if (item == runtime_option.backend) { - is_supported = true; - break; - } - } - } else if (use_timvx) { - for (auto& item : valid_timvx_backends) { - if (item == runtime_option.backend) { - is_supported = true; - break; - } - } - }else if(use_ipu) { - for (auto& item : valid_ipu_backends) { - if (item == runtime_option.backend) { - is_supported = true; - break; - } - } - } else { - for (auto& item : valid_cpu_backends) { - if (item == runtime_option.backend) { - is_supported = true; - break; - } - } + } else if (use_rknpu) { + if (!IsSupported(valid_rknpu_backends, runtime_option.backend)) { + FDERROR << "The valid rknpu backends of model " << ModelName() << " are " << Str(valid_rknpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; } - - if (is_supported) { - runtime_ = std::shared_ptr(new Runtime()); - if (!runtime_->Init(runtime_option)) { - return false; - } - runtime_initialized_ = true; - return true; - } else { - FDWARNING << ModelName() << " is not supported with backend " - << Str(runtime_option.backend) << "." << std::endl; - if (use_gpu) { - FDASSERT(valid_gpu_backends.size() > 0, - "There's no valid gpu backend for %s.", ModelName().c_str()); - FDWARNING << "FastDeploy will choose " << Str(valid_gpu_backends[0]) - << " for model inference." << std::endl; - } else { - FDASSERT(valid_cpu_backends.size() > 0, - "There's no valid cpu backend for %s.", ModelName().c_str()); - FDWARNING << "FastDeploy will choose " << Str(valid_cpu_backends[0]) - << " for model inference." << std::endl; - } + } else if (use_timvx) { + if (!IsSupported(valid_timvx_backends, runtime_option.backend)) { + FDERROR << "The valid timvx backends of model " << ModelName() << " are " << Str(valid_timvx_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; + } + } else if(use_ipu) { + if (!IsSupported(valid_ipu_backends, runtime_option.backend)) { + FDERROR << "The valid ipu backends of model " << ModelName() << " are " << Str(valid_ipu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; + } + } else { + if (!IsSupported(valid_cpu_backends, runtime_option.backend)) { + FDERROR << "The valid cpu backends of model " << ModelName() << " are " << Str(valid_cpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; } } + runtime_ = std::shared_ptr(new Runtime()); + if (!runtime_->Init(runtime_option)) { + return false; + } + runtime_initialized_ = true; + return true; +} + +bool FastDeployModel::InitRuntimeWithSpecifiedDevice() { if (runtime_option.device == Device::CPU) { return CreateCpuBackend(); } else if (runtime_option.device == Device::GPU) { @@ -130,10 +111,26 @@ bool FastDeployModel::InitRuntime() { return false; #endif } - FDERROR << "Only support CPU/GPU/NPU now." << std::endl; + FDERROR << "Only support CPU/GPU/IPU/RKNPU/TIMVX now." << std::endl; return false; } +bool FastDeployModel::InitRuntime() { + FDASSERT( + CheckModelFormat(runtime_option.model_file, runtime_option.model_format), + "ModelFormatCheck Failed."); + if (runtime_initialized_) { + FDERROR << "The model is already initialized, cannot be initliazed again." + << std::endl; + return false; + } + if (runtime_option.backend != Backend::UNKNOWN) { + return InitRuntimeWithSpecifiedBackend(); + } + + return InitRuntimeWithSpecifiedDevice(); +} + bool FastDeployModel::CreateCpuBackend() { if (valid_cpu_backends.size() == 0) { FDERROR << "There's no valid cpu backends for model: " << ModelName() diff --git a/fastdeploy/fastdeploy_model.h b/fastdeploy/fastdeploy_model.h index 21c4a44a1..717639694 100755 --- a/fastdeploy/fastdeploy_model.h +++ b/fastdeploy/fastdeploy_model.h @@ -41,12 +41,10 @@ class FASTDEPLOY_DECL FastDeployModel { std::vector valid_gpu_backends = {Backend::ORT}; /** Model's valid ipu backends. This member defined all the ipu backends have successfully tested for the model */ - std::vector valid_ipu_backends = {Backend::PDINFER}; + std::vector valid_ipu_backends = {}; /** Model's valid timvx backends. This member defined all the timvx backends have successfully tested for the model */ std::vector valid_timvx_backends = {}; - - /** Model's valid hardware backends. This member defined all the gpu backends have successfully tested for the model */ std::vector valid_rknpu_backends = {}; @@ -116,20 +114,22 @@ class FASTDEPLOY_DECL FastDeployModel { protected: virtual bool InitRuntime(); - virtual bool CreateCpuBackend(); - virtual bool CreateGpuBackend(); - virtual bool CreateIpuBackend(); - virtual bool CreateRKNPUBackend(); - virtual bool CreateTimVXBackend(); bool initialized = false; - std::vector valid_external_backends_; // Reused input tensors std::vector reused_input_tensors_; // Reused output tensors std::vector reused_output_tensors_; private: + bool InitRuntimeWithSpecifiedBackend(); + bool InitRuntimeWithSpecifiedDevice(); + bool CreateCpuBackend(); + bool CreateGpuBackend(); + bool CreateIpuBackend(); + bool CreateRKNPUBackend(); + bool CreateTimVXBackend(); + std::shared_ptr runtime_; bool runtime_initialized_ = false; // whether to record inference time diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 73edafc43..fc519fcff 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -97,7 +97,7 @@ std::string Str(const Backend& b) { }else if (b == Backend::OPENVINO) { return "Backend::OPENVINO"; } else if (b == Backend::LITE) { - return "Backend::LITE"; + return "Backend::PDLITE"; } return "UNKNOWN-Backend"; } @@ -116,9 +116,10 @@ std::ostream& operator<<(std::ostream& out, const Backend& backend) { }else if (backend == Backend::POROS) { out << "Backend::POROS"; } else if (backend == Backend::LITE) { - out << "Backend::LITE"; + out << "Backend::PDLITE"; + } else { + out << "UNKNOWN-Backend"; } - out << "UNKNOWN-Backend"; return out; } diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index 160c9ae85..94ee0e119 100755 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -23,10 +23,16 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, const std::string& config_file, const RuntimeOption& custom_option, const ModelFormat& model_format) : preprocessor_(config_file) { - valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER, - Backend::LITE}; - valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; - valid_timvx_backends = {Backend::LITE}; + if (model_format == ModelFormat::PADDLE) { + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER, + Backend::LITE}; + valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; + valid_timvx_backends = {Backend::LITE}; + valid_ipu_backends = {Backend::PDINFER}; + } else if (model_format == ModelFormat::ONNX) { + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/picodet.cc b/fastdeploy/vision/detection/ppdet/picodet.cc index abde7eca4..9b67db4a7 100644 --- a/fastdeploy/vision/detection/ppdet/picodet.cc +++ b/fastdeploy/vision/detection/ppdet/picodet.cc @@ -24,7 +24,7 @@ PicoDet::PicoDet(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 7b00b0734..f23911994 100644 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -103,16 +103,11 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { int input_height = input_shape[2].as(); int input_width = input_shape[3].as(); if (input_height == -1 || input_width == -1) { - FDWARNING << "The exported PaddleSeg model is with dynamic shape input, " - << "which is not supported by ONNX Runtime and Tensorrt. " - << "Only OpenVINO and Paddle Inference are available now. " - << "For using ONNX Runtime or Tensorrt, " - << "Please refer to " - "https://github.com/PaddlePaddle/PaddleSeg/blob/develop/" - "docs/model_export.md" - << " to export model with fixed input shape." << std::endl; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER}; + FDWARNING << "Some exportd PaddleSeg models with dynamic shape may " + "not be able inference with ONNX Runtime/TensorRT, if error " + "happend, please try to change to use Paddle " + "Inference/OpenVINO backends instead, or export model with " + "fixed input shape." << std::endl; } if (input_height != -1 && input_width != -1 && !yml_contain_resize_op) { processors_.push_back( diff --git a/python/setup.py b/python/setup.py index 2f3183222..a411a12be 100755 --- a/python/setup.py +++ b/python/setup.py @@ -45,9 +45,15 @@ import multiprocessing with open(os.path.join(TOP_DIR, "python", "requirements.txt")) as fin: REQUIRED_PACKAGES = fin.read() +if os.getenv("BUILD_ON_CPU", "OFF") == "ON": + os.environ["ENABLE_PADDLE_BACKEND"] = "ON" + os.environ["ENABLE_ORT_BACKEND"] = "ON" + os.environ["ENABLE_OPENVINO_BACKEND"] = "ON" + os.environ["ENABLE_VISION"] = "ON" + os.environ["ENABLE_TEXT"] = "ON" + os.environ["WITH_GPU"] = "OFF" + setup_configs = dict() -setup_configs["ENABLE_PADDLE_FRONTEND"] = os.getenv("ENABLE_PADDLE_FRONTEND", - "ON") setup_configs["ENABLE_RKNPU2_BACKEND"] = os.getenv("ENABLE_RKNPU2_BACKEND", "OFF") setup_configs["ENABLE_ORT_BACKEND"] = os.getenv("ENABLE_ORT_BACKEND", "OFF")