Merge branch 'develop' into set_stream_infer-shareExData

benchmark/cpp/benchmark_ppyolov8.cc

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "fastdeploy/benchmark/utils.h"
-#include "fastdeploy/vision.h"
 #include "flags.h"
+#include "macros.h"
+#include "option.h"
 
 #ifdef WIN32
 const char sep = '\\';
@@ -22,104 +22,24 @@ const char sep = '\\';
 const char sep = '/';
 #endif
 
-bool RunModel(std::string model_dir, std::string image_file, size_t warmup,
+int main(int argc, char* argv[]) {
-              size_t repeats, size_t dump_period, std::string cpu_mem_file_name,
+  google::ParseCommandLineFlags(&argc, &argv, true);
-              std::string gpu_mem_file_name) {
+  auto im = cv::imread(FLAGS_image);
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option)) {
     PrintUsage();
     return false;
   }
-  auto model_file = model_dir + sep + "model.pdmodel";
+  auto model_file = FLAGS_model + sep + "model.pdmodel";
-  auto params_file = model_dir + sep + "model.pdiparams";
+  auto params_file = FLAGS_model + sep + "model.pdiparams";
-  auto config_file = model_dir + sep + "infer_cfg.yml";
+  auto config_file = FLAGS_model + sep + "infer_cfg.yml";
-
+  auto model_ppyolov8 = fastdeploy::vision::detection::PaddleYOLOv8(
-  if (FLAGS_profile_mode == "runtime") {
-    option.EnableProfiling(FLAGS_include_h2d_d2h, repeats, warmup);
-  }
-  auto model = fastdeploy::vision::detection::PaddleYOLOv8(
       model_file, params_file, config_file, option);
-  if (!model.Initialized()) {
+  fastdeploy::vision::DetectionResult res;
-    std::cerr << "Failed to initialize." << std::endl;
+  BENCHMARK_MODEL(model_ppyolov8, model_ppyolov8.Predict(im, &res))
-    return false;
+  auto vis_im = fastdeploy::vision::VisDetection(im, res);
-  }
+  cv::imwrite("vis_result.jpg", vis_im);
-  auto im = cv::imread(image_file);
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  // For Runtime
-  if (FLAGS_profile_mode == "runtime") {
-    fastdeploy::vision::DetectionResult res;
-    if (!model.Predict(im, &res)) {
-      std::cerr << "Failed to predict." << std::endl;
-      return false;
-    }
-    double profile_time = model.GetProfileTime() * 1000;
-    std::cout << "Runtime(ms): " << profile_time << "ms." << std::endl;
-    auto vis_im = fastdeploy::vision::VisDetection(im, res);
-    cv::imwrite("vis_result.jpg", vis_im);
-    std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  } else {
-    // For End2End
-    // Step1: warm up for warmup times
-    std::cout << "Warmup " << warmup << " times..." << std::endl;
-    for (int i = 0; i < warmup; i++) {
-      fastdeploy::vision::DetectionResult res;
-      if (!model.Predict(im, &res)) {
-        std::cerr << "Failed to predict." << std::endl;
-        return false;
-      }
-    }
-    std::vector<float> end2end_statis;
-    // Step2: repeat for repeats times
-    std::cout << "Counting time..." << std::endl;
-    fastdeploy::TimeCounter tc;
-    fastdeploy::vision::DetectionResult res;
-    for (int i = 0; i < repeats; i++) {
-      if (FLAGS_collect_memory_info && i % dump_period == 0) {
-        fastdeploy::benchmark::DumpCurrentCpuMemoryUsage(cpu_mem_file_name);
-#if defined(WITH_GPU)
-        fastdeploy::benchmark::DumpCurrentGpuMemoryUsage(gpu_mem_file_name,
-                                                         FLAGS_device_id);
-#endif
-      }
-      tc.Start();
-      if (!model.Predict(im, &res)) {
-        std::cerr << "Failed to predict." << std::endl;
-        return false;
-      }
-      tc.End();
-      end2end_statis.push_back(tc.Duration() * 1000);
-    }
-    float end2end = std::accumulate(end2end_statis.end() - repeats,
-                                    end2end_statis.end(), 0.f) /
-                    repeats;
-    std::cout << "End2End(ms): " << end2end << "ms." << std::endl;
-    auto vis_im = fastdeploy::vision::VisDetection(im, res);
-    cv::imwrite("vis_result.jpg", vis_im);
-    std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  }
-
-  return true;
-}
-
-int main(int argc, char* argv[]) {
-  google::ParseCommandLineFlags(&argc, &argv, true);
-  int repeats = FLAGS_repeat;
-  int warmup = FLAGS_warmup;
-  int dump_period = FLAGS_dump_period;
-  std::string cpu_mem_file_name = "result_cpu.txt";
-  std::string gpu_mem_file_name = "result_gpu.txt";
-  // Run model
-  if (RunModel(FLAGS_model, FLAGS_image, warmup, repeats, dump_period,
-               cpu_mem_file_name, gpu_mem_file_name) != true) {
-    exit(1);
-  }
-  if (FLAGS_collect_memory_info) {
-    float cpu_mem = fastdeploy::benchmark::GetCpuMemoryUsage(cpu_mem_file_name);
-    std::cout << "cpu_pss_mb: " << cpu_mem << "MB." << std::endl;
-#if defined(WITH_GPU)
-    float gpu_mem = fastdeploy::benchmark::GetGpuMemoryUsage(gpu_mem_file_name);
-    std::cout << "gpu_pss_mb: " << gpu_mem << "MB." << std::endl;
-#endif
-  }
   return 0;
 }

benchmark/cpp/benchmark_yolov5.cc

@@ -12,96 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "fastdeploy/benchmark/utils.h"
-#include "fastdeploy/vision.h"
 #include "flags.h"
+#include "macros.h"
+#include "option.h"
 
-bool RunModel(std::string model_file, std::string image_file, size_t warmup,
+int main(int argc, char* argv[]) {
-              size_t repeats, size_t sampling_interval) {
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  auto im = cv::imread(FLAGS_image);
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option)) {
     PrintUsage();
     return false;
   }
-  if (FLAGS_profile_mode == "runtime") {
+  auto model_yolov5 =
-    option.EnableProfiling(FLAGS_include_h2d_d2h, repeats, warmup);
+      fastdeploy::vision::detection::YOLOv5(FLAGS_model, "", option);
-  }
+  fastdeploy::vision::DetectionResult res;
-  auto model = fastdeploy::vision::detection::YOLOv5(model_file, "", option);
+  BENCHMARK_MODEL(model_yolov5, model_yolov5.Predict(im, &res))
-  if (!model.Initialized()) {
+  auto vis_im = fastdeploy::vision::VisDetection(im, res);
-    std::cerr << "Failed to initialize." << std::endl;
+  cv::imwrite("vis_result.jpg", vis_im);
-    return false;
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  }
-  auto im = cv::imread(image_file);
-  // For collect memory info
-  fastdeploy::benchmark::ResourceUsageMonitor resource_moniter(
-      sampling_interval, FLAGS_device_id);
-  if (FLAGS_collect_memory_info) {
-    resource_moniter.Start();
-  }
-  // For Runtime
-  if (FLAGS_profile_mode == "runtime") {
-    fastdeploy::vision::DetectionResult res;
-    if (!model.Predict(im, &res)) {
-      std::cerr << "Failed to predict." << std::endl;
-      return false;
-    }
-    double profile_time = model.GetProfileTime() * 1000;
-    std::cout << "Runtime(ms): " << profile_time << "ms." << std::endl;
-    auto vis_im = fastdeploy::vision::VisDetection(im, res);
-    cv::imwrite("vis_result.jpg", vis_im);
-    std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  } else {
-    // For End2End
-    // Step1: warm up for warmup times
-    std::cout << "Warmup " << warmup << " times..." << std::endl;
-    for (int i = 0; i < warmup; i++) {
-      fastdeploy::vision::DetectionResult res;
-      if (!model.Predict(im, &res)) {
-        std::cerr << "Failed to predict." << std::endl;
-        return false;
-      }
-    }
-    // Step2: repeat for repeats times
-    std::cout << "Counting time..." << std::endl;
-    std::cout << "Repeat " << repeats << " times..." << std::endl;
-    fastdeploy::vision::DetectionResult res;
-    fastdeploy::TimeCounter tc;
-    tc.Start();
-    for (int i = 0; i < repeats; i++) {
-      if (!model.Predict(im, &res)) {
-        std::cerr << "Failed to predict." << std::endl;
-        return false;
-      }
-    }
-    tc.End();
-    double end2end = tc.Duration() / repeats * 1000;
-    std::cout << "End2End(ms): " << end2end << "ms." << std::endl;
-    auto vis_im = fastdeploy::vision::VisDetection(im, res);
-    cv::imwrite("vis_result.jpg", vis_im);
-    std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
-  }
-  if (FLAGS_collect_memory_info) {
-    float cpu_mem = resource_moniter.GetMaxCpuMem();
-    float gpu_mem = resource_moniter.GetMaxGpuMem();
-    float gpu_util = resource_moniter.GetMaxGpuUtil();
-    std::cout << "cpu_pss_mb: " << cpu_mem << "MB." << std::endl;
-    std::cout << "gpu_pss_mb: " << gpu_mem << "MB." << std::endl;
-    std::cout << "gpu_util: " << gpu_util << std::endl;
-    resource_moniter.Stop();
-  }
-
-  return true;
-}
-
-int main(int argc, char* argv[]) {
-  google::ParseCommandLineFlags(&argc, &argv, true);
-  int repeats = FLAGS_repeat;
-  int warmup = FLAGS_warmup;
-  int sampling_interval = FLAGS_sampling_interval;
-  // Run model
-  if (!RunModel(FLAGS_model, FLAGS_image, warmup, repeats, sampling_interval)) {
-    exit(1);
-  }
   return 0;
 }

benchmark/cpp/flags.h

@@ -15,7 +15,6 @@
 #pragma once
 
 #include "gflags/gflags.h"
-#include "fastdeploy/utils/perf.h"
 
 DEFINE_string(model, "", "Directory of the inference model.");
 DEFINE_string(image, "", "Path of the image file.");
@@ -49,75 +48,3 @@ void PrintUsage() {
   std::cout << "Default value of backend: default" << std::endl;
   std::cout << "Default value of use_fp16: false" << std::endl;
 }
-
-bool CreateRuntimeOption(fastdeploy::RuntimeOption* option) {
-  if (FLAGS_device == "gpu") {
-    option->UseGpu(FLAGS_device_id);
-    if (FLAGS_backend == "ort") {
-      option->UseOrtBackend();
-    } else if (FLAGS_backend == "paddle") {
-      option->UsePaddleInferBackend();
-    } else if (FLAGS_backend == "trt" || FLAGS_backend == "paddle_trt") {
-      option->UseTrtBackend();
-      if (FLAGS_backend == "paddle_trt") {
-        option->EnablePaddleToTrt();
-      }
-      if (FLAGS_use_fp16) {
-        option->EnableTrtFP16();
-      }
-    } else if (FLAGS_backend == "default") {
-      return true;
-    } else {
-      std::cout << "While inference with GPU, only support "
-                   "default/ort/paddle/trt/paddle_trt now, "
-                << FLAGS_backend << " is not supported." << std::endl;
-      return false;
-    }
-  } else if (FLAGS_device == "cpu") {
-    option->SetCpuThreadNum(FLAGS_cpu_thread_nums);
-    if (FLAGS_backend == "ort") {
-      option->UseOrtBackend();
-    } else if (FLAGS_backend == "ov") {
-      option->UseOpenVINOBackend();
-    } else if (FLAGS_backend == "paddle") {
-      option->UsePaddleInferBackend();
-    } else if (FLAGS_backend == "lite") {
-      option->UsePaddleLiteBackend();
-      if (FLAGS_use_fp16) {
-        option->EnableLiteFP16();
-      }
-    } else if (FLAGS_backend == "default") {
-      return true;
-    } else {
-      std::cout << "While inference with CPU, only support "
-                   "default/ort/ov/paddle/lite now, "
-                << FLAGS_backend << " is not supported." << std::endl;
-      return false;
-    }
-  } else if (FLAGS_device == "xpu") {
-    option->UseKunlunXin(FLAGS_device_id);
-    if (FLAGS_backend == "ort") {
-      option->UseOrtBackend();
-    } else if (FLAGS_backend == "paddle") {
-      option->UsePaddleInferBackend();
-    } else if (FLAGS_backend == "lite") {
-      option->UsePaddleLiteBackend();
-      if (FLAGS_use_fp16) {
-        option->EnableLiteFP16();
-      }
-    } else if (FLAGS_backend == "default") {
-      return true;
-    } else {
-      std::cout << "While inference with XPU, only support "
-                   "default/ort/paddle/lite now, "
-                << FLAGS_backend << " is not supported." << std::endl;
-      return false;
-    }
-  } else {
-    std::cerr << "Only support device CPU/GPU/XPU now, " << FLAGS_device
-              << " is not supported." << std::endl;
-    return false;
-  }
-
-  return true;
-}

benchmark/cpp/macros.h

@@ -0,0 +1,70 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "fastdeploy/benchmark/utils.h"
+#include "fastdeploy/utils/perf.h"
+
+#define BENCHMARK_MODEL(MODEL_NAME, BENCHMARK_FUNC)                         \
+{                                                                           \
+  std::cout << "====" << #MODEL_NAME << "====" << std::endl;                \
+  if (!MODEL_NAME.Initialized()) {                                          \
+    std::cerr << "Failed to initialize." << std::endl;                      \
+    return 0;                                                               \
+  }                                                                         \
+  auto __im__ = cv::imread(FLAGS_image);                                    \
+  fastdeploy::benchmark::ResourceUsageMonitor __resource_moniter__(         \
+      FLAGS_sampling_interval, FLAGS_device_id);                            \
+  if (FLAGS_collect_memory_info) {                                          \
+    __resource_moniter__.Start();                                           \
+  }                                                                         \
+  if (FLAGS_profile_mode == "runtime") {                                    \
+    if (!BENCHMARK_FUNC) {                                                  \
+      std::cerr << "Failed to predict." << std::endl;                       \
+      return 0;                                                             \
+    }                                                                       \
+    double __profile_time__ = MODEL_NAME.GetProfileTime() * 1000;           \
+    std::cout << "Runtime(ms): " << __profile_time__ << "ms." << std::endl; \
+  } else {                                                                  \
+    std::cout << "Warmup " << FLAGS_warmup << " times..." << std::endl;     \
+    for (int __i__ = 0; __i__ < FLAGS_warmup; __i__++) {                    \
+      if (!BENCHMARK_FUNC) {                                                \
+        std::cerr << "Failed to predict." << std::endl;                     \
+        return 0;                                                           \
+      }                                                                     \
+    }                                                                       \
+    std::cout << "Counting time..." << std::endl;                           \
+    std::cout << "Repeat " << FLAGS_repeat << " times..." << std::endl;     \
+    fastdeploy::TimeCounter __tc__;                                         \
+    __tc__.Start();                                                         \
+    for (int __i__ = 0; __i__ < FLAGS_repeat; __i__++) {                    \
+      if (!BENCHMARK_FUNC) {                                                \
+        std::cerr << "Failed to predict." << std::endl;                     \
+        return 0;                                                           \
+      }                                                                     \
+    }                                                                       \
+    __tc__.End();                                                           \
+    double __end2end__ = __tc__.Duration() / FLAGS_repeat * 1000;           \
+    std::cout << "End2End(ms): " << __end2end__ << "ms." << std::endl;      \
+  }                                                                         \
+  if (FLAGS_collect_memory_info) {                                          \
+    float __cpu_mem__ = __resource_moniter__.GetMaxCpuMem();                \
+    float __gpu_mem__ = __resource_moniter__.GetMaxGpuMem();                \
+    float __gpu_util__ = __resource_moniter__.GetMaxGpuUtil();              \
+    std::cout << "cpu_pss_mb: " << __cpu_mem__ << "MB." << std::endl;       \
+    std::cout << "gpu_pss_mb: " << __gpu_mem__ << "MB." << std::endl;       \
+    std::cout << "gpu_util: " << __gpu_util__ << std::endl;                 \
+    __resource_moniter__.Stop();                                            \
+  }                                                                         \
+}

benchmark/cpp/option.h

@@ -0,0 +1,92 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision.h"
+
+static bool CreateRuntimeOption(fastdeploy::RuntimeOption* option) {
+  if (FLAGS_profile_mode == "runtime") {
+    option->EnableProfiling(FLAGS_include_h2d_d2h, FLAGS_repeat, FLAGS_warmup);
+  }
+  if (FLAGS_device == "gpu") {
+    option->UseGpu(FLAGS_device_id);
+    if (FLAGS_backend == "ort") {
+      option->UseOrtBackend();
+    } else if (FLAGS_backend == "paddle") {
+      option->UsePaddleInferBackend();
+    } else if (FLAGS_backend == "trt" || FLAGS_backend == "paddle_trt") {
+      option->UseTrtBackend();
+      if (FLAGS_backend == "paddle_trt") {
+        option->EnablePaddleToTrt();
+      }
+      if (FLAGS_use_fp16) {
+        option->EnableTrtFP16();
+      }
+    } else if (FLAGS_backend == "default") {
+      return true;
+    } else {
+      std::cout << "While inference with GPU, only support "
+                   "default/ort/paddle/trt/paddle_trt now, "
+                << FLAGS_backend << " is not supported." << std::endl;
+      return false;
+    }
+  } else if (FLAGS_device == "cpu") {
+    option->SetCpuThreadNum(FLAGS_cpu_thread_nums);
+    if (FLAGS_backend == "ort") {
+      option->UseOrtBackend();
+    } else if (FLAGS_backend == "ov") {
+      option->UseOpenVINOBackend();
+    } else if (FLAGS_backend == "paddle") {
+      option->UsePaddleInferBackend();
+    } else if (FLAGS_backend == "lite") {
+      option->UsePaddleLiteBackend();
+      if (FLAGS_use_fp16) {
+        option->EnableLiteFP16();
+      }
+    } else if (FLAGS_backend == "default") {
+      return true;
+    } else {
+      std::cout << "While inference with CPU, only support "
+                   "default/ort/ov/paddle/lite now, "
+                << FLAGS_backend << " is not supported." << std::endl;
+      return false;
+    }
+  } else if (FLAGS_device == "xpu") {
+    option->UseKunlunXin(FLAGS_device_id);
+    if (FLAGS_backend == "ort") {
+      option->UseOrtBackend();
+    } else if (FLAGS_backend == "paddle") {
+      option->UsePaddleInferBackend();
+    } else if (FLAGS_backend == "lite") {
+      option->UsePaddleLiteBackend();
+      if (FLAGS_use_fp16) {
+        option->EnableLiteFP16();
+      }
+    } else if (FLAGS_backend == "default") {
+      return true;
+    } else {
+      std::cout << "While inference with XPU, only support "
+                   "default/ort/paddle/lite now, "
+                << FLAGS_backend << " is not supported." << std::endl;
+      return false;
+    }
+  } else {
+    std::cerr << "Only support device CPU/GPU/XPU now, " << FLAGS_device
+              << " is not supported." << std::endl;
+    return false;
+  }
+
+  return true;
+}

fastdeploy/runtime/option_pybind.cc

@@ -35,6 +35,7 @@ void BindOption(pybind11::module& m) {
       .def(pybind11::init())
       .def("set_model_path", &RuntimeOption::SetModelPath)
       .def("set_model_buffer", &RuntimeOption::SetModelBuffer)
+      .def("set_encryption_key", &RuntimeOption::SetEncryptionKey)
       .def("use_gpu", &RuntimeOption::UseGpu)
       .def("use_cpu", &RuntimeOption::UseCpu)
       .def("use_rknpu2", &RuntimeOption::UseRKNPU2)

fastdeploy/runtime/runtime.cc

@@ -104,7 +104,33 @@ bool AutoSelectBackend(RuntimeOption& option) {
 
 bool Runtime::Init(const RuntimeOption& _option) {
   option = _option;
-
+  // decrypt encrypted model
+  if ("" != option.encryption_key_) {
+#ifdef ENABLE_ENCRYPTION
+    if (option.model_from_memory_) {
+      option.model_file = Decrypt(option.model_file, option.encryption_key_);
+      if (!(option.params_file.empty())) {
+        option.params_file =
+            Decrypt(option.params_file, option.encryption_key_);
+      }
+    } else {
+      std::string model_buffer = "";
+      FDASSERT(ReadBinaryFromFile(option.model_file, &model_buffer),
+               "Fail to read binary from model file");
+      option.model_file = Decrypt(model_buffer, option.encryption_key_);
+      if (!(option.params_file.empty())) {
+        std::string params_buffer = "";
+        FDASSERT(ReadBinaryFromFile(option.params_file, &params_buffer),
+                 "Fail to read binary from parameter file");
+        option.params_file = Decrypt(params_buffer, option.encryption_key_);
+      }
+      option.model_from_memory_ = true;
+    }
+#else
+    FDERROR << "The FastDeploy didn't compile with encryption function."
+            << std::endl;
+#endif
+  }
   // Choose default backend by model format and device if backend is not
   // specified
   if (option.backend == Backend::UNKNOWN) {

fastdeploy/runtime/runtime.h

@@ -23,6 +23,9 @@
 #include "fastdeploy/core/fd_tensor.h"
 #include "fastdeploy/runtime/runtime_option.h"
 #include "fastdeploy/utils/perf.h"
+#ifdef ENABLE_ENCRYPTION
+#include "fastdeploy/encryption/include/decrypt.h"
+#endif
 
 /** \brief All C++ FastDeploy APIs are defined inside this namespace
 *

fastdeploy/runtime/runtime_option.cc

@@ -36,6 +36,15 @@ void RuntimeOption::SetModelBuffer(const std::string& model_buffer,
   model_from_memory_ = true;
 }
 
+void RuntimeOption::SetEncryptionKey(const std::string& encryption_key) {
+#ifdef ENABLE_ENCRYPTION
+  encryption_key_ = encryption_key;
+#else
+  FDERROR << "The FastDeploy didn't compile with encryption function."
+          << std::endl;
+#endif
+}
+
 void RuntimeOption::UseGpu(int gpu_id) {
 #ifdef WITH_GPU
   device = Device::GPU;

fastdeploy/runtime/runtime_option.h

@@ -59,6 +59,12 @@ struct FASTDEPLOY_DECL RuntimeOption {
                       const std::string& params_buffer = "",
                       const ModelFormat& format = ModelFormat::PADDLE);
 
+  /** \brief When loading encrypted model, encryption_key is required to decrypte model
+   *
+   * \param[in] encryption_key The key for decrypting model
+   */
+  void SetEncryptionKey(const std::string& encryption_key);
+
   /// Use cpu to inference, the runtime will inference on CPU by default
   void UseCpu();
   /// Use Nvidia GPU to inference
@@ -179,6 +185,8 @@ struct FASTDEPLOY_DECL RuntimeOption {
   /// format of input model
   ModelFormat model_format = ModelFormat::PADDLE;
 
+  std::string encryption_key_ = "";
+
   // for cpu inference
   // default will let the backend choose their own default value
   int cpu_thread_num = -1;

python/fastdeploy/runtime.py

@@ -195,6 +195,12 @@ class RuntimeOption:
         return self._option.set_model_buffer(model_buffer, params_buffer,
                                              model_format)
 
+    def set_encryption_key(self, encryption_key):
+        """When loading encrypted model, encryption_key is required to decrypte model
+        :param encryption_key: (str)The key for decrypting model
+        """
+        return self._option.set_encryption_key(encryption_key)
+
     def use_gpu(self, device_id=0):
         """Inference with Nvidia GPU
 

tutorials/encrypt_model/README.md

@@ -0,0 +1,46 @@
+English | [中文](README_CN.md)
+
+# FastDeploy generates an encrypted model
+
+This directory provides `encrypt.py` to quickly complete the encryption of the model and parameter files of ResNet50_vd
+
+## encryption
+```bash
+# Download deployment example code
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd  FastDeploy/tutorials/encrypt_model
+
+# Download the ResNet50_vd model file
+wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz
+tar -xvf ResNet50_vd_infer.tgz
+
+python encrypt.py --model_file ResNet50_vd_infer/inference.pdmodel  --params_file ResNet50_vd_infer/inference.pdiparams --encrypted_model_dir ResNet50_vd_infer_encrypt
+```
+>> **Note** After the encryption is completed, the ResNet50_vd_infer_encrypt folder will be generated, including `__model__.encrypted`, `__params__.encrypted`, `encryption_key.txt` three files, where `encryption_key.txt` contains the encrypted key. At the same time, you need to copy the `inference_cls.yaml` configuration file in the original folder to the ResNet50_vd_infer_encrypt folder for subsequent deployment
+
+### Python encryption interface
+
+Use the encrypted interface through the following interface settings
+```python
+import fastdeploy as fd
+import os
+# when key is not given, key will be automatically generated.
+# otherwise, the file will be encrypted by specific key
+encrypted_model, key = fd.encryption.encrypt(model_file.read())
+encrypted_params, key= fd.encryption.encrypt(params_file.read(), key)
+```
+
+### FastDeploy deployment encryption model (decryption)
+
+Through the setting of the following interface, FastDeploy can deploy the encryption model
+```python
+import fastdeploy as fd
+option = fd.RuntimeOption()
+option.set_encryption_key(key)
+```
+
+```C++
+fastdeploy::RuntimeOption option;
+option.SetEncryptionKey(key)
+```
+>> **Note** For more details about RuntimeOption, please refer to [RuntimeOption Python Documentation](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/python/html/runtime_option.html), [ RuntimeOption C++ Documentation](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/cpp/html/structfastdeploy_1_1RuntimeOption.html)

tutorials/encrypt_model/README_CN.md

@@ -0,0 +1,48 @@
+[English](README.md) | 中文
+
+# 使用FastDeploy生成加密模型
+
+本目录下提供`encrypt.py`快速完成ResNet50_vd的模型和参数文件加密
+
+FastDeploy支持对称加密的方案，通过调用OpenSSL中的对称加密算法（AES）对模型进行加密并产生密钥
+
+## 加密
+```bash
+#下载加密示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd  FastDeploy/tutorials/encrypt_model
+
+# 下载ResNet50_vd模型文件
+wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz
+tar -xvf ResNet50_vd_infer.tgz
+
+python encrypt.py --model_file ResNet50_vd_infer/inference.pdmodel  --params_file ResNet50_vd_infer/inference.pdiparams --encrypted_model_dir ResNet50_vd_infer_encrypt
+```
+>> **注意** 加密完成后会生成ResNet50_vd_infer_encrypt文件夹，包含`__model__.encrypted`,`__params__.encrypted`,`encryption_key.txt`三个文件，其中`encryption_key.txt`包含加密后的秘钥，同时需要将原文件夹中的、`inference_cls.yaml`配置文件 拷贝至ResNet50_vd_infer_encrypt文件夹，以便后续部署使用
+
+### Python加密接口
+
+通过如下接口的设定，使用加密接口（解密）
+```python
+import fastdeploy as fd
+import os
+# when key is not given, key will be automatically generated.
+# otherwise, the file will be encrypted by specific key
+encrypted_model, key = fd.encryption.encrypt(model_file.read())
+encrypted_params, key= fd.encryption.encrypt(params_file.read(), key)
+```
+
+### FastDeploy 部署加密模型
+
+通过如下接口的设定，完成加密模型的推理
+```python
+import fastdeploy as fd
+option = fd.RuntimeOption()
+option.set_encryption_key(key)
+```
+
+```C++
+fastdeploy::RuntimeOption option;
+option.SetEncryptionKey(key)
+```
+>> **注意** RuntimeOption的更多详细信息，请参考[RuntimeOption Python文档](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/python/html/runtime_option.html)，[RuntimeOption C++文档](https://www.paddlepaddle.org.cn/fastdeploy-api-doc/cpp/html/structfastdeploy_1_1RuntimeOption.html)

tutorials/encrypt_model/encrypt.py

@@ -0,0 +1,47 @@
+import fastdeploy as fd
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--encrypted_model_dir",
+        required=False,
+        help="Path of model directory.")
+    parser.add_argument(
+        "--model_file", required=True, help="Path of model file directory.")
+    parser.add_argument(
+        "--params_file",
+        required=True,
+        help="Path of parameters file directory.")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    model_buffer = open(args.model_file, 'rb')
+    params_buffer = open(args.params_file, 'rb')
+    encrypted_model, key = fd.encryption.encrypt(model_buffer.read())
+    # use the same key to encrypt parameter file
+    encrypted_params, key = fd.encryption.encrypt(params_buffer.read(), key)
+    encrypted_model_dir = "encrypt_model_dir"
+    if args.encrypted_model_dir:
+        encrypted_model_dir = args.encrypted_model_dir
+    model_buffer.close()
+    params_buffer.close()
+    os.mkdir(encrypted_model_dir)
+    with open(os.path.join(encrypted_model_dir, "__model__.encrypted"),
+              "w") as f:
+        f.write(encrypted_model)
+
+    with open(os.path.join(encrypted_model_dir, "__params__.encrypted"),
+              "w") as f:
+        f.write(encrypted_params)
+
+    with open(os.path.join(encrypted_model_dir, "encryption_key.txt"),
+              "w") as f:
+        f.write(key)
+    print("encryption key: ", key)
+    print("encryption success")