apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 09:07:10 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Backend] optimize cpu usage on openvino backend (#1531)

Browse Source 
* optimiza cpu usage on openvino backend

conver the hint varible to string

* update with affinity mode set-up

---------

Co-authored-by: Ethan <yicheng@intel.com>
Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
Ethan Yang
2023-03-17 10:24:25 +08:00
committed by GitHub
parent 6343b0db47
commit 4f48d84c27
4 changed files with 103 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
fastdeploy/runtime/backends/openvino/ov_backend.cc
View File

@@ -203,10 +203,10 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
}
ov::AnyMap properties;
if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
}
if (option_.device == "CPU") {
if (option_.hint == "UNDEFINED") {
if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
}
if (option_.num_streams == -1) {
properties["NUM_STREAMS"] = ov::streams::AUTO;
} else if (option_.num_streams == -2) {
@@ -214,17 +214,32 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
} else if (option_.num_streams > 0) {
properties["NUM_STREAMS"] = option_.num_streams;
}
} else {
if (option_.num_streams != 0) {
FDWARNING << "NUM_STREAMS only available on device CPU, currently the "
"device is set as "
<< option_.device << ", the NUM_STREAMS will be ignored."
<< std::endl;
FDINFO << "number of streams:" << option_.num_streams << "." << std::endl;
if (option_.affinity == "YES") {
properties["AFFINITY"] = "CORE";
} else if (option_.affinity == "NO") {
properties["AFFINITY"] = "NONE";
} else if (option_.affinity == "NUMA") {
properties["AFFINITY"] = "NUMA";
} else if (option_.affinity == "HYBRID_AWARE") {
properties["AFFINITY"] = "HYBRID_AWARE";
}
FDINFO << "affinity:" << option_.affinity << "." << std::endl;
} else if (option_.hint == "LATENCY") {
properties.emplace(
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
} else if (option_.hint == "THROUGHPUT") {
properties.emplace(
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
} else if (option_.hint == "CUMULATIVE_THROUGHPUT") {
properties.emplace(ov::hint::performance_mode(
ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
}
FDINFO << "Compile OpenVINO model on device_name:" << option.device << "."
<< std::endl;
compiled_model_ = core_.compile_model(model, option.device, properties);
request_ = compiled_model_.create_infer_request();
@@ -332,10 +347,10 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
}
ov::AnyMap properties;
if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
}
if (option_.device == "CPU") {
if (option_.hint == "UNDEFINED") {
if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
}
if (option_.num_streams == -1) {
properties["NUM_STREAMS"] = ov::streams::AUTO;
} else if (option_.num_streams == -2) {
@@ -343,13 +358,27 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
} else if (option_.num_streams > 0) {
properties["NUM_STREAMS"] = option_.num_streams;
}
} else {
if (option_.num_streams != 0) {
FDWARNING << "NUM_STREAMS only available on device CPU, currently the "
"device is set as "
<< option_.device << ", the NUM_STREAMS will be ignored."
<< std::endl;
FDINFO << "number of streams:" << option_.num_streams << "." << std::endl;
if (option_.affinity == "YES") {
properties["AFFINITY"] = "CORE";
} else if (option_.affinity == "NO") {
properties["AFFINITY"] = "NONE";
} else if (option_.affinity == "NUMA") {
properties["AFFINITY"] = "NUMA";
} else if (option_.affinity == "HYBRID_AWARE") {
properties["AFFINITY"] = "HYBRID_AWARE";
}
FDINFO << "affinity:" << option_.affinity << "." << std::endl;
} else if (option_.hint == "LATENCY") {
properties.emplace(
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
} else if (option_.hint == "THROUGHPUT") {
properties.emplace(
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
} else if (option_.hint == "CUMULATIVE_THROUGHPUT") {
properties.emplace(ov::hint::performance_mode(
ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
}
FDINFO << "Compile OpenVINO model on device_name:" << option.device << "."
@@ -384,7 +413,8 @@ bool OpenVINOBackend::Infer(std::vector<FDTensor>& inputs,
}
RUNTIME_PROFILE_LOOP_BEGIN(1)
request_.infer();
request_.start_async();
request_.wait();
RUNTIME_PROFILE_LOOP_END
outputs->resize(output_infos_.size());