add 'GetOutputInfos' and 'GetInputInfos' interface (#232)

add GetOutputInfos GetInputInfos
This commit is contained in:
heliqi
2022-09-15 00:09:31 -05:00
committed by GitHub
parent 6963296a89
commit 0805ead0ed
13 changed files with 125 additions and 42 deletions

View File

@@ -14,13 +14,14 @@
#pragma once #pragma once
#include "fastdeploy/backends/common/multiclass_nms.h"
#include "fastdeploy/core/fd_tensor.h"
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <string> #include <string>
#include <vector> #include <vector>
#include "fastdeploy/backends/common/multiclass_nms.h"
#include "fastdeploy/core/fd_tensor.h"
namespace fastdeploy { namespace fastdeploy {
struct TensorInfo { struct TensorInfo {
@@ -56,6 +57,8 @@ class BaseBackend {
virtual int NumOutputs() const = 0; virtual int NumOutputs() const = 0;
virtual TensorInfo GetInputInfo(int index) = 0; virtual TensorInfo GetInputInfo(int index) = 0;
virtual TensorInfo GetOutputInfo(int index) = 0; virtual TensorInfo GetOutputInfo(int index) = 0;
virtual std::vector<TensorInfo> GetInputInfos() = 0;
virtual std::vector<TensorInfo> GetOutputInfos() = 0;
virtual bool Infer(std::vector<FDTensor>& inputs, virtual bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs) = 0; std::vector<FDTensor>* outputs) = 0;
}; };

View File

@@ -107,19 +107,25 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
// So here will reorder it's inputs and outputs // So here will reorder it's inputs and outputs
std::string model_content; std::string model_content;
ReadBinaryFromFile(model_file, &model_content); ReadBinaryFromFile(model_file, &model_content);
auto reader = paddle2onnx::PaddleReader(model_content.c_str(), model_content.size()); auto reader =
paddle2onnx::PaddleReader(model_content.c_str(), model_content.size());
if (reader.num_inputs != input_infos.size()) { if (reader.num_inputs != input_infos.size()) {
FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl; FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs
<< " not equal to the number of inputs from OpenVINO:"
<< input_infos.size() << "." << std::endl;
return false; return false;
} }
if (reader.num_outputs != output_infos.size()) { if (reader.num_outputs != output_infos.size()) {
FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl; FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs
<< " not equal to the number of outputs from OpenVINO:"
<< output_infos.size() << "." << std::endl;
return false; return false;
} }
for (int i = 0; i < reader.num_inputs; ++i) { for (int i = 0; i < reader.num_inputs; ++i) {
auto iter = input_infos.find(std::string(reader.inputs[i].name)); auto iter = input_infos.find(std::string(reader.inputs[i].name));
if (iter == input_infos.end()) { if (iter == input_infos.end()) {
FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl; FDERROR << "Cannot find input name:" << reader.inputs[i].name
<< " from OpenVINO model." << std::endl;
return false; return false;
} }
input_infos_.push_back(iter->second); input_infos_.push_back(iter->second);
@@ -127,7 +133,8 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
for (int i = 0; i < reader.num_outputs; ++i) { for (int i = 0; i < reader.num_outputs; ++i) {
auto iter = output_infos.find(std::string(reader.outputs[i].name)); auto iter = output_infos.find(std::string(reader.outputs[i].name));
if (iter == output_infos.end()) { if (iter == output_infos.end()) {
FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl; FDERROR << "Cannot find output name:" << reader.outputs[i].name
<< " from OpenVINO model." << std::endl;
return false; return false;
} }
output_infos_.push_back(iter->second); output_infos_.push_back(iter->second);
@@ -146,6 +153,14 @@ TensorInfo OpenVINOBackend::GetInputInfo(int index) {
return input_infos_[index]; return input_infos_[index];
} }
std::vector<TensorInfo> OpenVINOBackend::GetInputInfos() {
return input_infos_;
}
std::vector<TensorInfo> OpenVINOBackend::GetOutputInfos() {
return output_infos_;
}
TensorInfo OpenVINOBackend::GetOutputInfo(int index) { TensorInfo OpenVINOBackend::GetOutputInfo(int index) {
FDASSERT(index < NumOutputs(), FDASSERT(index < NumOutputs(),
"The index: %d should less than the number of outputs: %d.", index, "The index: %d should less than the number of outputs: %d.", index,
@@ -181,19 +196,25 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
// So here will reorder it's inputs and outputs // So here will reorder it's inputs and outputs
std::string model_content; std::string model_content;
ReadBinaryFromFile(model_file, &model_content); ReadBinaryFromFile(model_file, &model_content);
auto reader = paddle2onnx::OnnxReader(model_content.c_str(), model_content.size()); auto reader =
paddle2onnx::OnnxReader(model_content.c_str(), model_content.size());
if (reader.num_inputs != input_infos.size()) { if (reader.num_inputs != input_infos.size()) {
FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl; FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs
<< " not equal to the number of inputs from OpenVINO:"
<< input_infos.size() << "." << std::endl;
return false; return false;
} }
if (reader.num_outputs != output_infos.size()) { if (reader.num_outputs != output_infos.size()) {
FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl; FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs
<< " not equal to the number of outputs from OpenVINO:"
<< output_infos.size() << "." << std::endl;
return false; return false;
} }
for (int i = 0; i < reader.num_inputs; ++i) { for (int i = 0; i < reader.num_inputs; ++i) {
auto iter = input_infos.find(std::string(reader.inputs[i].name)); auto iter = input_infos.find(std::string(reader.inputs[i].name));
if (iter == input_infos.end()) { if (iter == input_infos.end()) {
FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl; FDERROR << "Cannot find input name:" << reader.inputs[i].name
<< " from OpenVINO model." << std::endl;
return false; return false;
} }
input_infos_.push_back(iter->second); input_infos_.push_back(iter->second);
@@ -201,7 +222,8 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
for (int i = 0; i < reader.num_outputs; ++i) { for (int i = 0; i < reader.num_outputs; ++i) {
auto iter = output_infos.find(std::string(reader.outputs[i].name)); auto iter = output_infos.find(std::string(reader.outputs[i].name));
if (iter == output_infos.end()) { if (iter == output_infos.end()) {
FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl; FDERROR << "Cannot find output name:" << reader.outputs[i].name
<< " from OpenVINO model." << std::endl;
return false; return false;
} }
output_infos_.push_back(iter->second); output_infos_.push_back(iter->second);

View File

@@ -51,6 +51,8 @@ class OpenVINOBackend : public BaseBackend {
TensorInfo GetInputInfo(int index) override; TensorInfo GetInputInfo(int index) override;
TensorInfo GetOutputInfo(int index) override; TensorInfo GetOutputInfo(int index) override;
std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfos() override;
private: private:
void InitTensorInfo(const std::vector<ov::Output<ov::Node>>& ov_outputs, void InitTensorInfo(const std::vector<ov::Output<ov::Node>>& ov_outputs,

View File

@@ -247,6 +247,16 @@ TensorInfo OrtBackend::GetInputInfo(int index) {
return info; return info;
} }
std::vector<TensorInfo> OrtBackend::GetInputInfos() {
auto size = inputs_desc_.size();
std::vector<TensorInfo> infos;
infos.reserve(size);
for (auto i = 0; i < size; i++) {
infos.emplace_back(GetInputInfo(i));
}
return infos;
}
TensorInfo OrtBackend::GetOutputInfo(int index) { TensorInfo OrtBackend::GetOutputInfo(int index) {
FDASSERT(index < NumOutputs(), FDASSERT(index < NumOutputs(),
"The index: %d should less than the number of outputs: %d.", index, "The index: %d should less than the number of outputs: %d.", index,
@@ -259,6 +269,14 @@ TensorInfo OrtBackend::GetOutputInfo(int index) {
return info; return info;
} }
std::vector<TensorInfo> OrtBackend::GetOutputInfos() {
std::vector<TensorInfo> infos;
for (auto i = 0; i < outputs_desc_.size(); i++) {
infos.emplace_back(GetOutputInfo(i));
}
return infos;
}
void OrtBackend::InitCustomOperators() { void OrtBackend::InitCustomOperators() {
#ifndef NON_64_PLATFORM #ifndef NON_64_PLATFORM
if (custom_operators_.size() == 0) { if (custom_operators_.size() == 0) {

View File

@@ -74,6 +74,8 @@ class OrtBackend : public BaseBackend {
TensorInfo GetInputInfo(int index); TensorInfo GetInputInfo(int index);
TensorInfo GetOutputInfo(int index); TensorInfo GetOutputInfo(int index);
std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfos() override;
static std::vector<OrtCustomOp*> custom_operators_; static std::vector<OrtCustomOp*> custom_operators_;
void InitCustomOperators(); void InitCustomOperators();

View File

@@ -85,7 +85,7 @@ TensorInfo PaddleBackend::GetInputInfo(int index) {
return inputs_desc_[index]; return inputs_desc_[index];
} }
std::vector<TensorInfo> PaddleBackend::GetInputInfo() { return inputs_desc_; } std::vector<TensorInfo> PaddleBackend::GetInputInfos() { return inputs_desc_; }
TensorInfo PaddleBackend::GetOutputInfo(int index) { TensorInfo PaddleBackend::GetOutputInfo(int index) {
FDASSERT(index < NumOutputs(), FDASSERT(index < NumOutputs(),
@@ -94,7 +94,9 @@ TensorInfo PaddleBackend::GetOutputInfo(int index) {
return outputs_desc_[index]; return outputs_desc_[index];
} }
std::vector<TensorInfo> PaddleBackend::GetOutputInfo() { return outputs_desc_; } std::vector<TensorInfo> PaddleBackend::GetOutputInfos() {
return outputs_desc_;
}
bool PaddleBackend::Infer(std::vector<FDTensor>& inputs, bool PaddleBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs) { std::vector<FDTensor>* outputs) {

View File

@@ -75,8 +75,8 @@ class PaddleBackend : public BaseBackend {
TensorInfo GetInputInfo(int index); TensorInfo GetInputInfo(int index);
TensorInfo GetOutputInfo(int index); TensorInfo GetOutputInfo(int index);
std::vector<TensorInfo> GetInputInfo(); std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfo(); std::vector<TensorInfo> GetOutputInfos() override;
private: private:
paddle_infer::Config config_; paddle_infer::Config config_;

View File

@@ -589,6 +589,14 @@ TensorInfo TrtBackend::GetInputInfo(int index) {
return info; return info;
} }
std::vector<TensorInfo> TrtBackend::GetInputInfos() {
std::vector<TensorInfo> infos;
for (auto i = 0; i < inputs_desc_.size(); i++) {
infos.emplace_back(GetInputInfo(i));
}
return infos;
}
TensorInfo TrtBackend::GetOutputInfo(int index) { TensorInfo TrtBackend::GetOutputInfo(int index) {
FDASSERT(index < NumOutputs(), FDASSERT(index < NumOutputs(),
"The index: %d should less than the number of outputs: %d.", index, "The index: %d should less than the number of outputs: %d.", index,
@@ -600,4 +608,13 @@ TensorInfo TrtBackend::GetOutputInfo(int index) {
info.dtype = GetFDDataType(outputs_desc_[index].dtype); info.dtype = GetFDDataType(outputs_desc_[index].dtype);
return info; return info;
} }
std::vector<TensorInfo> TrtBackend::GetOutputInfos() {
std::vector<TensorInfo> infos;
for (auto i = 0; i < outputs_desc_.size(); i++) {
infos.emplace_back(GetOutputInfo(i));
}
return infos;
}
} // namespace fastdeploy } // namespace fastdeploy

View File

@@ -72,6 +72,8 @@ class TrtBackend : public BaseBackend {
int NumOutputs() const { return outputs_desc_.size(); } int NumOutputs() const { return outputs_desc_.size(); }
TensorInfo GetInputInfo(int index); TensorInfo GetInputInfo(int index);
TensorInfo GetOutputInfo(int index); TensorInfo GetOutputInfo(int index);
std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfos() override;
~TrtBackend() { ~TrtBackend() {
if (parser_) { if (parser_) {

View File

@@ -188,5 +188,33 @@ void FDTensor::PrintInfo(const std::string& prefix) {
<< ", min=" << min << std::endl; << ", min=" << min << std::endl;
} }
bool FDTensor::AllocFn(size_t nbytes) {
if (device == Device::GPU) {
#ifdef WITH_GPU
return FDDeviceAllocator()(&buffer_, nbytes);
#else
FDASSERT(false,
"The FastDeploy FDTensor allocator didn't compile under "
"-DWITH_GPU=ON,"
"so this is an unexpected problem happend.");
#endif
}
return FDHostAllocator()(&buffer_, nbytes);
}
void FDTensor::FreeFn() {
if (external_data_ptr != nullptr) external_data_ptr = nullptr;
if (buffer_ != nullptr) {
if (device == Device::GPU) {
#ifdef WITH_GPU
FDDeviceFree()(buffer_);
#endif
} else {
FDHostFree()(buffer_);
}
buffer_ = nullptr;
}
}
FDTensor::FDTensor(const std::string& tensor_name) { name = tensor_name; } FDTensor::FDTensor(const std::string& tensor_name) { name = tensor_name; }
} // namespace fastdeploy } // namespace fastdeploy

View File

@@ -98,33 +98,9 @@ struct FASTDEPLOY_DECL FDTensor {
// prefix will also be printed as tag // prefix will also be printed as tag
void PrintInfo(const std::string& prefix = "TensorInfo: "); void PrintInfo(const std::string& prefix = "TensorInfo: ");
bool AllocFn(size_t nbytes) { bool AllocFn(size_t nbytes);
if (device == Device::GPU) {
#ifdef WITH_GPU
return FDDeviceAllocator()(&buffer_, nbytes);
#else
FDASSERT(false,
"The FastDeploy FDTensor allocator didn't compile under "
"-DWITH_GPU=ON,"
"so this is an unexpected problem happend.");
#endif
}
return FDHostAllocator()(&buffer_, nbytes);
}
void FreeFn() { void FreeFn();
if (external_data_ptr != nullptr) external_data_ptr = nullptr;
if (buffer_ != nullptr) {
if (device == Device::GPU) {
#ifdef WITH_GPU
FDDeviceFree()(buffer_);
#endif
} else {
FDHostFree()(buffer_);
}
buffer_ = nullptr;
}
}
FDTensor() {} FDTensor() {}
explicit FDTensor(const std::string& tensor_name); explicit FDTensor(const std::string& tensor_name);

View File

@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
#include "fastdeploy/fastdeploy_runtime.h" #include "fastdeploy/fastdeploy_runtime.h"
#include "fastdeploy/utils/unique_ptr.h" #include "fastdeploy/utils/unique_ptr.h"
#include "fastdeploy/utils/utils.h" #include "fastdeploy/utils/utils.h"
@@ -297,6 +298,14 @@ TensorInfo Runtime::GetOutputInfo(int index) {
return backend_->GetOutputInfo(index); return backend_->GetOutputInfo(index);
} }
std::vector<TensorInfo> Runtime::GetInputInfos() {
return backend_->GetInputInfos();
}
std::vector<TensorInfo> Runtime::GetOutputInfos() {
return backend_->GetOutputInfos();
}
bool Runtime::Infer(std::vector<FDTensor>& input_tensors, bool Runtime::Infer(std::vector<FDTensor>& input_tensors,
std::vector<FDTensor>* output_tensors) { std::vector<FDTensor>* output_tensors) {
return backend_->Infer(input_tensors, output_tensors); return backend_->Infer(input_tensors, output_tensors);

View File

@@ -165,6 +165,8 @@ struct FASTDEPLOY_DECL Runtime {
int NumOutputs() { return backend_->NumOutputs(); } int NumOutputs() { return backend_->NumOutputs(); }
TensorInfo GetInputInfo(int index); TensorInfo GetInputInfo(int index);
TensorInfo GetOutputInfo(int index); TensorInfo GetOutputInfo(int index);
std::vector<TensorInfo> GetInputInfos();
std::vector<TensorInfo> GetOutputInfos();
RuntimeOption option; RuntimeOption option;