mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-06 00:57:33 +08:00
add 'GetOutputInfos' and 'GetInputInfos' interface (#232)
add GetOutputInfos GetInputInfos
This commit is contained in:
@@ -14,13 +14,14 @@
|
|||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "fastdeploy/backends/common/multiclass_nms.h"
|
|
||||||
#include "fastdeploy/core/fd_tensor.h"
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "fastdeploy/backends/common/multiclass_nms.h"
|
||||||
|
#include "fastdeploy/core/fd_tensor.h"
|
||||||
|
|
||||||
namespace fastdeploy {
|
namespace fastdeploy {
|
||||||
|
|
||||||
struct TensorInfo {
|
struct TensorInfo {
|
||||||
@@ -56,6 +57,8 @@ class BaseBackend {
|
|||||||
virtual int NumOutputs() const = 0;
|
virtual int NumOutputs() const = 0;
|
||||||
virtual TensorInfo GetInputInfo(int index) = 0;
|
virtual TensorInfo GetInputInfo(int index) = 0;
|
||||||
virtual TensorInfo GetOutputInfo(int index) = 0;
|
virtual TensorInfo GetOutputInfo(int index) = 0;
|
||||||
|
virtual std::vector<TensorInfo> GetInputInfos() = 0;
|
||||||
|
virtual std::vector<TensorInfo> GetOutputInfos() = 0;
|
||||||
virtual bool Infer(std::vector<FDTensor>& inputs,
|
virtual bool Infer(std::vector<FDTensor>& inputs,
|
||||||
std::vector<FDTensor>* outputs) = 0;
|
std::vector<FDTensor>* outputs) = 0;
|
||||||
};
|
};
|
||||||
|
@@ -107,19 +107,25 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
|
|||||||
// So here will reorder it's inputs and outputs
|
// So here will reorder it's inputs and outputs
|
||||||
std::string model_content;
|
std::string model_content;
|
||||||
ReadBinaryFromFile(model_file, &model_content);
|
ReadBinaryFromFile(model_file, &model_content);
|
||||||
auto reader = paddle2onnx::PaddleReader(model_content.c_str(), model_content.size());
|
auto reader =
|
||||||
|
paddle2onnx::PaddleReader(model_content.c_str(), model_content.size());
|
||||||
if (reader.num_inputs != input_infos.size()) {
|
if (reader.num_inputs != input_infos.size()) {
|
||||||
FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl;
|
FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs
|
||||||
|
<< " not equal to the number of inputs from OpenVINO:"
|
||||||
|
<< input_infos.size() << "." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (reader.num_outputs != output_infos.size()) {
|
if (reader.num_outputs != output_infos.size()) {
|
||||||
FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl;
|
FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs
|
||||||
|
<< " not equal to the number of outputs from OpenVINO:"
|
||||||
|
<< output_infos.size() << "." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < reader.num_inputs; ++i) {
|
for (int i = 0; i < reader.num_inputs; ++i) {
|
||||||
auto iter = input_infos.find(std::string(reader.inputs[i].name));
|
auto iter = input_infos.find(std::string(reader.inputs[i].name));
|
||||||
if (iter == input_infos.end()) {
|
if (iter == input_infos.end()) {
|
||||||
FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl;
|
FDERROR << "Cannot find input name:" << reader.inputs[i].name
|
||||||
|
<< " from OpenVINO model." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
input_infos_.push_back(iter->second);
|
input_infos_.push_back(iter->second);
|
||||||
@@ -127,7 +133,8 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
|
|||||||
for (int i = 0; i < reader.num_outputs; ++i) {
|
for (int i = 0; i < reader.num_outputs; ++i) {
|
||||||
auto iter = output_infos.find(std::string(reader.outputs[i].name));
|
auto iter = output_infos.find(std::string(reader.outputs[i].name));
|
||||||
if (iter == output_infos.end()) {
|
if (iter == output_infos.end()) {
|
||||||
FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl;
|
FDERROR << "Cannot find output name:" << reader.outputs[i].name
|
||||||
|
<< " from OpenVINO model." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
output_infos_.push_back(iter->second);
|
output_infos_.push_back(iter->second);
|
||||||
@@ -146,6 +153,14 @@ TensorInfo OpenVINOBackend::GetInputInfo(int index) {
|
|||||||
return input_infos_[index];
|
return input_infos_[index];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> OpenVINOBackend::GetInputInfos() {
|
||||||
|
return input_infos_;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> OpenVINOBackend::GetOutputInfos() {
|
||||||
|
return output_infos_;
|
||||||
|
}
|
||||||
|
|
||||||
TensorInfo OpenVINOBackend::GetOutputInfo(int index) {
|
TensorInfo OpenVINOBackend::GetOutputInfo(int index) {
|
||||||
FDASSERT(index < NumOutputs(),
|
FDASSERT(index < NumOutputs(),
|
||||||
"The index: %d should less than the number of outputs: %d.", index,
|
"The index: %d should less than the number of outputs: %d.", index,
|
||||||
@@ -181,19 +196,25 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
|
|||||||
// So here will reorder it's inputs and outputs
|
// So here will reorder it's inputs and outputs
|
||||||
std::string model_content;
|
std::string model_content;
|
||||||
ReadBinaryFromFile(model_file, &model_content);
|
ReadBinaryFromFile(model_file, &model_content);
|
||||||
auto reader = paddle2onnx::OnnxReader(model_content.c_str(), model_content.size());
|
auto reader =
|
||||||
|
paddle2onnx::OnnxReader(model_content.c_str(), model_content.size());
|
||||||
if (reader.num_inputs != input_infos.size()) {
|
if (reader.num_inputs != input_infos.size()) {
|
||||||
FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl;
|
FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs
|
||||||
|
<< " not equal to the number of inputs from OpenVINO:"
|
||||||
|
<< input_infos.size() << "." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (reader.num_outputs != output_infos.size()) {
|
if (reader.num_outputs != output_infos.size()) {
|
||||||
FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl;
|
FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs
|
||||||
|
<< " not equal to the number of outputs from OpenVINO:"
|
||||||
|
<< output_infos.size() << "." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < reader.num_inputs; ++i) {
|
for (int i = 0; i < reader.num_inputs; ++i) {
|
||||||
auto iter = input_infos.find(std::string(reader.inputs[i].name));
|
auto iter = input_infos.find(std::string(reader.inputs[i].name));
|
||||||
if (iter == input_infos.end()) {
|
if (iter == input_infos.end()) {
|
||||||
FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl;
|
FDERROR << "Cannot find input name:" << reader.inputs[i].name
|
||||||
|
<< " from OpenVINO model." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
input_infos_.push_back(iter->second);
|
input_infos_.push_back(iter->second);
|
||||||
@@ -201,7 +222,8 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
|
|||||||
for (int i = 0; i < reader.num_outputs; ++i) {
|
for (int i = 0; i < reader.num_outputs; ++i) {
|
||||||
auto iter = output_infos.find(std::string(reader.outputs[i].name));
|
auto iter = output_infos.find(std::string(reader.outputs[i].name));
|
||||||
if (iter == output_infos.end()) {
|
if (iter == output_infos.end()) {
|
||||||
FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl;
|
FDERROR << "Cannot find output name:" << reader.outputs[i].name
|
||||||
|
<< " from OpenVINO model." << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
output_infos_.push_back(iter->second);
|
output_infos_.push_back(iter->second);
|
||||||
|
@@ -51,6 +51,8 @@ class OpenVINOBackend : public BaseBackend {
|
|||||||
|
|
||||||
TensorInfo GetInputInfo(int index) override;
|
TensorInfo GetInputInfo(int index) override;
|
||||||
TensorInfo GetOutputInfo(int index) override;
|
TensorInfo GetOutputInfo(int index) override;
|
||||||
|
std::vector<TensorInfo> GetInputInfos() override;
|
||||||
|
std::vector<TensorInfo> GetOutputInfos() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void InitTensorInfo(const std::vector<ov::Output<ov::Node>>& ov_outputs,
|
void InitTensorInfo(const std::vector<ov::Output<ov::Node>>& ov_outputs,
|
||||||
|
@@ -247,6 +247,16 @@ TensorInfo OrtBackend::GetInputInfo(int index) {
|
|||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> OrtBackend::GetInputInfos() {
|
||||||
|
auto size = inputs_desc_.size();
|
||||||
|
std::vector<TensorInfo> infos;
|
||||||
|
infos.reserve(size);
|
||||||
|
for (auto i = 0; i < size; i++) {
|
||||||
|
infos.emplace_back(GetInputInfo(i));
|
||||||
|
}
|
||||||
|
return infos;
|
||||||
|
}
|
||||||
|
|
||||||
TensorInfo OrtBackend::GetOutputInfo(int index) {
|
TensorInfo OrtBackend::GetOutputInfo(int index) {
|
||||||
FDASSERT(index < NumOutputs(),
|
FDASSERT(index < NumOutputs(),
|
||||||
"The index: %d should less than the number of outputs: %d.", index,
|
"The index: %d should less than the number of outputs: %d.", index,
|
||||||
@@ -259,6 +269,14 @@ TensorInfo OrtBackend::GetOutputInfo(int index) {
|
|||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> OrtBackend::GetOutputInfos() {
|
||||||
|
std::vector<TensorInfo> infos;
|
||||||
|
for (auto i = 0; i < outputs_desc_.size(); i++) {
|
||||||
|
infos.emplace_back(GetOutputInfo(i));
|
||||||
|
}
|
||||||
|
return infos;
|
||||||
|
}
|
||||||
|
|
||||||
void OrtBackend::InitCustomOperators() {
|
void OrtBackend::InitCustomOperators() {
|
||||||
#ifndef NON_64_PLATFORM
|
#ifndef NON_64_PLATFORM
|
||||||
if (custom_operators_.size() == 0) {
|
if (custom_operators_.size() == 0) {
|
||||||
|
@@ -74,6 +74,8 @@ class OrtBackend : public BaseBackend {
|
|||||||
|
|
||||||
TensorInfo GetInputInfo(int index);
|
TensorInfo GetInputInfo(int index);
|
||||||
TensorInfo GetOutputInfo(int index);
|
TensorInfo GetOutputInfo(int index);
|
||||||
|
std::vector<TensorInfo> GetInputInfos() override;
|
||||||
|
std::vector<TensorInfo> GetOutputInfos() override;
|
||||||
static std::vector<OrtCustomOp*> custom_operators_;
|
static std::vector<OrtCustomOp*> custom_operators_;
|
||||||
void InitCustomOperators();
|
void InitCustomOperators();
|
||||||
|
|
||||||
|
@@ -85,7 +85,7 @@ TensorInfo PaddleBackend::GetInputInfo(int index) {
|
|||||||
return inputs_desc_[index];
|
return inputs_desc_[index];
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<TensorInfo> PaddleBackend::GetInputInfo() { return inputs_desc_; }
|
std::vector<TensorInfo> PaddleBackend::GetInputInfos() { return inputs_desc_; }
|
||||||
|
|
||||||
TensorInfo PaddleBackend::GetOutputInfo(int index) {
|
TensorInfo PaddleBackend::GetOutputInfo(int index) {
|
||||||
FDASSERT(index < NumOutputs(),
|
FDASSERT(index < NumOutputs(),
|
||||||
@@ -94,7 +94,9 @@ TensorInfo PaddleBackend::GetOutputInfo(int index) {
|
|||||||
return outputs_desc_[index];
|
return outputs_desc_[index];
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<TensorInfo> PaddleBackend::GetOutputInfo() { return outputs_desc_; }
|
std::vector<TensorInfo> PaddleBackend::GetOutputInfos() {
|
||||||
|
return outputs_desc_;
|
||||||
|
}
|
||||||
|
|
||||||
bool PaddleBackend::Infer(std::vector<FDTensor>& inputs,
|
bool PaddleBackend::Infer(std::vector<FDTensor>& inputs,
|
||||||
std::vector<FDTensor>* outputs) {
|
std::vector<FDTensor>* outputs) {
|
||||||
|
@@ -75,8 +75,8 @@ class PaddleBackend : public BaseBackend {
|
|||||||
|
|
||||||
TensorInfo GetInputInfo(int index);
|
TensorInfo GetInputInfo(int index);
|
||||||
TensorInfo GetOutputInfo(int index);
|
TensorInfo GetOutputInfo(int index);
|
||||||
std::vector<TensorInfo> GetInputInfo();
|
std::vector<TensorInfo> GetInputInfos() override;
|
||||||
std::vector<TensorInfo> GetOutputInfo();
|
std::vector<TensorInfo> GetOutputInfos() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
paddle_infer::Config config_;
|
paddle_infer::Config config_;
|
||||||
|
@@ -589,6 +589,14 @@ TensorInfo TrtBackend::GetInputInfo(int index) {
|
|||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> TrtBackend::GetInputInfos() {
|
||||||
|
std::vector<TensorInfo> infos;
|
||||||
|
for (auto i = 0; i < inputs_desc_.size(); i++) {
|
||||||
|
infos.emplace_back(GetInputInfo(i));
|
||||||
|
}
|
||||||
|
return infos;
|
||||||
|
}
|
||||||
|
|
||||||
TensorInfo TrtBackend::GetOutputInfo(int index) {
|
TensorInfo TrtBackend::GetOutputInfo(int index) {
|
||||||
FDASSERT(index < NumOutputs(),
|
FDASSERT(index < NumOutputs(),
|
||||||
"The index: %d should less than the number of outputs: %d.", index,
|
"The index: %d should less than the number of outputs: %d.", index,
|
||||||
@@ -600,4 +608,13 @@ TensorInfo TrtBackend::GetOutputInfo(int index) {
|
|||||||
info.dtype = GetFDDataType(outputs_desc_[index].dtype);
|
info.dtype = GetFDDataType(outputs_desc_[index].dtype);
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> TrtBackend::GetOutputInfos() {
|
||||||
|
std::vector<TensorInfo> infos;
|
||||||
|
for (auto i = 0; i < outputs_desc_.size(); i++) {
|
||||||
|
infos.emplace_back(GetOutputInfo(i));
|
||||||
|
}
|
||||||
|
return infos;
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace fastdeploy
|
} // namespace fastdeploy
|
||||||
|
@@ -72,6 +72,8 @@ class TrtBackend : public BaseBackend {
|
|||||||
int NumOutputs() const { return outputs_desc_.size(); }
|
int NumOutputs() const { return outputs_desc_.size(); }
|
||||||
TensorInfo GetInputInfo(int index);
|
TensorInfo GetInputInfo(int index);
|
||||||
TensorInfo GetOutputInfo(int index);
|
TensorInfo GetOutputInfo(int index);
|
||||||
|
std::vector<TensorInfo> GetInputInfos() override;
|
||||||
|
std::vector<TensorInfo> GetOutputInfos() override;
|
||||||
|
|
||||||
~TrtBackend() {
|
~TrtBackend() {
|
||||||
if (parser_) {
|
if (parser_) {
|
||||||
|
@@ -188,5 +188,33 @@ void FDTensor::PrintInfo(const std::string& prefix) {
|
|||||||
<< ", min=" << min << std::endl;
|
<< ", min=" << min << std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool FDTensor::AllocFn(size_t nbytes) {
|
||||||
|
if (device == Device::GPU) {
|
||||||
|
#ifdef WITH_GPU
|
||||||
|
return FDDeviceAllocator()(&buffer_, nbytes);
|
||||||
|
#else
|
||||||
|
FDASSERT(false,
|
||||||
|
"The FastDeploy FDTensor allocator didn't compile under "
|
||||||
|
"-DWITH_GPU=ON,"
|
||||||
|
"so this is an unexpected problem happend.");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
return FDHostAllocator()(&buffer_, nbytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
void FDTensor::FreeFn() {
|
||||||
|
if (external_data_ptr != nullptr) external_data_ptr = nullptr;
|
||||||
|
if (buffer_ != nullptr) {
|
||||||
|
if (device == Device::GPU) {
|
||||||
|
#ifdef WITH_GPU
|
||||||
|
FDDeviceFree()(buffer_);
|
||||||
|
#endif
|
||||||
|
} else {
|
||||||
|
FDHostFree()(buffer_);
|
||||||
|
}
|
||||||
|
buffer_ = nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
FDTensor::FDTensor(const std::string& tensor_name) { name = tensor_name; }
|
FDTensor::FDTensor(const std::string& tensor_name) { name = tensor_name; }
|
||||||
} // namespace fastdeploy
|
} // namespace fastdeploy
|
||||||
|
@@ -98,33 +98,9 @@ struct FASTDEPLOY_DECL FDTensor {
|
|||||||
// prefix will also be printed as tag
|
// prefix will also be printed as tag
|
||||||
void PrintInfo(const std::string& prefix = "TensorInfo: ");
|
void PrintInfo(const std::string& prefix = "TensorInfo: ");
|
||||||
|
|
||||||
bool AllocFn(size_t nbytes) {
|
bool AllocFn(size_t nbytes);
|
||||||
if (device == Device::GPU) {
|
|
||||||
#ifdef WITH_GPU
|
|
||||||
return FDDeviceAllocator()(&buffer_, nbytes);
|
|
||||||
#else
|
|
||||||
FDASSERT(false,
|
|
||||||
"The FastDeploy FDTensor allocator didn't compile under "
|
|
||||||
"-DWITH_GPU=ON,"
|
|
||||||
"so this is an unexpected problem happend.");
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
return FDHostAllocator()(&buffer_, nbytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
void FreeFn() {
|
void FreeFn();
|
||||||
if (external_data_ptr != nullptr) external_data_ptr = nullptr;
|
|
||||||
if (buffer_ != nullptr) {
|
|
||||||
if (device == Device::GPU) {
|
|
||||||
#ifdef WITH_GPU
|
|
||||||
FDDeviceFree()(buffer_);
|
|
||||||
#endif
|
|
||||||
} else {
|
|
||||||
FDHostFree()(buffer_);
|
|
||||||
}
|
|
||||||
buffer_ = nullptr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
FDTensor() {}
|
FDTensor() {}
|
||||||
explicit FDTensor(const std::string& tensor_name);
|
explicit FDTensor(const std::string& tensor_name);
|
||||||
|
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "fastdeploy/fastdeploy_runtime.h"
|
#include "fastdeploy/fastdeploy_runtime.h"
|
||||||
|
|
||||||
#include "fastdeploy/utils/unique_ptr.h"
|
#include "fastdeploy/utils/unique_ptr.h"
|
||||||
#include "fastdeploy/utils/utils.h"
|
#include "fastdeploy/utils/utils.h"
|
||||||
|
|
||||||
@@ -297,6 +298,14 @@ TensorInfo Runtime::GetOutputInfo(int index) {
|
|||||||
return backend_->GetOutputInfo(index);
|
return backend_->GetOutputInfo(index);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> Runtime::GetInputInfos() {
|
||||||
|
return backend_->GetInputInfos();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<TensorInfo> Runtime::GetOutputInfos() {
|
||||||
|
return backend_->GetOutputInfos();
|
||||||
|
}
|
||||||
|
|
||||||
bool Runtime::Infer(std::vector<FDTensor>& input_tensors,
|
bool Runtime::Infer(std::vector<FDTensor>& input_tensors,
|
||||||
std::vector<FDTensor>* output_tensors) {
|
std::vector<FDTensor>* output_tensors) {
|
||||||
return backend_->Infer(input_tensors, output_tensors);
|
return backend_->Infer(input_tensors, output_tensors);
|
||||||
|
@@ -165,6 +165,8 @@ struct FASTDEPLOY_DECL Runtime {
|
|||||||
int NumOutputs() { return backend_->NumOutputs(); }
|
int NumOutputs() { return backend_->NumOutputs(); }
|
||||||
TensorInfo GetInputInfo(int index);
|
TensorInfo GetInputInfo(int index);
|
||||||
TensorInfo GetOutputInfo(int index);
|
TensorInfo GetOutputInfo(int index);
|
||||||
|
std::vector<TensorInfo> GetInputInfos();
|
||||||
|
std::vector<TensorInfo> GetOutputInfos();
|
||||||
|
|
||||||
RuntimeOption option;
|
RuntimeOption option;
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user