apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-07 09:31:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Other] Optimize code style (#1032)

Browse Source 
* Optimize code

* optimize code

* optimize code

* fix compile error
This commit is contained in:
Jason
2023-01-03 19:54:12 +08:00
committed by GitHub
parent ab49b41080
commit f51697d745
31 changed files with 594 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fastdeploy/backends/backend.h
View File

@@ -19,7 +19,6 @@
#include <string>
#include <vector>
#include "fastdeploy/backends/common/multiclass_nms.h"
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/core/fd_type.h"

25
fastdeploy/backends/op_cuda_kernels/adaptive_pool2d_kernel.cu
View File

@@ -2,7 +2,9 @@
namespace fastdeploy {
__global__ void CudaCastKernel(const float* in, float* out, int edge, int out_bc_offset, int in_bc_offset, int ih, int iw, int oh, int ow, bool is_avg) {
__global__ void CudaCastKernel(const float* in, float* out, int edge,
int out_bc_offset, int in_bc_offset, int ih,
int iw, int oh, int ow, bool is_avg) {
int position = blockDim.x * blockIdx.x + threadIdx.x;
if (position >= edge) {
return;
@@ -14,38 +16,41 @@ __global__ void CudaCastKernel(const float* in, float* out, int edge, int out_b
int hend = ceilf(static_cast<float>((h + 1) * ih) / oh);
int wstart = floorf(static_cast<float>(w * iw) / ow);
int wend = ceilf(static_cast<float>((w + 1) * iw) / ow);
if(is_avg) {
if (is_avg) {
out[position] = 0.0;
} else {
out[position] = in[offset * in_bc_offset + hstart * iw + wstart];
}
for (int h = hstart; h < hend; ++h) {
for (int w = wstart; w < wend; ++w) {
for (int w = wstart; w < wend; ++w) {
int input_idx = h * iw + w;
if(is_avg) {
if (is_avg) {
out[position] = out[position] + in[offset * in_bc_offset + input_idx];
} else {
out[position] = max(out[position], in[offset * in_bc_offset + input_idx]);
out[position] =
max(out[position], in[offset * in_bc_offset + input_idx]);
}
}
}
out[position] = out[position] / ((hend - hstart) * (wend - wstart));
}
void CudaAdaptivePool(const std::vector<int64_t>& input_dims, const std::vector<int64_t>& output_dims, float* output, const float* input, void* compute_stream, const std::string& pooling_type){
void CudaAdaptivePool(const std::vector<int64_t>& input_dims,
const std::vector<int64_t>& output_dims, float* output,
const float* input, void* compute_stream,
const std::string& pooling_type) {
auto casted_compute_stream = reinterpret_cast<cudaStream_t>(compute_stream);
int out_bc_offset = output_dims[2] * output_dims[3];
int in_bc_offset = input_dims[2] * input_dims[3];
int jobs = 1;
for(int i : output_dims) {
for (int i : output_dims) {
jobs *= i;
}
bool is_avg = pooling_type == "avg";
int threads = 256;
int blocks = ceil(jobs / static_cast<float>(threads));
CudaCastKernel<<<blocks, threads, 0, casted_compute_stream>>>(
input,
output,
jobs, out_bc_offset, in_bc_offset, int(input_dims[2]), int(input_dims[3]), int(output_dims[2]), int(output_dims[3]), is_avg);
input, output, jobs, out_bc_offset, in_bc_offset, int(input_dims[2]),
int(input_dims[3]), int(output_dims[2]), int(output_dims[3]), is_avg);
}
} // namespace fastdeploy

11
fastdeploy/backends/op_cuda_kernels/adaptive_pool2d_kernel.h
View File

@@ -15,21 +15,18 @@
#pragma once
#include <cstdint>
#include <cuda.h>
#include <cuda_runtime.h>
#include <cstdint>
#include <iostream>
#include <vector>
#include <math.h>
#include <vector>
namespace fastdeploy {
void CudaAdaptivePool(const std::vector<int64_t>& input_dims,
const std::vector<int64_t>& output_dims,
float* output,
const float* input,
void* compute_stream,
const std::vector<int64_t>& output_dims, float* output,
const float* input, void* compute_stream,
const std::string& pooling_type);
} // namespace fastdeploy

17
fastdeploy/backends/openvino/ov_backend.cc
View File

@@ -341,8 +341,7 @@ int OpenVINOBackend::NumInputs() const { return input_infos_.size(); }
int OpenVINOBackend::NumOutputs() const { return output_infos_.size(); }
bool OpenVINOBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool copy_to_fd) {
std::vector<FDTensor>* outputs, bool copy_to_fd) {
if (inputs.size() != input_infos_.size()) {
FDERROR << "[OpenVINOBackend] Size of the inputs(" << inputs.size()
<< ") should keep same with the inputs of this model("
@@ -365,19 +364,17 @@ bool OpenVINOBackend::Infer(std::vector<FDTensor>& inputs,
auto out_tensor_shape = out_tensor.get_shape();
std::vector<int64_t> shape(out_tensor_shape.begin(),
out_tensor_shape.end());
if(copy_to_fd) {
if (copy_to_fd) {
(*outputs)[i].Resize(shape,
OpenVINODataTypeToFD(out_tensor.get_element_type()),
output_infos_[i].name,
Device::CPU);
output_infos_[i].name, Device::CPU);
memcpy((*outputs)[i].MutableData(), out_tensor.data(),
(*outputs)[i].Nbytes());
(*outputs)[i].Nbytes());
} else {
(*outputs)[i].name = output_infos_[i].name;
(*outputs)[i].SetExternalData(shape,
OpenVINODataTypeToFD(out_tensor.get_element_type()),
out_tensor.data(),
Device::CPU);
(*outputs)[i].SetExternalData(
shape, OpenVINODataTypeToFD(out_tensor.get_element_type()),
out_tensor.data(), Device::CPU);
}
}
return true;

3
fastdeploy/backends/openvino/ov_backend.h
View File

@@ -47,8 +47,7 @@ class OpenVINOBackend : public BaseBackend {
InitFromOnnx(const std::string& model_file,
const OpenVINOBackendOption& option = OpenVINOBackendOption());
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs,
bool copy_to_fd = true) override;
int NumInputs() const override;

61
fastdeploy/backends/ort/ops/adaptive_pool2d.cc
View File

@@ -25,30 +25,38 @@ struct OrtTensorDimensions : std::vector<int64_t> {
}
};
void AdaptivePool2dKernel::CpuAdaptivePool(const std::vector<int64_t>& input_size, const std::vector<int64_t>& output_size, const float* input_data, float* output_data){
void AdaptivePool2dKernel::CpuAdaptivePool(
const std::vector<int64_t>& input_size,
const std::vector<int64_t>& output_size, const float* input_data,
float* output_data) {
int64_t in_bc_offset = input_size[2] * input_size[3];
int64_t out_bc_offset = output_size[2] * output_size[3];
for (int64_t b = 0; b < output_size[0] ; b++) {
for (int64_t c = 0; c < output_size[1] ; c++) {
for(int64_t h = 0; h < output_size[2]; h++){
int64_t hstart = std::floor( static_cast<float>(h * input_size[2]) / output_size[2]);
int64_t hend = std::ceil(static_cast<float>((h + 1) * input_size[2]) / output_size[2]);
for(int64_t w = 0; w < output_size[3]; w++){
int64_t wstart = std::floor(static_cast<float>(w * input_size[3]) / output_size[3]);
int64_t wend = std::ceil(static_cast<float>((w + 1) * input_size[3]) / output_size[3]);
for (int64_t b = 0; b < output_size[0]; b++) {
for (int64_t c = 0; c < output_size[1]; c++) {
for (int64_t h = 0; h < output_size[2]; h++) {
int64_t hstart =
std::floor(static_cast<float>(h * input_size[2]) / output_size[2]);
int64_t hend = std::ceil(static_cast<float>((h + 1) * input_size[2]) /
output_size[2]);
for (int64_t w = 0; w < output_size[3]; w++) {
int64_t wstart = std::floor(static_cast<float>(w * input_size[3]) /
output_size[3]);
int64_t wend = std::ceil(static_cast<float>((w + 1) * input_size[3]) /
output_size[3]);
int64_t out_offset = h * output_size[3] + w;
output_data[out_offset] = 0;
for(auto i = hstart; i < hend; i++){
for(auto j = wstart; j< wend; j++){
if(pooling_type_ == "avg"){
for (auto i = hstart; i < hend; i++) {
for (auto j = wstart; j < wend; j++) {
if (pooling_type_ == "avg") {
output_data[out_offset] += input_data[i * input_size[3] + j];
}
if(pooling_type_ == "max"){
output_data[out_offset] = std::max(output_data[out_offset], input_data[i * input_size[3] + j]);
if (pooling_type_ == "max") {
output_data[out_offset] = std::max(
output_data[out_offset], input_data[i * input_size[3] + j]);
}
}
}
if(pooling_type_ == "avg"){
if (pooling_type_ == "avg") {
output_data[out_offset] /= ((hend - hstart) * (wend - wstart));
}
}
@@ -68,7 +76,7 @@ void AdaptivePool2dKernel::Compute(OrtKernelContext* context) {
OrtTensorDimensions input_dim(ort_, input);
output_size_[0] = input_dim[0];
std::vector<int64_t> input_size;
for(auto i: input_dim){
for (auto i : input_dim) {
input_size.push_back(i);
}
@@ -76,14 +84,15 @@ void AdaptivePool2dKernel::Compute(OrtKernelContext* context) {
context, 0, output_size_.data(), output_size_.size());
float* output_data = ort_.GetTensorMutableData<float>(output);
if(!strcmp(this->provider_, "CUDAExecutionProvider")){
if (!strcmp(this->provider_, "CUDAExecutionProvider")) {
#ifdef WITH_GPU
auto compute_stream = ort_.KernelContext_GetGPUComputeStream(context);
CudaAdaptivePool(input_size, output_size_, output_data, input_data, compute_stream, pooling_type_);
CudaAdaptivePool(input_size, output_size_, output_data, input_data,
compute_stream, pooling_type_);
#else
FDWARNING << "FastDeploy didn't compile with WITH_GPU. "
<< "Will force to use CPU to run." << std::endl;
CpuAdaptivePool(input_size, output_size_, input_data, output_data);
FDWARNING << "FastDeploy didn't compile with WITH_GPU. "
<< "Will force to use CPU to run." << std::endl;
CpuAdaptivePool(input_size, output_size_, input_data, output_data);
#endif
} else {
CpuAdaptivePool(input_size, output_size_, input_data, output_data);
@@ -91,9 +100,13 @@ void AdaptivePool2dKernel::Compute(OrtKernelContext* context) {
}
void AdaptivePool2dKernel::GetAttribute(const OrtKernelInfo* info) {
pooling_type_ = ort_.KernelInfoGetAttribute<std::string>(info, "pooling_type");
output_size_ = ort_.KernelInfoGetAttribute<std::vector<int64_t>>(info, "output_size");
FDASSERT(output_size_.size() == 4 && output_size_[2] > 0 && output_size_[3] > 0, "The output size of adaptive pool must be positive.");
pooling_type_ =
ort_.KernelInfoGetAttribute<std::string>(info, "pooling_type");
output_size_ =
ort_.KernelInfoGetAttribute<std::vector<int64_t>>(info, "output_size");
FDASSERT(output_size_.size() == 4 && output_size_[2] > 0 &&
output_size_[3] > 0,
"The output size of adaptive pool must be positive.");
}
} // namespace fastdeploy

23
fastdeploy/backends/ort/ops/adaptive_pool2d.h
View File

@@ -14,12 +14,12 @@
#pragma once
#include <map>
#include <string>
#include <algorithm>
#include <cmath>
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/utils/utils.h"
#include <algorithm>
#include <cmath>
#include <map>
#include <string>
#ifndef NON_64_PLATFORM
#include "onnxruntime_cxx_api.h" // NOLINT
@@ -38,9 +38,8 @@ struct AdaptivePool2dKernel {
const char* provider_;
public:
AdaptivePool2dKernel(Ort::CustomOpApi ort,
const OrtKernelInfo* info,
const char* provider)
AdaptivePool2dKernel(Ort::CustomOpApi ort, const OrtKernelInfo* info,
const char* provider)
: ort_(ort) {
GetAttribute(info);
provider_ = provider;
@@ -51,9 +50,8 @@ struct AdaptivePool2dKernel {
void Compute(OrtKernelContext* context);
void CpuAdaptivePool(const std::vector<int64_t>& input_size,
const std::vector<int64_t>& output_size,
const float* input_data,
float* output_data);
const std::vector<int64_t>& output_size,
const float* input_data, float* output_data);
};
struct AdaptivePool2dOp
@@ -77,9 +75,8 @@ struct AdaptivePool2dOp
return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
}
const char* GetExecutionProviderType() const {
return provider_;
}
const char* GetExecutionProviderType() const { return provider_; }
private:
const char* provider_;
};

2
fastdeploy/backends/ort/ops/multiclass_nms.cc
View File

@@ -15,9 +15,9 @@
#ifndef NON_64_PLATFORM
#include "fastdeploy/backends/ort/ops/multiclass_nms.h"
#include <algorithm>
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/utils/utils.h"
#include <algorithm>
namespace fastdeploy {

36
fastdeploy/backends/ort/ort_backend.cc
View File

@@ -16,8 +16,8 @@
#include <memory>
#include "fastdeploy/backends/ort/ops/multiclass_nms.h"
#include "fastdeploy/backends/ort/ops/adaptive_pool2d.h"
#include "fastdeploy/backends/ort/ops/multiclass_nms.h"
#include "fastdeploy/backends/ort/utils.h"
#include "fastdeploy/core/float16.h"
#include "fastdeploy/utils/utils.h"
@@ -64,7 +64,7 @@ void OrtBackend::BuildOption(const OrtBackendOption& option) {
} else {
OrtCUDAProviderOptions cuda_options;
cuda_options.device_id = option.gpu_id;
if(option.external_stream_) {
if (option.external_stream_) {
cuda_options.has_user_compute_stream = 1;
cuda_options.user_compute_stream = option.external_stream_;
}
@@ -94,8 +94,8 @@ bool OrtBackend::InitFromPaddle(const std::string& model_file,
if (!paddle2onnx::Export(model_file.c_str(), params_file.c_str(),
&model_content_ptr, &model_content_size, 11, true,
verbose, true, true, true, ops.data(),
2, "onnxruntime", nullptr, 0, "", &save_external)) {
verbose, true, true, true, ops.data(), 2,
"onnxruntime", nullptr, 0, "", &save_external)) {
FDERROR << "Error occured while export PaddlePaddle to ONNX format."
<< std::endl;
return false;
@@ -105,11 +105,11 @@ bool OrtBackend::InitFromPaddle(const std::string& model_file,
model_content_ptr + model_content_size);
delete[] model_content_ptr;
model_content_ptr = nullptr;
if(save_external){
if (save_external) {
std::string model_file_name = "model.onnx";
std::fstream f(model_file_name, std::ios::out);
FDASSERT(f.is_open(), "Can not open file: %s to save model.",
model_file_name.c_str());
model_file_name.c_str());
f << onnx_model_proto;
f.close();
return InitFromOnnx(model_file_name, option, false);
@@ -182,7 +182,7 @@ bool OrtBackend::InitFromOnnx(const std::string& model_file,
}
void OrtBackend::OrtValueToFDTensor(const Ort::Value& value, FDTensor* tensor,
const std::string& name, bool copy_to_fd) {
const std::string& name, bool copy_to_fd) {
const auto info = value.GetTensorTypeAndShapeInfo();
const auto data_type = info.GetElementType();
size_t numel = info.GetElementCount();
@@ -216,15 +216,13 @@ void OrtBackend::OrtValueToFDTensor(const Ort::Value& value, FDTensor* tensor,
memcpy(tensor->MutableData(), value_ptr, numel);
} else {
tensor->name = name;
tensor->SetExternalData(
shape, dtype,
const_cast<void*>(value_ptr), Device::CPU);
tensor->SetExternalData(shape, dtype, const_cast<void*>(value_ptr),
Device::CPU);
}
}
bool OrtBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool copy_to_fd) {
std::vector<FDTensor>* outputs, bool copy_to_fd) {
if (inputs.size() != inputs_desc_.size()) {
FDERROR << "[OrtBackend] Size of the inputs(" << inputs.size()
<< ") should keep same with the inputs of this model("
@@ -256,8 +254,8 @@ bool OrtBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<Ort::Value> ort_outputs = binding_->GetOutputValues();
outputs->resize(ort_outputs.size());
for (size_t i = 0; i < ort_outputs.size(); ++i) {
OrtValueToFDTensor(ort_outputs[i], &((*outputs)[i]),
outputs_desc_[i].name, copy_to_fd);
OrtValueToFDTensor(ort_outputs[i], &((*outputs)[i]), outputs_desc_[i].name,
copy_to_fd);
}
return true;
@@ -310,11 +308,13 @@ void OrtBackend::InitCustomOperators() {
if (custom_operators_.size() == 0) {
MultiClassNmsOp* multiclass_nms = new MultiClassNmsOp{};
custom_operators_.push_back(multiclass_nms);
if(option_.use_gpu){
AdaptivePool2dOp* adaptive_pool2d = new AdaptivePool2dOp{"CUDAExecutionProvider"};
if (option_.use_gpu) {
AdaptivePool2dOp* adaptive_pool2d =
new AdaptivePool2dOp{"CUDAExecutionProvider"};
custom_operators_.push_back(adaptive_pool2d);
}else{
AdaptivePool2dOp* adaptive_pool2d = new AdaptivePool2dOp{"CPUExecutionProvider"};
} else {
AdaptivePool2dOp* adaptive_pool2d =
new AdaptivePool2dOp{"CPUExecutionProvider"};
custom_operators_.push_back(adaptive_pool2d);
}
}

4
fastdeploy/backends/ort/ort_backend.h
View File

@@ -18,6 +18,7 @@
#include <memory>
#include <string>
#include <vector>
#include <map>
#include "fastdeploy/backends/backend.h"
#include "onnxruntime_cxx_api.h" // NOLINT
@@ -67,8 +68,7 @@ class OrtBackend : public BaseBackend {
const OrtBackendOption& option = OrtBackendOption(),
bool from_memory_buffer = false);
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs,
bool copy_to_fd = true) override;
int NumInputs() const override { return inputs_desc_.size(); }

7
fastdeploy/backends/paddle/paddle_backend.cc
View File

@@ -104,7 +104,8 @@ bool PaddleBackend::InitFromPaddle(const std::string& model_file,
std::string contents;
if (option.model_from_memory_) {
config_.SetModelBuffer(model_file.c_str(), option.model_buffer_size_, params_file.c_str(), option.params_buffer_size_);
config_.SetModelBuffer(model_file.c_str(), option.model_buffer_size_,
params_file.c_str(), option.params_buffer_size_);
contents = model_file;
} else {
config_.SetModel(model_file, params_file);
@@ -182,7 +183,9 @@ bool PaddleBackend::InitFromPaddle(const std::string& model_file,
FDINFO << "Start generating shape range info file." << std::endl;
paddle_infer::Config analysis_config;
if (option.model_from_memory_) {
analysis_config.SetModelBuffer(model_file.c_str(), option.model_buffer_size_, params_file.c_str(), option.params_buffer_size_);
analysis_config.SetModelBuffer(
model_file.c_str(), option.model_buffer_size_, params_file.c_str(),
option.params_buffer_size_);
} else {
analysis_config.SetModel(model_file, params_file);
}

34
fastdeploy/backends/paddle/util.cc
View File

@@ -30,24 +30,24 @@ void ShareTensorFromFDTensor(paddle_infer::Tensor* tensor,
auto place = ConvertFDDeviceToPlace(fd_tensor.device);
if (fd_tensor.dtype == FDDataType::FP32) {
if (place == paddle_infer::PlaceType::kGPU) {
tensor->ShareExternalData(static_cast<const float*>(fd_tensor.Data()),
shape, place);
tensor->ShareExternalData(static_cast<const float*>(fd_tensor.Data()),
shape, place);
} else {
tensor->CopyFromCpu(static_cast<const float*>(fd_tensor.Data()));
}
return;
} else if (fd_tensor.dtype == FDDataType::INT32) {
if (place == paddle_infer::PlaceType::kGPU) {
tensor->ShareExternalData(static_cast<const int32_t*>(fd_tensor.Data()),
shape, place);
tensor->ShareExternalData(static_cast<const int32_t*>(fd_tensor.Data()),
shape, place);
} else {
tensor->CopyFromCpu(static_cast<const int32_t*>(fd_tensor.Data()));
}
return;
} else if (fd_tensor.dtype == FDDataType::INT64) {
if (place == paddle_infer::PlaceType::kGPU) {
tensor->ShareExternalData(static_cast<const int64_t*>(fd_tensor.Data()),
shape, place);
tensor->ShareExternalData(static_cast<const int64_t*>(fd_tensor.Data()),
shape, place);
} else {
tensor->CopyFromCpu(static_cast<const int64_t*>(fd_tensor.Data()));
}
@@ -62,13 +62,12 @@ void ShareTensorFromFDTensor(paddle_infer::Tensor* tensor,
}
void PaddleTensorToFDTensor(std::unique_ptr<paddle_infer::Tensor>& tensor,
FDTensor* fd_tensor,
bool copy_to_fd) {
FDTensor* fd_tensor, bool copy_to_fd) {
auto fd_dtype = PaddleDataTypeToFD(tensor->type());
std::vector<int64_t> shape;
auto tmp_shape = tensor->shape();
shape.assign(tmp_shape.begin(), tmp_shape.end());
if(copy_to_fd) {
if (copy_to_fd) {
fd_tensor->Resize(shape, fd_dtype, tensor->name());
if (fd_tensor->dtype == FDDataType::FP32) {
tensor->CopyToCpu(static_cast<float*>(fd_tensor->MutableData()));
@@ -81,7 +80,7 @@ void PaddleTensorToFDTensor(std::unique_ptr<paddle_infer::Tensor>& tensor,
return;
}
FDASSERT(false, "Unexpected data type(%s) while infer with PaddleBackend.",
Str(fd_tensor->dtype).c_str());
Str(fd_tensor->dtype).c_str());
} else {
paddle_infer::PlaceType place;
int size = 0;
@@ -99,17 +98,17 @@ void PaddleTensorToFDTensor(std::unique_ptr<paddle_infer::Tensor>& tensor,
} else if (fd_dtype == FDDataType::UINT8) {
out_data = tensor->data<uint8_t>(&place, &size);
} else {
FDASSERT(false, "Unexpected data type(%s) while infer shared with PaddleBackend.",
FDASSERT(
false,
"Unexpected data type(%s) while infer shared with PaddleBackend.",
Str(fd_dtype).c_str());
}
Device device = Device::CPU;
if(place == paddle_infer::PlaceType::kGPU) {
if (place == paddle_infer::PlaceType::kGPU) {
device = Device::GPU;
}
fd_tensor->name = tensor->name();
fd_tensor->SetExternalData(
shape, fd_dtype,
out_data, device);
fd_tensor->SetExternalData(shape, fd_dtype, out_data, device);
}
}
@@ -153,7 +152,10 @@ FDDataType ReaderDataTypeToFD(int32_t dtype) {
} else if (dtype == 6) {
fd_dtype = FDDataType::FP16;
} else {
FDASSERT(false, "Unexpected data type: %d while call ReaderDataTypeToFD in PaddleBackend.", dtype);
FDASSERT(false,
"Unexpected data type: %d while call ReaderDataTypeToFD in "
"PaddleBackend.",
dtype);
}
return fd_dtype;
}

103
fastdeploy/backends/poros/common/compile.h
View File

@@ -14,14 +14,14 @@
#pragma once
#include <string>
#include <algorithm>
#include <unordered_map>
#include <set>
#include <string>
#include <unordered_map>
#include "torch/script.h"
#include "iengine.h"
#include "poros_module.h"
#include "torch/script.h"
namespace baidu {
namespace mirana {
@@ -36,28 +36,29 @@ namespace poros {
* @return porosmodule
* @retval !nullptr => succeed nullptr => failed
**/
std::unique_ptr<PorosModule> Compile(const torch::jit::Module& module,
const std::vector<std::vector<c10::IValue> >& prewarm_datas,
std::unique_ptr<PorosModule>
Compile(const torch::jit::Module& module,
const std::vector<std::vector<c10::IValue>>& prewarm_datas,
const PorosOptions& options);
class Compiler {
public:
typedef std::unordered_map<const torch::jit::Node*, IEngine*> engine_map_t;
typedef std::vector<std::vector<c10::IValue> > ivalue_vec_t;
public:
typedef std::unordered_map<const torch::jit::Node*, IEngine*> engine_map_t;
typedef std::vector<std::vector<c10::IValue>> ivalue_vec_t;
Compiler() : _origin_module(NULL) {}
~Compiler();
Compiler() : _origin_module(NULL) {}
~Compiler();
/**
/**
* @brief initial Compiler
*
* @param [in] options : poros options
* @return int
* @retval 0 => succeed <0 => failed
**/
int init(const PorosOptions& options);
int init(const PorosOptions& options);
/**
/**
* @brief compile whole graph
*
* @param [in] origin_module
@@ -66,13 +67,12 @@ public:
* @return int
* @retval 0 => succeed <0 => failed
**/
int compile(const torch::jit::Module& origin_module,
const ivalue_vec_t& prewarm_datas,
torch::jit::Module* optimized_module);
int compile(const torch::jit::Module& origin_module,
const ivalue_vec_t& prewarm_datas,
torch::jit::Module* optimized_module);
private:
/**
private:
/**
* @brief preprocess this calculation graph
*
* @param [in] prewarm_datas : ivalue_vec_t, vector of IValue
@@ -80,23 +80,25 @@ private:
* @return int
* @retval 0 => succeed <0 => failed
**/
int preprocess_graph(const ivalue_vec_t& prewarm_datas, std::shared_ptr<torch::jit::Graph>& graph);
int preprocess_graph(const ivalue_vec_t& prewarm_datas,
std::shared_ptr<torch::jit::Graph>& graph);
/**
/**
* @brief segement this calculation graph
*
* @param [in/out] graph
* @return int
* @retval 0 => succeed <0 => failed
**/
int segment_graph(std::shared_ptr<torch::jit::Graph>& graph);
int segment_graph(std::shared_ptr<torch::jit::Graph>& graph);
// Split subgraph（block)
// The divided subgraph, as a subgraph, is associated with the block
int segment_block(torch::jit::Block& block, IEngine* engine, int current_depth);
// Split subgraph（block)
// The divided subgraph, as a subgraph, is associated with the block
int segment_block(torch::jit::Block& block, IEngine* engine,
int current_depth);
// Subgraph optimization
/**
// Subgraph optimization
/**
* @brief Subgraph optimization
*
* @param [in] prewarm_datas : ivalue_vec_t, vector of IValue
@@ -105,15 +107,15 @@ private:
* @return int
* @retval 0 => succeed <0 => failed
**/
int optimize_subgraph(const ivalue_vec_t& prewarm_datas,
const std::shared_ptr<torch::jit::Graph>& opt_graph,
torch::jit::Module* optimized_module);
int optimize_subgraph(const ivalue_vec_t& prewarm_datas,
const std::shared_ptr<torch::jit::Graph>& opt_graph,
torch::jit::Module* optimized_module);
// Subgraph optimization(block)
int optimize_subblock(torch::jit::Block* block,
torch::jit::Module* optimized_module);
// Subgraph optimization(block)
int optimize_subblock(torch::jit::Block* block,
torch::jit::Module* optimized_module);
/**
/**
* @brief Compile the subgraph into a new graph based on the engine
*
* @param [in] engine : The engine used by the subgraph
@@ -121,32 +123,32 @@ private:
* @return [out] module : Transformed model
* @retval 0 => succeed <0 => failed
**/
int transform(IEngine* engine, torch::jit::Node& subgraph_node,
torch::jit::Module& module);
int transform(IEngine* engine, torch::jit::Node& subgraph_node,
torch::jit::Module& module);
/**
/**
* @brief Select engine based on subgraph and options
*
* @param [in] node : Jit Node
* @return int
* @retval 0 => succeed <0 => failed
**/
IEngine* select_engine(const torch::jit::Node* n);
IEngine* select_engine(const torch::jit::Node* n);
/**
/**
* @brief destory
*
* @return void
**/
void close();
void close();
private:
int _max_segment_depth{5}; // Maximum subgraph segmentation depth
ivalue_vec_t _prewarm_datas; // Prewarm datas
PorosOptions _options;
engine_map_t _engine_map; // The engine used to record the subgraph
const torch::jit::Module* _origin_module; // Origin_module
std::atomic<int> _engine_index = {0}; // Record engine index
private:
int _max_segment_depth{5}; // Maximum subgraph segmentation depth
ivalue_vec_t _prewarm_datas; // Prewarm datas
PorosOptions _options;
engine_map_t _engine_map; // The engine used to record the subgraph
const torch::jit::Module* _origin_module; // Origin_module
std::atomic<int> _engine_index = {0}; // Record engine index
};
/**
@@ -158,9 +160,10 @@ private:
* @return optimized_module
* @retval !nullptr => succeed nullptr => failed
**/
std::unique_ptr<torch::jit::Module> CompileGraph(const torch::jit::Module& module,
const std::vector<std::vector<c10::IValue> >& prewarm_datas,
const PorosOptions& options);
std::unique_ptr<torch::jit::Module>
CompileGraph(const torch::jit::Module& module,
const std::vector<std::vector<c10::IValue>>& prewarm_datas,
const PorosOptions& options);
} // namespace poros
} // namespace mirana

45
fastdeploy/backends/poros/common/iengine.h
View File

@@ -17,9 +17,9 @@
#include <string>
//from pytorch
#include "torch/script.h"
#include "torch/csrc/jit/ir/ir.h"
#include "ATen/core/interned_strings.h"
#include "torch/csrc/jit/ir/ir.h"
#include "torch/script.h"
#include "plugin_create.h"
@@ -28,50 +28,51 @@ namespace mirana {
namespace poros {
struct PorosGraph {
torch::jit::Graph* graph = NULL;
torch::jit::Node* node = NULL;
torch::jit::Graph* graph = NULL;
torch::jit::Node* node = NULL;
};
typedef uint64_t EngineID;
class IEngine : public IPlugin, public torch::CustomClassHolder{
public:
virtual ~IEngine() {}
class IEngine : public IPlugin, public torch::CustomClassHolder {
public:
virtual ~IEngine() {}
/**
/**
* @brief init, initialization must be successful if the init is successful
* @return int
* @retval 0 => success, <0 => fail
**/
virtual int init() = 0;
virtual int init() = 0;
/**
/**
* @brief During compilation, the subgraph is converted into the graph structure of the corresponding engine and stored inside the engine, so that the execute_engine at runtime can be called
* @param [in] sub_graph : subgraph
* @return [res]int
* @retval 0 => success, <0 => fail
**/
virtual int transform(const PorosGraph& sub_graph) = 0;
virtual int transform(const PorosGraph& sub_graph) = 0;
/**
/**
* @brief Subgraph execution period logic
* @param [in] inputs : input tensor
* @return [res] output tensor
**/
virtual std::vector<at::Tensor> excute_engine(const std::vector<at::Tensor>& inputs) = 0;
virtual std::vector<at::Tensor>
excute_engine(const std::vector<at::Tensor>& inputs) = 0;
virtual void register_module_attribute(const std::string& name, torch::jit::Module& module) = 0;
virtual void register_module_attribute(const std::string& name,
torch::jit::Module& module) = 0;
// Logo
virtual const std::string who_am_i() = 0;
// Logo
virtual const std::string who_am_i() = 0;
// Whether the node is supported by the current engine
bool is_node_supported(const torch::jit::Node* node);
public:
std::pair<uint64_t, uint64_t> _num_io; // Number of input/output parameters
EngineID _id;
// Whether the node is supported by the current engine
bool is_node_supported(const torch::jit::Node* node);
public:
std::pair<uint64_t, uint64_t> _num_io; // Number of input/output parameters
EngineID _id;
};
} // namespace poros

40
fastdeploy/backends/poros/common/plugin_create.h
View File

@@ -14,52 +14,56 @@
#pragma once
#include <unordered_map>
#include <string>
#include <unordered_map>
namespace baidu {
namespace mirana {
namespace poros {
class IPlugin {
public:
virtual ~IPlugin() {}
virtual const std::string who_am_i() = 0;
public:
virtual ~IPlugin() {}
virtual const std::string who_am_i() = 0;
};
typedef IPlugin* (*plugin_creator_t)();
typedef std::unordered_map<std::string, plugin_creator_t> plugin_creator_map_t;
IPlugin* create_plugin(const std::string& plugin_name);
IPlugin* create_plugin(const std::string& plugin_name, const plugin_creator_map_t& plugin_creator_map);
IPlugin* create_plugin(const std::string& plugin_name,
const plugin_creator_map_t& plugin_creator_map);
void create_all_plugins(const plugin_creator_map_t& plugin_creator_map,
std::unordered_map<std::string, IPlugin*>& plugin_m);
std::unordered_map<std::string, IPlugin*>& plugin_m);
//void create_all_plugins(std::unordered_map<std::string, IPlugin*>& plugin_m);
template <typename PluginType>
IPlugin* default_plugin_creator() {
return new (std::nothrow)PluginType;
template <typename PluginType> IPlugin* default_plugin_creator() {
return new (std::nothrow) PluginType;
}
void register_plugin_creator(const std::string& plugin_name, plugin_creator_t creator);
void register_plugin_creator(const std::string& plugin_name,
plugin_creator_t creator, plugin_creator_map_t& plugin_creator_map);
plugin_creator_t creator);
void register_plugin_creator(const std::string& plugin_name,
plugin_creator_t creator,
plugin_creator_map_t& plugin_creator_map);
template <typename PluginType>
void register_plugin_class(const std::string& plugin_name) {
return register_plugin_creator(plugin_name, default_plugin_creator<PluginType>);
return register_plugin_creator(plugin_name,
default_plugin_creator<PluginType>);
}
// This version is recommended
template <typename PluginType>
void register_plugin_class(const std::string& plugin_name, plugin_creator_map_t& plugin_creator_map) {
return register_plugin_creator(plugin_name, default_plugin_creator<PluginType>, plugin_creator_map);
void register_plugin_class(const std::string& plugin_name,
plugin_creator_map_t& plugin_creator_map) {
return register_plugin_creator(
plugin_name, default_plugin_creator<PluginType>, plugin_creator_map);
}
}//poros
}//mirana
}//baidu
} // namespace poros
} // namespace mirana
} // namespace baidu
/* vim: set ts=4 sw=4 sts=4 tw=100 */

52
fastdeploy/backends/poros/common/poros_module.h
View File

@@ -14,53 +14,45 @@
#pragma once
#include <string>
#include "torch/script.h"
#include "torch/csrc/jit/jit_log.h"
#include "torch/script.h"
#include <string>
// #include "ATen/Context.h"
namespace baidu {
namespace mirana {
namespace poros {
enum Device : int8_t {
GPU = 0,
CPU,
XPU,
UNKNOW
};
enum Device : int8_t { GPU = 0, CPU, XPU, UNKNOW };
struct PorosOptions {
Device device = GPU;
bool debug = false;
bool use_fp16 = false;
bool is_dynamic = false;
bool long_to_int = true;
uint64_t max_workspace_size = 1ULL << 30;
int32_t device_id = -1;
int32_t unconst_ops_thres = -1;
bool use_nvidia_tf32 = false;
Device device = GPU;
bool debug = false;
bool use_fp16 = false;
bool is_dynamic = false;
bool long_to_int = true;
uint64_t max_workspace_size = 1ULL << 30;
int32_t device_id = -1;
int32_t unconst_ops_thres = -1;
bool use_nvidia_tf32 = false;
};
class PorosModule : public torch::jit::Module {
public:
PorosModule(torch::jit::Module module) : torch::jit::Module(module) {
}
~PorosModule() = default;
public:
PorosModule(torch::jit::Module module) : torch::jit::Module(module) {}
~PorosModule() = default;
void to_device(Device device){
_options.device = device;
}
//c10::IValue forward(std::vector<c10::IValue> inputs);
//void save(const std::string& filename);
public:
PorosOptions _options;
void to_device(Device device) { _options.device = device; }
//c10::IValue forward(std::vector<c10::IValue> inputs);
//void save(const std::string& filename);
public:
PorosOptions _options;
};
//via porosmodule.save
std::unique_ptr<PorosModule> Load(const std::string& filename, const PorosOptions& options);
std::unique_ptr<PorosModule> Load(const std::string& filename,
const PorosOptions& options);
} // namespace poros
} // namespace mirana

3
fastdeploy/backends/poros/poros_backend.cc
View File

@@ -188,8 +188,7 @@ bool PorosBackend::InitFromPoros(const std::string& model_file,
}
bool PorosBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool copy_to_fd) {
std::vector<FDTensor>* outputs, bool copy_to_fd) {
// Convert FD Tensor to PyTorch Tensor
std::vector<torch::jit::IValue> poros_inputs;
bool is_backend_cuda =

9
fastdeploy/backends/poros/poros_backend.h
View File

@@ -74,9 +74,9 @@ class PorosBackend : public BaseBackend {
void BuildOption(const PorosBackendOption& option);
bool InitFromTorchScript(
const std::string& model_file,
const PorosBackendOption& option = PorosBackendOption());
bool
InitFromTorchScript(const std::string& model_file,
const PorosBackendOption& option = PorosBackendOption());
bool InitFromPoros(const std::string& model_file,
const PorosBackendOption& option = PorosBackendOption());
@@ -85,8 +85,7 @@ class PorosBackend : public BaseBackend {
std::vector<std::vector<FDTensor>>& prewarm_tensors,
const PorosBackendOption& option = PorosBackendOption());
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs,
bool copy_to_fd = true) override;
int NumInputs() const { return _numinputs; }

57
fastdeploy/backends/poros/utils.cc
View File

@@ -23,32 +23,32 @@ namespace fastdeploy {
std::string AtType2String(const at::ScalarType& dtype) {
std::string out;
switch (dtype) {
case at::kByte:
out = "at::kByte";
break;
case at::kChar:
out = "at::kChar";
break;
case at::kShort:
out = "at::kShort";
break;
case at::kInt:
out = "at::kInt";
break;
case at::kLong:
out = "at::kLong";
break;
case at::kHalf:
out = "at::kHalf";
break;
case at::kFloat:
out = "at::kFloat";
break;
case at::kDouble:
out = "at::kDouble";
break;
default:
out = "at::UNKNOWN";
case at::kByte:
out = "at::kByte";
break;
case at::kChar:
out = "at::kChar";
break;
case at::kShort:
out = "at::kShort";
break;
case at::kInt:
out = "at::kInt";
break;
case at::kLong:
out = "at::kLong";
break;
case at::kHalf:
out = "at::kHalf";
break;
case at::kFloat:
out = "at::kFloat";
break;
case at::kDouble:
out = "at::kDouble";
break;
default:
out = "at::UNKNOWN";
}
return out;
}
@@ -129,9 +129,8 @@ at::Tensor CreatePorosValue(FDTensor& tensor, bool is_backend_cuda) {
numel * sizeof(double));
}
} else {
FDASSERT(false,
"Unrecognized data type while calling "
"PorosBackend::CreatePorosValue().");
FDASSERT(false, "Unrecognized data type while calling "
"PorosBackend::CreatePorosValue().");
}
return poros_value;
}

58
fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.cc
View File

@@ -27,14 +27,14 @@ RKNPU2Backend::~RKNPU2Backend() {
for (uint32_t i = 0; i < io_num.n_input; i++) {
rknn_destroy_mem(ctx, input_mems_[i]);
}
if(input_mems_ != nullptr){
if (input_mems_ != nullptr) {
free(input_mems_);
}
for (uint32_t i = 0; i < io_num.n_output; i++) {
rknn_destroy_mem(ctx, output_mems_[i]);
}
if(output_mems_ != nullptr){
if (output_mems_ != nullptr) {
free(output_mems_);
}
}
@@ -173,16 +173,15 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
// create input tensor memory
// rknn_tensor_mem* input_mems[io_num.n_input];
input_mems_ = (rknn_tensor_mem**)malloc(sizeof(rknn_tensor_mem*) * io_num.n_input);
input_mems_ =
(rknn_tensor_mem**)malloc(sizeof(rknn_tensor_mem*) * io_num.n_input);
// get input info and copy to input tensor info
for (uint32_t i = 0; i < io_num.n_input; i++) {
input_attrs_[i].index = i;
// query info
ret = rknn_query(ctx,
RKNN_QUERY_INPUT_ATTR,
&(input_attrs_[i]),
ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs_[i]),
sizeof(rknn_tensor_attr));
DumpTensorAttr(input_attrs_[i]);
@@ -190,12 +189,12 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
printf("rknn_init error! ret=%d\n", ret);
return false;
}
if((input_attrs_[i].fmt != RKNN_TENSOR_NHWC) &&
(input_attrs_[i].fmt != RKNN_TENSOR_UNDEFINED)){
FDERROR << "rknpu2_backend only support input format is NHWC or UNDEFINED" << std::endl;
if ((input_attrs_[i].fmt != RKNN_TENSOR_NHWC) &&
(input_attrs_[i].fmt != RKNN_TENSOR_UNDEFINED)) {
FDERROR << "rknpu2_backend only support input format is NHWC or UNDEFINED"
<< std::endl;
}
// copy input_attrs_ to input tensor info
std::string temp_name = input_attrs_[i].name;
std::vector<int> temp_shape{};
@@ -203,25 +202,28 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
for (int j = 0; j < input_attrs_[i].n_dims; j++) {
temp_shape[j] = (int)input_attrs_[i].dims[j];
}
FDDataType temp_dtype = fastdeploy::RKNPU2Backend::RknnTensorTypeToFDDataType(input_attrs_[i].type);
FDDataType temp_dtype =
fastdeploy::RKNPU2Backend::RknnTensorTypeToFDDataType(
input_attrs_[i].type);
TensorInfo temp_input_info = {temp_name, temp_shape, temp_dtype};
inputs_desc_[i] = temp_input_info;
}
// Get detailed output parameters
output_attrs_ = (rknn_tensor_attr*)malloc(sizeof(rknn_tensor_attr) * io_num.n_output);
output_attrs_ =
(rknn_tensor_attr*)malloc(sizeof(rknn_tensor_attr) * io_num.n_output);
memset(output_attrs_, 0, io_num.n_output * sizeof(rknn_tensor_attr));
outputs_desc_.resize(io_num.n_output);
// Create output tensor memory
output_mems_ = (rknn_tensor_mem**)malloc(sizeof(rknn_tensor_mem*) * io_num.n_output);;
output_mems_ =
(rknn_tensor_mem**)malloc(sizeof(rknn_tensor_mem*) * io_num.n_output);
;
for (uint32_t i = 0; i < io_num.n_output; i++) {
output_attrs_[i].index = i;
// query info
ret = rknn_query(ctx,
RKNN_QUERY_OUTPUT_ATTR,
&(output_attrs_[i]),
ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs_[i]),
sizeof(rknn_tensor_attr));
DumpTensorAttr(output_attrs_[i]);
@@ -233,7 +235,7 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
// If the output dimension is 3, the runtime will automatically change it to 4.
// Obviously, this is wrong, and manual correction is required here.
int n_dims = output_attrs_[i].n_dims;
if((n_dims == 4) && (output_attrs_[i].dims[3] == 1)){
if ((n_dims == 4) && (output_attrs_[i].dims[3] == 1)) {
n_dims--;
}
@@ -292,8 +294,7 @@ std::vector<TensorInfo> RKNPU2Backend::GetOutputInfos() {
}
bool RKNPU2Backend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool copy_to_fd) {
std::vector<FDTensor>* outputs, bool copy_to_fd) {
int ret = RKNN_SUCC;
// Judge whether the input and output size are the same
if (inputs.size() != inputs_desc_.size()) {
@@ -303,15 +304,17 @@ bool RKNPU2Backend::Infer(std::vector<FDTensor>& inputs,
return false;
}
if(!this->infer_init){
if (!this->infer_init) {
for (uint32_t i = 0; i < io_num.n_input; i++) {
// Judge whether the input and output types are the same
rknn_tensor_type input_type =
fastdeploy::RKNPU2Backend::FDDataTypeToRknnTensorType(inputs[i].dtype);
fastdeploy::RKNPU2Backend::FDDataTypeToRknnTensorType(
inputs[i].dtype);
if (input_type != input_attrs_[i].type) {
FDWARNING << "The input tensor type != model's inputs type."
<< "The input_type need " << get_type_string(input_attrs_[i].type)
<< ",but inputs["<< i << "].type is " << get_type_string(input_type)
<< "The input_type need "
<< get_type_string(input_attrs_[i].type) << ",but inputs["
<< i << "].type is " << get_type_string(input_type)
<< std::endl;
}
@@ -319,10 +322,11 @@ bool RKNPU2Backend::Infer(std::vector<FDTensor>& inputs,
input_attrs_[i].type = input_type;
input_attrs_[i].size = inputs[0].Nbytes();
input_attrs_[i].size_with_stride = inputs[0].Nbytes();
if(input_attrs_[i].type == RKNN_TENSOR_FLOAT16 ||
input_attrs_[i].type == RKNN_TENSOR_FLOAT32){
if (input_attrs_[i].type == RKNN_TENSOR_FLOAT16 ||
input_attrs_[i].type == RKNN_TENSOR_FLOAT32) {
FDINFO << "The input model is not a quantitative model. "
"Close the normalize operation." << std::endl;
"Close the normalize operation."
<< std::endl;
}
input_mems_[i] = rknn_create_mem(ctx, inputs[i].Nbytes());
@@ -474,4 +478,4 @@ RKNPU2Backend::FDDataTypeToRknnTensorType(fastdeploy::FDDataType type) {
FDERROR << "rknn_tensor_type don't support this type" << std::endl;
return RKNN_TENSOR_TYPE_MAX;
}
} // namespace fastdeploy
} // namespace fastdeploy

7
fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h
View File

@@ -14,9 +14,9 @@
#pragma once
#include "fastdeploy/backends/backend.h"
#include "fastdeploy/core/fd_tensor.h"
#include "rknn_api.h" // NOLINT
#include "fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h"
#include "fastdeploy/core/fd_tensor.h"
#include "rknn_api.h" // NOLINT
#include <cstring>
#include <iostream>
#include <memory>
@@ -71,8 +71,7 @@ class RKNPU2Backend : public BaseBackend {
TensorInfo GetOutputInfo(int index) override;
std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfos() override;
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs,
bool copy_to_fd = true) override;
private:

6
fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h
View File

@@ -24,9 +24,9 @@ typedef enum _rknpu2_cpu_name {
/*! RKNPU2 core mask for mobile device. */
typedef enum _rknpu2_core_mask {
RKNN_NPU_CORE_AUTO = 0, //< default, run on NPU core randomly.
RKNN_NPU_CORE_0 = 1, //< run on NPU core 0.
RKNN_NPU_CORE_1 = 2, //< run on NPU core 1.
RKNN_NPU_CORE_2 = 4, //< run on NPU core 2.
RKNN_NPU_CORE_0 = 1, //< run on NPU core 0.
RKNN_NPU_CORE_1 = 2, //< run on NPU core 1.
RKNN_NPU_CORE_2 = 4, //< run on NPU core 2.
RKNN_NPU_CORE_0_1 =
RKNN_NPU_CORE_0 | RKNN_NPU_CORE_1, //< run on NPU core 1 and core 2.
RKNN_NPU_CORE_0_1_2 =

153
fastdeploy/backends/tensorrt/ops/adaptive_pool2d.cc
View File

@@ -17,108 +17,106 @@
namespace fastdeploy {
nvinfer1::PluginFieldCollection AdaptivePool2dPluginCreator::mFC{};
std::vector<nvinfer1::PluginField> AdaptivePool2dPluginCreator::mPluginAttributes;
std::vector<nvinfer1::PluginField>
AdaptivePool2dPluginCreator::mPluginAttributes;
pluginStatus_t AdaptivePool2dInference(cudaStream_t stream, int32_t n, const void* input, void* output);
pluginStatus_t AdaptivePool2dInference(cudaStream_t stream, int32_t n,
const void* input, void* output);
AdaptivePool2d::AdaptivePool2d(std::vector<int32_t> output_size, std::string pooling_type) {
AdaptivePool2d::AdaptivePool2d(std::vector<int32_t> output_size,
std::string pooling_type) {
output_size_ = output_size;
pooling_type_ = pooling_type;
}
AdaptivePool2d::AdaptivePool2d(const void* buffer, size_t length) {
const char *d = reinterpret_cast<const char*>(buffer), *a = d;
output_size_.resize(4);
for(int64_t i =0 ; i < 4; i++){
output_size_[i] =read<int32_t>(d);
}
if(read<int32_t>(d) == 0){
pooling_type_ = "avg";
}else{
pooling_type_ = "max";
}
FDASSERT(d == a + length, "deserialize failed.");
const char *d = reinterpret_cast<const char*>(buffer), *a = d;
output_size_.resize(4);
for (int64_t i = 0; i < 4; i++) {
output_size_[i] = read<int32_t>(d);
}
if (read<int32_t>(d) == 0) {
pooling_type_ = "avg";
} else {
pooling_type_ = "max";
}
FDASSERT(d == a + length, "deserialize failed.");
}
int AdaptivePool2d::getNbOutputs() const noexcept {
return 1;
}
int AdaptivePool2d::getNbOutputs() const noexcept { return 1; }
nvinfer1::DimsExprs AdaptivePool2d::getOutputDimensions(
int outputIndex, const nvinfer1::DimsExprs* inputs,
int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept {
int outputIndex, const nvinfer1::DimsExprs* inputs, int nbInputs,
nvinfer1::IExprBuilder& exprBuilder) noexcept {
try {
nvinfer1::DimsExprs output(inputs[0]);
output.d[2] = exprBuilder.constant(static_cast<int32_t>(output_size_[2]));
output.d[3] = exprBuilder.constant(static_cast<int32_t>(output_size_[3]));
return output;
}
catch (const std::exception& e) {
FDASSERT(false, "getOutputDimensions failed: %s.",e.what());
} catch (const std::exception& e) {
FDASSERT(false, "getOutputDimensions failed: %s.", e.what());
}
return nvinfer1::DimsExprs{};
}
int AdaptivePool2d::enqueue(const nvinfer1::PluginTensorDesc* inputDesc,
const nvinfer1::PluginTensorDesc* outputDesc,
const void* const* inputs,
void* const* outputs,
void* workspace,
cudaStream_t stream) noexcept {
const void* const* inputs, void* const* outputs,
void* workspace, cudaStream_t stream) noexcept {
if (inputDesc[0].type != nvinfer1::DataType::kFLOAT) {
return -1;
return -1;
}
auto const* data = static_cast<float const*>(inputs[0]);
auto* result = static_cast<float*>(outputs[0]);
int nums = outputDesc[0].dims.d[0] * outputDesc[0].dims.d[1] * outputDesc[0].dims.d[2]* outputDesc[0].dims.d[3];
int nums = outputDesc[0].dims.d[0] * outputDesc[0].dims.d[1] *
outputDesc[0].dims.d[2] * outputDesc[0].dims.d[3];
std::vector<int64_t> input_size, output_size;
for(int i =0; i< 4; i++){
for (int i = 0; i < 4; i++) {
input_size.push_back(inputDesc[0].dims.d[i]);
output_size.push_back(outputDesc[0].dims.d[i]);
}
CudaAdaptivePool(input_size, output_size, result, data, stream, pooling_type_);
CudaAdaptivePool(input_size, output_size, result, data, stream,
pooling_type_);
return cudaPeekAtLastError();
}
size_t AdaptivePool2d::getSerializationSize() const noexcept {
return 5 * sizeof(int32_t) ;
return 5 * sizeof(int32_t);
}
void AdaptivePool2d::serialize(void* buffer) const noexcept {
char *d = reinterpret_cast<char*>(buffer), *a = d;
for(int64_t i=0; i< 4; i++){
for (int64_t i = 0; i < 4; i++) {
write(d, output_size_[i]);
}
int32_t pooling_type_val = 0;
if(pooling_type_ != "avg"){
if (pooling_type_ != "avg") {
pooling_type_val = 1;
}
write(d, pooling_type_val);
FDASSERT(d == a + getSerializationSize(), "d == a + getSerializationSize()");
}
nvinfer1::DataType AdaptivePool2d::getOutputDataType(
int index, const nvinfer1::DataType* inputType, int nbInputs) const noexcept {
nvinfer1::DataType
AdaptivePool2d::getOutputDataType(int index,
const nvinfer1::DataType* inputType,
int nbInputs) const noexcept {
return inputType[0];
}
bool AdaptivePool2d::supportsFormatCombination(
int pos, const nvinfer1::PluginTensorDesc* inOut, int nbInputs, int nbOutputs) noexcept {
int pos, const nvinfer1::PluginTensorDesc* inOut, int nbInputs,
int nbOutputs) noexcept {
return (inOut[pos].format == nvinfer1::PluginFormat::kLINEAR);
}
int AdaptivePool2d::initialize() noexcept {
return 0;
}
int AdaptivePool2d::initialize() noexcept { return 0; }
void AdaptivePool2d::terminate() noexcept {
return;
}
void AdaptivePool2d::terminate() noexcept { return; }
size_t AdaptivePool2d::getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs,
int nbInputs,
const nvinfer1::PluginTensorDesc* outputs,
int nbOutputs) const noexcept {
size_t AdaptivePool2d::getWorkspaceSize(
const nvinfer1::PluginTensorDesc* inputs, int nbInputs,
const nvinfer1::PluginTensorDesc* outputs, int nbOutputs) const noexcept {
return 0;
}
@@ -126,33 +124,32 @@ const char* AdaptivePool2d::getPluginType() const noexcept {
return "AdaptivePool2d";
}
const char* AdaptivePool2d::getPluginVersion() const noexcept {
return "1";
}
const char* AdaptivePool2d::getPluginVersion() const noexcept { return "1"; }
void AdaptivePool2d::destroy() noexcept {
void AdaptivePool2d::destroy() noexcept { return; }
void AdaptivePool2d::configurePlugin(
const nvinfer1::DynamicPluginTensorDesc* in, int nbInputs,
const nvinfer1::DynamicPluginTensorDesc* out, int nbOutputs) noexcept {
return;
}
void AdaptivePool2d::configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in, int nbInputs,
const nvinfer1::DynamicPluginTensorDesc* out, int nbOutputs) noexcept {
return;
}
nvinfer1::IPluginV2DynamicExt* AdaptivePool2d::clone() const noexcept {
try{
nvinfer1::IPluginV2DynamicExt* plugin = new AdaptivePool2d(output_size_, pooling_type_);
plugin->setPluginNamespace(mNamespace.c_str());
return plugin;
}
catch (std::exception const& e){
FDASSERT(false, "clone failed: %s.",e.what());
try {
nvinfer1::IPluginV2DynamicExt* plugin =
new AdaptivePool2d(output_size_, pooling_type_);
plugin->setPluginNamespace(mNamespace.c_str());
return plugin;
} catch (std::exception const& e) {
FDASSERT(false, "clone failed: %s.", e.what());
}
return nullptr;
}
AdaptivePool2dPluginCreator::AdaptivePool2dPluginCreator() {
mPluginAttributes.clear();
mPluginAttributes.emplace_back(nvinfer1::PluginField("output_size", nullptr, nvinfer1::PluginFieldType::kINT32, 4));
mPluginAttributes.emplace_back(nvinfer1::PluginField("pooling_type", nullptr, nvinfer1::PluginFieldType::kCHAR, 3));
mPluginAttributes.emplace_back(nvinfer1::PluginField(
"output_size", nullptr, nvinfer1::PluginFieldType::kINT32, 4));
mPluginAttributes.emplace_back(nvinfer1::PluginField(
"pooling_type", nullptr, nvinfer1::PluginFieldType::kCHAR, 3));
mFC.nbFields = mPluginAttributes.size();
mFC.fields = mPluginAttributes.data();
@@ -166,17 +163,18 @@ const char* AdaptivePool2dPluginCreator::getPluginVersion() const noexcept {
return "1";
}
const nvinfer1::PluginFieldCollection* AdaptivePool2dPluginCreator::getFieldNames() noexcept {
const nvinfer1::PluginFieldCollection*
AdaptivePool2dPluginCreator::getFieldNames() noexcept {
return &mFC;
}
nvinfer1::IPluginV2DynamicExt* AdaptivePool2dPluginCreator::createPlugin(const char* name,
const nvinfer1::PluginFieldCollection* fc) noexcept {
try{
nvinfer1::IPluginV2DynamicExt* AdaptivePool2dPluginCreator::createPlugin(
const char* name, const nvinfer1::PluginFieldCollection* fc) noexcept {
try {
const nvinfer1::PluginField* fields = fc->fields;
auto const dims = static_cast<int32_t const*>(fields[0].data);
output_size_.resize(4);
for(int64_t i = 0; i < 4; i++){
for (int64_t i = 0; i < 4; i++) {
output_size_[i] = dims[i];
}
@@ -184,23 +182,20 @@ nvinfer1::IPluginV2DynamicExt* AdaptivePool2dPluginCreator::createPlugin(const c
std::string pooling_type(pooling_type_ptr, 3);
pooling_type_ = pooling_type;
return new AdaptivePool2d(output_size_, pooling_type_);
}
catch (std::exception const& e){
FDASSERT(false, "createPlugin failed: %s.",e.what());
} catch (std::exception const& e) {
FDASSERT(false, "createPlugin failed: %s.", e.what());
}
return nullptr;
}
nvinfer1::IPluginV2DynamicExt* AdaptivePool2dPluginCreator::deserializePlugin(const char* name,
const void* serialData,
size_t serialLength) noexcept {
try{
nvinfer1::IPluginV2DynamicExt* AdaptivePool2dPluginCreator::deserializePlugin(
const char* name, const void* serialData, size_t serialLength) noexcept {
try {
return new AdaptivePool2d(serialData, serialLength);
}
catch (std::exception const& e){
FDASSERT(false, "deserializePlugin failed: %s.",e.what());
} catch (std::exception const& e) {
FDASSERT(false, "deserializePlugin failed: %s.", e.what());
}
return nullptr;
}
} // namespace fastdeploy
} // namespace fastdeploy

109
fastdeploy/backends/tensorrt/ops/adaptive_pool2d.h
View File

@@ -13,98 +13,93 @@
// limitations under the License.
#pragma once
#include "common.h" // NOLINT
#include "fastdeploy/backends/op_cuda_kernels/adaptive_pool2d_kernel.h"
#include "common.h" // NOLINT
namespace fastdeploy {
class AdaptivePool2d : public BasePlugin {
public:
AdaptivePool2d(std::vector<int32_t> output_size, std::string pooling_type);
AdaptivePool2d(std::vector<int32_t> output_size, std::string pooling_type);
AdaptivePool2d(const void* buffer, size_t length);
AdaptivePool2d(const void* buffer, size_t length);
~AdaptivePool2d() override = default;
~AdaptivePool2d() override = default;
int getNbOutputs() const noexcept override;
int getNbOutputs() const noexcept override;
nvinfer1::DimsExprs getOutputDimensions(
int outputIndex,
const nvinfer1::DimsExprs* inputs,
int nbInputs,
nvinfer1::IExprBuilder& exprBuilder) noexcept override;
nvinfer1::DimsExprs
getOutputDimensions(int outputIndex, const nvinfer1::DimsExprs* inputs,
int nbInputs,
nvinfer1::IExprBuilder& exprBuilder) noexcept override;
nvinfer1::DataType getOutputDataType(
int index,
const nvinfer1::DataType* inputType,
int nbInputs) const noexcept override;
nvinfer1::DataType getOutputDataType(int index,
const nvinfer1::DataType* inputType,
int nbInputs) const noexcept override;
bool supportsFormatCombination(
int pos,
const nvinfer1::PluginTensorDesc* inOut,
int nbInputs,
int nbOutputs) noexcept override;
bool supportsFormatCombination(int pos,
const nvinfer1::PluginTensorDesc* inOut,
int nbInputs, int nbOutputs) noexcept override;
int initialize() noexcept override;
int initialize() noexcept override;
void terminate() noexcept override;
void terminate() noexcept override;
size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs,
int nbInputs,
const nvinfer1::PluginTensorDesc* outputs,
int nbOutputs) const noexcept override;
size_t getWorkspaceSize(const nvinfer1::PluginTensorDesc* inputs,
int nbInputs,
const nvinfer1::PluginTensorDesc* outputs,
int nbOutputs) const noexcept override;
int enqueue(const nvinfer1::PluginTensorDesc* inputDesc,
const nvinfer1::PluginTensorDesc* outputDesc,
const void* const* inputs,
void* const* outputs,
void* workspace,
cudaStream_t stream) noexcept override;
int enqueue(const nvinfer1::PluginTensorDesc* inputDesc,
const nvinfer1::PluginTensorDesc* outputDesc,
const void* const* inputs, void* const* outputs, void* workspace,
cudaStream_t stream) noexcept override;
size_t getSerializationSize() const noexcept override;
size_t getSerializationSize() const noexcept override;
void serialize(void* buffer) const noexcept override;
void serialize(void* buffer) const noexcept override;
const char* getPluginType() const noexcept override;
const char* getPluginType() const noexcept override;
const char* getPluginVersion() const noexcept override;
void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in,
int nbInputs,
const nvinfer1::DynamicPluginTensorDesc* out,
int nbOutputs) noexcept override;
void destroy() noexcept override;
const char* getPluginVersion() const noexcept override;
void configurePlugin(const nvinfer1::DynamicPluginTensorDesc* in,
int nbInputs,
const nvinfer1::DynamicPluginTensorDesc* out,
int nbOutputs) noexcept override;
void destroy() noexcept override;
nvinfer1::IPluginV2DynamicExt* clone() const noexcept override;
nvinfer1::IPluginV2DynamicExt* clone() const noexcept override;
private:
std::vector<int32_t> output_size_;
std::string pooling_type_;
std::vector<int32_t> output_size_;
std::string pooling_type_;
};
class AdaptivePool2dPluginCreator : public BaseCreator {
public:
AdaptivePool2dPluginCreator();
AdaptivePool2dPluginCreator();
~AdaptivePool2dPluginCreator() override = default;
~AdaptivePool2dPluginCreator() override = default;
const char* getPluginName() const noexcept override;
const char* getPluginName() const noexcept override;
const char* getPluginVersion() const noexcept override;
const char* getPluginVersion() const noexcept override;
const nvinfer1::PluginFieldCollection* getFieldNames() noexcept override;
const nvinfer1::PluginFieldCollection* getFieldNames() noexcept override;
nvinfer1::IPluginV2DynamicExt* createPlugin(const char* name,
const nvinfer1::PluginFieldCollection* fc) noexcept override;
nvinfer1::IPluginV2DynamicExt*
createPlugin(const char* name,
const nvinfer1::PluginFieldCollection* fc) noexcept override;
nvinfer1::IPluginV2DynamicExt* deserializePlugin(const char* name,
const void* serialData,
size_t serialLength) noexcept override;
nvinfer1::IPluginV2DynamicExt*
deserializePlugin(const char* name, const void* serialData,
size_t serialLength) noexcept override;
private:
static nvinfer1::PluginFieldCollection mFC;
static std::vector<nvinfer1::PluginField> mPluginAttributes;
std::vector<int32_t> output_size_;
std::string pooling_type_;
static nvinfer1::PluginFieldCollection mFC;
static std::vector<nvinfer1::PluginField> mPluginAttributes;
std::vector<int32_t> output_size_;
std::string pooling_type_;
};
REGISTER_TENSORRT_PLUGIN(AdaptivePool2dPluginCreator);

52
fastdeploy/backends/tensorrt/ops/common.h
View File

@@ -17,40 +17,40 @@
#include "NvInferPlugin.h"
#include "NvInferRuntimeCommon.h"
#include "fastdeploy/utils/utils.h"
#include <cstring>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <vector>
#include <memory>
#include <cstring>
#include <sstream>
namespace fastdeploy {
class BasePlugin : public nvinfer1::IPluginV2DynamicExt {
protected:
void setPluginNamespace(const char* libNamespace) noexcept override {
mNamespace = libNamespace;
}
void setPluginNamespace(const char* libNamespace) noexcept override {
mNamespace = libNamespace;
}
const char* getPluginNamespace() const noexcept override {
return mNamespace.c_str();
}
const char* getPluginNamespace() const noexcept override {
return mNamespace.c_str();
}
std::string mNamespace;
std::string mNamespace;
};
class BaseCreator : public nvinfer1::IPluginCreator {
public:
void setPluginNamespace(const char* libNamespace) noexcept override {
mNamespace = libNamespace;
}
void setPluginNamespace(const char* libNamespace) noexcept override {
mNamespace = libNamespace;
}
const char* getPluginNamespace() const noexcept override {
return mNamespace.c_str();
}
const char* getPluginNamespace() const noexcept override {
return mNamespace.c_str();
}
protected:
std::string mNamespace;
std::string mNamespace;
};
typedef enum {
@@ -62,19 +62,17 @@ typedef enum {
} pluginStatus_t;
// Write values into buffer
template <typename T>
void write(char*& buffer, const T& val) {
std::memcpy(buffer, &val, sizeof(T));
buffer += sizeof(T);
template <typename T> void write(char*& buffer, const T& val) {
std::memcpy(buffer, &val, sizeof(T));
buffer += sizeof(T);
}
// Read values from buffer
template <typename T>
T read(const char*& buffer) {
T val{};
std::memcpy(&val, buffer, sizeof(T));
buffer += sizeof(T);
return val;
template <typename T> T read(const char*& buffer) {
T val{};
std::memcpy(&val, buffer, sizeof(T));
buffer += sizeof(T);
return val;
}
} // namespace fastdeploy

163
fastdeploy/backends/tensorrt/trt_backend.cc
View File

@@ -134,9 +134,9 @@ bool TrtBackend::InitFromPaddle(const std::string& model_file,
int calibration_cache_size = 0;
if (!paddle2onnx::Export(model_file.c_str(), params_file.c_str(),
&model_content_ptr, &model_content_size, 11, true,
verbose, true, true, true, ops.data(),
1, "tensorrt",
&calibration_cache_ptr, &calibration_cache_size, "", &save_external_)) {
verbose, true, true, true, ops.data(), 1, "tensorrt",
&calibration_cache_ptr, &calibration_cache_size, "",
&save_external_)) {
FDERROR << "Error occured while export PaddlePaddle to ONNX format."
<< std::endl;
return false;
@@ -152,11 +152,11 @@ bool TrtBackend::InitFromPaddle(const std::string& model_file,
calibration_str_ = calibration_str;
delete[] calibration_cache_ptr;
}
if(save_external_){
if (save_external_) {
model_file_name_ = "model.onnx";
std::fstream f(model_file_name_, std::ios::out);
FDASSERT(f.is_open(), "Can not open file: %s to save model.",
model_file_name_.c_str());
model_file_name_.c_str());
f << onnx_model_proto;
f.close();
return InitFromOnnx(model_file_name_, option, false);
@@ -215,13 +215,14 @@ bool TrtBackend::InitFromOnnx(const std::string& model_file,
outputs_desc_.resize(onnx_reader.num_outputs);
for (int i = 0; i < onnx_reader.num_inputs; ++i) {
std::string name(onnx_reader.inputs[i].name);
std::vector<int64_t> shape(
onnx_reader.inputs[i].shape,
onnx_reader.inputs[i].shape + onnx_reader.inputs[i].rank);
std::vector<int64_t> shape(onnx_reader.inputs[i].shape,
onnx_reader.inputs[i].shape +
onnx_reader.inputs[i].rank);
inputs_desc_[i].name = name;
inputs_desc_[i].shape.assign(shape.begin(), shape.end());
inputs_desc_[i].dtype = ReaderDtypeToTrtDtype(onnx_reader.inputs[i].dtype);
inputs_desc_[i].original_dtype = ReaderDtypeToFDDtype(onnx_reader.inputs[i].dtype);
inputs_desc_[i].original_dtype =
ReaderDtypeToFDDtype(onnx_reader.inputs[i].dtype);
auto info = ShapeRangeInfo(shape);
info.name = name;
auto iter_min = option.min_shape.find(name);
@@ -237,9 +238,9 @@ bool TrtBackend::InitFromOnnx(const std::string& model_file,
for (int i = 0; i < onnx_reader.num_outputs; ++i) {
std::string name(onnx_reader.outputs[i].name);
std::vector<int64_t> shape(
onnx_reader.outputs[i].shape,
onnx_reader.outputs[i].shape + onnx_reader.outputs[i].rank);
std::vector<int64_t> shape(onnx_reader.outputs[i].shape,
onnx_reader.outputs[i].shape +
onnx_reader.outputs[i].rank);
outputs_desc_[i].name = name;
outputs_desc_[i].shape.assign(shape.begin(), shape.end());
outputs_desc_[i].dtype =
@@ -252,10 +253,10 @@ bool TrtBackend::InitFromOnnx(const std::string& model_file,
stream_ = reinterpret_cast<cudaStream_t>(option_.external_stream_);
} else {
FDASSERT(cudaStreamCreate(&stream_) == 0,
"[ERROR] Error occurs while calling cudaStreamCreate().");
"[ERROR] Error occurs while calling cudaStreamCreate().");
}
if(save_external_){
if (save_external_) {
onnx_content.clear();
onnx_content = model_file_name_;
}
@@ -283,8 +284,7 @@ int TrtBackend::ShapeRangeInfoUpdated(const std::vector<FDTensor>& inputs) {
}
bool TrtBackend::Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool copy_to_fd) {
std::vector<FDTensor>* outputs, bool copy_to_fd) {
if (inputs.size() != NumInputs()) {
FDERROR << "Require " << NumInputs() << "inputs, but get " << inputs.size()
<< "." << std::endl;
@@ -297,7 +297,8 @@ bool TrtBackend::Infer(std::vector<FDTensor>& inputs,
<< "TensorRT engine will be rebuilt once shape range information "
"changed, this may take lots of time, you can set a proper shape "
"range before loading model to avoid rebuilding process. refer "
"https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/en/faq/"
"https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/en/"
"faq/"
"tensorrt_tricks.md for more details."
<< std::endl;
BuildTrtEngine();
@@ -314,38 +315,42 @@ bool TrtBackend::Infer(std::vector<FDTensor>& inputs,
for (size_t i = 0; i < outputs->size(); ++i) {
// if the final output tensor's dtype is different from the model output tensor's dtype,
// then we need cast the data to the final output's dtype
auto model_output_dtype = GetFDDataType(outputs_device_buffer_[(*outputs)[i].name].dtype());
auto model_output_dtype =
GetFDDataType(outputs_device_buffer_[(*outputs)[i].name].dtype());
if ((*outputs)[i].dtype != model_output_dtype) {
FDTensor output_tensor;
output_tensor.SetExternalData((*outputs)[i].shape, model_output_dtype,
outputs_device_buffer_[(*outputs)[i].name].data(),
Device::GPU);
output_tensor.SetExternalData(
(*outputs)[i].shape, model_output_dtype,
outputs_device_buffer_[(*outputs)[i].name].data(), Device::GPU);
casted_output_tensors_[(*outputs)[i].name].Resize((*outputs)[i].shape, (*outputs)[i].dtype,
(*outputs)[i].name, Device::GPU);
function::CudaCast(output_tensor, &casted_output_tensors_[(*outputs)[i].name], stream_);
if(!copy_to_fd) {
(*outputs)[i].SetExternalData((*outputs)[i].shape, model_output_dtype,
casted_output_tensors_[(*outputs)[i].name].MutableData(),
Device::GPU, option_.gpu_id);
casted_output_tensors_[(*outputs)[i].name].Resize(
(*outputs)[i].shape, (*outputs)[i].dtype, (*outputs)[i].name,
Device::GPU);
function::CudaCast(output_tensor,
&casted_output_tensors_[(*outputs)[i].name], stream_);
if (!copy_to_fd) {
(*outputs)[i].SetExternalData(
(*outputs)[i].shape, model_output_dtype,
casted_output_tensors_[(*outputs)[i].name].MutableData(),
Device::GPU, option_.gpu_id);
}
} else {
casted_output_tensors_[(*outputs)[i].name].SetExternalData(
(*outputs)[i].shape, model_output_dtype,
outputs_device_buffer_[(*outputs)[i].name].data(),
Device::GPU);
outputs_device_buffer_[(*outputs)[i].name].data(), Device::GPU);
}
}
if (copy_to_fd) {
for (size_t i = 0; i < outputs->size(); ++i) {
FDASSERT(cudaMemcpyAsync((*outputs)[i].Data(),
casted_output_tensors_[(*outputs)[i].name].Data(),
(*outputs)[i].Nbytes(), cudaMemcpyDeviceToHost,
stream_) == 0,
"[ERROR] Error occurs while copy memory from GPU to CPU.");
FDASSERT(
cudaMemcpyAsync((*outputs)[i].Data(),
casted_output_tensors_[(*outputs)[i].name].Data(),
(*outputs)[i].Nbytes(), cudaMemcpyDeviceToHost,
stream_) == 0,
"[ERROR] Error occurs while copy memory from GPU to CPU.");
}
FDASSERT(cudaStreamSynchronize(stream_) == cudaSuccess,
"[ERROR] Error occurs while sync cuda stream.");
"[ERROR] Error occurs while sync cuda stream.");
}
return true;
@@ -356,10 +361,12 @@ void TrtBackend::GetInputOutputInfo() {
std::unordered_map<std::string, FDDataType> inputs_original_dtype_map;
std::unordered_map<std::string, FDDataType> outputs_original_dtype_map;
for (size_t i = 0; i < inputs_desc_.size(); ++i) {
inputs_original_dtype_map[inputs_desc_[i].name] = inputs_desc_[i].original_dtype;
inputs_original_dtype_map[inputs_desc_[i].name] =
inputs_desc_[i].original_dtype;
}
for (size_t i = 0; i < outputs_desc_.size(); ++i) {
outputs_original_dtype_map[outputs_desc_[i].name] = outputs_desc_[i].original_dtype;
outputs_original_dtype_map[outputs_desc_[i].name] =
outputs_desc_[i].original_dtype;
}
// Re-read the tensor infos from TRT model and write into inputs_desc_ and outputs_desc_
@@ -373,12 +380,18 @@ void TrtBackend::GetInputOutputInfo() {
auto shape = ToVec(engine_->getBindingDimensions(i));
auto dtype = engine_->getBindingDataType(i);
if (engine_->bindingIsInput(i)) {
auto original_dtype = inputs_original_dtype_map.count(name) ? inputs_original_dtype_map[name] : GetFDDataType(dtype);
inputs_desc_.emplace_back(TrtValueInfo{name, shape, dtype, original_dtype});
auto original_dtype = inputs_original_dtype_map.count(name)
? inputs_original_dtype_map[name]
: GetFDDataType(dtype);
inputs_desc_.emplace_back(
TrtValueInfo{name, shape, dtype, original_dtype});
inputs_device_buffer_[name] = FDDeviceBuffer(dtype);
} else {
auto original_dtype = outputs_original_dtype_map.count(name) ? outputs_original_dtype_map[name] : GetFDDataType(dtype);
outputs_desc_.emplace_back(TrtValueInfo{name, shape, dtype, original_dtype});
auto original_dtype = outputs_original_dtype_map.count(name)
? outputs_original_dtype_map[name]
: GetFDDataType(dtype);
outputs_desc_.emplace_back(
TrtValueInfo{name, shape, dtype, original_dtype});
outputs_device_buffer_[name] = FDDeviceBuffer(dtype);
casted_output_tensors_[name] = FDTensor();
}
@@ -391,7 +404,8 @@ void TrtBackend::SetInputs(const std::vector<FDTensor>& inputs) {
for (const auto& item : inputs) {
// auto idx = engine_->getBindingIndex(item.name.c_str());
auto iter = io_name_index_.find(item.name);
FDASSERT(iter != io_name_index_.end(), "TRTBackend SetInputs not find name:%s", item.name.c_str());
FDASSERT(iter != io_name_index_.end(),
"TRTBackend SetInputs not find name:%s", item.name.c_str());
auto idx = iter->second;
std::vector<int> shape(item.shape.begin(), item.shape.end());
auto dims = ToDims(shape);
@@ -424,9 +438,8 @@ void TrtBackend::SetInputs(const std::vector<FDTensor>& inputs) {
"Error occurs while copy memory from CPU to GPU.");
} else {
FDASSERT(cudaMemcpyAsync(inputs_device_buffer_[item.name].data(),
item.Data(),
item.Nbytes(), cudaMemcpyHostToDevice,
stream_) == 0,
item.Data(), item.Nbytes(),
cudaMemcpyHostToDevice, stream_) == 0,
"Error occurs while copy memory from CPU to GPU.");
}
}
@@ -443,7 +456,9 @@ void TrtBackend::AllocateOutputsBuffer(std::vector<FDTensor>* outputs,
for (size_t i = 0; i < outputs_desc_.size(); ++i) {
// auto idx = engine_->getBindingIndex(outputs_desc_[i].name.c_str());
auto idx_iter = io_name_index_.find(outputs_desc_[i].name);
FDASSERT(idx_iter != io_name_index_.end(), "TRTBackend Outputs not find name:%s", outputs_desc_[i].name.c_str());
FDASSERT(idx_iter != io_name_index_.end(),
"TRTBackend Outputs not find name:%s",
outputs_desc_[i].name.c_str());
auto idx = idx_iter->second;
auto output_dims = context_->getBindingDimensions(idx);
@@ -464,16 +479,15 @@ void TrtBackend::AllocateOutputsBuffer(std::vector<FDTensor>* outputs,
// set user's outputs info
std::vector<int64_t> shape(output_dims.d,
output_dims.d + output_dims.nbDims);
if(copy_to_fd) {
if (copy_to_fd) {
(*outputs)[ori_idx].is_pinned_memory = option_.enable_pinned_memory;
(*outputs)[ori_idx].Resize(shape, outputs_desc_[i].original_dtype,
outputs_desc_[i].name);
} else {
(*outputs)[ori_idx].name = outputs_desc_[i].name;
(*outputs)[ori_idx].SetExternalData(
shape, outputs_desc_[i].original_dtype,
bindings_[idx], Device::GPU,
option_.gpu_id);
shape, outputs_desc_[i].original_dtype, bindings_[idx], Device::GPU,
option_.gpu_id);
}
}
}
@@ -587,7 +601,8 @@ bool TrtBackend::BuildTrtEngine() {
if (option_.serialize_file != "") {
FDINFO << "Serialize TensorRTEngine to local file "
<< option_.serialize_file << "." << std::endl;
std::ofstream engine_file(option_.serialize_file.c_str(), std::ios::binary | std::ios::out);
std::ofstream engine_file(option_.serialize_file.c_str(),
std::ios::binary | std::ios::out);
if (!engine_file) {
FDERROR << "Failed to open " << option_.serialize_file << " to write."
<< std::endl;
@@ -628,10 +643,11 @@ bool TrtBackend::CreateTrtEngineFromOnnx(const std::string& onnx_model_buffer) {
return false;
}
bool model_parser;
if(save_external_){
model_parser=!parser_->parseFromFile(onnx_model_buffer.c_str(), 0);
}else{
model_parser = !parser_->parse(onnx_model_buffer.data(), onnx_model_buffer.size());
if (save_external_) {
model_parser = !parser_->parseFromFile(onnx_model_buffer.c_str(), 0);
} else {
model_parser =
!parser_->parse(onnx_model_buffer.data(), onnx_model_buffer.size());
}
if (model_parser) {
FDERROR << "Failed to parse ONNX model by TensorRT." << std::endl;
@@ -665,7 +681,8 @@ bool TrtBackend::CreateTrtEngineFromOnnx(const std::string& onnx_model_buffer) {
"should be noticed that FastDeploy will rebuild the engine while "
"new input shape is out of the collected shape range, this may "
"bring some time consuming problem, refer "
"https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/en/faq/"
"https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/en/"
"faq/"
"tensorrt_tricks.md for more details."
<< std::endl;
initialized_ = true;
@@ -721,27 +738,24 @@ std::vector<TensorInfo> TrtBackend::GetOutputInfos() {
return infos;
}
std::unique_ptr<BaseBackend> TrtBackend::Clone(void *stream, int device_id) {
std::unique_ptr<BaseBackend> TrtBackend::Clone(void* stream, int device_id) {
std::unique_ptr<BaseBackend> new_backend = utils::make_unique<TrtBackend>();
auto casted_backend = dynamic_cast<TrtBackend*>(new_backend.get());
if(device_id > 0 && device_id != option_.gpu_id) {
if (device_id > 0 && device_id != option_.gpu_id) {
auto clone_option = option_;
clone_option.gpu_id = device_id;
clone_option.external_stream_ = stream;
if (option_.model_format == ModelFormat::ONNX) {
FDASSERT(casted_backend->InitFromOnnx(option_.model_file, clone_option),
"Clone model from ONNX failed while initialize TrtBackend.");
"Clone model from ONNX failed while initialize TrtBackend.");
} else {
FDASSERT(casted_backend->InitFromPaddle(option_.model_file,
option_.params_file, clone_option),
"Clone model from Paddle failed while initialize TrtBackend.");
FDASSERT(casted_backend->InitFromPaddle(
option_.model_file, option_.params_file, clone_option),
"Clone model from Paddle failed while initialize TrtBackend.");
}
FDWARNING << "The target device id:"
<< device_id
<< " is different from current device id:"
<< option_.gpu_id
<< ", cannot share memory with current engine."
<< std::endl;
FDWARNING << "The target device id:" << device_id
<< " is different from current device id:" << option_.gpu_id
<< ", cannot share memory with current engine." << std::endl;
return new_backend;
}
cudaSetDevice(option_.gpu_id);
@@ -750,12 +764,15 @@ std::unique_ptr<BaseBackend> TrtBackend::Clone(void *stream, int device_id) {
casted_backend->stream_ = reinterpret_cast<cudaStream_t>(stream);
} else {
FDASSERT(cudaStreamCreate(&casted_backend->stream_) == 0,
"[ERROR] Error occurs while clone calling cudaStreamCreate().");
"[ERROR] Error occurs while clone calling cudaStreamCreate().");
}
casted_backend->inputs_desc_.assign(inputs_desc_.begin(), inputs_desc_.end());
casted_backend->outputs_desc_.assign(outputs_desc_.begin(), outputs_desc_.end());
casted_backend->outputs_order_.insert(outputs_order_.begin(), outputs_order_.end());
casted_backend->shape_range_info_.insert(shape_range_info_.begin(), shape_range_info_.end());
casted_backend->outputs_desc_.assign(outputs_desc_.begin(),
outputs_desc_.end());
casted_backend->outputs_order_.insert(outputs_order_.begin(),
outputs_order_.end());
casted_backend->shape_range_info_.insert(shape_range_info_.begin(),
shape_range_info_.end());
casted_backend->engine_ = engine_;
casted_backend->context_ = std::shared_ptr<nvinfer1::IExecutionContext>(
casted_backend->engine_->createExecutionContext());

7
fastdeploy/backends/tensorrt/trt_backend.h
View File

@@ -58,7 +58,7 @@ namespace fastdeploy {
struct TrtValueInfo {
std::string name;
std::vector<int> shape;
nvinfer1::DataType dtype; // dtype of TRT model
nvinfer1::DataType dtype; // dtype of TRT model
FDDataType original_dtype; // dtype of original ONNX/Paddle model
};
@@ -97,8 +97,7 @@ class TrtBackend : public BaseBackend {
bool InitFromOnnx(const std::string& model_file,
const TrtBackendOption& option = TrtBackendOption(),
bool from_memory_buffer = false);
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs,
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs,
bool copy_to_fd = true) override;
int NumInputs() const { return inputs_desc_.size(); }
@@ -107,7 +106,7 @@ class TrtBackend : public BaseBackend {
TensorInfo GetOutputInfo(int index);
std::vector<TensorInfo> GetInputInfos() override;
std::vector<TensorInfo> GetOutputInfos() override;
std::unique_ptr<BaseBackend> Clone(void *stream = nullptr,
std::unique_ptr<BaseBackend> Clone(void* stream = nullptr,
int device_id = -1) override;
~TrtBackend() {

30
fastdeploy/backends/tensorrt/utils.h
View File

@@ -32,17 +32,15 @@
namespace fastdeploy {
struct FDInferDeleter {
template <typename T>
void operator()(T* obj) const {
template <typename T> void operator()(T* obj) const {
if (obj) {
delete obj;
// obj->destroy();
// obj->destroy();
}
}
};
template <typename T>
using FDUniquePtr = std::unique_ptr<T, FDInferDeleter>;
template <typename T> using FDUniquePtr = std::unique_ptr<T, FDInferDeleter>;
int64_t Volume(const nvinfer1::Dims& d);
@@ -72,17 +70,13 @@ std::ostream& operator<<(std::ostream& out, const std::vector<T>& vec) {
return out;
}
template <typename AllocFunc, typename FreeFunc>
class FDGenericBuffer {
template <typename AllocFunc, typename FreeFunc> class FDGenericBuffer {
public:
//!
//! \brief Construct an empty buffer.
//!
explicit FDGenericBuffer(nvinfer1::DataType type = nvinfer1::DataType::kFLOAT)
: mSize(0),
mCapacity(0),
mType(type),
mBuffer(nullptr),
: mSize(0), mCapacity(0), mType(type), mBuffer(nullptr),
mExternal_buffer(nullptr) {}
//!
@@ -104,9 +98,7 @@ class FDGenericBuffer {
}
FDGenericBuffer(FDGenericBuffer&& buf)
: mSize(buf.mSize),
mCapacity(buf.mCapacity),
mType(buf.mType),
: mSize(buf.mSize), mCapacity(buf.mCapacity), mType(buf.mType),
mBuffer(buf.mBuffer) {
buf.mSize = 0;
buf.mCapacity = 0;
@@ -133,7 +125,8 @@ class FDGenericBuffer {
//! \brief Returns pointer to underlying array.
//!
void* data() {
if (mExternal_buffer != nullptr) return mExternal_buffer;
if (mExternal_buffer != nullptr)
return mExternal_buffer;
return mBuffer;
}
@@ -141,7 +134,8 @@ class FDGenericBuffer {
//! \brief Returns pointer to underlying array.
//!
const void* data() const {
if (mExternal_buffer != nullptr) return mExternal_buffer;
if (mExternal_buffer != nullptr)
return mExternal_buffer;
return mBuffer;
}
@@ -213,8 +207,8 @@ class FDGenericBuffer {
};
using FDDeviceBuffer = FDGenericBuffer<FDDeviceAllocator, FDDeviceFree>;
using FDDeviceHostBuffer = FDGenericBuffer<FDDeviceHostAllocator,
FDDeviceHostFree>;
using FDDeviceHostBuffer =
FDGenericBuffer<FDDeviceHostAllocator, FDDeviceHostFree>;
class FDTrtLogger : public nvinfer1::ILogger {
public:

14
fastdeploy/backends/common/multiclass_nms.cc → fastdeploy/vision/detection/ppdet/multiclass_nms.cc
View File

@@ -12,13 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/backends/common/multiclass_nms.h"
#include "fastdeploy/vision/detection/ppdet/multiclass_nms.h"
#include <algorithm>
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/utils/utils.h"
namespace fastdeploy {
namespace backend {
namespace vision {
namespace detection {
template <class T>
bool SortScorePairDescend(const std::pair<float, T>& pair1,
const std::pair<float, T>& pair2) {
@@ -79,7 +80,7 @@ float JaccardOverlap(const float* box1, const float* box2,
}
}
void MultiClassNMS::FastNMS(const float* boxes, const float* scores,
void PaddleMultiClassNMS::FastNMS(const float* boxes, const float* scores,
const int& num_boxes,
std::vector<int>* keep_indices) {
std::vector<std::pair<float, int>> sorted_indices;
@@ -109,7 +110,7 @@ void MultiClassNMS::FastNMS(const float* boxes, const float* scores,
}
}
int MultiClassNMS::NMSForEachSample(
int PaddleMultiClassNMS::NMSForEachSample(
const float* boxes, const float* scores, int num_boxes, int num_classes,
std::map<int, std::vector<int>>* keep_indices) {
for (int i = 0; i < num_classes; ++i) {
@@ -152,7 +153,7 @@ int MultiClassNMS::NMSForEachSample(
return num_det;
}
void MultiClassNMS::Compute(const float* boxes_data, const float* scores_data,
void PaddleMultiClassNMS::Compute(const float* boxes_data, const float* scores_data,
const std::vector<int64_t>& boxes_dim,
const std::vector<int64_t>& scores_dim) {
int score_size = scores_dim.size();
@@ -220,5 +221,6 @@ void MultiClassNMS::Compute(const float* boxes_data, const float* scores_data,
}
}
}
} // namespace backend
} // namespace detection
} // namespace vision
} // namespace fastdeploy

9
fastdeploy/backends/common/multiclass_nms.h → fastdeploy/vision/detection/ppdet/multiclass_nms.h
View File

@@ -18,8 +18,9 @@
#include <vector>
namespace fastdeploy {
namespace backend {
struct MultiClassNMS {
namespace vision {
namespace detection {
struct PaddleMultiClassNMS {
int64_t background_label = -1;
int64_t keep_top_k = -1;
float nms_eta;
@@ -40,6 +41,6 @@ struct MultiClassNMS {
const std::vector<int64_t>& boxes_dim,
const std::vector<int64_t>& scores_dim);
};
} // namespace backend
} // namespace detection
} // namespace vision
} // namespace fastdeploy

3
fastdeploy/vision/detection/ppdet/postprocessor.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/detection/ppdet/postprocessor.h"
#include "fastdeploy/vision/detection/ppdet/multiclass_nms.h"
#include "fastdeploy/vision/utils/utils.h"
namespace fastdeploy {
@@ -176,7 +177,7 @@ bool PaddleDetPostprocessor::ProcessUnDecodeResults(
return false;
}
backend::MultiClassNMS nms;
PaddleMultiClassNMS nms;
nms.background_label = -1;
nms.keep_top_k = 100;
nms.nms_eta = 1.0;