apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 17:17:14 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Integrate FalconCV to boost image processing (#332)

Browse Source 
* Rename GetCpuMat to GetOpenCVMat

* refine code structure
This commit is contained in:
Jason
2022-10-16 14:49:57 +08:00
committed by GitHub
parent c7fbc0375c
commit 24c8fdc27d
25 changed files with 267 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fastdeploy/backends/tensorrt/utils.h
View File

@@ -35,7 +35,8 @@ struct FDInferDeleter {
template <typename T>
void operator()(T* obj) const {
if (obj) {
obj->destroy();
delete obj;
// obj->destroy();
}
}
};

2
fastdeploy/vision/classification/ppcls/model.cc
View File

@@ -108,7 +108,7 @@ bool PaddleClasModel::Preprocess(Mat* mat, FDTensor* output) {
int height = mat->Height();
output->name = InputInfoOfRuntime(0).name;
output->SetExternalData({1, channel, height, width}, FDDataType::FP32,
mat->GetCpuMat()->ptr());
mat->GetOpenCVMat()->ptr());
return true;
}

7
fastdeploy/vision/common/processors/base.h
View File

@@ -34,9 +34,10 @@ class Processor {
virtual bool ImplByOpenCV(Mat* mat) = 0;
// virtual bool ImplByFalconCV(Mat* mat) {
// return ImplByOpenCV(mat);
// }
virtual bool ImplByFalconCV(Mat* mat) {
FDASSERT(false, "%s is not implemented with FalconCV, please use OpenCV instead.", Name().c_str());
return false;
}
virtual bool operator()(Mat* mat,
ProcLib lib = ProcLib::OPENCV);

2
fastdeploy/vision/common/processors/cast.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool Cast::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int c = im->channels();
if (dtype_ == "float") {
if (im->type() != CV_32FC(c)) {

2
fastdeploy/vision/common/processors/center_crop.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool CenterCrop::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int height = static_cast<int>(im->rows);
int width = static_cast<int>(im->cols);
if (height < height_ || width < width_) {

4
fastdeploy/vision/common/processors/color_space_convert.cc
View File

@@ -17,7 +17,7 @@
namespace fastdeploy {
namespace vision {
bool BGR2RGB::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Mat new_im;
cv::cvtColor(*im, new_im, cv::COLOR_BGR2RGB);
mat->SetMat(new_im);
@@ -25,7 +25,7 @@ bool BGR2RGB::ImplByOpenCV(Mat* mat) {
}
bool RGB2BGR::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Mat new_im;
cv::cvtColor(*im, new_im, cv::COLOR_RGB2BGR);
mat->SetMat(new_im);

2
fastdeploy/vision/common/processors/convert.cc
View File

@@ -29,7 +29,7 @@ Convert::Convert(const std::vector<float>& alpha,
}
bool Convert::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
std::vector<cv::Mat> split_im;
cv::split(*im, split_im);
for (int c = 0; c < im->channels(); c++) {

2
fastdeploy/vision/common/processors/crop.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool Crop::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int height = static_cast<int>(im->rows);
int width = static_cast<int>(im->cols);
if (height < height_ + offset_h_ || width < width_ + offset_w_) {

2
fastdeploy/vision/common/processors/hwc2chw.cc
View File

@@ -22,7 +22,7 @@ bool HWC2CHW::ImplByOpenCV(Mat* mat) {
<< std::endl;
return false;
}
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Mat im_clone = im->clone();
int rh = im->rows;
int rw = im->cols;

2
fastdeploy/vision/common/processors/limit_by_stride.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool LimitByStride::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
int rw = origin_w - origin_w % stride_;

2
fastdeploy/vision/common/processors/limit_long.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool LimitLong::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
int im_size_max = std::max(origin_w, origin_h);

2
fastdeploy/vision/common/processors/limit_short.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool LimitShort::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
int im_size_min = std::min(origin_w, origin_h);

84
fastdeploy/vision/common/processors/mat.cc
View File

@@ -12,17 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/common/processors/mat.h"
#include "fastdeploy/vision/common/processors/utils.h"
#include "fastdeploy/utils/utils.h"
namespace fastdeploy {
namespace vision {
cv::Mat* Mat::GetCpuMat() {
return &cpu_mat;
void* Mat::Data() {
if (mat_type == ProcLib::FALCONCV) {
#ifdef ENABLE_FALCONCV
return fcv_mat.data();
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
#endif
}
return cpu_mat.ptr();
}
void Mat::ShareWithTensor(FDTensor* tensor) {
tensor->SetExternalData({Channels(), Height(), Width()}, Type(),
GetCpuMat()->ptr());
Data());
tensor->device = Device::CPU;
if (layout == Layout::HWC) {
tensor->shape = {Height(), Width(), Channels()};
@@ -30,8 +39,7 @@ void Mat::ShareWithTensor(FDTensor* tensor) {
}
bool Mat::CopyToTensor(FDTensor* tensor) {
cv::Mat* im = GetCpuMat();
int total_bytes = im->total() * im->elemSize();
int total_bytes = Height() * Width() * Channels() * FDDataTypeSize(Type());
if (total_bytes != tensor->Nbytes()) {
FDERROR << "While copy Mat to Tensor, requires the memory size be same, "
"but now size of Tensor = "
@@ -39,53 +47,52 @@ bool Mat::CopyToTensor(FDTensor* tensor) {
<< std::endl;
return false;
}
memcpy(tensor->MutableData(), im->ptr(), im->total() * im->elemSize());
memcpy(tensor->MutableData(), Data(), total_bytes);
return true;
}
void Mat::PrintInfo(const std::string& flag) {
cv::Mat* im = GetCpuMat();
cv::Scalar mean = cv::mean(*im);
if (mat_type == ProcLib::FALCONCV) {
#ifdef ENABLE_FALCONCV
fcv::Scalar mean = fcv::mean(fcv_mat);
std::cout << flag << ": "
<< "Channel=" << Channels() << ", height=" << Height()
<< ", width=" << Width() << ", mean=";
<< "DataType=" << Type() << ", "
<< "Channel=" << Channels() << ", "
<< "Height=" << Height() << ", "
<< "Width=" << Width() << ", "
<< "Mean=";
for (int i = 0; i < Channels(); ++i) {
std::cout << mean[i] << " ";
}
std::cout << std::endl;
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
#endif
} else {
cv::Scalar mean = cv::mean(cpu_mat);
std::cout << flag << ": "
<< "DataType=" << Type() << ", "
<< "Channel=" << Channels() << ", "
<< "Height=" << Height() << ", "
<< "Width=" << Width() << ", "
<< "Mean=";
for (int i = 0; i < Channels(); ++i) {
std::cout << mean[i] << " ";
}
std::cout << std::endl;
}
}
FDDataType Mat::Type() {
int type = -1;
type = cpu_mat.type();
if (type < 0) {
FDASSERT(false,
"While calling Mat::Type(), get negative value, which is not "
"expected!.");
}
type = type % 8;
if (type == 0) {
return FDDataType::UINT8;
} else if (type == 1) {
return FDDataType::INT8;
} else if (type == 2) {
FDASSERT(false,
"While calling Mat::Type(), get UINT16 type which is not "
"supported now.");
} else if (type == 3) {
return FDDataType::INT16;
} else if (type == 4) {
return FDDataType::INT32;
} else if (type == 5) {
return FDDataType::FP32;
} else if (type == 6) {
return FDDataType::FP64;
} else {
FDASSERT(
false,
"While calling Mat::Type(), get type = %d, which is not expected!.",
type);
if (mat_type == ProcLib::FALCONCV) {
#ifdef ENABLE_FALCONCV
return FalconCVDataTypeToFD(fcv_mat.type());
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
#endif
}
return OpenCVDataTypeToFD(cpu_mat.type());
}
Mat CreateFromTensor(const FDTensor& tensor) {
@@ -139,5 +146,6 @@ Mat CreateFromTensor(const FDTensor& tensor) {
return mat;
}
} // namespace vision
} // namespace fastdeploy

20
fastdeploy/vision/common/processors/mat.h
View File

@@ -22,10 +22,14 @@
#include "opencv2/cudawarping.hpp"
#endif
namespace fcv {
class Mat;
}
namespace fastdeploy {
namespace vision {
enum ProcLib { DEFAULT, OPENCV};
enum ProcLib { DEFAULT, OPENCV, FALCONCV};
enum Layout { HWC, CHW };
struct FASTDEPLOY_DECL Mat {
@@ -47,13 +51,25 @@ struct FASTDEPLOY_DECL Mat {
mat_type = ProcLib::OPENCV;
}
inline cv::Mat* GetOpenCVMat() {
FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure.");
return &cpu_mat;
}
#ifdef ENABLE_FALCONCV
void SetMat(const fcv::Mat& mat) {
fcv_mat = mat;
mat_type = Proclib::FALCONCV;
}
inline fcv::Mat* GetFalconCVMat() {
FDASSERT(mat_type == ProcLib::FALCONCV, "Met non fcv::Mat data strucure.");
return &fcv_mat;
}
#endif
void* Data();
private:
int channels;
int height;
@@ -70,8 +86,6 @@ struct FASTDEPLOY_DECL Mat {
return &cpu_mat;
}
cv::Mat* GetCpuMat();
FDDataType Type();
int Channels() const { return channels; }
int Width() const { return width; }

2
fastdeploy/vision/common/processors/normalize.cc
View File

@@ -53,7 +53,7 @@ Normalize::Normalize(const std::vector<float>& mean,
}
bool Normalize::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
std::vector<cv::Mat> split_im;
cv::split(*im, split_im);
for (int c = 0; c < im->channels(); c++) {

2
fastdeploy/vision/common/processors/pad.cc
View File

@@ -34,7 +34,7 @@ bool Pad::ImplByOpenCV(Mat* mat) {
<< std::endl;
return false;
}
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Scalar value;
if (value_.size() == 1) {
value = cv::Scalar(value_[0]);

2
fastdeploy/vision/common/processors/pad_to_size.cc
View File

@@ -56,7 +56,7 @@ bool PadToSize::ImplByOpenCV(Mat* mat) {
return true;
}
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Scalar value;
if (value_.size() == 1) {
value = cv::Scalar(value_[0]);

2
fastdeploy/vision/common/processors/resize.cc
View File

@@ -22,7 +22,7 @@ bool Resize::ImplByOpenCV(Mat* mat) {
FDERROR << "Resize: The format of input is not HWC." << std::endl;
return false;
}
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
if (width_ > 0 && height_ > 0) {

2
fastdeploy/vision/common/processors/resize_by_long.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool ResizeByLong::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
double scale = GenerateScale(origin_w, origin_h);

2
fastdeploy/vision/common/processors/resize_by_short.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
bool ResizeByShort::ImplByOpenCV(Mat* mat) {
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
int origin_w = im->cols;
int origin_h = im->rows;
double scale = GenerateScale(origin_w, origin_h);

2
fastdeploy/vision/common/processors/stride_pad.cc
View File

@@ -45,7 +45,7 @@ bool StridePad::ImplByOpenCV(Mat* mat) {
if (pad_h == 0 && pad_w == 0) {
return true;
}
cv::Mat* im = mat->GetCpuMat();
cv::Mat* im = mat->GetOpenCVMat();
cv::Scalar value;
if (value_.size() == 1) {
value = cv::Scalar(value_[0]);

138
fastdeploy/vision/common/processors/utils.cc Normal file
View File

@@ -0,0 +1,138 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/common/processors/utils.h"
namespace fastdeploy {
namespace vision {
FDDataType OpenCVDataTypeToFD(int type) {
type = type % 8;
if (type == 0) {
return FDDataType::UINT8;
} else if (type == 1) {
return FDDataType::INT8;
} else if (type == 2) {
FDASSERT(false,
"While calling OpenCVDataTypeToFD(), get UINT16 type which is not "
"supported now.");
} else if (type == 3) {
return FDDataType::INT16;
} else if (type == 4) {
return FDDataType::INT32;
} else if (type == 5) {
return FDDataType::FP32;
} else if (type == 6) {
return FDDataType::FP64;
} else {
FDASSERT(false, "While calling OpenCVDataTypeToFD(), get type = %d, which is not expected.", type);
}
}
#ifdef ENABLE_FALCONCV
FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type) {
if (type == fcv::FCVImageType::GRAY_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_BGR_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_RGB_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_BGR_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_RGB_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PLANAR_BGR_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PLANAR_RGB_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PLANAR_BGRA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PLANAR_RGBA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PLANAR_BGR_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PLANAR_RGB_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PLANAR_BGRA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PLANAR_RGBA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_BGRA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_RGBA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_BGRA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_RGBA_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_BGR565_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_RGB565_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::GRAY_S32) {
return FDDataType::INT32;
} else if (type == fcv::FCVImageType::GRAY_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_BGR_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_RGB_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_BGR_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_RGB_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_BGRA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_RGBA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_BGRA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::PACKAGE_RGBA_F32) {
return FDDataType::FP32;
} else if (type == fcv::FCVImageType::GRAY_F64) {
return FDDataType::FP64;
}
FDASSERT(false, "While calling FalconDataTypeToFD(), get unexpected type:" + std::to_string(int(type)) + ".");
return FDDataType::UNKNOWN;
}
fcv::FCVImageType CreateFalconDataCVType(FDDataType type, int channel) {
FDASSERT(channel == 1 || channel == 3 || channel == 4,
"Only support channel be 1/3/4 in Falcon.");
if (type == FDDataType::UINT8) {
if (channel == 1) {
return fcv::FCVImageType::GRAY_U8;
} else if (channel == 3) {
return fcv::FCVImageType::PACKAGE_BGR_U8;
} else {
return fcv::FCVImageType::PACKAGE_BGRA_U8;
}
} else if (type == FDDataType::FP32) {
if (channel == 1) {
return fcv::FCVImageType::GRAY_F32;
} else if (channel == 3) {
return fcv::FCVImageType::PACKAGE_BGR_F32;
} else {
return fcv::FCVImageType::PACKAGE_BGRA_F32;
}
}
FDASSERT(false, "Data type of " + Str(type) + " is not supported.");
return fcv::FCVImageType::PACKAGE_BGR_F32;
}
#endif
} // namespace vision
} // namespace fastdeploy

34
fastdeploy/vision/common/processors/utils.h Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "fastdeploy/utils/utils.h"
#include "fastdeploy/core/fd_tensor.h"
namespace fastdeploy {
namespace vision {
// Convert data type of opencv to FDDataType
FDDataType OpenCVDataTypeToFD(int type);
#ifdef ENABLE_FALCONCV
// Convert data type of falconcv to FDDataType
FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type);
// Create data type of falconcv by FDDataType
fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel = 1);
#endif
} // namespace vision
} // namespace fastdeploy

2
fastdeploy/vision/matting/contrib/modnet.cc
View File

@@ -118,7 +118,7 @@ bool MODNet::Postprocess(
int numel = ipt_h * ipt_w;
int nbytes = numel * sizeof(float);
result->Resize(numel);
std::memcpy(result->alpha.data(), alpha_resized.GetCpuMat()->data, nbytes);
std::memcpy(result->alpha.data(), alpha_resized.GetOpenCVMat()->data, nbytes);
return true;
}

2
fastdeploy/vision/matting/ppmatting/ppmatting.cc
View File

@@ -196,7 +196,7 @@ bool PPMatting::Postprocess(
int numel = iter_ipt->second[0] * iter_ipt->second[1];
int nbytes = numel * sizeof(float);
result->Resize(numel);
std::memcpy(result->alpha.data(), mat.GetCpuMat()->data, nbytes);
std::memcpy(result->alpha.data(), mat.GetOpenCVMat()->data, nbytes);
return true;
}