apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-28 02:42:21 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

[FlyCV] optimize the integration of FlyCV (#492)

Browse Source 
* [Backend] fix lite backend save model error

* [Backend] fixed typos

* [FlyCV] optimize the integration of FlyCV

* [cmake] close some tests options

* [cmake] close some test option

* [FlyCV] remove un-need warnings

* [FlyCV] remove un-need GetMat method

* [FlyCV] optimize FlyCV codes

* [cmake] remove un-need cmake function in examples/CMakelists

* [cmake] support gflags for Android
This commit is contained in:
DefTruth
2022-11-04 09:19:03 +08:00
committed by GitHub
parent 5b1fecd668
commit a36d49a973
56 changed files with 670 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -38,3 +38,5 @@ coverage
*.local
yalc.*
.yalc
examples/vision/collect_quantize_cc.sh
examples/vision/tests_quantize

41
cmake/gflags.cmake
View File

@@ -38,7 +38,42 @@ ENDIF(WIN32)
INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
ExternalProject_Add(
if(ANDROID)
ExternalProject_Add(
extern_gflags
${EXTERNAL_PROJECT_LOG_ARGS}
${SHALLOW_CLONE}
GIT_REPOSITORY ${GFLAGS_REPOSITORY}
GIT_TAG ${GFLAGS_TAG}
PREFIX ${GFLAGS_PREFIX_DIR}
UPDATE_COMMAND ""
BUILD_COMMAND ${BUILD_COMMAND}
INSTALL_COMMAND ${INSTALL_COMMAND}
CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-DANDROID_ABI=${ANDROID_ABI}
-DANDROID_NDK=${ANDROID_NDK}
-DANDROID_PLATFORM=${ANDROID_PLATFORM}
-DANDROID_STL=c++_static
-DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
)
else()
ExternalProject_Add(
extern_gflags
${EXTERNAL_PROJECT_LOG_ARGS}
${SHALLOW_CLONE}
@@ -66,8 +101,8 @@ ExternalProject_Add(
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
)
)
endif()
ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES})
ADD_DEPENDENCIES(gflags extern_gflags)

29
examples/CMakeLists.txt
View File

@@ -37,12 +37,6 @@ function(config_fastdeploy_executable_link_flags TARGET_NAME)
endif()
endfunction()
# Usage: add_fastdeploy_executable_cc_files(xxx_var vision detection)
function(add_fastdeploy_executable_cc_files CC_FILES_VAR FIELD SUB_FIELD)
file(GLOB_RECURSE _EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/${FIELD}/${SUB_FIELD}/*/cpp/*.cc)
set(${CC_FILES_VAR} ${_EXAMPLE_SRCS} PARENT_SCOPE)
endfunction()
set(EXAMPLES_NUM 0)
function(add_fastdeploy_executable FIELD CC_FILE)
# temp target name/file var in function scope
@@ -55,7 +49,11 @@ function(add_fastdeploy_executable FIELD CC_FILE)
add_executable(${TEMP_TARGET_NAME} ${TEMP_TARGET_FILE})
target_link_libraries(${TEMP_TARGET_NAME} PUBLIC fastdeploy)
if(TARGET gflags)
if(NOT ANDROID)
target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags pthread)
else()
target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags)
endif()
endif()
config_fastdeploy_executable_link_flags(${TEMP_TARGET_NAME})
math(EXPR _EXAMPLES_NUM "${EXAMPLES_NUM} + 1")
@@ -78,23 +76,14 @@ if(BUILD_EXAMPLES AND ENABLE_VISION)
if(EXISTS ${PROJECT_SOURCE_DIR}/examples/vision)
message(STATUS "")
message(STATUS "*************FastDeploy Vision Examples Summary**********")
set(ALL_VISION_SUD_FIELDS classification
detection
facedet
faceid
keypointdetection
matting
ocr
segmentation)
if(NOT ANDROID)
list(APPEND ALL_VISION_SUD_FIELDS tracking)
file(GLOB_RECURSE ALL_VISION_EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/*/*/cpp/*.cc)
if(ANDROID)
file(GLOB_RECURSE TRACKING_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/tracking/*/cpp/*.cc)
list(REMOVE_ITEM ALL_VISION_EXAMPLE_SRCS ${TRACKING_SRCS})
endif()
foreach(_SUB_FIELD ${ALL_VISION_SUD_FIELDS})
add_fastdeploy_executable_cc_files(_SUB_CC_FILES vision ${_SUB_FIELD})
foreach(_CC_FILE ${_SUB_CC_FILES})
foreach(_CC_FILE ${ALL_VISION_EXAMPLE_SRCS})
add_fastdeploy_executable(vision ${_CC_FILE})
endforeach()
endforeach()
message(STATUS " [FastDeploy Executable Path] : ${EXECUTABLE_OUTPUT_PATH}")
endif()
endif()

3
fastdeploy/vision/classification/ppcls/model.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/classification/ppcls/model.h"
#include "fastdeploy/vision/utils/utils.h"
#include "yaml-cpp/yaml.h"
@@ -108,7 +109,7 @@ bool PaddleClasModel::Preprocess(Mat* mat, FDTensor* output) {
int height = mat->Height();
output->name = InputInfoOfRuntime(0).name;
output->SetExternalData({1, channel, height, width}, FDDataType::FP32,
mat->GetOpenCVMat()->ptr());
mat->Data());
return true;
}

113
fastdeploy/vision/common/processors/base.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/common/processors/base.h"
#include "fastdeploy/utils/utils.h"
namespace fastdeploy {
@@ -21,27 +22,18 @@ namespace vision {
ProcLib Processor::default_lib = ProcLib::DEFAULT;
bool Processor::operator()(Mat* mat, ProcLib lib) {
// if default_lib is set
// then use default_lib
ProcLib target = lib;
if (default_lib != ProcLib::DEFAULT) {
if (lib == ProcLib::DEFAULT) {
target = default_lib;
}
if (target == ProcLib::FLYCV) {
#ifdef ENABLE_FLYCV
if (mat->mat_type != ProcLib::FLYCV) {
if (mat->layout != Layout::HWC) {
FDERROR << "Cannot convert cv::Mat to fcv::Mat while layout is not HWC." << std::endl;
}
fcv::Mat fcv_mat = ConvertOpenCVMatToFalconCV(*(mat->GetOpenCVMat()));
mat->SetMat(fcv_mat);
}
return ImplByFalconCV(mat);
return ImplByFlyCV(mat);
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV.");
FDASSERT(false, "FastDeploy didn't compile with FlyCV.");
#endif
}
// DEFAULT & OPENCV
return ImplByOpenCV(mat);
}
@@ -63,5 +55,100 @@ void DisableFlyCV() {
<< Processor::default_lib << std::endl;
}
cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor) {
FDDataType type = tensor.dtype;
FDASSERT(tensor.shape.size() == 3,
"When create FD Mat from tensor, tensor shape should be 3-Dim, HWC "
"layout");
int64_t height = tensor.shape[0];
int64_t width = tensor.shape[1];
int64_t channel = tensor.shape[2];
cv::Mat ocv_mat;
// reference to outside FDTensor, zero copy
switch (type) {
case FDDataType::UINT8:
ocv_mat = cv::Mat(height, width, CV_8UC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT8:
ocv_mat = cv::Mat(height, width, CV_8SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT16:
ocv_mat = cv::Mat(height, width, CV_16SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT32:
ocv_mat = cv::Mat(height, width, CV_32SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP32:
ocv_mat = cv::Mat(height, width, CV_32FC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP64:
ocv_mat = cv::Mat(height, width, CV_64FC(channel),
const_cast<void*>(tensor.Data()));
break;
default:
FDASSERT(false,
"Tensor type %d is not supported While calling "
"CreateFDMatFromTensor.",
type);
break;
}
return ocv_mat;
}
#ifdef ENABLE_FLYCV
fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor) {
FDDataType type = tensor.dtype;
FDASSERT(tensor.shape.size() == 3,
"When create FD Mat from tensor, tensor shape should be 3-Dim, HWC "
"layout");
int64_t height = tensor.shape[0];
int64_t width = tensor.shape[1];
int64_t channel = tensor.shape[2];
fcv::Mat fcv_mat;
auto fcv_type = CreateFlyCVDataType(type, static_cast<int>(channel));
switch (type) {
case FDDataType::UINT8:
fcv_mat =
fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP32:
fcv_mat =
fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP64:
fcv_mat =
fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
break;
default:
FDASSERT(false,
"Tensor type %d is not supported While calling "
"CreateFDMatFromTensor.",
type);
break;
}
return fcv_mat;
}
#endif
Mat CreateFDMatFromTensor(const FDTensor& tensor) {
if (Processor::default_lib == ProcLib::FLYCV) {
#ifdef ENABLE_FLYCV
fcv::Mat fcv_mat = CreateFlyCVMatFromTensor(tensor);
Mat mat = Mat(fcv_mat);
return mat;
#else
FDASSERT(false, "FastDeploy didn't compiled with FlyCV!");
#endif
}
cv::Mat ocv_mat = CreateOpenCVMatFromTensor(tensor);
Mat mat = Mat(ocv_mat);
return mat;
}
} // namespace vision
} // namespace fastdeploy

21
fastdeploy/vision/common/processors/base.h
View File

@@ -22,7 +22,9 @@
namespace fastdeploy {
namespace vision {
/*! @brief Enable using FlyCV to process image while deploy vision models. Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will fallback to using OpenCV in other platform
/*! @brief Enable using FlyCV to process image while deploy vision models.
* Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will
* fallback to using OpenCV in other platform
*/
FASTDEPLOY_DECL void EnableFlyCV();
@@ -41,16 +43,19 @@ class FASTDEPLOY_DECL Processor {
virtual bool ImplByOpenCV(Mat* mat) = 0;
virtual bool ImplByFalconCV(Mat* mat) {
FDASSERT(false,
"%s is not implemented with FalconCV, please use OpenCV instead.",
Name().c_str());
return false;
virtual bool ImplByFlyCV(Mat* mat) {
return ImplByOpenCV(mat);
}
virtual bool operator()(Mat* mat,
ProcLib lib = ProcLib::OPENCV);
virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
};
// Create OpenCV/FlyCV/FD Mat from FD Tensor
cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor);
#ifdef ENABLE_FLYCV
fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor);
#endif
Mat CreateFDMatFromTensor(const FDTensor& tensor);
} // namespace vision
} // namespace fastdeploy

10
fastdeploy/vision/common/processors/cast.cc
View File

@@ -36,8 +36,8 @@ bool Cast::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Cast::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool Cast::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
if (dtype_ == "float" && mat->Type() == FDDataType::FP32) {
return true;
}
@@ -46,18 +46,18 @@ bool Cast::ImplByFalconCV(Mat* mat) {
}
if (mat->layout != Layout::HWC) {
FDERROR
<< "While using Falcon to cast image, the image must be layout of HWC."
<< "While using FlyCV to cast image, the image must be layout of HWC."
<< std::endl;
return false;
}
if (dtype_ == "float") {
fcv::Mat new_im;
auto fcv_type = CreateFalconCVDataType(FDDataType::FP32, im->channels());
auto fcv_type = CreateFlyCVDataType(FDDataType::FP32, im->channels());
im->convert_to(new_im, fcv_type);
mat->SetMat(new_im);
} else if (dtype_ == "double") {
fcv::Mat new_im;
auto fcv_type = CreateFalconCVDataType(FDDataType::FP64, im->channels());
auto fcv_type = CreateFlyCVDataType(FDDataType::FP64, im->channels());
im->convert_to(new_im, fcv_type);
mat->SetMat(new_im);
} else {

8
fastdeploy/vision/common/processors/cast.h
View File

@@ -24,15 +24,13 @@ class FASTDEPLOY_DECL Cast : public Processor {
explicit Cast(const std::string& dtype = "float") : dtype_(dtype) {}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Cast"; }
static bool Run(Mat* mat, const std::string& dtype,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
std::string GetDtype() const {
return dtype_;
}
std::string GetDtype() const { return dtype_; }
private:
std::string dtype_;

4
fastdeploy/vision/common/processors/center_crop.cc
View File

@@ -36,8 +36,8 @@ bool CenterCrop::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool CenterCrop::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool CenterCrop::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int height = static_cast<int>(im->height());
int width = static_cast<int>(im->width());
if (height < height_ || width < width_) {

4
fastdeploy/vision/common/processors/center_crop.h
View File

@@ -24,12 +24,12 @@ class FASTDEPLOY_DECL CenterCrop : public Processor {
CenterCrop(int width, int height) : height_(height), width_(width) {}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "CenterCrop"; }
static bool Run(Mat* mat, const int& width, const int& height,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int height_;

14
fastdeploy/vision/common/processors/color_space_convert.cc
View File

@@ -25,10 +25,11 @@ bool BGR2RGB::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool BGR2RGB::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool BGR2RGB::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
if (im->channels() != 3) {
FDERROR << "[BGR2RGB] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl;
FDERROR << "[BGR2RGB] The channel of input image must be 3, but not it's "
<< im->channels() << "." << std::endl;
return false;
}
fcv::Mat new_im;
@@ -47,10 +48,11 @@ bool RGB2BGR::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool RGB2BGR::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool RGB2BGR::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
if (im->channels() != 3) {
FDERROR << "[RGB2BGR] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl;
FDERROR << "[RGB2BGR] The channel of input image must be 3, but not it's "
<< im->channels() << "." << std::endl;
return false;
}
fcv::Mat new_im;

8
fastdeploy/vision/common/processors/color_space_convert.h
View File

@@ -23,22 +23,22 @@ class FASTDEPLOY_DECL BGR2RGB : public Processor {
public:
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
virtual std::string Name() { return "BGR2RGB"; }
static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV);
static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
};
class FASTDEPLOY_DECL RGB2BGR : public Processor {
public:
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "RGB2BGR"; }
static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV);
static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
};
} // namespace vision
} // namespace fastdeploy

9
fastdeploy/vision/common/processors/convert.cc
View File

@@ -40,9 +40,9 @@ bool Convert::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Convert::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
FDASSERT(im->channels() == 3, "Only support 3-channels image in FalconCV.");
bool Convert::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
FDASSERT(im->channels() == 3, "Only support 3-channels image in FlyCV.");
std::vector<float> mean(3, 0);
std::vector<float> std(3, 0);
for (size_t i = 0; i < 3; ++i) {
@@ -50,7 +50,8 @@ bool Convert::ImplByFalconCV(Mat* mat) {
mean[i] = -1 * beta_[i] * std[i];
}
fcv::Mat new_im;
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(), new_im, true);
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(),
new_im, true);
mat->SetMat(new_im);
return true;
}

4
fastdeploy/vision/common/processors/convert.h
View File

@@ -24,7 +24,7 @@ class FASTDEPLOY_DECL Convert : public Processor {
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Convert"; }
@@ -32,7 +32,7 @@ class FASTDEPLOY_DECL Convert : public Processor {
// The default behavior is the same as OpenCV's convertTo method.
static bool Run(Mat* mat, const std::vector<float>& alpha,
const std::vector<float>& beta,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
std::vector<float> alpha_;

4
fastdeploy/vision/common/processors/crop.cc
View File

@@ -37,8 +37,8 @@ bool Crop::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Crop::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool Crop::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int height = static_cast<int>(im->height());
int width = static_cast<int>(im->width());
if (height < height_ + offset_h_ || width < width_ + offset_w_) {

4
fastdeploy/vision/common/processors/crop.h
View File

@@ -31,12 +31,12 @@ class FASTDEPLOY_DECL Crop : public Processor {
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Crop"; }
static bool Run(Mat* mat, int offset_w, int offset_h, int width, int height,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int offset_w_;

14
fastdeploy/vision/common/processors/hwc2chw.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/common/processors/hwc2chw.h"
#include "fastdeploy/function/transpose.h"
namespace fastdeploy {
@@ -41,18 +42,21 @@ bool HWC2CHW::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool HWC2CHW::ImplByFalconCV(Mat* mat) {
bool HWC2CHW::ImplByFlyCV(Mat* mat) {
if (mat->layout != Layout::HWC) {
FDERROR << "HWC2CHW: The input data is not Layout::HWC format!" << std::endl;
FDERROR << "HWC2CHW: The input data is not Layout::HWC format!"
<< std::endl;
return false;
}
if (mat->Type() != FDDataType::FP32) {
FDERROR << "HWC2CHW: Only support float data while use FalconCV, but now it's " << mat->Type() << "." << std::endl;
FDERROR << "HWC2CHW: Only support float data while use FlyCV, but now it's "
<< mat->Type() << "." << std::endl;
return false;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
fcv::Mat new_im;
fcv::normalize_to_submean_to_reorder(*im, {0.0, 0.0, 0.0}, {1.0, 1.0, 1.0}, std::vector<uint32_t>(), new_im, false);
fcv::normalize_to_submean_to_reorder(*im, {0.0, 0.0, 0.0}, {1.0, 1.0, 1.0},
std::vector<uint32_t>(), new_im, false);
mat->SetMat(new_im);
mat->layout = Layout::CHW;
return true;

4
fastdeploy/vision/common/processors/hwc2chw.h
View File

@@ -23,11 +23,11 @@ class FASTDEPLOY_DECL HWC2CHW : public Processor {
public:
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "HWC2CHW"; }
static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV);
static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
};
} // namespace vision
} // namespace fastdeploy

2
fastdeploy/vision/common/processors/letter_box.h Executable file → Normal file
View File

@@ -33,7 +33,7 @@ class LetterBoxResize : public Processor {
static bool Run(Mat* mat, const std::vector<int>& target_size,
const std::vector<float>& color,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
std::vector<int> target_size_;

8
fastdeploy/vision/common/processors/limit_by_stride.cc
View File

@@ -38,8 +38,8 @@ bool LimitByStride::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool LimitByStride::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool LimitByStride::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int origin_w = im->width();
int origin_h = im->height();
int rw = origin_w - origin_w % stride_;
@@ -59,16 +59,14 @@ bool LimitByStride::ImplByFalconCV(Mat* mat) {
} else if (interp_ == 2) {
interp_method = fcv::InterpolationType::INTER_CUBIC;
} else {
FDERROR << "LimitByStride: Only support interp_ be 0/1/2 with FalconCV, but "
FDERROR << "LimitByStride: Only support interp_ be 0/1/2 with FlyCV, but "
"now it's "
<< interp_ << "." << std::endl;
return false;
}
fcv::Mat new_im;
FDERROR << "Before " << im->width() << " " << im->height() << std::endl;
fcv::resize(*im, new_im, fcv::Size(rw, rh), 0, 0, interp_method);
FDERROR << "After " << new_im.width() << " " << new_im.height() << std::endl;
mat->SetMat(new_im);
mat->SetWidth(new_im.width());
mat->SetHeight(new_im.height());

4
fastdeploy/vision/common/processors/limit_by_stride.h
View File

@@ -29,12 +29,12 @@ class FASTDEPLOY_DECL LimitByStride : public Processor {
// Resize Mat* mat to make the size divisible by stride_.
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "LimitByStride"; }
static bool Run(Mat* mat, int stride = 32, int interp = 1,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int interp_;

12
fastdeploy/vision/common/processors/limit_long.cc
View File

@@ -39,8 +39,8 @@ bool LimitLong::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool LimitLong::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool LimitLong::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int origin_w = im->width();
int origin_h = im->height();
int im_size_max = std::max(origin_w, origin_h);
@@ -64,16 +64,13 @@ bool LimitLong::ImplByFalconCV(Mat* mat) {
} else if (interp_ == 2) {
interp_method = fcv::InterpolationType::INTER_CUBIC;
} else {
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but "
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but "
"now it's "
<< interp_ << "." << std::endl;
return false;
}
fcv::Mat new_im;
FDERROR << "origin " << im->width() << " " << im->height() << std::endl;
FDERROR << "scale " << scale << std::endl;
fcv::resize(*im, new_im, fcv::Size(), scale, scale, interp_method);
FDERROR << "after " << new_im.width() << " " << new_im.height() << std::endl;
mat->SetMat(new_im);
mat->SetWidth(new_im.width());
mat->SetHeight(new_im.height());
@@ -82,7 +79,8 @@ bool LimitLong::ImplByFalconCV(Mat* mat) {
}
#endif
bool LimitLong::Run(Mat* mat, int max_long, int min_long, int interp, ProcLib lib) {
bool LimitLong::Run(Mat* mat, int max_long, int min_long, int interp,
ProcLib lib) {
auto l = LimitLong(max_long, min_long, interp);
return l(mat, lib);
}

4
fastdeploy/vision/common/processors/limit_long.h
View File

@@ -34,12 +34,12 @@ class FASTDEPLOY_DECL LimitLong : public Processor {
// to min_long_, while scale the short edge proportionally.
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "LimitLong"; }
static bool Run(Mat* mat, int max_long = -1, int min_long = -1,
int interp = 1, ProcLib lib = ProcLib::OPENCV);
int interp = 1, ProcLib lib = ProcLib::DEFAULT);
int GetMaxLong() const { return max_long_; }
private:

9
fastdeploy/vision/common/processors/limit_short.cc
View File

@@ -41,8 +41,8 @@ bool LimitShort::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool LimitShort::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool LimitShort::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int origin_w = im->width();
int origin_h = im->height();
int im_size_min = std::min(origin_w, origin_h);
@@ -65,7 +65,7 @@ bool LimitShort::ImplByFalconCV(Mat* mat) {
} else if (interp_ == 2) {
interp_method = fcv::InterpolationType::INTER_CUBIC;
} else {
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but "
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but "
"now it's "
<< interp_ << "." << std::endl;
return false;
@@ -81,7 +81,8 @@ bool LimitShort::ImplByFalconCV(Mat* mat) {
}
#endif
bool LimitShort::Run(Mat* mat, int max_short, int min_short, int interp, ProcLib lib) {
bool LimitShort::Run(Mat* mat, int max_short, int min_short, int interp,
ProcLib lib) {
auto l = LimitShort(max_short, min_short, interp);
return l(mat, lib);
}

4
fastdeploy/vision/common/processors/limit_short.h
View File

@@ -34,12 +34,12 @@ class LimitShort : public Processor {
// to min_short_, while scale the long edge proportionally.
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "LimitShort"; }
static bool Run(Mat* mat, int max_short = -1, int min_short = -1,
int interp = 1, ProcLib lib = ProcLib::OPENCV);
int interp = 1, ProcLib lib = ProcLib::DEFAULT);
int GetMaxShort() const { return max_short_; }
private:

76
fastdeploy/vision/common/processors/mat.cc
View File

@@ -12,8 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/common/processors/mat.h"
#include "fastdeploy/vision/common/processors/utils.h"
#include "fastdeploy/utils/utils.h"
#include "fastdeploy/vision/common/processors/utils.h"
namespace fastdeploy {
namespace vision {
@@ -23,15 +24,16 @@ void* Mat::Data() {
#ifdef ENABLE_FLYCV
return fcv_mat.data();
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
FDASSERT(false,
"FastDeploy didn't compile with FlyCV, but met data type with "
"fcv::Mat.");
#endif
}
return cpu_mat.ptr();
}
void Mat::ShareWithTensor(FDTensor* tensor) {
tensor->SetExternalData({Channels(), Height(), Width()}, Type(),
Data());
tensor->SetExternalData({Channels(), Height(), Width()}, Type(), Data());
tensor->device = Device::CPU;
if (layout == Layout::HWC) {
tensor->shape = {Height(), Width(), Channels()};
@@ -66,7 +68,9 @@ void Mat::PrintInfo(const std::string& flag) {
}
std::cout << std::endl;
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
FDASSERT(false,
"FastDeploy didn't compile with FlyCV, but met data type with "
"fcv::Mat.");
#endif
} else {
cv::Scalar mean = cv::mean(cpu_mat);
@@ -87,66 +91,17 @@ FDDataType Mat::Type() {
int type = -1;
if (mat_type == ProcLib::FLYCV) {
#ifdef ENABLE_FLYCV
return FalconCVDataTypeToFD(fcv_mat.type());
return FlyCVDataTypeToFD(fcv_mat.type());
#else
FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat.");
FDASSERT(false,
"FastDeploy didn't compile with FlyCV, but met data type with "
"fcv::Mat.");
#endif
}
return OpenCVDataTypeToFD(cpu_mat.type());
}
Mat CreateFromTensor(const FDTensor& tensor) {
int type = tensor.dtype;
cv::Mat temp_mat;
FDASSERT(tensor.shape.size() == 3,
"When create FD Mat from tensor, tensor shape should be 3-Dim, HWC "
"layout");
int64_t height = tensor.shape[0];
int64_t width = tensor.shape[1];
int64_t channel = tensor.shape[2];
switch (type) {
case FDDataType::UINT8:
temp_mat = cv::Mat(height, width, CV_8UC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT8:
temp_mat = cv::Mat(height, width, CV_8SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT16:
temp_mat = cv::Mat(height, width, CV_16SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::INT32:
temp_mat = cv::Mat(height, width, CV_32SC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP32:
temp_mat = cv::Mat(height, width, CV_32FC(channel),
const_cast<void*>(tensor.Data()));
break;
case FDDataType::FP64:
temp_mat = cv::Mat(height, width, CV_64FC(channel),
const_cast<void*>(tensor.Data()));
break;
default:
FDASSERT(
false,
"Tensor type %d is not supported While calling CreateFromTensor.",
type);
break;
}
Mat mat = Mat(temp_mat);
return mat;
}
std::ostream& operator<<(std::ostream& out,const ProcLib& p) {
std::ostream& operator<<(std::ostream& out, const ProcLib& p) {
switch (p) {
case ProcLib::DEFAULT:
out << "ProcLib::DEFAULT";
@@ -163,8 +118,5 @@ std::ostream& operator<<(std::ostream& out,const ProcLib& p) {
return out;
}
} // namespace vision
} // namespace fastdeploy

73
fastdeploy/vision/common/processors/mat.h
View File

@@ -13,19 +13,19 @@
// limitations under the License.
#pragma once
#include "fastdeploy/core/fd_tensor.h"
#include "opencv2/core/core.hpp"
#include "fastdeploy/vision/common/processors/utils.h"
#include "opencv2/core/core.hpp"
namespace fastdeploy {
namespace vision {
enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV};
enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV };
enum Layout { HWC, CHW };
FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ProcLib& p);
struct FASTDEPLOY_DECL Mat {
explicit Mat(cv::Mat& mat) {
explicit Mat(const cv::Mat& mat) {
cpu_mat = mat;
layout = Layout::HWC;
height = cpu_mat.rows;
@@ -34,7 +34,18 @@ struct FASTDEPLOY_DECL Mat {
mat_type = ProcLib::OPENCV;
}
// careful if you use this interface
#ifdef ENABLE_FLYCV
explicit Mat(const fcv::Mat& mat) {
fcv_mat = mat;
layout = Layout::HWC;
height = fcv_mat.height();
width = fcv_mat.width();
channels = fcv_mat.channels();
mat_type = ProcLib::FLYCV;
}
#endif
// Careful if you use this interface
// this only used if you don't want to write
// the original data, and write to a new cv::Mat
// then replace the old cv::Mat of this structure
@@ -43,15 +54,23 @@ struct FASTDEPLOY_DECL Mat {
mat_type = ProcLib::OPENCV;
}
inline cv::Mat* GetOpenCVMat() {
FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure.");
cv::Mat* GetOpenCVMat() {
if (mat_type == ProcLib::OPENCV) {
return &cpu_mat;
} else if (mat_type == ProcLib::FLYCV) {
#ifdef ENABLE_FLYCV
// Just a reference to fcv_mat, zero copy. After you
// call this method, cpu_mat and fcv_mat will point
// to the same memory buffer.
cpu_mat = ConvertFlyCVMatToOpenCV(fcv_mat);
mat_type = ProcLib::OPENCV;
return &cpu_mat;
#else
FDASSERT(false, "FastDeploy didn't compiled with FlyCV!");
#endif
} else {
FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT");
}
inline const cv::Mat* GetOpenCVMat() const {
FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure.");
return &cpu_mat;
}
#ifdef ENABLE_FLYCV
@@ -60,9 +79,19 @@ struct FASTDEPLOY_DECL Mat {
mat_type = ProcLib::FLYCV;
}
inline fcv::Mat* GetFalconCVMat() {
FDASSERT(mat_type == ProcLib::FLYCV, "Met non fcv::Mat data strucure.");
fcv::Mat* GetFlyCVMat() {
if (mat_type == ProcLib::FLYCV) {
return &fcv_mat;
} else if (mat_type == ProcLib::OPENCV) {
// Just a reference to cpu_mat, zero copy. After you
// call this method, fcv_mat and cpu_mat will point
// to the same memory buffer.
fcv_mat = ConvertOpenCVMatToFlyCV(cpu_mat);
mat_type = ProcLib::FLYCV;
return &fcv_mat;
} else {
FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT");
}
}
#endif
@@ -73,17 +102,11 @@ struct FASTDEPLOY_DECL Mat {
int height;
int width;
cv::Mat cpu_mat;
#ifdef ENABLE_FLYCV
fcv::Mat fcv_mat;
#endif
public:
template<typename T>
T* GetMat() {
return &cpu_mat;
}
FDDataType Type();
int Channels() const { return channels; }
int Width() const { return width; }
@@ -97,18 +120,16 @@ struct FASTDEPLOY_DECL Mat {
// Only support copy to cpu tensor now
bool CopyToTensor(FDTensor* tensor);
// debug functions
// TODO(jiangjiajun) Develop a right process pipeline with c++ is not a easy
// things
// Will add more debug function here to help debug processed image
// This function will print shape / mean of each channels of the Mat
// Debug functions
// TODO(jiangjiajun) Develop a right process pipeline with c++
// is not a easy things, Will add more debug function here to
// help debug processed image. This function will print shape
// and mean of each channels of the Mat
void PrintInfo(const std::string& flag);
ProcLib mat_type = ProcLib::OPENCV;
Layout layout = Layout::HWC;
};
Mat CreateFromTensor(const FDTensor& tensor);
} // namespace vision
} // namespace fastdeploy

14
fastdeploy/vision/common/processors/normalize.cc
View File

@@ -65,10 +65,11 @@ bool Normalize::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Normalize::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool Normalize::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
if (im->channels() != 3) {
FDERROR << "Only supports 3-channels image in FalconCV, but now it's " << im->channels() << "." << std::endl;
FDERROR << "Only supports 3-channels image in FlyCV, but now it's "
<< im->channels() << "." << std::endl;
return false;
}
@@ -78,14 +79,15 @@ bool Normalize::ImplByFalconCV(Mat* mat) {
std[i] = 1.0 / alpha_[i];
mean[i] = -1 * beta_[i] * std[i];
}
fcv::Mat new_im(im->width(), im->height(), fcv::FCVImageType::PACKAGE_BGR_F32);
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(), new_im, true);
fcv::Mat new_im(im->width(), im->height(),
fcv::FCVImageType::PACKAGE_BGR_F32);
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(),
new_im, true);
mat->SetMat(new_im);
return true;
}
#endif
bool Normalize::Run(Mat* mat, const std::vector<float>& mean,
const std::vector<float>& std, bool is_scale,
const std::vector<float>& min,

12
fastdeploy/vision/common/processors/normalize.h
View File

@@ -26,7 +26,7 @@ class FASTDEPLOY_DECL Normalize : public Processor {
const std::vector<float>& max = std::vector<float>());
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Normalize"; }
@@ -44,14 +44,10 @@ class FASTDEPLOY_DECL Normalize : public Processor {
const std::vector<float>& std, bool is_scale = true,
const std::vector<float>& min = std::vector<float>(),
const std::vector<float>& max = std::vector<float>(),
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
std::vector<float> GetAlpha() const {
return alpha_;
}
std::vector<float> GetBeta() const {
return beta_;
}
std::vector<float> GetAlpha() const { return alpha_; }
std::vector<float> GetBeta() const { return beta_; }
private:
std::vector<float> alpha_;

20
fastdeploy/vision/common/processors/normalize_and_permute.cc
View File

@@ -17,9 +17,9 @@
namespace fastdeploy {
namespace vision {
NormalizeAndPermute::NormalizeAndPermute(const std::vector<float>& mean,
const std::vector<float>& std, bool is_scale,
const std::vector<float>& std,
bool is_scale,
const std::vector<float>& min,
const std::vector<float>& max) {
FDASSERT(mean.size() == std.size(),
@@ -65,7 +65,10 @@ bool NormalizeAndPermute::ImplByOpenCV(Mat* mat) {
}
cv::Mat res(origin_h, origin_w, CV_32FC(im->channels()));
for (int i = 0; i < im->channels(); ++i) {
cv::extractChannel(split_im[i], cv::Mat(origin_h, origin_w, CV_32FC1, res.ptr() + i * origin_h * origin_w * 4), 0);
cv::extractChannel(split_im[i],
cv::Mat(origin_h, origin_w, CV_32FC1,
res.ptr() + i * origin_h * origin_w * 4),
0);
}
mat->SetMat(res);
@@ -74,14 +77,15 @@ bool NormalizeAndPermute::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool NormalizeAndPermute::ImplByFalconCV(Mat* mat) {
bool NormalizeAndPermute::ImplByFlyCV(Mat* mat) {
if (mat->layout != Layout::HWC) {
FDERROR << "Only supports input with HWC layout." << std::endl;
return false;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
if (im->channels() != 3) {
FDERROR << "Only supports 3-channels image in FalconCV, but now it's " << im->channels() << "." << std::endl;
FDERROR << "Only supports 3-channels image in FlyCV, but now it's "
<< im->channels() << "." << std::endl;
return false;
}
std::vector<float> mean(3, 0);
@@ -91,14 +95,14 @@ bool NormalizeAndPermute::ImplByFalconCV(Mat* mat) {
mean[i] = -1 * beta_[i] * std[i];
}
fcv::Mat new_im;
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(), new_im, false);
fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector<uint32_t>(),
new_im, false);
mat->SetMat(new_im);
mat->layout = Layout::CHW;
return true;
}
#endif
bool NormalizeAndPermute::Run(Mat* mat, const std::vector<float>& mean,
const std::vector<float>& std, bool is_scale,
const std::vector<float>& min,

7
fastdeploy/vision/common/processors/normalize_and_permute.h
View File

@@ -21,13 +21,12 @@ namespace vision {
class FASTDEPLOY_DECL NormalizeAndPermute : public Processor {
public:
NormalizeAndPermute(const std::vector<float>& mean,
const std::vector<float>& std,
bool is_scale = true,
const std::vector<float>& std, bool is_scale = true,
const std::vector<float>& min = std::vector<float>(),
const std::vector<float>& max = std::vector<float>());
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "NormalizeAndPermute"; }
@@ -45,7 +44,7 @@ class FASTDEPLOY_DECL NormalizeAndPermute : public Processor {
const std::vector<float>& std, bool is_scale = true,
const std::vector<float>& min = std::vector<float>(),
const std::vector<float>& max = std::vector<float>(),
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
void SetAlpha(const std::vector<float>& alpha) {
alpha_.clear();

7
fastdeploy/vision/common/processors/pad.cc
View File

@@ -53,7 +53,7 @@ bool Pad::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Pad::ImplByFalconCV(Mat* mat) {
bool Pad::ImplByFlyCV(Mat* mat) {
if (mat->layout != Layout::HWC) {
FDERROR << "Pad: The input data must be Layout::HWC format!" << std::endl;
return false;
@@ -70,7 +70,7 @@ bool Pad::ImplByFalconCV(Mat* mat) {
<< std::endl;
return false;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
fcv::Scalar value;
if (value_.size() == 1) {
value = fcv::Scalar(value_[0]);
@@ -92,8 +92,7 @@ bool Pad::ImplByFalconCV(Mat* mat) {
#endif
bool Pad::Run(Mat* mat, const int& top, const int& bottom, const int& left,
const int& right, const std::vector<float>& value,
ProcLib lib) {
const int& right, const std::vector<float>& value, ProcLib lib) {
auto p = Pad(top, bottom, left, right, value);
return p(mat, lib);
}

4
fastdeploy/vision/common/processors/pad.h
View File

@@ -31,13 +31,13 @@ class FASTDEPLOY_DECL Pad : public Processor {
}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Pad"; }
static bool Run(Mat* mat, const int& top, const int& bottom, const int& left,
const int& right, const std::vector<float>& value,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int top_;

9
fastdeploy/vision/common/processors/pad_to_size.cc
View File

@@ -76,7 +76,7 @@ bool PadToSize::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool PadToSize::ImplByFalconCV(Mat* mat) {
bool PadToSize::ImplByFlyCV(Mat* mat) {
if (width_ == -1 || height_ == -1) {
return true;
}
@@ -115,7 +115,7 @@ bool PadToSize::ImplByFalconCV(Mat* mat) {
return true;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
fcv::Scalar value;
if (value_.size() == 1) {
value = fcv::Scalar(value_[0]);
@@ -128,8 +128,9 @@ bool PadToSize::ImplByFalconCV(Mat* mat) {
}
fcv::Mat new_im;
// top, bottom, left, right
fcv::copy_make_border(*im, new_im, 0, height_ - origin_h, 0, width_ - origin_w,
fcv::BorderTypes::BORDER_CONSTANT, value);
fcv::copy_make_border(*im, new_im, 0, height_ - origin_h, 0,
width_ - origin_w, fcv::BorderTypes::BORDER_CONSTANT,
value);
mat->SetMat(new_im);
mat->SetHeight(height_);
mat->SetWidth(width_);

4
fastdeploy/vision/common/processors/pad_to_size.h
View File

@@ -29,13 +29,13 @@ class FASTDEPLOY_DECL PadToSize : public Processor {
}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "PadToSize"; }
static bool Run(Mat* mat, int width, int height,
const std::vector<float>& value,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int width_;

9
fastdeploy/vision/common/processors/resize.cc
View File

@@ -55,12 +55,12 @@ bool Resize::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool Resize::ImplByFalconCV(Mat* mat) {
bool Resize::ImplByFlyCV(Mat* mat) {
if (mat->layout != Layout::HWC) {
FDERROR << "Resize: The format of input is not HWC." << std::endl;
return false;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
int origin_w = im->width();
int origin_h = im->height();
@@ -79,7 +79,7 @@ bool Resize::ImplByFalconCV(Mat* mat) {
} else if (interp_ == 2) {
interp_method = fcv::InterpolationType::INTER_CUBIC;
} else {
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but "
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but "
"now it's "
<< interp_ << "." << std::endl;
return false;
@@ -99,7 +99,8 @@ bool Resize::ImplByFalconCV(Mat* mat) {
mat->SetHeight(new_im.height());
} else if (scale_w_ > 0 && scale_h_ > 0) {
fcv::Mat new_im;
fcv::resize(*im, new_im, fcv::Size(0, 0), scale_w_, scale_h_, interp_method);
fcv::resize(*im, new_im, fcv::Size(0, 0), scale_w_, scale_h_,
interp_method);
mat->SetMat(new_im);
mat->SetWidth(new_im.width());
mat->SetHeight(new_im.height());

4
fastdeploy/vision/common/processors/resize.h
View File

@@ -33,13 +33,13 @@ class FASTDEPLOY_DECL Resize : public Processor {
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "Resize"; }
static bool Run(Mat* mat, int width, int height, float scale_w = -1.0,
float scale_h = -1.0, int interp = 1, bool use_scale = false,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
bool SetWidthAndHeight(int width, int height) {
width_ = width;

2
fastdeploy/vision/common/processors/resize_by_long.h
View File

@@ -33,7 +33,7 @@ class ResizeByLong : public Processor {
static bool Run(Mat* mat, int target_size, int interp = 1,
bool use_scale = true, int max_size = -1,
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
double GenerateScale(const int origin_w, const int origin_h);

6
fastdeploy/vision/common/processors/resize_by_short.cc
View File

@@ -37,8 +37,8 @@ bool ResizeByShort::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool ResizeByShort::ImplByFalconCV(Mat* mat) {
fcv::Mat* im = mat->GetFalconCVMat();
bool ResizeByShort::ImplByFlyCV(Mat* mat) {
fcv::Mat* im = mat->GetFlyCVMat();
int origin_w = im->width();
int origin_h = im->height();
double scale = GenerateScale(origin_w, origin_h);
@@ -51,7 +51,7 @@ bool ResizeByShort::ImplByFalconCV(Mat* mat) {
} else if (interp_ == 2) {
interp_method = fcv::InterpolationType::INTER_CUBIC;
} else {
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but "
FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but "
"now it's "
<< interp_ << "." << std::endl;
return false;

4
fastdeploy/vision/common/processors/resize_by_short.h
View File

@@ -30,14 +30,14 @@ class FASTDEPLOY_DECL ResizeByShort : public Processor {
}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "ResizeByShort"; }
static bool Run(Mat* mat, int target_size, int interp = 1,
bool use_scale = true,
const std::vector<int>& max_hw = std::vector<int>(),
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
double GenerateScale(const int origin_w, const int origin_h);

7
fastdeploy/vision/common/processors/stride_pad.cc
View File

@@ -64,7 +64,7 @@ bool StridePad::ImplByOpenCV(Mat* mat) {
}
#ifdef ENABLE_FLYCV
bool StridePad::ImplByFalconCV(Mat* mat) {
bool StridePad::ImplByFlyCV(Mat* mat) {
if (mat->layout != Layout::HWC) {
FDERROR << "StridePad: The input data must be Layout::HWC format!"
<< std::endl;
@@ -92,7 +92,7 @@ bool StridePad::ImplByFalconCV(Mat* mat) {
if (pad_h == 0 && pad_w == 0) {
return true;
}
fcv::Mat* im = mat->GetFalconCVMat();
fcv::Mat* im = mat->GetFlyCVMat();
fcv::Scalar value;
if (value_.size() == 1) {
value = fcv::Scalar(value_[0]);
@@ -105,7 +105,8 @@ bool StridePad::ImplByFalconCV(Mat* mat) {
}
fcv::Mat new_im;
// top, bottom, left, right
fcv::copy_make_border(*im, new_im, 0, pad_h, 0, pad_w, fcv::BorderTypes::BORDER_CONSTANT, value);
fcv::copy_make_border(*im, new_im, 0, pad_h, 0, pad_w,
fcv::BorderTypes::BORDER_CONSTANT, value);
mat->SetMat(new_im);
mat->SetHeight(new_im.height());
mat->SetWidth(new_im.width());

4
fastdeploy/vision/common/processors/stride_pad.h
View File

@@ -28,13 +28,13 @@ class FASTDEPLOY_DECL StridePad : public Processor {
}
bool ImplByOpenCV(Mat* mat);
#ifdef ENABLE_FLYCV
bool ImplByFalconCV(Mat* mat);
bool ImplByFlyCV(Mat* mat);
#endif
std::string Name() { return "StridePad"; }
static bool Run(Mat* mat, int stride,
const std::vector<float>& value = std::vector<float>(),
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
int stride_ = 32;

34
fastdeploy/vision/common/processors/transform.h
View File

@@ -20,24 +20,25 @@
#include "fastdeploy/vision/common/processors/convert.h"
#include "fastdeploy/vision/common/processors/crop.h"
#include "fastdeploy/vision/common/processors/hwc2chw.h"
#include "fastdeploy/vision/common/processors/letter_box.h"
#include "fastdeploy/vision/common/processors/limit_by_stride.h"
#include "fastdeploy/vision/common/processors/limit_long.h"
#include "fastdeploy/vision/common/processors/limit_short.h"
#include "fastdeploy/vision/common/processors/normalize.h"
#include "fastdeploy/vision/common/processors/normalize_and_permute.h"
#include "fastdeploy/vision/common/processors/pad.h"
#include "fastdeploy/vision/common/processors/pad_to_size.h"
#include "fastdeploy/vision/common/processors/resize.h"
#include "fastdeploy/vision/common/processors/resize_by_long.h"
#include "fastdeploy/vision/common/processors/resize_by_short.h"
#include "fastdeploy/vision/common/processors/stride_pad.h"
#include "fastdeploy/vision/common/processors/normalize_and_permute.h"
#include "fastdeploy/vision/common/processors/warp_affine.h"
#include "fastdeploy/vision/common/processors/letter_box.h"
namespace fastdeploy {
namespace vision {
inline void FuseNormalizeCast(std::vector<std::shared_ptr<Processor>>* processors) {
inline void FuseNormalizeCast(
std::vector<std::shared_ptr<Processor>>* processors) {
// Fuse Normalize and Cast<Float>
int cast_index = -1;
for (size_t i = 0; i < processors->size(); ++i) {
@@ -45,7 +46,8 @@ inline void FuseNormalizeCast(std::vector<std::shared_ptr<Processor>>* processor
if (i == 0) {
continue;
}
if ((*processors)[i - 1]->Name() != "Normalize" && (*processors)[i - 1]->Name() != "NormalizeAndPermute") {
if ((*processors)[i - 1]->Name() != "Normalize" &&
(*processors)[i - 1]->Name() != "NormalizeAndPermute") {
continue;
}
cast_index = i;
@@ -55,15 +57,18 @@ inline void FuseNormalizeCast(std::vector<std::shared_ptr<Processor>>* processor
return;
}
std::cout << dynamic_cast<Cast*>((*processors)[cast_index].get())->GetDtype() << "-----" << std::endl;
if (dynamic_cast<Cast*>((*processors)[cast_index].get())->GetDtype() != "float") {
if (dynamic_cast<Cast*>((*processors)[cast_index].get())->GetDtype() !=
"float") {
return;
}
processors->erase(processors->begin() + cast_index);
FDINFO << (*processors)[cast_index - 1]->Name() << " and Cast are fused to " << (*processors)[cast_index - 1]->Name() << " in preprocessing pipeline." << std::endl;
FDINFO << (*processors)[cast_index - 1]->Name() << " and Cast are fused to "
<< (*processors)[cast_index - 1]->Name()
<< " in preprocessing pipeline." << std::endl;
}
inline void FuseNormalizeHWC2CHW(std::vector<std::shared_ptr<Processor>>* processors) {
inline void FuseNormalizeHWC2CHW(
std::vector<std::shared_ptr<Processor>>* processors) {
// Fuse Normalize and HWC2CHW to NormalizeAndPermute
int hwc2chw_index = -1;
for (size_t i = 0; i < processors->size(); ++i) {
@@ -83,10 +88,12 @@ inline void FuseNormalizeHWC2CHW(std::vector<std::shared_ptr<Processor>>* proces
}
// Get alpha and beta of Normalize
std::vector<float> alpha = dynamic_cast<Normalize*>(
(*processors)[hwc2chw_index - 1].get())->GetAlpha();
std::vector<float> beta = dynamic_cast<Normalize*>(
(*processors)[hwc2chw_index - 1].get())->GetBeta();
std::vector<float> alpha =
dynamic_cast<Normalize*>((*processors)[hwc2chw_index - 1].get())
->GetAlpha();
std::vector<float> beta =
dynamic_cast<Normalize*>((*processors)[hwc2chw_index - 1].get())
->GetBeta();
// Delete Normalize and HWC2CHW
processors->erase(processors->begin() + hwc2chw_index);
@@ -104,7 +111,8 @@ inline void FuseNormalizeHWC2CHW(std::vector<std::shared_ptr<Processor>>* proces
processor->SetAlpha(alpha);
processor->SetBeta(beta);
FDINFO << "Normalize and HWC2CHW are fused to NormalizeAndPermute "
" in preprocessing pipeline." << std::endl;
" in preprocessing pipeline."
<< std::endl;
}
inline void FuseTransforms(

63
fastdeploy/vision/common/processors/utils.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/utils/utils.h"
#include "fastdeploy/vision/common/processors/utils.h"
namespace fastdeploy {
@@ -37,12 +38,15 @@ FDDataType OpenCVDataTypeToFD(int type) {
} else if (type == 6) {
return FDDataType::FP64;
} else {
FDASSERT(false, "While calling OpenCVDataTypeToFD(), get type = %d, which is not expected.", type);
FDASSERT(false,
"While calling OpenCVDataTypeToFD(), get type = %d, which is not "
"expected.",
type);
}
}
#ifdef ENABLE_FLYCV
FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type) {
FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type) {
if (type == fcv::FCVImageType::GRAY_U8) {
return FDDataType::UINT8;
} else if (type == fcv::FCVImageType::PACKAGE_BGR_U8) {
@@ -104,13 +108,14 @@ FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type) {
} else if (type == fcv::FCVImageType::GRAY_F64) {
return FDDataType::FP64;
}
FDASSERT(false, "While calling FalconDataTypeToFD(), get unexpected type:%d.", int(type));
FDASSERT(false, "While calling FlyCVDataTypeToFD(), get unexpected type:%d.",
int(type));
return FDDataType::UNKNOWN1;
}
fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel) {
fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel) {
FDASSERT(channel == 1 || channel == 3 || channel == 4,
"Only support channel be 1/3/4 in Falcon.");
"Only support channel be 1/3/4 in FlyCV.");
if (type == FDDataType::UINT8) {
if (channel == 1) {
return fcv::FCVImageType::GRAY_U8;
@@ -132,18 +137,54 @@ fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel) {
return fcv::FCVImageType::PACKAGE_BGR_F32;
}
fcv::Mat ConvertOpenCVMatToFalconCV(cv::Mat& im) {
int CreateOpenCVDataType(FDDataType type, int channel) {
FDASSERT(channel == 1 || channel == 3 || channel == 4,
"Only support channel be 1/3/4 in OpenCV.");
if (type == FDDataType::UINT8) {
if (channel == 1) {
return CV_8UC1;
} else if (channel == 3) {
return CV_8UC3;
} else {
return CV_8UC4;
}
} else if (type == FDDataType::FP32) {
if (channel == 1) {
return CV_32FC1;
} else if (channel == 3) {
return CV_32FC3;
} else {
return CV_32FC4;
}
}
FDASSERT(false, "Data type of %s is not supported.", Str(type).c_str());
return CV_32FC3;
}
fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im) {
int type = im.type() % 8;
// 0: uint8; 5: float32; 6: float64
if (type != 0 && type != 5 && type != 6) {
FDASSERT(false, "Only support type of uint8/float/double, but now it's %d.", im.type());
FDASSERT(false, "Only support type of uint8/float/double, but now it's %d.",
im.type());
}
auto fcv_type = CreateFalconCVDataType(OpenCVDataTypeToFD(im.type()), im.channels());
return fcv::Mat(im.cols, im.rows, fcv_type, im.ptr());
auto fcv_type =
CreateFlyCVDataType(OpenCVDataTypeToFD(im.type()), im.channels());
return fcv::Mat(im.cols, im.rows, fcv_type, im.ptr()); // reference only
}
cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim) {
auto fd_dtype = FlyCVDataTypeToFD(fim.type());
if (fd_dtype != FDDataType::UINT8 && fd_dtype != FDDataType::FP32 &&
fd_dtype != FDDataType::FP64) {
FDASSERT(false, "Only support type of uint8/float/double, but now it's %s.",
Str(fd_dtype).c_str());
}
auto ocv_type = CreateOpenCVDataType(fd_dtype, fim.channels());
return cv::Mat(fim.height(), fim.width(), ocv_type,
fim.data()); // reference only
}
#endif
} // namespace vision
} // namespace fastdeploy

12
fastdeploy/vision/common/processors/utils.h
View File

@@ -14,9 +14,9 @@
#pragma once
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/utils/utils.h"
#include "opencv2/core/core.hpp"
#include "fastdeploy/core/fd_tensor.h"
#ifdef ENABLE_FLYCV
#include "flycv.h" // NOLINT
@@ -27,14 +27,18 @@ namespace vision {
// Convert data type of opencv to FDDataType
FDDataType OpenCVDataTypeToFD(int type);
// Create data type of opencv by FDDataType
int CreateOpenCVDataType(FDDataType type, int channel = 1);
#ifdef ENABLE_FLYCV
// Convert data type of flycv to FDDataType
FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type);
FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type);
// Create data type of flycv by FDDataType
fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel = 1);
fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel = 1);
// Convert cv::Mat to fcv::Mat
fcv::Mat ConvertOpenCVMatToFalconCV(cv::Mat& im);
fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im);
// Convert fcv::Mat to fcv::mat
cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim);
#endif
} // namespace vision

15
fastdeploy/vision/common/processors/warp_affine.h
View File

@@ -21,9 +21,7 @@ namespace vision {
class WarpAffine : public Processor {
public:
WarpAffine(const cv::Mat& trans_matrix,
int width, int height,
int interp = 1,
WarpAffine(const cv::Mat& trans_matrix, int width, int height, int interp = 1,
int border_mode = 0,
const cv::Scalar& borderValue = cv::Scalar()) {
trans_matrix_ = trans_matrix;
@@ -37,7 +35,7 @@ class WarpAffine : public Processor {
bool ImplByOpenCV(Mat* mat);
std::string Name() { return "WarpAffine"; }
bool SetTransformMatrix(const cv::Mat &trans_matrix) {
bool SetTransformMatrix(const cv::Mat& trans_matrix) {
trans_matrix_ = trans_matrix;
return true;
}
@@ -46,13 +44,10 @@ class WarpAffine : public Processor {
return std::make_tuple(width_, height_);
}
static bool Run(Mat* mat,
const cv::Mat& trans_matrix,
int width, int height,
int interp = 1,
int border_mode = 0,
static bool Run(Mat* mat, const cv::Mat& trans_matrix, int width, int height,
int interp = 1, int border_mode = 0,
const cv::Scalar& borderValue = cv::Scalar(),
ProcLib lib = ProcLib::OPENCV);
ProcLib lib = ProcLib::DEFAULT);
private:
cv::Mat trans_matrix_;

45
fastdeploy/vision/detection/contrib/yolov5.cc
View File

@@ -66,7 +66,7 @@ YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file,
valid_cpu_backends = {Backend::OPENVINO, Backend::ORT};
valid_gpu_backends = {Backend::ORT, Backend::TRT};
} else {
valid_cpu_backends = {Backend::PDINFER, Backend::ORT};
valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE};
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
}
runtime_option = custom_option;
@@ -185,10 +185,13 @@ void YOLOv5::UseCudaPreprocessing(int max_image_size) {
is_scale_up_ = true;
if (input_img_cuda_buffer_host_ == nullptr) {
// prepare input data cache in GPU pinned memory
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3));
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_,
max_image_size * 3));
// prepare input data cache in GPU device memory
CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size_[0] * size_[1] * sizeof(float)));
CUDA_CHECK(
cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_,
3 * size_[0] * size_[1] * sizeof(float)));
}
#else
FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON."
@@ -197,15 +200,17 @@ void YOLOv5::UseCudaPreprocessing(int max_image_size) {
#endif
}
bool YOLOv5::CudaPreprocess(Mat* mat, FDTensor* output,
bool YOLOv5::CudaPreprocess(
Mat* mat, FDTensor* output,
std::map<std::string, std::array<float, 2>>* im_info,
const std::vector<int>& size,
const std::vector<float> padding_value,
bool is_mini_pad, bool is_no_pad, bool is_scale_up,
int stride, float max_wh, bool multi_label) {
const std::vector<int>& size, const std::vector<float> padding_value,
bool is_mini_pad, bool is_no_pad, bool is_scale_up, int stride,
float max_wh, bool multi_label) {
#ifdef ENABLE_CUDA_PREPROCESS
if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) {
FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl;
FDERROR << "Preprocessing with CUDA is only available when the arguments "
"satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)."
<< std::endl;
return false;
}
@@ -219,14 +224,15 @@ bool YOLOv5::CudaPreprocess(Mat* mat, FDTensor* output,
int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels();
memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size);
CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_,
input_img_cuda_buffer_host_,
src_img_buf_size, cudaMemcpyHostToDevice, stream));
input_img_cuda_buffer_host_, src_img_buf_size,
cudaMemcpyHostToDevice, stream));
utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(),
mat->Height(), input_tensor_cuda_buffer_device_,
size[0], size[1], padding_value, stream);
// Record output shape of preprocessed image
(*im_info)["output_shape"] = {static_cast<float>(size[0]), static_cast<float>(size[1])};
(*im_info)["output_shape"] = {static_cast<float>(size[0]),
static_cast<float>(size[1])};
output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32,
input_tensor_cuda_buffer_device_);
@@ -339,22 +345,21 @@ bool YOLOv5::Postprocess(
bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold,
float nms_iou_threshold) {
Mat mat(*im);
std::map<std::string, std::array<float, 2>> im_info;
if (use_cuda_preprocessing_) {
if (!CudaPreprocess(&mat, &reused_input_tensors[0], &im_info, size_, padding_value_,
is_mini_pad_, is_no_pad_, is_scale_up_, stride_, max_wh_,
multi_label_)) {
if (!CudaPreprocess(&mat, &reused_input_tensors[0], &im_info, size_,
padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_,
stride_, max_wh_, multi_label_)) {
FDERROR << "Failed to preprocess input image." << std::endl;
return false;
}
} else {
if (!Preprocess(&mat, &reused_input_tensors[0], &im_info, size_, padding_value_,
is_mini_pad_, is_no_pad_, is_scale_up_, stride_, max_wh_,
multi_label_)) {
if (!Preprocess(&mat, &reused_input_tensors[0], &im_info, size_,
padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_,
stride_, max_wh_, multi_label_)) {
FDERROR << "Failed to preprocess input image." << std::endl;
return false;
}

26
fastdeploy/vision/detection/contrib/yolov6.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/detection/contrib/yolov6.h"
#include "fastdeploy/utils/perf.h"
#include "fastdeploy/vision/utils/utils.h"
#ifdef ENABLE_CUDA_PREPROCESS
@@ -69,7 +70,7 @@ YOLOv6::YOLOv6(const std::string& model_file, const std::string& params_file,
valid_cpu_backends = {Backend::OPENVINO, Backend::ORT};
valid_gpu_backends = {Backend::ORT, Backend::TRT};
} else {
valid_cpu_backends = {Backend::PDINFER, Backend::ORT};
valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE};
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
}
runtime_option = custom_option;
@@ -175,10 +176,13 @@ void YOLOv6::UseCudaPreprocessing(int max_image_size) {
is_scale_up = true;
if (input_img_cuda_buffer_host_ == nullptr) {
// prepare input data cache in GPU pinned memory
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3));
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_,
max_image_size * 3));
// prepare input data cache in GPU device memory
CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size[0] * size[1] * sizeof(float)));
CUDA_CHECK(
cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_,
3 * size[0] * size[1] * sizeof(float)));
}
#else
FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON."
@@ -187,11 +191,14 @@ void YOLOv6::UseCudaPreprocessing(int max_image_size) {
#endif
}
bool YOLOv6::CudaPreprocess(Mat* mat, FDTensor* output,
bool YOLOv6::CudaPreprocess(
Mat* mat, FDTensor* output,
std::map<std::string, std::array<float, 2>>* im_info) {
#ifdef ENABLE_CUDA_PREPROCESS
if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) {
FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl;
FDERROR << "Preprocessing with CUDA is only available when the arguments "
"satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)."
<< std::endl;
return false;
}
@@ -205,14 +212,15 @@ bool YOLOv6::CudaPreprocess(Mat* mat, FDTensor* output,
int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels();
memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size);
CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_,
input_img_cuda_buffer_host_,
src_img_buf_size, cudaMemcpyHostToDevice, stream));
input_img_cuda_buffer_host_, src_img_buf_size,
cudaMemcpyHostToDevice, stream));
utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(),
mat->Height(), input_tensor_cuda_buffer_device_,
size[0], size[1], padding_value, stream);
// Record output shape of preprocessed image
(*im_info)["output_shape"] = {static_cast<float>(size[0]), static_cast<float>(size[1])};
(*im_info)["output_shape"] = {static_cast<float>(size[0]),
static_cast<float>(size[1])};
output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32,
input_tensor_cuda_buffer_device_);

26
fastdeploy/vision/detection/contrib/yolov7.cc
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/detection/contrib/yolov7.h"
#include "fastdeploy/utils/perf.h"
#include "fastdeploy/vision/utils/utils.h"
#ifdef ENABLE_CUDA_PREPROCESS
@@ -67,7 +68,7 @@ YOLOv7::YOLOv7(const std::string& model_file, const std::string& params_file,
valid_cpu_backends = {Backend::OPENVINO, Backend::ORT};
valid_gpu_backends = {Backend::ORT, Backend::TRT};
} else {
valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE};
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
}
runtime_option = custom_option;
@@ -173,10 +174,13 @@ void YOLOv7::UseCudaPreprocessing(int max_image_size) {
is_scale_up = true;
if (input_img_cuda_buffer_host_ == nullptr) {
// prepare input data cache in GPU pinned memory
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3));
CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_,
max_image_size * 3));
// prepare input data cache in GPU device memory
CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size[0] * size[1] * sizeof(float)));
CUDA_CHECK(
cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3));
CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_,
3 * size[0] * size[1] * sizeof(float)));
}
#else
FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON."
@@ -185,11 +189,14 @@ void YOLOv7::UseCudaPreprocessing(int max_image_size) {
#endif
}
bool YOLOv7::CudaPreprocess(Mat* mat, FDTensor* output,
bool YOLOv7::CudaPreprocess(
Mat* mat, FDTensor* output,
std::map<std::string, std::array<float, 2>>* im_info) {
#ifdef ENABLE_CUDA_PREPROCESS
if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) {
FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl;
FDERROR << "Preprocessing with CUDA is only available when the arguments "
"satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)."
<< std::endl;
return false;
}
@@ -203,14 +210,15 @@ bool YOLOv7::CudaPreprocess(Mat* mat, FDTensor* output,
int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels();
memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size);
CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_,
input_img_cuda_buffer_host_,
src_img_buf_size, cudaMemcpyHostToDevice, stream));
input_img_cuda_buffer_host_, src_img_buf_size,
cudaMemcpyHostToDevice, stream));
utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(),
mat->Height(), input_tensor_cuda_buffer_device_,
size[0], size[1], padding_value, stream);
// Record output shape of preprocessed image
(*im_info)["output_shape"] = {static_cast<float>(size[0]), static_cast<float>(size[1])};
(*im_info)["output_shape"] = {static_cast<float>(size[0]),
static_cast<float>(size[1])};
output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32,
input_tensor_cuda_buffer_device_);

3
fastdeploy/vision/matting/contrib/modnet.cc Executable file → Normal file
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/matting/contrib/modnet.h"
#include "fastdeploy/utils/perf.h"
#include "fastdeploy/vision/utils/utils.h"
@@ -118,7 +119,7 @@ bool MODNet::Postprocess(
int numel = ipt_h * ipt_w;
int nbytes = numel * sizeof(float);
result->Resize(numel);
std::memcpy(result->alpha.data(), alpha_resized.GetOpenCVMat()->data, nbytes);
std::memcpy(result->alpha.data(), alpha_resized.Data(), nbytes);
return true;
}

17
fastdeploy/vision/matting/contrib/rvm.cc Executable file → Normal file
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/matting/contrib/rvm.h"
#include "fastdeploy/utils/perf.h"
#include "fastdeploy/vision/utils/utils.h"
@@ -22,7 +23,8 @@ namespace vision {
namespace matting {
RobustVideoMatting::RobustVideoMatting(const std::string& model_file, const std::string& params_file,
RobustVideoMatting::RobustVideoMatting(const std::string& model_file,
const std::string& params_file,
const RuntimeOption& custom_option,
const ModelFormat& model_format) {
if (model_format == ModelFormat::ONNX) {
@@ -52,7 +54,8 @@ bool RobustVideoMatting::Initialize() {
return true;
}
bool RobustVideoMatting::Preprocess(Mat* mat, FDTensor* output,
bool RobustVideoMatting::Preprocess(
Mat* mat, FDTensor* output,
std::map<std::string, std::array<int, 2>>* im_info) {
// Resize
int resize_w = size[0];
@@ -98,7 +101,7 @@ bool RobustVideoMatting::Postprocess(
// update context
if (video_mode) {
for (size_t i = 0; i < 4; ++i) {
FDTensor& rki = infer_result.at(i+2);
FDTensor& rki = infer_result.at(i + 2);
dynamic_inputs_dims_[i] = rki.shape;
dynamic_inputs_datas_[i].resize(rki.Numel());
memcpy(dynamic_inputs_datas_[i].data(), rki.Data(),
@@ -139,8 +142,8 @@ bool RobustVideoMatting::Postprocess(
int numel = in_h * in_w;
int nbytes = numel * sizeof(float);
result->Resize(numel);
memcpy(result->alpha.data(), alpha_resized.GetOpenCVMat()->data, nbytes);
memcpy(result->foreground.data(), fgr_resized.GetOpenCVMat()->data, nbytes);
memcpy(result->alpha.data(), alpha_resized.Data(), nbytes);
memcpy(result->foreground.data(), fgr_resized.Data(), nbytes);
return true;
}
@@ -154,7 +157,9 @@ bool RobustVideoMatting::Predict(cv::Mat* im, MattingResult* result) {
im_info["output_shape"] = {mat.Height(), mat.Width()};
// convert vector to FDTensor
for (size_t i = 1; i < inputs_nums; ++i) {
input_tensors[i].SetExternalData(dynamic_inputs_dims_[i-1], FDDataType::FP32, dynamic_inputs_datas_[i-1].data());
input_tensors[i].SetExternalData(dynamic_inputs_dims_[i - 1],
FDDataType::FP32,
dynamic_inputs_datas_[i - 1].data());
input_tensors[i].device = Device::CPU;
}
if (!Preprocess(&mat, &input_tensors[0], &im_info)) {

10
fastdeploy/vision/matting/ppmatting/ppmatting.cc Executable file → Normal file
View File

@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/matting/ppmatting/ppmatting.h"
#include "fastdeploy/vision/utils/utils.h"
#include "yaml-cpp/yaml.h"
@@ -171,11 +172,11 @@ bool PPMatting::Postprocess(
std::vector<int64_t> dim{0, 2, 3, 1};
Transpose(alpha_tensor, &alpha_tensor, dim);
alpha_tensor.Squeeze(0);
Mat mat = CreateFromTensor(alpha_tensor);
Mat mat = CreateFDMatFromTensor(alpha_tensor);
auto iter_ipt = im_info.find("input_shape");
auto iter_out = im_info.find("output_shape");
if (is_fixed_input_shape_){
if (is_fixed_input_shape_) {
double scale_h = static_cast<double>(iter_out->second[0]) /
static_cast<double>(iter_ipt->second[0]);
double scale_w = static_cast<double>(iter_out->second[1]) /
@@ -188,7 +189,8 @@ bool PPMatting::Postprocess(
Crop::Run(&mat, 0, 0, size_before_pad_w, size_before_pad_h);
}
Resize::Run(&mat, iter_ipt->second[1], iter_ipt->second[0]);
Resize::Run(&mat, iter_ipt->second[1], iter_ipt->second[0], -1.0f, -1.0f, 1,
false, ProcLib::OPENCV);
result->Clear();
// note: must be setup shape before Resize
@@ -197,7 +199,7 @@ bool PPMatting::Postprocess(
int numel = iter_ipt->second[0] * iter_ipt->second[1];
int nbytes = numel * sizeof(float);
result->Resize(numel);
std::memcpy(result->alpha.data(), mat.GetOpenCVMat()->data, nbytes);
std::memcpy(result->alpha.data(), mat.Data(), nbytes);
return true;
}

25
fastdeploy/vision/segmentation/ppseg/model.cc Executable file → Normal file
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#include "fastdeploy/vision/segmentation/ppseg/model.h"
#include "fastdeploy/vision/utils/utils.h"
#include "yaml-cpp/yaml.h"
@@ -26,7 +27,8 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file,
const RuntimeOption& custom_option,
const ModelFormat& model_format) {
config_file_ = config_file;
valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE};
valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT,
Backend::LITE};
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
valid_rknpu_backends = {Backend::RKNPU2};
runtime_option = custom_option;
@@ -68,7 +70,7 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() {
FDASSERT(op.IsMap(),
"Require the transform information in yaml be Map type.");
if (op["type"].as<std::string>() == "Normalize") {
if(!(this->disable_normalize_and_permute)){
if (!(this->disable_normalize_and_permute)) {
std::vector<float> mean = {0.5, 0.5, 0.5};
std::vector<float> std = {0.5, 0.5, 0.5};
if (op["mean"]) {
@@ -105,9 +107,10 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() {
<< "which is not supported by ONNX Runtime and Tensorrt. "
<< "Only OpenVINO and Paddle Inference are available now. "
<< "For using ONNX Runtime or Tensorrt, "
<< "Please refer to https://github.com/PaddlePaddle/PaddleSeg/blob/develop/docs/model_export.md"
<< " to export model with fixed input shape."
<< std::endl;
<< "Please refer to "
"https://github.com/PaddlePaddle/PaddleSeg/blob/develop/"
"docs/model_export.md"
<< " to export model with fixed input shape." << std::endl;
valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE};
valid_gpu_backends = {Backend::PDINFER};
}
@@ -132,7 +135,7 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() {
<< "." << std::endl;
}
}
if(!(this->disable_normalize_and_permute)){
if (!(this->disable_normalize_and_permute)) {
processors_.push_back(std::make_shared<HWC2CHW>());
}
return true;
@@ -260,7 +263,7 @@ bool PaddleSegModel::Postprocess(
infer_result->shape, FDDataType::FP32,
static_cast<void*>(fp32_result_buffer->data()));
}
mat = new Mat(CreateFromTensor(*infer_result));
mat = new Mat(CreateFDMatFromTensor(*infer_result));
Resize::Run(mat, ipt_w, ipt_h, -1.0f, -1.0f, 1);
mat->ShareWithTensor(&new_infer_result);
result->shape = new_infer_result.shape;
@@ -361,11 +364,13 @@ bool PaddleSegModel::Predict(cv::Mat* im, SegmentationResult* result) {
return true;
}
void PaddleSegModel::DisableNormalizeAndPermute(){
void PaddleSegModel::DisableNormalizeAndPermute() {
this->disable_normalize_and_permute = true;
// the DisableNormalizeAndPermute function will be invalid if the configuration file is loaded during preprocessing
// the DisableNormalizeAndPermute function will be invalid if the
// configuration file is loaded during preprocessing
if (!BuildPreprocessPipelineFromConfig()) {
FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl;
FDERROR << "Failed to build preprocess pipeline from configuration file."
<< std::endl;
}
}

3
fastdeploy/vision/tracking/pptracking/trajectory.h
View File

@@ -75,7 +75,8 @@ class FASTDEPLOY_DECL Trajectory : public TKalmanFilter {
int timestamp,
bool update_embedding = true);
virtual void activate(int &cnt, int timestamp);
virtual void reactivate(Trajectory *traj, int &cnt, int timestamp, bool newid = false);
virtual void reactivate(Trajectory *traj, int &cnt,
int timestamp, bool newid = false);
virtual void mark_lost(void);
virtual void mark_removed(void);

6
fastdeploy/vision/utils/crop_image.cc
View File

@@ -18,7 +18,7 @@ namespace fastdeploy {
namespace vision {
namespace utils {
bool CropImageByBox(const Mat& src_im, Mat* dst_im,
bool CropImageByBox(Mat& src_im, Mat* dst_im,
const std::vector<float>& box, std::vector<float>* center,
std::vector<float>* scale, const float expandratio) {
const cv::Mat* img = src_im.GetOpenCVMat();
@@ -35,8 +35,8 @@ bool CropImageByBox(const Mat& src_im, Mat* dst_im,
if (half_h * 3 > half_w * 4) {
half_w = half_h * 0.75;
}
int crop_xmin =std::max(0, static_cast<int>(centerx - half_w));
int crop_ymin =std::max(0, static_cast<int>(centery - half_h));
int crop_xmin = std::max(0, static_cast<int>(centerx - half_w));
int crop_ymin = std::max(0, static_cast<int>(centery - half_h));
int crop_xmax = std::min(img->cols - 1, static_cast<int>(centerx + half_w));
int crop_ymax = std::min(img->rows - 1, static_cast<int>(centery + half_h));

13
fastdeploy/vision/utils/utils.h
View File

@@ -17,6 +17,7 @@
#include <opencv2/opencv.hpp>
#include <set>
#include <vector>
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/utils/utils.h"
#include "fastdeploy/vision/common/result.h"
@@ -43,7 +44,7 @@ std::vector<int32_t> TopKIndices(const T* array, int array_size, int topk) {
std::vector<int32_t> res(topk);
std::set<int32_t> searched;
for (int32_t i = 0; i < topk; ++i) {
T min = -99999999;
T min = static_cast<T>(-99999999);
for (int32_t j = 0; j < array_size; ++j) {
if (searched.find(j) != searched.end()) {
continue;
@@ -75,12 +76,13 @@ FASTDEPLOY_DECL float CosineSimilarity(const std::vector<float>& a,
const std::vector<float>& b,
bool normalized = true);
bool CropImageByBox(const Mat& src_im, Mat* dst_im,
bool CropImageByBox(Mat& src_im, Mat* dst_im,
const std::vector<float>& box, std::vector<float>* center,
std::vector<float>* scale, const float expandratio = 0.3);
/**
* Function: for keypoint detection model, fine positioning of keypoints in postprocess
* Function: for keypoint detection model, fine positioning of keypoints in
* postprocess
* Parameters:
* heatmap: model inference results for keypoint detection models
* dim: shape information of the inference result
@@ -89,8 +91,9 @@ bool CropImageByBox(const Mat& src_im, Mat* dst_im,
* py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions
* index: index information of heatmap pixels
* ch: channel
* Paper reference: DARK postpocessing, Zhang et al. Distribution-Aware Coordinate
* Representation for Human Pose Estimation (CVPR 2020).
* Paper reference: DARK postpocessing, Zhang et al.
* Distribution-Aware Coordinate Representation for Human Pose Estimation (CVPR
* 2020).
*/
void DarkParse(const std::vector<float>& heatmap, const std::vector<int>& dim,
std::vector<float>* coords, const int px, const int py,