diff --git a/.gitignore b/.gitignore index 2edbc3103..0bc8ba218 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,6 @@ coverage yalc.* .yalc examples/vision/collect_quantize_cc.sh -examples/vision/tests_quantize \ No newline at end of file +examples/vision/tests_quantize +fastdeploy/LICENSE +fastdeploy/ThirdPartyNotices.txt \ No newline at end of file diff --git a/examples/vision/sr/README.md b/examples/vision/sr/README.md new file mode 100644 index 000000000..88f9e7777 --- /dev/null +++ b/examples/vision/sr/README.md @@ -0,0 +1,9 @@ +# sr 模型部署 + +FastDeploy目前支持如下超分模型部署 + +| 模型 | 说明 | 模型格式 | 版本 | +|:-----------------------------------------|:----------------------|:-------|:----------------------------------------------------------------------------------| +| [PaddleGAN/BasicVSR](./basicvsr) | BasicVSR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | +| [PaddleGAN/EDVR](./edvr) | EDVR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | +| [PaddleGAN/PP-MSVSR](./ppmsvsr) | PP-MSVSR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | diff --git a/examples/vision/sr/basicvsr/README.md b/examples/vision/sr/basicvsr/README.md new file mode 100644 index 000000000..a077a3cbf --- /dev/null +++ b/examples/vision/sr/basicvsr/README.md @@ -0,0 +1,28 @@ +# BasicVSR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [BasicVSR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的BasicVSR导出成部署模型，导出BasicVSR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:----------------------------------------------------------------------------|:-------|:----- | :------ | +| [BasicVSR](https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tgz) | 30.1MB | - | + +**注意**：非常不建议在没有独立显卡的设备上运行该模型 + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/basicvsr/cpp/CMakeLists.txt b/examples/vision/sr/basicvsr/cpp/CMakeLists.txt new file mode 100644 index 000000000..93540a7e8 --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/basicvsr/cpp/README.md b/examples/vision/sr/basicvsr/cpp/README.md new file mode 100644 index 000000000..ff5e17a99 --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/README.md @@ -0,0 +1,74 @@ +# BasicVSR C++部署示例 + +本目录下提供`infer.cc`快速完成BasicVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +以Linux上BasicVSR推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/basicvsr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载BasicVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tar +tar -xvf BasicVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## BasicVSR C++接口 + +### BasicVSR类 + +```c++ +fastdeploy::vision::sr::BasicVSR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +BasicVSR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> BasicVSR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/basicvsr/cpp/infer.cc b/examples/vision/sr/basicvsr/cpp/infer.cc new file mode 100644 index 000000000..603179bec --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/infer.cc @@ -0,0 +1,297 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::BasicVSR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::sr::BasicVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + // use paddle-TRT + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::BasicVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/basicvsr/python/README.md b/examples/vision/sr/basicvsr/python/README.md new file mode 100644 index 000000000..d5594b70a --- /dev/null +++ b/examples/vision/sr/basicvsr/python/README.md @@ -0,0 +1,61 @@ +# BasicVSR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成BasicVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/basicvsr/python + +# 下载BasicVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tar +tar -xvf BasicVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## BasicVSR Python接口 + +```python +fd.vision.sr.BasicVSR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +BasicVSR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> BasicVSR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [BasicVSR 模型介绍](..) +- [BasicVSR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/basicvsr/python/infer.py b/examples/vision/sr/basicvsr/python/infer.py new file mode 100644 index 000000000..8b1a9dbb7 --- /dev/null +++ b/examples/vision/sr/basicvsr/python/infer.py @@ -0,0 +1,86 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.BasicVSR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,2,3,720,1280] +# 所以导出模型相当重要(最关键的是根据netron查看网络输出shape) +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +reach_end = False +while capture.isOpened(): + imgs = [] + for i in range(args.frame_num): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/sr/edvr/README.md b/examples/vision/sr/edvr/README.md new file mode 100644 index 000000000..398715cd5 --- /dev/null +++ b/examples/vision/sr/edvr/README.md @@ -0,0 +1,28 @@ +# EDVR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [EDVR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的EDVR导出成部署模型，导出EDVR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:--------------------------------------------------------------------------------|:-------|:----- | :------ | +| [EDVR](https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tgz) | 14.9MB | - | + +**注意**：非常不建议在没有独立显卡的设备上运行该模型 + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/edvr/cpp/CMakeLists.txt b/examples/vision/sr/edvr/cpp/CMakeLists.txt new file mode 100644 index 000000000..93540a7e8 --- /dev/null +++ b/examples/vision/sr/edvr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/edvr/cpp/README.md b/examples/vision/sr/edvr/cpp/README.md new file mode 100644 index 000000000..c40b8cf83 --- /dev/null +++ b/examples/vision/sr/edvr/cpp/README.md @@ -0,0 +1,75 @@ +# EDVR C++部署示例 + +本目录下提供`infer.cc`快速完成EDVR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上EDVR推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/edvr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载EDVR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tar +tar -xvf EDVR_M_wo_tsa_SRx4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## EDVR C++接口 + +### EDVR类 + +```c++ +fastdeploy::vision::sr::EDVR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +EDVR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> EDVR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/edvr/cpp/infer.cc b/examples/vision/sr/edvr/cpp/infer.cc new file mode 100644 index 000000000..3e9e81ca9 --- /dev/null +++ b/examples/vision/sr/edvr/cpp/infer.cc @@ -0,0 +1,279 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::EDVR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::sr::EDVR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + // use paddle-TRT + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::EDVR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/edvr/python/README.md b/examples/vision/sr/edvr/python/README.md new file mode 100644 index 000000000..83f0a01e3 --- /dev/null +++ b/examples/vision/sr/edvr/python/README.md @@ -0,0 +1,61 @@ +# EDVR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成EDVR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/edvr/python + +# 下载VSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tar +tar -xvf EDVR_M_wo_tsa_SRx4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## EDVR Python接口 + +```python +fd.vision.sr.EDVR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +EDVR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> EDVR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [EDVR 模型介绍](..) +- [EDVR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/edvr/python/infer.py b/examples/vision/sr/edvr/python/infer.py new file mode 100644 index 000000000..fe30bcf4f --- /dev/null +++ b/examples/vision/sr/edvr/python/infer.py @@ -0,0 +1,89 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.EDVR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,3,720,1280](注意此处与PP-MSVSR不同) +# 所以导出模型相当重要 +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +imgs = [] +while capture.isOpened(): + ret, frame = capture.read() + if frame_id < args.frame_num and frame is not None: + imgs.append(frame) + frame_id += 1 + continue + # 始终保持imgs队列中具有frame_num帧 + imgs.pop(0) + imgs.append(frame) + frame_id += 1 + # 视频读取完毕退出 + if not ret: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/sr/ppmsvsr/README.md b/examples/vision/sr/ppmsvsr/README.md new file mode 100644 index 000000000..944e61446 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/README.md @@ -0,0 +1,27 @@ +# PP-MSVSR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PP-MSVSR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的PP-MSVSR导出成部署模型，导出PP-MSVSR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:----------------------------------------------------------------------------|:------|:----- | :------ | +| [PP-MSVSR](https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tgz) | 8.8MB | - | + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt b/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt new file mode 100644 index 000000000..93540a7e8 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/ppmsvsr/cpp/README.md b/examples/vision/sr/ppmsvsr/cpp/README.md new file mode 100644 index 000000000..2130ffe7e --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/README.md @@ -0,0 +1,75 @@ +# VSR C++部署示例 + +本目录下提供`infer.cc`快速完成PP-MSVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上 PP-MSVSR 推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/ppmsvsr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载PP-MSVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tar +tar -xvf PP-MSVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## PP-MSVSR C++接口 + +### PPMSVSR类 + +```c++ +fastdeploy::vision::sr::PPMSVSR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +PP-MSVSR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> PPMSVSR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/ppmsvsr/cpp/infer.cc b/examples/vision/sr/ppmsvsr/cpp/infer.cc new file mode 100644 index 000000000..2267b83a8 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/infer.cc @@ -0,0 +1,298 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::PPMSVSR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + // use paddle-TRT + option.UseGpu(); + option.UseTrtBackend(); + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::PPMSVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + auto model = fastdeploy::vision::sr::PPMSVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/ppmsvsr/python/README.md b/examples/vision/sr/ppmsvsr/python/README.md new file mode 100644 index 000000000..8e4c736ae --- /dev/null +++ b/examples/vision/sr/ppmsvsr/python/README.md @@ -0,0 +1,61 @@ +# PP-MSVSR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成PP-MSVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/ppmsvsr/python + +# 下载VSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tar +tar -xvf PP-MSVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## VSR Python接口 + +```python +fd.vision.sr.PPMSVSR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +PP-MSVSR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> PPMSVSR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [PP-MSVSR 模型介绍](..) +- [PP-MSVSR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/ppmsvsr/python/infer.py b/examples/vision/sr/ppmsvsr/python/infer.py new file mode 100644 index 000000000..de7ab851e --- /dev/null +++ b/examples/vision/sr/ppmsvsr/python/infer.py @@ -0,0 +1,86 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,2,3,720,1280] +# 所以导出模型相当重要 +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +reach_end = False +while capture.isOpened(): + imgs = [] + for i in range(args.frame_num): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/tracking/pptracking/cpp/infer.cc b/examples/vision/tracking/pptracking/cpp/infer.cc index 58b4d4b61..0d47e386b 100644 --- a/examples/vision/tracking/pptracking/cpp/infer.cc +++ b/examples/vision/tracking/pptracking/cpp/infer.cc @@ -48,7 +48,7 @@ void CpuInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding // if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &recorder); @@ -91,7 +91,7 @@ void GpuInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding //if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &trail_recorder); @@ -135,7 +135,7 @@ void TrtInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding // if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &recorder); diff --git a/examples/vision/tracking/pptracking/python/README.md b/examples/vision/tracking/pptracking/python/README.md index d3b943759..48f8300c9 100644 --- a/examples/vision/tracking/pptracking/python/README.md +++ b/examples/vision/tracking/pptracking/python/README.md @@ -30,7 +30,7 @@ python infer.py --model fairmot_hrnetv2_w18_dlafpn_30e_576x320 --video person.mp fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE) ``` -PP-Tracking模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) +PP-Tracking模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/cpp/README.md) **参数** @@ -42,7 +42,7 @@ PP-Tracking模型加载和初始化，其中model_file, params_file以及config_ ### predict函数 -> ```python +> ```python > PPTracking.predict(frame) > ``` > diff --git a/examples/vision/tracking/pptracking/python/infer.py b/examples/vision/tracking/pptracking/python/infer.py index 378d89bc1..768509ab8 100644 --- a/examples/vision/tracking/pptracking/python/infer.py +++ b/examples/vision/tracking/pptracking/python/infer.py @@ -21,8 +21,7 @@ def parse_arguments(): import argparse import ast parser = argparse.ArgumentParser() - parser.add_argument( - "--model", required=True, help="Path of PaddleSeg model.") + parser.add_argument("--model", required=True, help="Path of model.") parser.add_argument( "--video", type=str, required=True, help="Path of test video file.") parser.add_argument( diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 581d3b91a..00a51cefb 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -55,6 +55,7 @@ #include "fastdeploy/vision/segmentation/ppseg/model.h" #include "fastdeploy/vision/tracking/pptracking/model.h" #include "fastdeploy/vision/headpose/contrib/fsanet.h" +#include "fastdeploy/vision/sr/ppsr/model.h" #endif #include "fastdeploy/vision/visualize/visualize.h" diff --git a/fastdeploy/vision/sr/ppsr/basicvsr.cc b/fastdeploy/vision/sr/ppsr/basicvsr.cc new file mode 100644 index 000000000..6f1691cc1 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/basicvsr.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/basicvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +BasicVSR::BasicVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/basicvsr.h b/fastdeploy/vision/sr/ppsr/basicvsr.h new file mode 100644 index 000000000..976372121 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/basicvsr.h @@ -0,0 +1,41 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +class FASTDEPLOY_DECL BasicVSR : public PPMSVSR{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g BasicVSR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g BasicVSR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + BasicVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained BasicVSR + std::string ModelName() const override { return "BasicVSR"; } +}; + +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/edvr.cc b/fastdeploy/vision/sr/ppsr/edvr.cc new file mode 100644 index 000000000..f9bb4e0cc --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/edvr.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/edvr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +EDVR::EDVR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} + +bool EDVR::Postprocess(std::vector& infer_results, std::vector& results){ + // group to image + // output_shape is [b, n, c, h, w] n = frame_nums b=1(default) + // b and n is dependence export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + auto output_shape = infer_results[0].shape; + // EDVR + int h_ = output_shape[2]; + int w_ = output_shape[3]; + int c_ = output_shape[1]; + int frame_num = 1; + float *out_data = static_cast(infer_results[0].Data()); + cv::Mat temp = cv::Mat::zeros(h_, w_, CV_32FC3); // RGB image + int pix_num = h_ * w_; + int frame_pix_num = pix_num * c_; + for (int frame = 0; frame < frame_num; frame++) { + int index = 0; + for (int h = 0; h < h_; ++h) { + for (int w = 0; w < w_; ++w) { + temp.at(h, w) = {out_data[2 * pix_num + index + frame_pix_num * frame], + out_data[pix_num + index + frame_pix_num * frame], + out_data[index + frame_pix_num * frame]}; + index += 1; + } + } + // tmp data type is float[0-1.0],convert to uint type + cv::Mat res = cv::Mat::zeros(temp.size(), CV_8UC3); + temp.convertTo(res, CV_8UC3, 255); + results.push_back(res); + } + return true; +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/edvr.h b/fastdeploy/vision/sr/ppsr/edvr.h new file mode 100644 index 000000000..1e460c028 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/edvr.h @@ -0,0 +1,45 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + + +class FASTDEPLOY_DECL EDVR : public PPMSVSR{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g EDVR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g EDVR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + EDVR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained EDVR + std::string ModelName() const override { return "EDVR"; } + + private: + bool Postprocess(std::vector& infer_results, + std::vector& results) override; +}; +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/model.h b/fastdeploy/vision/sr/ppsr/model.h new file mode 100644 index 000000000..1d5398335 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/model.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" +#include "fastdeploy/vision/sr/ppsr/edvr.h" +#include "fastdeploy/vision/sr/ppsr/basicvsr.h" diff --git a/fastdeploy/vision/sr/ppsr/ppmsvsr.cc b/fastdeploy/vision/sr/ppsr/ppmsvsr.cc new file mode 100644 index 000000000..888f7609a --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppmsvsr.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +PPMSVSR::PPMSVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} + +bool PPMSVSR::Initialize(){ + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + mean_ = {0., 0., 0.}; + scale_ = {1., 1., 1.}; + return true; +} + +bool PPMSVSR::Preprocess(Mat* mat, std::vector& output) { + + BGR2RGB::Run(mat); + Normalize::Run(mat, mean_, scale_, true); + HWC2CHW::Run(mat); + // Csat float + float* ptr = static_cast(mat->Data()); + size_t size = mat->Width() * mat->Height() * mat->Channels(); + output = std::vector(ptr, ptr + size); + return true; +} + +bool PPMSVSR::Predict(std::vector& imgs, std::vector& results) { + + // Theoretically, the more frame nums there are, the better the result will be, + // but it will lead to a significant increase in memory + int frame_num = imgs.size(); + int rows = imgs[0].rows; + int cols = imgs[0].cols; + int channels = imgs[0].channels(); + std::vector input_tensors; + input_tensors.resize(1); + std::vector all_data_temp; + for (int i = 0; i < frame_num; i++) { + Mat mat(imgs[i]); + std::vector data_temp; + Preprocess(&mat, data_temp); + all_data_temp.insert(all_data_temp.end(), data_temp.begin(), data_temp.end()); + } + // share memory in order to avoid memory copy, data type must be float32 + input_tensors[0].SetExternalData({1 ,frame_num , channels, rows, cols}, FDDataType::FP32, + all_data_temp.data()); + input_tensors[0].shape = {1, frame_num, channels, rows, cols}; + input_tensors[0].name = InputInfoOfRuntime(0).name; + std::vector output_tensors; + if (!Infer(input_tensors, &output_tensors)) { + FDERROR << "Failed to inference." << std::endl; + return false; + } + if (!Postprocess(output_tensors, results)) { + FDERROR << "Failed to post process." << std::endl; + return false; + } + return true; +} + +bool PPMSVSR::Postprocess(std::vector& infer_results, std::vector& results){ + // group to image + // output_shape is [b, n, c, h, w] n = frame_nums b=1(default) + // b and n is dependence export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + auto output_shape = infer_results[0].shape; + // PP-MSVSR + int h_ = output_shape[3]; + int w_ = output_shape[4]; + int c_ = output_shape[2]; + int frame_num = output_shape[1]; + + float *out_data = static_cast(infer_results[0].Data()); + cv::Mat temp = cv::Mat::zeros(h_, w_, CV_32FC3); // RGB image + int pix_num = h_ * w_; + int frame_pix_num = pix_num * c_; + for (int frame = 0; frame < frame_num; frame++) { + int index = 0; + for (int h = 0; h < h_; ++h) { + for (int w = 0; w < w_; ++w) { + temp.at(h, w) = {out_data[2 * pix_num + index + frame_pix_num * frame], + out_data[pix_num + index + frame_pix_num * frame], + out_data[index + frame_pix_num * frame]}; + index += 1; + } + } + // tmp data type is float[0-1.0],convert to uint type + cv::Mat res = cv::Mat::zeros(temp.size(), CV_8UC3); + temp.convertTo(res, CV_8UC3, 255); + results.push_back(res); + } + return true; +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/ppmsvsr.h b/fastdeploy/vision/sr/ppsr/ppmsvsr.h new file mode 100644 index 000000000..49cdfd03f --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppmsvsr.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/fastdeploy_model.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + + +class FASTDEPLOY_DECL PPMSVSR:public FastDeployModel{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g PPMSVSR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g PPMSVSR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + PPMSVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained PP-MSVSR。 + std::string ModelName() const override { return "PPMSVSR"; } + /** + * get super resolution frame sequence + * @param[in] imgs origin frame sequences + * @param[in] results super resolution frame sequence + * @return true if the prediction successed, otherwise false + */ + virtual bool Predict(std::vector& imgs, + std::vector& results); + + protected: + PPMSVSR(){}; + + virtual bool Initialize(); + + virtual bool Preprocess(Mat* mat, std::vector& output); + + virtual bool Postprocess(std::vector& infer_results, + std::vector& results); + + std::vector mean_; + std::vector scale_; +}; +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc b/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc new file mode 100644 index 000000000..5688281c6 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc @@ -0,0 +1,70 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy{ +void BindPPSR(pybind11::module &m) { + pybind11::class_(m, "PPMSVSR") + .def(pybind11::init()) + .def("predict", [](vision::sr::PPMSVSR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); + pybind11::class_(m, "EDVR") + .def(pybind11::init()) + .def("predict", [](vision::sr::EDVR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); + pybind11::class_(m, "BasicVSR") + .def(pybind11::init()) + .def("predict", [](vision::sr::BasicVSR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/sr_pybind.cc b/fastdeploy/vision/sr/sr_pybind.cc new file mode 100644 index 000000000..03be14df3 --- /dev/null +++ b/fastdeploy/vision/sr/sr_pybind.cc @@ -0,0 +1,25 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { + + void BindPPSR(pybind11::module& m); + + void BindSR(pybind11::module& m) { + auto sr_module = m.def_submodule("sr", "sr(super resolution) submodule"); + BindPPSR(sr_module); + } +} // namespace fastdeploy diff --git a/fastdeploy/vision/vision_pybind.cc b/fastdeploy/vision/vision_pybind.cc index a1fc6cac6..c2cacae5c 100755 --- a/fastdeploy/vision/vision_pybind.cc +++ b/fastdeploy/vision/vision_pybind.cc @@ -27,6 +27,7 @@ void BindOcr(pybind11::module& m); void BindTracking(pybind11::module& m); void BindKeyPointDetection(pybind11::module& m); void BindHeadPose(pybind11::module& m); +void BindSR(pybind11::module& m); #ifdef ENABLE_VISION_VISUALIZE void BindVisualize(pybind11::module& m); #endif @@ -142,6 +143,7 @@ void BindVision(pybind11::module& m) { BindTracking(m); BindKeyPointDetection(m); BindHeadPose(m); + BindSR(m); #ifdef ENABLE_VISION_VISUALIZE BindVisualize(m); #endif diff --git a/python/fastdeploy/vision/__init__.py b/python/fastdeploy/vision/__init__.py index 047591ccd..a5531a8a9 100755 --- a/python/fastdeploy/vision/__init__.py +++ b/python/fastdeploy/vision/__init__.py @@ -24,6 +24,7 @@ from . import facealign from . import faceid from . import ocr from . import headpose +from . import sr from . import evaluation from .utils import fd_result_to_json from .visualize import * diff --git a/python/fastdeploy/vision/sr/__init__.py b/python/fastdeploy/vision/sr/__init__.py new file mode 100644 index 000000000..d056cce11 --- /dev/null +++ b/python/fastdeploy/vision/sr/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from .ppsr import PPMSVSR, EDVR, BasicVSR diff --git a/python/fastdeploy/vision/sr/ppsr/__init__.py b/python/fastdeploy/vision/sr/ppsr/__init__.py new file mode 100644 index 000000000..38e462d06 --- /dev/null +++ b/python/fastdeploy/vision/sr/ppsr/__init__.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class PPMSVSR(FastDeployModel): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a VSR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g PPMSVSR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g PPMSVSR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "PPMSVSR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.PPMSVSR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "PPMSVSR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) + + +class EDVR(PPMSVSR): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a EDVR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g EDVR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g EDVR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "EDVR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.EDVR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "EDVR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) + + +class BasicVSR(PPMSVSR): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a EDVR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g BasicVSR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g BasicVSR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "BasicVSR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.BasicVSR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "BasicVSR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) diff --git a/tests/models/test_basicvsr.py b/tests/models/test_basicvsr.py new file mode 100644 index 000000000..479343444 --- /dev/null +++ b/tests/models/test_basicvsr.py @@ -0,0 +1,71 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_basicvsr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/basicvsr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/basicvsr/BasicVSR_reds_x4" + # use default backend + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=rc.test_option) + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + frame_id = 0 + reach_end = False + t = 0 + while capture.isOpened(): + imgs = [] + for i in range(2): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + if t < 10: + ret = pickle.load( + open("./resources/basicvsr/frame_" + str(t) + ".pkl", + "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-02 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + frame_id += 1 + if t >= 10: + break + capture.release() diff --git a/tests/models/test_edvr.py b/tests/models/test_edvr.py new file mode 100644 index 000000000..a9f9517e7 --- /dev/null +++ b/tests/models/test_edvr.py @@ -0,0 +1,76 @@ +test_pptracking.py # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_edvr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/edvr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/edvr/EDVR_M_wo_tsa_SRx4" + # use default backend + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.EDVR( + model_file, params_file, runtime_option=rc.test_option) + + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + # Create VideoWriter for output + frame_id = 0 + imgs = [] + t = 0 + while capture.isOpened(): + ret, frame = capture.read() + if frame_id < 5 and frame is not None: + imgs.append(frame) + frame_id += 1 + continue + # 始终保持imgs队列中具有frame_num帧 + imgs.pop(0) + imgs.append(frame) + frame_id += 1 + # 视频读取完毕退出 + if not ret: + break + results = model.predict(imgs) + for item in results: + if frame_id <= 4: + continue + if t < 10: + ret = pickle.load( + open("./resources/edvr/frame_" + str(t) + ".pkl", "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-03 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + if t >= 10: + break + capture.release() diff --git a/tests/models/test_ppmsvsr.py b/tests/models/test_ppmsvsr.py new file mode 100644 index 000000000..09cfe4585 --- /dev/null +++ b/tests/models/test_ppmsvsr.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_ppmsvsr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppmsvsr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/ppmsvsr/MSVSR_reds_x4" + # use default backend + # runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=rc.test_option) + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + frame_id = 0 + reach_end = False + t = 0 + while capture.isOpened(): + imgs = [] + for i in range(2): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + if t < 10: + ret = pickle.load( + open("./resources/ppmsvsr/frame_" + str(t) + ".pkl", "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-03 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + frame_id += 1 + if t >= 10: + break + capture.release()