From 9ebe62527637bc3124f117dde0ec815b2a99faed Mon Sep 17 00:00:00 2001 From: li-zhenyun Date: Sat, 3 Sep 2022 01:47:35 +0800 Subject: [PATCH] Video-ReID --- Video-ReID/README.md | 265 ++++++++++++++ Video-ReID/config/coco.names | 81 +++++ Video-ReID/config/yolov3_tf_bs1_fp16.cfg | 10 + Video-ReID/get_gallery_features.py | 145 ++++++++ Video-ReID/image.py | 131 +++++++ Video-ReID/models/OSNet/aipp.config | 21 ++ Video-ReID/models/YOLOv3/tf_aipp.cfg | 22 ++ Video-ReID/pipeline/gallery.pipeline | 109 ++++++ Video-ReID/pipeline/image.pipeline | 185 ++++++++++ Video-ReID/pipeline/video.pipeline | 215 +++++++++++ .../plugins/PluginFeatureMatch/CMakeLists.txt | 46 +++ .../Plugin_FeatureMatch.cpp | 335 ++++++++++++++++++ .../PluginFeatureMatch/Plugin_FeatureMatch.h | 111 ++++++ .../plugins/PluginFeatureMatch/build.sh | 36 ++ Video-ReID/run.sh | 30 ++ Video-ReID/video.py | 83 +++++ 16 files changed, 1825 insertions(+) create mode 100644 Video-ReID/README.md create mode 100644 Video-ReID/config/coco.names create mode 100644 Video-ReID/config/yolov3_tf_bs1_fp16.cfg create mode 100644 Video-ReID/get_gallery_features.py create mode 100644 Video-ReID/image.py create mode 100644 Video-ReID/models/OSNet/aipp.config create mode 100644 Video-ReID/models/YOLOv3/tf_aipp.cfg create mode 100644 Video-ReID/pipeline/gallery.pipeline create mode 100644 Video-ReID/pipeline/image.pipeline create mode 100644 Video-ReID/pipeline/video.pipeline create mode 100644 Video-ReID/plugins/PluginFeatureMatch/CMakeLists.txt create mode 100644 Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.cpp create mode 100644 Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.h create mode 100644 Video-ReID/plugins/PluginFeatureMatch/build.sh create mode 100644 Video-ReID/run.sh create mode 100644 Video-ReID/video.py diff --git a/Video-ReID/README.md b/Video-ReID/README.md new file mode 100644 index 0000000..ec5c1da --- /dev/null +++ b/Video-ReID/README.md @@ -0,0 +1,265 @@ +# MindXSDK 视频行人重识别 + +## 1 简介 +本开发样例基于MindX SDK实现了对图片和视频流的行人重识别（Person Re-identification, ReID），支持检索给定照片、视频中的行人ID。其主要流程为： +- 构建行人特征库：将底库图片调整大小，利用目标检测模型YOLOv3推理，检测图片中的行人，检测结果经过抠图与调整大小，再利用OSNet模型提取图片中每个行人的特征向量并保存，特征向量用于与后续的待查询图片或者视频中的行人作比较。 +- 对于查询图片或视频帧：利用目标检测模型YOLOv3推理，检测图片中的行人，检测结果经过抠图与调整大小，再利用OSNet模型提取图片中每个行人的特征向量。 +- 行人检索：将查询图片中行人的特征向量与底库中的特征向量做比较，为每个查询图片中的行人检索最有可能的ID，通过识别框和文字信息进行可视化标记。 +- 如果输入是图片，最终得到标记过的图片文件，如果是视频流，得到标记过的H264格式视频文件 + +## 2 目录结构 +本工程名称为Video-ReID，工程目录如下图所示： +``` +video-ReID +|---- config +| | |---- coco.names +| | |---- yolov3_tf_bs1_fp16.cfg +|---- data +| |---- gallery // 行人底库图片文件夹 +| |---- query +| |---- images // 查询场景图片文件夹 +| |---- videos // 查询场景视频文件夹,也可根据推流时实际环境配置存放位置 +|---- models // 目标检测、OSNet模型与配置文件夹 +| |---- OSNet +| | | |---- aipp.config +|---- YOLOv3 +| | | |---- tf_aipp.config +|---- pipeline // 流水线配置文件夹 +| | |---- image.pipeline +| | |---- gallery.pipeline +| | |---- video.pipeline +|---- plugins // 自定义插件目录 +| |---- PluginFeatureMatch +|---- output // 结果保存文件夹 +| |---- gallery +| |---- query +| |---- images +| |---- videos //需自行创建 +|---- image.py +|---- gallery.py +|---- video.py +|---- run.sh +|---- README.md +``` +> 由于无法在Gitee上创建空文件夹，请按照该工程目录，自行创建output文件夹、data文件夹与其内部的文件夹 + +## 3 依赖 +| 软件名称 | 版本 | +| :--------: | :------: | +|ubuntu 18.04|18.04.1 LTS | +|CANN|5.0.4| +|MindX SDK|2.0.4| +|Python| 3.9.2| +|numpy | 1.21.0 | +|opencv_python|4.5.2| + +- 设置环境变量（请确认install_path路径是否正确） +``` +#执行如下命令 +. ${SDK-path}/set_env.sh +. ${ascend_toolkit_path}/set_env.sh +``` + +请注意MindX SDK使用python版本为3.9.2，如出现无法找到python对应lib库请在root下安装python3.9开发库 +``` +apt-get install libpython3.9 +``` + +- 推理中涉及到第三方软件依赖如下表所示。 + +| 依赖软件 | 版本 | 说明 | 使用教程 | +| -------- | ---------- | -------------------------------- | ------------------------------------------------------------ | +| live555 | 1.10 | 实现视频转 rstp 进行推流 | [链接](https://gitee.com/ascend/mindxsdk-referenceapps/blob/master/docs/参考资料/Live555离线视频转RTSP说明文档.md) | +| ffmpeg | 2022-06-27 | 实现 mp4 格式视频转为264格式视频 | [链接](https://gitee.com/ascend/mindxsdk-referenceapps/blob/master/docs/参考资料/pc端ffmpeg安装教程.md) | + +## 4 模型转换 +行人重识别先采用了yolov3模型将图片中的行人检测出来，然后利用OsNet模型获取行人的特征向量。由于yolov3模型和OsNet模型分别是基于Pytorch和Tensorflow的深度模型，我们需要借助ATC工具分别将其转换成对应的.om模型。 + +### 4.1 yolov3的模型转换： + +**步骤1** 获取yolov3的原始模型(.pb文件)和相应的配置文件(.cfg文件) +      [原始模型下载链接](https://www.hiascend.com/zh/software/modelzoo/models/detail/1/ba2a4c054a094ef595da288ecbc7d7b4) + +**步骤2** 将获取到的yolov3模型.pb文件和.cfg文件存放至：“项目所在目录/models/YOLOv3/” + +**步骤3** .om模型转换 +进入“项目所在目录/models/YOLOv3” + +- 使用ATC将.pb文件转成为.om文件 +``` +atc --model=yolov3_tf.pb --framework=3 --output=yolov3 --output_type=FP32 --soc_version=Ascend310 --input_shape="input:1,416,416,3" --out_nodes="yolov3/yolov3_head/Conv_6/BiasAdd:0;yolov3/yolov3_head/Conv_14/BiasAdd:0;yolov3/yolov3_head/Conv_22/BiasAdd:0" --log=info --insert_op_conf=tf_aipp.cfg +``` +- 执行完模型转换脚本后，若提示如下信息说明模型转换成功，可以在该路径下找到名为yolov3.om模型文件。 +（可以通过修改output参数来重命名这个.om文件） +``` +ATC run success, welcome to the next use. +``` + +### 4.2 OSNet的模型转换 + +#### 4.2.1 模型概述 +      [OSNet论文地址](https://arxiv.org/pdf/1905.00953.pdf) +      [OSNet代码地址](https://github.com/KaiyangZhou/deep-person-reid) + +#### 4.2.2 模型转换步骤 + +**步骤1** 从ModelZoo源码包中获取OSNet的onnx模型文件(osnet_x1_0.onnx) +      [权重文件源码包下载链接](https://www.hiascend.com/zh/software/modelzoo/models/detail/1/43a754e306c6461d86dafced5046121f) + +**步骤2** 将获取到的onnx模型存放至：“项目所在目录/models/OSNet/” + +**步骤3** .om模型转换 +进入“项目所在目录/models/OSNet” + +- 使用ATC将.onnx文件转成为.om文件 +``` +atc --framework=5 --model=./osnet_x1_0.onnx --input_format=NCHW --insert_op_conf=./aipp.config --input_shape="image:-1,3,256,128" --dynamic_batch_size="1,2,3,4,5,6,7,8" --output=osnet --log=debug --soc_version=Ascend310 +// dynamic参数为支持的动态batchsize,可根据实际图片中可能出现的行人数目更改 +``` +- 执行完模型转换脚本后，若提示如下信息说明模型转换成功，可以在该路径下找到名为yolov3.om模型文件。 +（可以通过修改output参数来重命名这个.om文件） +``` +ATC run success, welcome to the next use. +``` +经过上述操作，可以在“项目所在目录/models”的子目录下找到yolov3.om模型和osnet.om模型，模型转换操作已全部完成 + +### 4.3 参考链接 +> 模型转换使用了ATC工具，如需更多信息请参考：[ATC工具使用指南-快速入门](https://support.huaweicloud.com/tg-cannApplicationDev330/atlasatc_16_0005.html) +> Yolov3模型转换的参考链接：[ATC YOLOv3(FP16)](https://www.hiascend.com/zh/software/modelzoo/models/detail/1/ba2a4c054a094ef595da288ecbc7d7b4) +> OSNet模型转换的参考链接：[OSNet](https://gitee.com/ascend/ModelZoo-PyTorch/tree/master/ACL_PyTorch/contrib/cv/classfication/OSNet) + +## 5 准备 + +### 5.1 数据 + +为适配网络输入以及性能要求，建议输入图片或视频流长宽比接近1：1，图像长宽均需为偶数，待检测行人范围像素面积大于5120且宽度大于32像素，长度大于16像素,且图像后缀为jpg/JPG。 +涉及文件夹 +> “项目所在目录/data/gallery”：用于存放制作行人底库的场景图片 +建议针对待检测的行人单人正面，侧面，背面各采集图片作为其底库。图片中行人清晰醒目且身体完整。 + +> “项目所在目录/data/query/images”：用于存放待查询行人图片 +可检测行人正面，侧面，背面。要求图片中行人清晰醒目且身体尽量完整。 + +> “项目所在目录/data/query/videos”：用于存放待查询行人视频 +可检测行人正面，侧面，背面。要求视频帧中行人清晰醒目且身体尽量完整。 + +### 5.2 编译插件 + +项目需要用到自定义插件进行特征匹配与重标定，自定义插件源码目录为“项目所在目录/plugins/PluginFeatureMatch” + +#### 5.2.1 编译插件 +``` +> bash build.sh +``` +编译后会在${MX_SDK_HOME}/lib/plugins/目录下生成libmxpi_featurematch.so文件 + +#### 5.2.2 为编译获得的.so文件授予640权限. + +编译参考[插件编译指南](https://support.huawei.com/enterprise/zh/doc/EDOC1100234263/21d24289) + +### 5.3 视频推流 +按照第 3 小结软件依赖安装 live555 和 ffmpeg，按照 Live555离线视频转RTSP说明文档 将 mp4 视频转换为 h264 格式。并将生成的 264 格式的视频上传到 live/mediaServer 目录下，然后修改 项目所在目录/pipeline 目录下的 video.pipeline 文件中 mxpi_rtspsrc0 的内容。 +``` + "mxpi_rtspsrc0": { + "props": { + "rtspUrl":"rtsp://xxx.xxx.xxx.xxx:xxxx/xxx.264", // 修改为自己开发板的地址和文件名 + "channelId": "0", + "timeout": "30" + }, + "factory": "mxpi_rtspsrc", + "next": "mxpi_videodecoder0" + }, +``` + +### 5.4 适用场景 + +项目适用于大部分行人目标较完整且醒目可见的场景，对于行人不完整程度高，或者行人底库中缺少同一行人对应机位的参照时，检测框或行人重标定可能会出现误差，请根据实际应用进行数据选择和处理。 +对于视频，每帧的处理时长约为200ms，建议拉流视频帧率在5左右，若出现内存不足等情况请适当降低帧率。 + +## 6 测试 + +### 6.1 获取om模型 +``` +步骤详见4： 模型转换 +``` +### 6.2 准备 +``` +步骤详见5： 准备 +``` +### 6.3 配置pipeline +根据所需场景，配置pipeline文件，调整路径参数等。 +``` + # 配置mxpi_tensorinfer插件的yolov3.om模型加载路径（三个pipeline均需配置） + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "models/YOLOv3/yolov3.om(这里根据你的命名或路径进行更改)" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + # 配置mxpi_objectpostprocessor插件的yolov3.cfg配置文件加载路径以及SDN的安装路径（三个pipeline均需配置） + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "config/yolov3_tf_bs1_fp16.cfg(这里根据你的命名或路径进行更改)", + "labelPath": "config/coco.names", + "postProcessLibPath": "libyolov3postprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_objectselector0" + }, + # 配置mxpi_tensorinfer插件的OsNet.om模型加载路径（三个pipeline均需配置） + "mxpi_tensorinfer1": { + "props": { + "dataSource": "mxpi_imagecrop0", + "dynamicStrategy": "Upper", + "modelPath": "models/OSNet/osnet.om", + "waitingTime": "1" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_featurematch0" + }, + +``` +### 6.4 执行 + +#### 6.4.1 构建行人特征库 +``` +bash run.sh gallery +``` +执行成功后会打印单张图片处理耗时并在output/gallery目录下生成.txt标签文件和.bin特征向量存储文件。 + +#### 6.4.2 图片查询 +``` +bash run.sh image +``` +执行成功后会打印单张图片处理耗时并在output/query/images目录下生成标记了Reid目标的图片输出。 +经测试，端到端处理耗时在200ms以内。 + +#### 6.4.3 视频查询 +``` +bash run.sh video 60 # 60s为处理视频流时长参数，可根据实际情况修改 +``` +执行成功后会在output/query/videos目录下获得标记了Reid目标的h264格式视频文件。 + +### 6.5 查看结果 +执行命令后，可在“项目所在目录/output”路径下查看结果。 + + +## 7 参考链接 +> 特定行人检索：[Person Search Demo](https://github.com/KaiyangZhou/deep-person-reid) + + +## 8 Q&A +### 8.1 在运行查询脚本时出现"[DVPP: image width out of range] The crop width of image No.[0] is out of range [32,4096]" +> 这里的错误是因为yolov3模型检测到的目标过小，调大“mxpi_objectselector0”插件的MinArea参数或者更新“项目所在目录/config/yolov3_tf_bs1_fp16.cfg”文件，将OBJECTNESS_THRESH适度调大可解决该问题 + +### 8.2 运行video.py时出现"[6003][stream change state fail] create stream(queryVideoProcess) failed." +> 可能是因为video.pipeline中filelink插件的保存路径文件夹未创建，请手动创建(output/query/videos). + +### 8.3 运行脚本时出现 streamInstance GetResult return nullptr. +> 可能是因为图像/视频里没有检测到行人，请更换有显著行人的图像输入。 + +### 8.4 视频处理时出现 "[Error code unknown] Failed to send frame.","[DVPP: decode H264 or H265 fail] Decode video failed.",且在销毁stream时出现 “Failed to destroy vdec channel.", "Failed to destroy stream:queryVideoProcess"或出现 "Malloc device memory failed." +> 此错误可能是因为视频帧率过高，请适当降低.264视频帧率。 diff --git a/Video-ReID/config/coco.names b/Video-ReID/config/coco.names new file mode 100644 index 0000000..6100f36 --- /dev/null +++ b/Video-ReID/config/coco.names @@ -0,0 +1,81 @@ +# This file is originally from https://github.com/pjreddie/darknet/blob/master/data/coco.names +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush \ No newline at end of file diff --git a/Video-ReID/config/yolov3_tf_bs1_fp16.cfg b/Video-ReID/config/yolov3_tf_bs1_fp16.cfg new file mode 100644 index 0000000..bcc1b49 --- /dev/null +++ b/Video-ReID/config/yolov3_tf_bs1_fp16.cfg @@ -0,0 +1,10 @@ +CLASS_NUM=80 +BIASES_NUM=18 +BIASES=10,13,16,30,33,23,30,61,62,45,59,119,116,90,156,198,373,326 +SCORE_THRESH=0.3 +OBJECTNESS_THRESH=0.3 +IOU_THRESH=0.45 +YOLO_TYPE=3 +ANCHOR_DIM=3 +MODEL_TYPE=0 +RESIZE_FLAG=0 \ No newline at end of file diff --git a/Video-ReID/get_gallery_features.py b/Video-ReID/get_gallery_features.py new file mode 100644 index 0000000..89c01ca --- /dev/null +++ b/Video-ReID/get_gallery_features.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) Huawei Technologies Co.,Ltd. 2012-2021 All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import os +import time +import numpy as np +import MxpiDataType_pb2 as MxpiDataType +from StreamManagerApi import StreamManagerApi, MxDataInput, StringVector + + +GALLERY_STREAM_NAME = b'galleryProcess' +IN_PLUGIN_ID = 0 +OUT_PLUGIN_ID = 0 + + +def initialize_stream(): + """ + Initialize stream galleryImageProcess for detecting and re-identifying persons in galley images + + :arg: + None + + :return: + Stream api + """ + + stream_pi = StreamManagerApi() + ret = stream_pi.InitManager() + if ret != 0: + error_message = "Failed to init Stream manager, ret=%s" % str(ret) + print(error_message) + exit() + + # creating stream based on json strings in the pipeline file: 'ReID.pipeline' + with open("pipeline/gallery.pipeline", 'rb') as f: + pipeline = f.read() + + ret = stream_pi.CreateMultipleStreams(pipeline) + if ret != 0: + error_message = "Failed to create Stream, ret=%s" % str(ret) + print(error_message) + exit() + + return stream_pi + + +def get_gallery_feature(input_dir, output_dir, stream_api): + """ + Extract the features of gallery images, save the feature vector and the Pids to files + + :arg: + imgPath: the directory of gallery images + outputDir: the directory of gallery output files + streamApi: stream api + + :return: + None + """ + + # constructing the results returned by the queryImageProcess stream + key_vec = StringVector() + key_vec.push_back(b"mxpi_tensorinfer1") + + # check the query file + if os.path.exists(input_dir) != 1: + error_message = 'The img dir does not exist.' + print(error_message) + exit() + if len(os.listdir(input_dir)) == 0: + error_message = 'The img file is empty.' + print(error_message) + exit() + if os.path.exists(output_dir) != 1: + root = os.getcwd() + os.makedirs(os.path.join(root, output_dir)) + features = [] + pids = [] + + # extract the features for all images in gallery file + for root, dirs, files in os.walk(input_dir): + for file in files: + if file.endswith('.jpg') or file.endswith('.JPG'): + data_input = MxDataInput() + file_path = os.path.join(root, file) + with open(file_path, 'rb') as f: + data_input.data = f.read() + start = time.time() + # send the prepared data to the stream + unique_id = stream_api.SendData(GALLERY_STREAM_NAME, IN_PLUGIN_ID, data_input) + if unique_id < 0: + error_message = 'Failed to send data to queryImageProcess stream.' + print(error_message) + exit() + # get infer result + infer_result = stream_api.GetProtobuf(GALLERY_STREAM_NAME, OUT_PLUGIN_ID, key_vec) + end = time.time() + print("time:", end-start) + # checking whether the infer results is valid or not + if infer_result.size() == 0: + error_message = 'Unable to get effective infer results, please check the stream log for details' + print(error_message) + exit() + + tensor_packages = MxpiDataType.MxpiTensorPackageList() + tensor_packages.ParseFromString(infer_result[0].messageBuf) + feature = np.frombuffer(tensor_packages.tensorPackageVec[0].tensorVec[0].dataStr, + dtype=np.float32) + features.append(feature) + pids.append(file.split('.')[0]) + else: + print('Input image only support jpg') + exit() + + features = np.array(features) + features.tofile(os.path.join(output_dir, 'gallery_features.bin')) + pids = np.array(pids).T + np.savetxt(os.path.join(output_dir, 'persons.txt'), pids, fmt='%s') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--galleryInputDir', type=str, default='data/gallery', help="Gallery File Path") + parser.add_argument('--galleryOutputDir', type=str, default='output/gallery', help="Gallery Features Output Path") + opt = parser.parse_args() + stream_manager_api = initialize_stream() + get_gallery_feature(opt.galleryInputDir, opt.galleryOutputDir, stream_manager_api) + + stream_manager_api.DestroyAllStreams() \ No newline at end of file diff --git a/Video-ReID/image.py b/Video-ReID/image.py new file mode 100644 index 0000000..b33eca4 --- /dev/null +++ b/Video-ReID/image.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) Huawei Technologies Co.,Ltd. 2012-2021 All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import stat +import os +import time +from StreamManagerApi import StreamManagerApi, MxDataInput + + +QUERY_STREAM_NAME = b'queryImageProcess' + +IN_PLUGIN_ID = 0 +OUT_PLUGIN_ID = 0 + + +def initialize_stream(): + """ + Initialize stream queryImageProcess for detecting and re-identifying persons in image + + :arg: + None + + :return: + Stream api + """ + + stream_pi = StreamManagerApi() + ret = stream_pi.InitManager() + if ret != 0: + error_message = "Failed to init Stream manager, ret=%s" % str(ret) + print(error_message) + exit() + + # creating stream based on json strings in the pipeline file: 'ReID.pipeline' + with open("pipeline/image.pipeline", 'rb') as f: + pipeline = f.read() + + ret = stream_pi.CreateMultipleStreams(pipeline) + if ret != 0: + error_message = "Failed to create Stream, ret=%s" % str(ret) + print(error_message) + exit() + + return stream_pi + + +def reid(input_dir, output_dir, stream_api): + + """ + detecting and re-identifying persons in videos + + :arg: + inputDir: the directory of query images + outputDir: the directory of output images + streamApi: stream api + + :return: + None + """ + + if os.path.exists(input_dir) != 1: + error_message = 'The img dir does not exist.' + print(error_message) + exit() + if len(os.listdir(input_dir)) == 0: + error_message = 'The img file is empty.' + print(error_message) + exit() + if os.path.exists(output_dir) != 1: + root = os.getcwd() + os.makedirs(os.path.join(root, output_dir)) + + for root, dirs, files in os.walk(input_dir): + for file in files: + if file.endswith('.jpg') or file.endswith('.JPG'): + data_input = MxDataInput() + file_path = os.path.join(root, file) + with open(file_path, 'rb') as f: + data_input.data = f.read() + start = time.time() + + unique_id = stream_api.SendData(QUERY_STREAM_NAME, IN_PLUGIN_ID, data_input) + if unique_id < 0: + error_message = 'Failed to send data to queryImageProcess stream.' + print(error_message) + exit() + + # get infer result + infer_result = stream_api.GetResult(QUERY_STREAM_NAME, OUT_PLUGIN_ID) + if infer_result.errorCode: + error_message = 'Unable to get effective infer results, please check the stream log for details' + print(error_message) + exit() + end = time.time() + out_path = os.path.join(output_dir, file) + flags = os.O_WRONLY | os.O_CREAT # Sets how files are read and written + modes = stat.S_IWUSR | stat.S_IRUSR # Set file permissions + with os.fdopen(os.open(out_path, flags, modes), 'wb') as f: + f.write(infer_result.data) + print("time:", end - start) + else: + print('Input image only support jpg') + exit() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--queryInputDir', type=str, default='data/query/images', help="Query Images Input Dir") + parser.add_argument('--queryOutputDir', type=str, default='output/query/images', help="Query Images Output Dir") + opt = parser.parse_args() + stream_manager_api = initialize_stream() + reid(opt.queryInputDir, opt.queryOutputDir, stream_manager_api) + + stream_manager_api.DestroyAllStreams() \ No newline at end of file diff --git a/Video-ReID/models/OSNet/aipp.config b/Video-ReID/models/OSNet/aipp.config new file mode 100644 index 0000000..c3e7317 --- /dev/null +++ b/Video-ReID/models/OSNet/aipp.config @@ -0,0 +1,21 @@ +aipp_op { + aipp_mode: static + input_format : YUV420SP_U8 + csc_switch : true + rbuv_swap_switch : false + matrix_r0c0 : 256 + matrix_r0c1 : 0 + matrix_r0c2 : 359 + matrix_r1c0 : 256 + matrix_r1c1 : -88 + matrix_r1c2 : -183 + matrix_r2c0 : 256 + matrix_r2c1 : 454 + matrix_r2c2 : 0 + input_bias_0 : 0 + input_bias_1 : 128 + input_bias_2 : 128 + + src_image_size_w: 128 + src_image_size_h: 256 +} diff --git a/Video-ReID/models/YOLOv3/tf_aipp.cfg b/Video-ReID/models/YOLOv3/tf_aipp.cfg new file mode 100644 index 0000000..0e688bf --- /dev/null +++ b/Video-ReID/models/YOLOv3/tf_aipp.cfg @@ -0,0 +1,22 @@ +aipp_op { + aipp_mode: static + input_format : YUV420SP_U8 + csc_switch : true + rbuv_swap_switch : false + matrix_r0c0 : 256 + matrix_r0c1 : 0 + matrix_r0c2 : 359 + matrix_r1c0 : 256 + matrix_r1c1 : -88 + matrix_r1c2 : -183 + matrix_r2c0 : 256 + matrix_r2c1 : 454 + matrix_r2c2 : 0 + input_bias_0 : 0 + input_bias_1 : 128 + input_bias_2 : 128 + var_reci_chn_0 : 0.003921568627451 + var_reci_chn_1 : 0.003921568627451 + var_reci_chn_2 : 0.003921568627451 +} + diff --git a/Video-ReID/pipeline/gallery.pipeline b/Video-ReID/pipeline/gallery.pipeline new file mode 100644 index 0000000..46b86c7 --- /dev/null +++ b/Video-ReID/pipeline/gallery.pipeline @@ -0,0 +1,109 @@ +{ + "galleryProcess": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_imagedecoder0" + }, + "mxpi_imagedecoder0": { + "factory": "mxpi_imagedecoder", + "next": "mxpi_imageresize0" + }, + "mxpi_imageresize0": { + "props": { + "resizeHeight": "416", + "resizeWidth": "416" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "models/YOLOv3/yolov3.om", + "waitingTime": "1" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "config/yolov3_tf_bs1_fp16.cfg", + "labelPath": "config/coco.names", + "postProcessLibPath": "libyolov3postprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_objectselector0" + }, + "mxpi_objectselector0": { + "props": { + "dataSource": "mxpi_objectpostprocessor0", + "FirstDetectionFilter": { + "Type": "Area", + "TopN": 1, + "BottomN": 0, + "MinArea": 0, + "MaxArea": 0, + "ConfThresh": 0.1 + } + }, + "factory": "mxpi_objectselector", + "next": "mxpi_distributor0" + }, + "mxpi_distributor0": { + "props": { + "dataSource": "mxpi_objectselector0", + "classIds": "0" + }, + "factory": "mxpi_distributor", + "next": "queue2" + }, + "queue2": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imagecrop0" + }, + "mxpi_imagecrop0":{ + "props":{ + "dataSource": "mxpi_distributor0_0", + "dataSourceImage": "mxpi_imageresize0", + "resizeHeight":"256", + "resizeWidth":"128" + }, + "factory":"mxpi_imagecrop", + "next":"mxpi_imageresize1" + }, + "mxpi_imageresize1": { + "props": { + "resizeHeight": "256", + "resizeWidth": "128" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer1" + }, + "mxpi_tensorinfer1": { + "props": { + "dataSource": "mxpi_imageresize1", + "dynamicStrategy": "Upper", + "modelPath": "models/OSNet/osnet.om", + "waitingTime": "1" + }, + "factory": "mxpi_tensorinfer", + "next": "appsink0" + }, + "appsink0": { + "props": { + "blocksize": "4096000" + }, + "factory": "appsink" + } + } +} \ No newline at end of file diff --git a/Video-ReID/pipeline/image.pipeline b/Video-ReID/pipeline/image.pipeline new file mode 100644 index 0000000..b16ff23 --- /dev/null +++ b/Video-ReID/pipeline/image.pipeline @@ -0,0 +1,185 @@ +{ + "queryImageProcess": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_imagedecoder0" + }, + "mxpi_imagedecoder0": { + "factory": "mxpi_imagedecoder", + "next": "tee0" + }, + "tee0": { + "factory": "tee", + "next": [ + "queue0", + "queue1" + ] + }, + "queue0": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imageresize0" + }, + "mxpi_imageresize0": { + "props": { + "resizeHeight": "416", + "resizeWidth": "416" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "models/YOLOv3/yolov3.om", + "waitingTime": "1" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "queue1": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_opencvosd0:0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "config/yolov3_tf_bs1_fp16.cfg", + "labelPath": "config/coco.names", + "postProcessLibPath": "libyolov3postprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_objectselector0" + }, + "mxpi_objectselector0": { + "props": { + "FirstDetectionFilter": { + "Type": "Area", + "TopN": 0, + "BottomN": 0, + "MinArea": 5120, + "MaxArea": 173056, + "ConfThresh": 0.4 + } + }, + "factory": "mxpi_objectselector", + "next": "mxpi_distributor0" + }, + "mxpi_distributor0": { + "props": { + "dataSource": "mxpi_objectselector0", + "classIds": "0" + }, + "factory": "mxpi_distributor", + "next": "tee1" + }, + "tee1": { + "factory": "tee", + "next": [ + "queue2", + "queue3" + ] + }, + "queue2": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imagecrop0" + }, + "queue3": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_featurematch0:1" + }, + "mxpi_imagecrop0":{ + "props":{ + "dataSource": "mxpi_distributor0_0", + "dataSourceImage": "mxpi_imageresize0", + "resizeHeight":"256", + "resizeWidth":"128" + }, + "factory":"mxpi_imagecrop", + "next":"mxpi_tensorinfer1" + }, + "mxpi_tensorinfer1": { + "props": { + "dataSource": "mxpi_imagecrop0", + "dynamicStrategy": "Upper", + "modelPath": "models/OSNet/osnet.om", + "waitingTime": "1" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_featurematch0" + }, + "mxpi_featurematch0": { + "props": { + "status": "1", + "querySource": "mxpi_tensorinfer1", + "objectSource": "mxpi_distributor0_0", + "galleryFeaturesPath": "output/gallery/gallery_features.bin", + "galleryIdsPath": "output/gallery/persons.txt", + "metric": "euclidean", + "threshold": "-1" + }, + "factory": "mxpi_featurematch", + "next": "mxpi_object2osdinstances0" + }, + "mxpi_object2osdinstances0": { + "props": { + "dataSource": "mxpi_featurematch0", + "colorMap":"255,100,100|100,255,100", + "fontFace": "1", + "fontScale": "0.8", + "fontThickness": "1", + "fontLineType": "8", + "rectThickness": "1", + "rectLineType": "8" + }, + "factory": "mxpi_object2osdinstances", + "next": "queue4" + }, + "queue4": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_opencvosd0:1" + }, + "mxpi_opencvosd0":{ + "factory":"mxpi_opencvosd", + "next":"mxpi_imageencoder0" + }, + + "mxpi_imageencoder0":{ + "factory":"mxpi_imageencoder", + "next":"queue5" + }, + "queue5": { + "props": { + "max-size-buffers": "100" + }, + "factory": "queue", + "next":"appsink0" + }, + "appsink0": { + "props": { + "blocksize": "4096000" + }, + "factory": "appsink" + } + } +} diff --git a/Video-ReID/pipeline/video.pipeline b/Video-ReID/pipeline/video.pipeline new file mode 100644 index 0000000..c0a2159 --- /dev/null +++ b/Video-ReID/pipeline/video.pipeline @@ -0,0 +1,215 @@ +{ + "queryVideoProcess": { + "stream_config": { + "deviceId": "0" + }, + "mxpi_rtspsrc0": { + "factory": "mxpi_rtspsrc", + "props": { + "rtspUrl":"rtsp://xxx.xxx.xxx.xxx:xxxx/xxx.264", + "channelId": "0", + "timeout": "30" + }, + "next": "mxpi_videodecoder0" + }, + "mxpi_videodecoder0": { + "factory": "mxpi_videodecoder", + "props": { + "inputVideoFormat": "H264", + "outputImageFormat": "YUV420SP_NV12" + }, + "next": "tee0" + }, + "tee0": { + "factory": "tee", + "next": [ + "queue0", + "queue1" + ] + }, + "queue0": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imageresize0" + }, + "mxpi_imageresize0": { + "props": { + "resizeHeight": "416", + "resizeWidth": "416" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer0" + }, + "queue1": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_opencvosd0:0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "models/YOLOv3/yolov3.om", + "waitingTime": "50" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "config/yolov3_tf_bs1_fp16.cfg", + "labelPath": "config/coco.names", + "postProcessLibPath": "libyolov3postprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_objectselector0" + }, + "mxpi_objectselector0": { + "props": { + "dataSource": "mxpi_objectpostprocessor0", + "FirstDetectionFilter": { + "Type": "Area", + "TopN": 0, + "BottomN": 0, + "MinArea": 5120, + "MaxArea": 173056, + "ConfThresh": 0.3 + } + }, + "factory": "mxpi_objectselector", + "next": "mxpi_distributor0" + }, + "mxpi_distributor0": { + "props": { + "dataSource": "mxpi_objectselector0", + "classIds": "0" + }, + "factory": "mxpi_distributor", + "next": "tee1" + }, + "tee1": { + "factory": "tee", + "next": [ + "queue2", + "queue3" + ] + }, + "queue2": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imagecrop0" + }, + "queue3": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_featurematch0:1" + }, + "mxpi_imagecrop0":{ + "props":{ + "dataSource": "mxpi_distributor0_0", + "dataSourceImage": "mxpi_imageresize0", + "resizeHeight":"256", + "resizeWidth":"128" + }, + "factory":"mxpi_imagecrop", + "next":"mxpi_imageresize1" + }, + "mxpi_imageresize1": { + "props": { + "resizeHeight": "256", + "resizeWidth": "128" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer1" + }, + "mxpi_tensorinfer1": { + "props": { + "dataSource": "mxpi_imageresize1", + "dynamicStrategy": "Upper", + "modelPath": "models/OSNet/osnet.om", + "waitingTime": "60" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_featurematch0" + }, + "mxpi_featurematch0": { + "props": { + "status": "1", + "querySource": "mxpi_tensorinfer1", + "objectSource": "mxpi_distributor0_0", + "galleryFeaturesPath": "output/gallery/gallery_features.bin", + "galleryIdsPath": "output/gallery/persons.txt", + "metric": "euclidean", + "threshold": "-1" + }, + "factory": "mxpi_featurematch", + "next": "mxpi_object2osdinstances0" + }, + "mxpi_object2osdinstances0": { + "props": { + "dataSource": "mxpi_featurematch0", + "colorMap":"255,100,100|100,255,100", + "fontFace": "1", + "fontScale": "0.8", + "fontThickness": "1", + "fontLineType": "8", + "rectThickness": "1", + "rectLineType": "8" + }, + "factory": "mxpi_object2osdinstances", + "next": "queue4" + }, + "queue4": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_opencvosd0:1" + }, + "mxpi_opencvosd0":{ + "factory":"mxpi_opencvosd", + "next":"queue6" + }, + "queue6": { + "props": { + "max-size-buffers": "200" + }, + "factory": "queue", + "next": "mxpi_imageresize2" + }, + "mxpi_imageresize2": { + "props": { + "dataSource": "mxpi_opencvosd0", + "resizeHeight": "416", + "resizeWidth": "416" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_videoencoder0" + }, + "mxpi_videoencoder0":{ + "props": { + "imageHeight": "416", + "imageWidth": "416", + "iFrameInterval": "200", + "fps": "1" + }, + "factory":"mxpi_videoencoder", + "next":"filesink0" + }, + "filesink0":{ + "props":{ + "blocksize":"40960000", + "location":"output/query/videos/reid.264" + }, + "factory":"filesink" + } + } +} diff --git a/Video-ReID/plugins/PluginFeatureMatch/CMakeLists.txt b/Video-ReID/plugins/PluginFeatureMatch/CMakeLists.txt new file mode 100644 index 0000000..ce56d11 --- /dev/null +++ b/Video-ReID/plugins/PluginFeatureMatch/CMakeLists.txt @@ -0,0 +1,46 @@ +cmake_minimum_required(VERSION 3.10) +project(mxpi_featurematch) + +set(CMAKE_CXX_STANDARD 11) + +set(PLUGIN_NAME "mxpi_featurematch") +set(TARGET_LIBRARY ${PLUGIN_NAME}) + +add_compile_options(-fPIC -fstack-protector-all -g -Wl,-z,relro,-z,now,-z -pie -Wall) +add_compile_options(-std=c++11 -Wno-deprecated-declarations) +add_compile_options("-DPLUGIN_NAME=${PLUGIN_NAME}") + +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0 -Dgoogle=mindxsdk_private) +add_definitions(-DENABLE_DVPP_INTERFACE) + + +set(MX_SDK_HOME "$ENV{MX_SDK_HOME}") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${MX_SDK_HOME}/lib/plugins/) + +include_directories(${MX_SDK_HOME}/include) +include_directories(${MX_SDK_HOME}/opensource/include) +include_directories(${MX_SDK_HOME}/opensource/include/gstreamer-1.0) +include_directories(${MX_SDK_HOME}/opensource/include/opencv4) +include_directories(${MX_SDK_HOME}/opensource/include/glib-2.0) +include_directories(${MX_SDK_HOME}/opensource/lib/glib-2.0/include) + + +link_directories(${MX_SDK_HOME}/lib) +link_directories(${MX_SDK_HOME}/opensource/lib) + +file(GLOB PLUGIN_SRC ./*.cpp) +message(${PLUGIN_SRC}) + +add_library(${TARGET_LIBRARY} SHARED ${PLUGIN_SRC}) +target_link_libraries(${TARGET_LIBRARY} + mxpidatatype + plugintoolkit + mxbase + streammanager + mindxsdk_protobuf + glib-2.0 + gstreamer-1.0 + gobject-2.0 + gstbase-1.0 + gmodule-2.0 + ) diff --git a/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.cpp b/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.cpp new file mode 100644 index 0000000..6cb1ab3 --- /dev/null +++ b/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.cpp @@ -0,0 +1,335 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "MxBase/Log/Log.h" +#include "MxBase/Tensor/TensorBase/TensorBase.h" +#include "Plugin_FeatureMatch.h" +#include +#include +#include + + +using namespace MxPlugins; +using namespace MxTools; +using namespace MxBase; +using namespace cv; + + +APP_ERROR PluginFeatureMatch::Init(std::map> &configParamMap) { + LogInfo << "Begin to initialize PluginFeatureMatch(" << pluginName_ << ")."; + // Get the property values by key + std::shared_ptr querySource = std::static_pointer_cast(configParamMap["querySource"]); + querySource_ = *querySource; + std::shared_ptr objectSource = std::static_pointer_cast(configParamMap["objectSource"]); + objectSource_ = *objectSource; + std::shared_ptr galleryFeaturesPath = std::static_pointer_cast(configParamMap["galleryFeaturesPath"]); + galleryFeaturesPath_ = *galleryFeaturesPath; + std::shared_ptr galleryIdsPath = std::static_pointer_cast(configParamMap["galleryIdsPath"]); + galleryIdsPath_ = *galleryIdsPath; + std::shared_ptr metric = std::static_pointer_cast(configParamMap["metric"]); + metric_ = *metric; + + std::shared_ptr threshold = std::static_pointer_cast(configParamMap["threshold"]); + threshold_ = *threshold; + + ReadGalleryFeatures(galleryFeaturesPath_, galleryIdsPath_, galleryFeatures, galleryIds); + LogInfo << "End to initialize PluginFeatureMatch(" << pluginName_ << ")."; + return APP_ERR_OK; +} + +APP_ERROR PluginFeatureMatch::DeInit() { + LogInfo << "Begin to deinitialize PluginFeatureMatch(" << pluginName_ << ")."; + LogInfo << "End to deinitialize PluginFeatureMatch(" << pluginName_ << ")."; + return APP_ERR_OK; +} + +APP_ERROR PluginFeatureMatch::CheckDataSource(MxTools::MxpiMetadataManager &mxpiMetadataManager, + MxTools::MxpiMetadataManager &mxpiMetadataManager1) { + if (mxpiMetadataManager.GetMetadata(querySource_) == nullptr) { + LogDebug << GetError(APP_ERR_METADATA_IS_NULL, pluginName_) + << "class metadata is null. please check" + << "Your property querySource(" << querySource_ << ")."; + return APP_ERR_METADATA_IS_NULL; + } + if (mxpiMetadataManager1.GetMetadata(objectSource_) == nullptr) { + LogDebug << GetError(APP_ERR_METADATA_IS_NULL, pluginName_) + << "class metadata is null. please check" + << "Your property objectSource(" << objectSource_ << ")."; + return APP_ERR_METADATA_IS_NULL; + } + return APP_ERR_OK; +} + +void EuclideanSquaredDistance(Mat queryFeatures, Mat galleryFeatures, Mat &distMat) { + int admm_beta = 1; + int admm_alpha = -2; + int square = 2; + Mat qdst1; + pow(queryFeatures, square, qdst1); + Mat qdst2; + reduce(qdst1, qdst2, 1, REDUCE_SUM); + + Mat qdst3 = Mat(qdst2.rows, galleryFeatures.rows, CV_32FC1); + for (int i = 0; i < galleryFeatures.rows; i++) { + Mat dstTemp = qdst3.col(i); + qdst2.copyTo(dstTemp); + } + + Mat gdst1; + pow(galleryFeatures, square, gdst1); + Mat gdst2; + reduce(gdst1, gdst2, 1, REDUCE_SUM); + + Mat gdst3 = Mat(gdst2.rows, queryFeatures.rows, CV_32FC1); + for (int i = 0; i < queryFeatures.rows;i++) { + Mat dstTemp = gdst3.col(i); + gdst2.copyTo(dstTemp); + } + + Mat gdst4; + transpose(gdst3, gdst4); + Mat dst1 = qdst3 +gdst4; + + Mat gdst5; + transpose(galleryFeatures, gdst5); + Mat dst2 = queryFeatures * gdst5; + + distMat = admm_beta * dst1 + admm_alpha * dst2; +} + +void CosineDistance(Mat queryFeatures, Mat galleryFeatures, Mat &distMat) { + Mat gdst1; + + transpose(galleryFeatures, gdst1); + distMat = queryFeatures * gdst1; +} + +APP_ERROR PluginFeatureMatch::ComputeDistance(MxpiTensorPackageList queryTensors, + Mat galleryFeatures, int tensorSize, Mat &distMat) { + Mat queryFeatures = Mat::zeros(tensorSize, queryTensors. + tensorpackagevec(0).tensorvec(0).tensorshape(1), CV_32FC1); + + for (int i = 0; i < tensorSize; i++) { + Mat dstTemp = queryFeatures.row(i); + auto vec = queryTensors.tensorpackagevec(i).tensorvec(0); + Mat feature = Mat(vec.tensorshape(0), vec.tensorshape(1), CV_32FC1, + (void *) vec.tensordataptr()); + feature.copyTo(dstTemp); + } + if (!(metric_.compare("euclidean"))) { + EuclideanSquaredDistance(queryFeatures, galleryFeatures, distMat); + } + else if (!(metric_.compare("cosine"))) { + CosineDistance(queryFeatures, galleryFeatures, distMat); + } + return APP_ERR_OK; +} + +void PluginFeatureMatch::GenerateOutput(MxpiObjectList srcObjectList, Mat distMat, + int tensorSize, std::vector galleryIds, + MxpiObjectList &dstMxpiObjectList) { + std::vector minValues = {}; + for (int i = 0; i < tensorSize; i++) { + auto minValue = std::min_element(distMat.ptr(i, 0), distMat.ptr(i, 0+distMat.cols)); + if (threshold_ == -1 || *minValue < threshold_) { + int minIndex = std::distance(distMat.ptr(i, 0), minValue); + auto objvec = srcObjectList.objectvec(i); + MxpiObject* dstMxpiObject = dstMxpiObjectList.add_objectvec(); + MxpiMetaHeader* dstMxpiMetaHeaderList = dstMxpiObject->add_headervec(); + dstMxpiMetaHeaderList->set_datasource(objectSource_); + dstMxpiMetaHeaderList->set_memberid(0); + + dstMxpiObject->set_x0(objvec.x0()); + dstMxpiObject->set_y0(objvec.y0()); + dstMxpiObject->set_x1(objvec.x1()); + dstMxpiObject->set_y1(objvec.y1()); + + // Generate ClassList + MxpiClass* dstMxpiClass = dstMxpiObject->add_classvec(); + MxpiMetaHeader* dstMxpiMetaHeaderList_c = dstMxpiClass->add_headervec(); + dstMxpiMetaHeaderList_c->set_datasource(objectSource_); + dstMxpiMetaHeaderList_c->set_memberid(0); + dstMxpiClass->set_classid(i); + dstMxpiClass->set_confidence(objvec.classvec(0).confidence()); + dstMxpiClass->set_classname(galleryIds[minIndex]); + } + } +} + +void PluginFeatureMatch::ReadGalleryFeatures(std::string featuresPath, + std::string idsPath, Mat &galleryFeatures, + std::vector &galleryIds) { + // 读取gallery的人名 + std::ifstream ifile(idsPath); + std::string str1; + while (std::getline(ifile, str1)) { + galleryIds.push_back(str1); + } + ifile.close(); + + // 读取gallery特征库 + int featureLen = 512; + const char* feaPath = featuresPath.c_str(); + FILE* fp = fopen(feaPath, "rb"); + galleryFeatures = Mat::zeros(int(galleryIds.size()), featureLen, CV_32FC1); + for (int i = 0; i < int(galleryIds.size()); i++) + { + for (int j = 0; j < featureLen; j++) + { + fread(&galleryFeatures.at(i, j), 1, sizeof(float), fp); + } + } + fclose(fp); + fp = NULL; +} + +APP_ERROR PluginFeatureMatch::Process(std::vector &mxpiBuffer) { + LogInfo << "Begin to process PluginFeatureMatch."; + // Get MxpiClassList from MxpiBuffer + MxpiBuffer *inputMxpiBuffer = mxpiBuffer[0]; + MxpiMetadataManager mxpiMetadataManager(*inputMxpiBuffer); + auto errorInfoPtr = mxpiMetadataManager.GetErrorInfo(); + MxpiBuffer *inputMxpiBuffer1 = mxpiBuffer[1]; + MxpiMetadataManager mxpiMetadataManager1(*inputMxpiBuffer1); + auto errorInfoPtr1 = mxpiMetadataManager1.GetErrorInfo(); + MxpiErrorInfo mxpiErrorInfo; + if (errorInfoPtr != nullptr | errorInfoPtr1 != nullptr) { + ErrorInfo_ << GetError(APP_ERR_COMM_FAILURE, pluginName_) + << "PluginFeatureMatch process is not implemented"; + mxpiErrorInfo.ret = APP_ERR_COMM_FAILURE; + mxpiErrorInfo.errorInfo = ErrorInfo_.str(); + LogError << "PluginFeatureMatch process is not implemented"; + return APP_ERR_COMM_FAILURE; + } + + // check data source + APP_ERROR ret = CheckDataSource(mxpiMetadataManager, mxpiMetadataManager1); + if (ret != APP_ERR_OK) { + SendData(0, *inputMxpiBuffer1); + return ret; + } + std::shared_ptr queryListMetadata = mxpiMetadataManager.GetMetadata(querySource_); + std::shared_ptr objectListMetadata = mxpiMetadataManager1.GetMetadata(objectSource_); + if (queryListMetadata == nullptr || objectListMetadata == nullptr) { + ErrorInfo_ << GetError(APP_ERR_METADATA_IS_NULL, pluginName_) << "Metadata is NULL, failed"; + mxpiErrorInfo.ret = APP_ERR_METADATA_IS_NULL; + mxpiErrorInfo.errorInfo = ErrorInfo_.str(); + return APP_ERR_METADATA_IS_NULL; + } + std::shared_ptr srcQueryListPtr = std::static_pointer_cast(queryListMetadata); + std::shared_ptr srcObjectListPtr = std::static_pointer_cast(objectListMetadata); + std::shared_ptr resultObjectListPtr = std::make_shared(); + int objectSize = (*srcObjectListPtr).objectvec_size(); + int tensorSize = (*srcQueryListPtr).tensorpackagevec_size(); + + Mat distMat = Mat(tensorSize, int(galleryIds.size()), CV_32FC1); + if (objectSize > 0 && objectSize == tensorSize) { + ret = ComputeDistance(*srcQueryListPtr, galleryFeatures, tensorSize, distMat); + } + + GenerateOutput(*srcObjectListPtr, distMat, tensorSize, galleryIds, *resultObjectListPtr); + ret = mxpiMetadataManager1.AddProtoMetadata(pluginName_, std::static_pointer_cast(resultObjectListPtr)); + if (ret != APP_ERR_OK) { + LogError << ErrorInfo_.str(); + SendMxpiErrorInfo(*inputMxpiBuffer1, pluginName_, ret, ErrorInfo_.str()); + SendData(0, *inputMxpiBuffer1); + } + // Send the data to downstream plugin + SendData(0, *inputMxpiBuffer1); + + LogInfo << "End to process PluginFeatureMatch(" << elementName_ << ")."; + return APP_ERR_OK; +} + +std::vector> PluginFeatureMatch::DefineProperties() { + std::vector> properties; + // Get the action category from previous plugin + auto querySource = std::make_shared>(ElementProperty { + STRING, + "querySource", + "queryFeatureSource", + "query infer output ", + "default", "NULL", "NULL" + }); + + auto objectSource = std::make_shared>(ElementProperty { + STRING, + "objectSource", + "queryobjectSource", + "detection infer postprocess output ", + "default", "NULL", "NULL" + }); + + // The action of interest file path + auto galleryFeaturesPath = std::make_shared>(ElementProperty { + STRING, + "galleryFeaturesPath", + "features of gallery file path", + "the path of gallery images features file", + "NULL", "NULL", "NULL" + }); + + auto galleryIdsPath = std::make_shared>(ElementProperty { + STRING, + "galleryIdsPath", + "id of gallery file path", + "the path of gallery person name file", + "NULL", "NULL", "NULL" + }); + + auto metric = std::make_shared>(ElementProperty { + STRING, + "metric", + "metric", + "the method of compute distance, select from 'euclidean', 'cosine'.", + "euclidean", "NULL", "NULL" + }); + + auto threshold = std::make_shared>(ElementProperty { + FLOAT, + "threshold", + "distanceThreshold", + "if distance is more than threshold,not matched gallery", + -1.0, -1.0, 1000.0 + }); + + properties.push_back(querySource); + properties.push_back(objectSource); + properties.push_back(galleryFeaturesPath); + properties.push_back(galleryIdsPath); + properties.push_back(metric); + properties.push_back(threshold); + return properties; +} + +MxpiPortInfo PluginFeatureMatch::DefineInputPorts() { + MxpiPortInfo inputPortInfo; + std::vector> value = {{"ANY"}, {"ANY"}}; + GenerateStaticInputPortsInfo(value, inputPortInfo); + return inputPortInfo; +} + +MxpiPortInfo PluginFeatureMatch::DefineOutputPorts() { + MxpiPortInfo outputPortInfo; + // Output: {{MxpiObjectList}} + std::vector> value = {{"ANY"}}; + GenerateStaticOutputPortsInfo(value, outputPortInfo); + return outputPortInfo; +} + +namespace { + MX_PLUGIN_GENERATE(PluginFeatureMatch) +} diff --git a/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.h b/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.h new file mode 100644 index 0000000..85bfbea --- /dev/null +++ b/Video-ReID/plugins/PluginFeatureMatch/Plugin_FeatureMatch.h @@ -0,0 +1,111 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDKMEMORY_PLUGINFEATUREMATCH_H +#define SDKMEMORY_PLUGINFEATUREMATCH_H + +#include "opencv2/opencv.hpp" +#include "opencv2/core/mat.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" +#include "MxBase/ErrorCode/ErrorCode.h" +#include "MxTools/PluginToolkit/base/MxPluginGenerator.h" +#include "MxTools/PluginToolkit/base/MxPluginBase.h" +#include "MxTools/PluginToolkit/metadata/MxpiMetadataManager.h" +#include "MxTools/Proto/MxpiDataType.pb.h" +#include "MxBase/ErrorCode/ErrorCode.h" + + +/** + * This plug is to recognize whether the object's action is a Violent Action and alarm. +*/ + +namespace MxPlugins { + class PluginFeatureMatch : public MxTools::MxPluginBase { + public: + /** + * @description: Init configs. + * @param configParamMap: config. + * @return: Error code. + */ + APP_ERROR Init(std::map> &configParamMap) override; + + /** + * @description: DeInit device. + * @return: Error code. + */ + APP_ERROR DeInit() override; + + /** + * @description: Plugin_FeatureMatch plugin process. + * @param mxpiBuffer: data receive from the previous. + * @return: Error code. + */ + APP_ERROR Process(std::vector &mxpiBuffer) override; + + /** + * @description: Plugin_FeatureMatch plugin define properties. + * @return: properties. + */ + static std::vector> DefineProperties(); + + /** + * @api + * @brief Define the number and data type of input ports. + * @return MxTools::MxpiPortInfo. + */ + static MxTools::MxpiPortInfo DefineInputPorts(); + + /** + * @api + * @brief Define the number and data type of output ports. + * @return MxTools::MxpiPortInfo. + */ + static MxTools::MxpiPortInfo DefineOutputPorts(); + + private: + /** + * @api + * @brief Check metadata. + * @param MxTools::MxpiMetadataManager. + * @return Error Code. + */ + APP_ERROR CheckDataSource(MxTools::MxpiMetadataManager &mxpiMetadataManager, + MxTools::MxpiMetadataManager &mxpiMetadataManager1); + + APP_ERROR ComputeDistance(MxTools::MxpiTensorPackageList queryTensors, + cv::Mat galleryFeatures, int tensorSize, cv::Mat &distMat); + + void GenerateOutput(MxTools::MxpiObjectList srcObjectList, cv::Mat distMat, + int tensorSize, std::vector galleryIds, + MxTools::MxpiObjectList &dstMxpiObjectList); + + void ReadGalleryFeatures(std::string featuresPath, + std::string idsPath, cv::Mat &galleryFeatures, std::vector &galleryIds); + + std::string querySource_ = ""; // previous plugin MxpiClassList + std::string objectSource_ = ""; + std::string galleryFeaturesPath_ = ""; + std::string galleryIdsPath_ = ""; + std::string metric_ = ""; + std::ostringstream ErrorInfo_; // Error Code + float threshold_ = 0.0; + cv::Mat galleryFeatures = cv::Mat(); + std::vector galleryIds = {}; + }; +} +#endif \ No newline at end of file diff --git a/Video-ReID/plugins/PluginFeatureMatch/build.sh b/Video-ReID/plugins/PluginFeatureMatch/build.sh new file mode 100644 index 0000000..006f86b --- /dev/null +++ b/Video-ReID/plugins/PluginFeatureMatch/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.mitations under the License. + +set -e + +current_folder="$( cd "$(dirname "$0")" ;pwd -P )" + +function build_plugin() { + build_path=$current_folder/build + if [ -d "$build_path" ]; then + rm -rf "$build_path" + else + echo "file $build_path is not exist." + fi + mkdir -p "$build_path" + cd "$build_path" + cmake .. + make -j + cd .. + exit 0 +} + +build_plugin +exit 0 \ No newline at end of file diff --git a/Video-ReID/run.sh b/Video-ReID/run.sh new file mode 100644 index 0000000..d0444d7 --- /dev/null +++ b/Video-ReID/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MODE=$1 +DURATION=$2 + + +if [ ${MODE} = "image" ]; then + python image.py +elif [ ${MODE} = "video" ]; then + python video.py --duration ${DURATION} +elif [ ${MODE} = "gallery" ]; then + python get_gallery_features.py +else + echo -e "The mode must be image or video or gallery" +fi + +exit 0 \ No newline at end of file diff --git a/Video-ReID/video.py b/Video-ReID/video.py new file mode 100644 index 0000000..5e4cb52 --- /dev/null +++ b/Video-ReID/video.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) Huawei Technologies Co.,Ltd. 2012-2021 All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import time +from StreamManagerApi import StreamManagerApi + + +def initialize_stream(): + """ + Initialize stream for detecting and re-identifying persons in video + + :arg: + None + + :return: + Stream api + """ + stream_api = StreamManagerApi() + ret = stream_api.InitManager() + if ret != 0: + error_message = "Failed to init Stream manager, ret=%s" % str(ret) + print(error_message) + exit() + + # creating stream based on json strings in the pipeline file: 'ReID.pipeline' + with open("pipeline/video.pipeline", 'rb') as f: + pipeline = f.read() + + ret = stream_api.CreateMultipleStreams(pipeline) + if ret != 0: + error_message = "Failed to create Stream, ret=%s" % str(ret) + print(error_message) + exit() + + return stream_api + + +def wait(duration): + + """ + Wait for destroy streams + + :arg: + duration: Duration of process video + + :return: + None + """ + + start = time.time() + + while True: + now = time.time() + cost_time = now - start + if cost_time >= duration: + break + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--duration', type=float, default= 30, help="Duration Of Process Video") + opt = parser.parse_args() + stream_manager_api = initialize_stream() + wait(opt.duration) + + stream_manager_api.DestroyAllStreams()