diff --git a/fastdeploy/vision/evaluation/segmentation.py b/fastdeploy/vision/evaluation/segmentation.py new file mode 100644 index 000000000..7e0d21e1a --- /dev/null +++ b/fastdeploy/vision/evaluation/segmentation.py @@ -0,0 +1,95 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tqdm import trange +import numpy as np +import collections +import os +import math +import time + + +def eval_segmentation(model, data_dir): + import cv2 + from utils import Cityscapes + from utils import f1_score, calculate_area, mean_iou, accuracy, kappa + assert os.path.isdir( + data_dir), "The image_file_path:{} is not a directory.".format( + data_dir) + eval_dataset = Cityscapes(dataset_root=data_dir, mode="val") + file_list = eval_dataset.file_list + image_num = eval_dataset.num_samples + num_classes = eval_dataset.num_classes + intersect_area_all = 0 + pred_area_all = 0 + label_area_all = 0 + conf_mat_all = [] + twenty_percent_image_num = math.ceil(image_num * 0.2) + start_time = 0 + end_time = 0 + average_inference_time = 0 + for image_label_path, i in zip(file_list, + trange( + image_num, desc="Inference Progress")): + if i == twenty_percent_image_num: + start_time = time.time() + im = cv2.imread(image_label_path[0]) + label = cv2.imread(image_label_path[1], cv2.IMREAD_GRAYSCALE) + result = model.predict(im) + if i == image_num - 1: + end_time = time.time() + average_inference_time = round( + (end_time - start_time) / (image_num - twenty_percent_image_num), + 4) + pred = np.array(result.label_map).reshape(result.shape[0], + result.shape[1]) + intersect_area, pred_area, label_area = calculate_area(pred, label, + num_classes) + intersect_area_all = intersect_area_all + intersect_area + pred_area_all = pred_area_all + pred_area + label_area_all = label_area_all + label_area + + class_iou, miou = mean_iou(intersect_area_all, pred_area_all, + label_area_all) + class_acc, oacc = accuracy(intersect_area_all, pred_area_all) + kappa_res = kappa(intersect_area_all, pred_area_all, label_area_all) + category_f1score = f1_score(intersect_area_all, pred_area_all, + label_area_all) + + eval_metrics = collections.OrderedDict( + zip([ + 'miou', 'category_iou', 'oacc', 'category_acc', 'kappa', + 'category_F1-score', 'average_inference_time(s)' + ], [ + miou, class_iou, oacc, class_acc, kappa_res, category_f1score, + average_inference_time + ])) + return eval_metrics + + +import fastdeploy as fd +#model = fd.vision.segmentation.PaddleSegModel("/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/unet/unet_Cityscapes/model.pdmodel", +# "/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/unet/unet_Cityscapes//model.pdiparams", +# "/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/unet/unet_Cityscapes/deploy.yaml") +# +option = fd.RuntimeOption() +option.use_paddle_backend() +option.use_gpu(3) +model = fd.vision.segmentation.PaddleSegModel( + "/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/PP-LiteSeg/output_no_static_size/model.pdmodel", + "/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/PP-LiteSeg/output_no_static_size/model.pdiparams", + "/huangjianhui/temp/FastDeploy/model_zoo/vision/ppseg/PP-LiteSeg/output_no_static_size/deploy.yaml", + option) + +result = eval_segmentation(model, "/huangjianhui/PaddleSeg/data/cityscapes/") diff --git a/fastdeploy/vision/evaluation/utils/__init__.py b/fastdeploy/vision/evaluation/utils/__init__.py index afa10c0e8..c6de72fd6 100644 --- a/fastdeploy/vision/evaluation/utils/__init__.py +++ b/fastdeploy/vision/evaluation/utils/__init__.py @@ -14,8 +14,10 @@ from . import fd_logging from .util import * -from .metrics import * +from .coco_metrics import * +from .seg_metrics import * from .json_results import * from .map_utils import * from .coco_utils import * from .coco import * +from .cityscapes import * diff --git a/fastdeploy/vision/evaluation/utils/cityscapes.py b/fastdeploy/vision/evaluation/utils/cityscapes.py new file mode 100644 index 000000000..407c52a3e --- /dev/null +++ b/fastdeploy/vision/evaluation/utils/cityscapes.py @@ -0,0 +1,74 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import glob +from . import fd_logging as logging + +#import fd_logging as logging + + +class Cityscapes(object): + """ + Cityscapes dataset `https://www.cityscapes-dataset.com/`. + The folder structure is as follow: + + cityscapes + | + |--leftImg8bit + | |--train + | |--val + | |--test + | + |--gtFine + | |--train + | |--val + | |--test + + Args: + dataset_root (str): Cityscapes dataset directory. + """ + NUM_CLASSES = 19 + + def __init__(self, dataset_root, mode): + self.dataset_root = dataset_root + self.file_list = list() + mode = mode.lower() + self.mode = mode + self.num_classes = self.NUM_CLASSES + self.ignore_index = 255 + + img_dir = os.path.join(self.dataset_root, 'leftImg8bit') + label_dir = os.path.join(self.dataset_root, 'gtFine') + if self.dataset_root is None or not os.path.isdir( + self.dataset_root) or not os.path.isdir( + img_dir) or not os.path.isdir(label_dir): + raise ValueError( + "The dataset is not Found or the folder structure is nonconfoumance." + ) + + label_files = sorted( + glob.glob( + os.path.join(label_dir, mode, '*', + '*_gtFine_labelTrainIds.png'))) + img_files = sorted( + glob.glob(os.path.join(img_dir, mode, '*', '*_leftImg8bit.png'))) + + self.file_list = [ + [img_path, label_path] + for img_path, label_path in zip(img_files, label_files) + ] + + self.num_samples = len(self.file_list) + logging.info("{} samples in file {}".format(self.num_samples, img_dir)) diff --git a/fastdeploy/vision/evaluation/utils/metrics.py b/fastdeploy/vision/evaluation/utils/coco_metrics.py similarity index 100% rename from fastdeploy/vision/evaluation/utils/metrics.py rename to fastdeploy/vision/evaluation/utils/coco_metrics.py diff --git a/fastdeploy/vision/evaluation/utils/seg_metrics.py b/fastdeploy/vision/evaluation/utils/seg_metrics.py new file mode 100644 index 000000000..6d5527820 --- /dev/null +++ b/fastdeploy/vision/evaluation/utils/seg_metrics.py @@ -0,0 +1,143 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + + +def f1_score(intersect_area, pred_area, label_area): + class_f1_sco = [] + for i in range(len(intersect_area)): + if pred_area[i] + label_area[i] == 0: + f1_sco = 0 + elif pred_area[i] == 0: + f1_sco = 0 + else: + prec = intersect_area[i] / pred_area[i] + rec = intersect_area[i] / label_area[i] + f1_sco = 2 * prec * rec / (prec + rec) + class_f1_sco.append(f1_sco) + return np.array(class_f1_sco) + + +def calculate_area(pred, label, num_classes, ignore_index=255): + """ + Calculate intersect, prediction and label area + + Args: + pred (np.ndarray): The prediction by model. + label (np.ndarray): The ground truth of image. + num_classes (int): The unique number of target classes. + ignore_index (int): Specifies a target value that is ignored. Default: 255. + + Returns: + Numpy Array: The intersection area of prediction and the ground on all class. + Numpy Array: The prediction area on all class. + Numpy Array: The ground truth area on all class + """ + if not pred.shape == label.shape: + raise ValueError('Shape of `pred` and `label should be equal, ' + 'but there are {} and {}.'.format(pred.shape, + label.shape)) + + mask = label != ignore_index + pred = pred + 1 + label = label + 1 + pred = pred * mask + label = label * mask + pred = np.eye(num_classes + 1)[pred] + label = np.eye(num_classes + 1)[label] + pred = pred[:, 1:] + label = label[:, 1:] + + pred_area = [] + label_area = [] + intersect_area = [] + + for i in range(num_classes): + pred_i = pred[:, :, i] + label_i = label[:, :, i] + pred_area_i = np.sum(pred_i) + label_area_i = np.sum(label_i) + intersect_area_i = np.sum(pred_i * label_i) + pred_area.append(pred_area_i) + label_area.append(label_area_i) + intersect_area.append(intersect_area_i) + return np.array(intersect_area), np.array(pred_area), np.array(label_area) + + +def mean_iou(intersect_area, pred_area, label_area): + """ + Calculate iou. + + Args: + intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes. + pred_area (np.ndarray): The prediction area on all classes. + label_area (np.ndarray): The ground truth area on all classes. + + Returns: + np.ndarray: iou on all classes. + float: mean iou of all classes. + """ + union = pred_area + label_area - intersect_area + class_iou = [] + for i in range(len(intersect_area)): + if union[i] == 0: + iou = 0 + else: + iou = intersect_area[i] / union[i] + class_iou.append(iou) + miou = np.mean(class_iou) + return np.array(class_iou), miou + + +def accuracy(intersect_area, pred_area): + """ + Calculate accuracy + + Args: + intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes.. + pred_area (np.ndarray): The prediction area on all classes. + + Returns: + np.ndarray: accuracy on all classes. + float: mean accuracy. + """ + class_acc = [] + for i in range(len(intersect_area)): + if pred_area[i] == 0: + acc = 0 + else: + acc = intersect_area[i] / pred_area[i] + class_acc.append(acc) + macc = np.sum(intersect_area) / np.sum(pred_area) + return np.array(class_acc), macc + + +def kappa(intersect_area, pred_area, label_area): + """ + Calculate kappa coefficient + + Args: + intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes.. + pred_area (np.ndarray): The prediction area on all classes. + label_area (np.ndarray): The ground truth area on all classes. + + Returns: + float: kappa coefficient. + """ + total_area = np.sum(label_area) + po = np.sum(intersect_area) / total_area + pe = np.sum(pred_area * label_area) / (total_area * total_area) + kappa = (po - pe) / (1 - pe) + return kappa