Modify file structure to separate python and cpp code (#223)

Modify code structure

python/fastdeploy/vision/evaluation/utils/__init__.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import fd_logging
+from .util import *
+from .coco_metrics import *
+from .seg_metrics import *
+from .json_results import *
+from .map_utils import *
+from .coco_utils import *
+from .coco import *
+from .cityscapes import *

python/fastdeploy/vision/evaluation/utils/cityscapes.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import glob
+from . import fd_logging as logging
+
+#import fd_logging as logging
+
+
+class Cityscapes(object):
+    """
+    Cityscapes dataset `https://www.cityscapes-dataset.com/`.
+    The folder structure is as follow:
+
+        cityscapes
+        |
+        |--leftImg8bit
+        |  |--train
+        |  |--val
+        |  |--test
+        |
+        |--gtFine
+        |  |--train
+        |  |--val
+        |  |--test
+
+    Args:
+        dataset_root (str): Cityscapes dataset directory.
+    """
+    NUM_CLASSES = 19
+
+    def __init__(self, dataset_root, mode):
+        self.dataset_root = dataset_root
+        self.file_list = list()
+        mode = mode.lower()
+        self.mode = mode
+        self.num_classes = self.NUM_CLASSES
+        self.ignore_index = 255
+
+        img_dir = os.path.join(self.dataset_root, 'leftImg8bit')
+        label_dir = os.path.join(self.dataset_root, 'gtFine')
+        if self.dataset_root is None or not os.path.isdir(
+                self.dataset_root) or not os.path.isdir(
+                    img_dir) or not os.path.isdir(label_dir):
+            raise ValueError(
+                "The dataset is not Found or the folder structure is nonconfoumance."
+            )
+
+        label_files = sorted(
+            glob.glob(
+                os.path.join(label_dir, mode, '*',
+                             '*_gtFine_labelTrainIds.png')))
+        img_files = sorted(
+            glob.glob(os.path.join(img_dir, mode, '*', '*_leftImg8bit.png')))
+
+        self.file_list = [
+            [img_path, label_path]
+            for img_path, label_path in zip(img_files, label_files)
+        ]
+
+        self.num_samples = len(self.file_list)
+        logging.info("{} samples in file {}".format(self.num_samples, img_dir))

python/fastdeploy/vision/evaluation/utils/coco.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import copy
+import os.path as osp
+import sys
+import numpy as np
+from . import fd_logging as logging
+from .util import is_pic, get_num_workers
+
+
+class CocoDetection(object):
+    """读取MSCOCO格式的检测数据集，并对样本进行相应的处理，该格式的数据集同样可以应用到实例分割模型的训练中。
+
+    Args:
+        data_dir (str): 数据集所在的目录路径。
+        ann_file (str): 数据集的标注文件，为一个独立的json格式文件。
+        num_workers (int|str): 数据集中样本在预处理过程中的线程或进程数。默认为'auto'。当设为'auto'时，根据
+            系统的实际CPU核数设置`num_workers`: 如果CPU核数的一半大于8，则`num_workers`为8，否则为CPU核数的一半。
+        shuffle (bool): 是否需要对数据集中样本打乱顺序。默认为False。
+        allow_empty (bool): 是否加载负样本。默认为False。
+        empty_ratio (float): 用于指定负样本占总样本数的比例。如果小于0或大于等于1，则保留全部的负样本。默认为1。
+    """
+
+    def __init__(self,
+                 data_dir,
+                 ann_file,
+                 num_workers='auto',
+                 shuffle=False,
+                 allow_empty=False,
+                 empty_ratio=1.):
+
+        from pycocotools.coco import COCO
+        self.data_dir = data_dir
+        self.data_fields = None
+        self.num_max_boxes = 1000
+        self.num_workers = get_num_workers(num_workers)
+        self.shuffle = shuffle
+        self.allow_empty = allow_empty
+        self.empty_ratio = empty_ratio
+        self.file_list = list()
+        neg_file_list = list()
+        self.labels = list()
+
+        coco = COCO(ann_file)
+        self.coco_gt = coco
+        img_ids = sorted(coco.getImgIds())
+        cat_ids = coco.getCatIds()
+        catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
+        cname2clsid = dict({
+            coco.loadCats(catid)[0]['name']: clsid
+            for catid, clsid in catid2clsid.items()
+        })
+        for label, cid in sorted(cname2clsid.items(), key=lambda d: d[1]):
+            self.labels.append(label)
+        logging.info("Starting to read file list from dataset...")
+
+        ct = 0
+        for img_id in img_ids:
+            is_empty = False
+            img_anno = coco.loadImgs(img_id)[0]
+            im_fname = osp.join(data_dir, img_anno['file_name'])
+            if not is_pic(im_fname):
+                continue
+            im_w = float(img_anno['width'])
+            im_h = float(img_anno['height'])
+            ins_anno_ids = coco.getAnnIds(imgIds=img_id, iscrowd=False)
+            instances = coco.loadAnns(ins_anno_ids)
+
+            bboxes = []
+            for inst in instances:
+                x, y, box_w, box_h = inst['bbox']
+                x1 = max(0, x)
+                y1 = max(0, y)
+                x2 = min(im_w - 1, x1 + max(0, box_w))
+                y2 = min(im_h - 1, y1 + max(0, box_h))
+                if inst['area'] > 0 and x2 >= x1 and y2 >= y1:
+                    inst['clean_bbox'] = [x1, y1, x2, y2]
+                    bboxes.append(inst)
+                else:
+                    logging.warning(
+                        "Found an invalid bbox in annotations: "
+                        "im_id: {}, area: {} x1: {}, y1: {}, x2: {}, y2: {}."
+                        .format(img_id, float(inst['area']), x1, y1, x2, y2))
+            num_bbox = len(bboxes)
+            if num_bbox == 0 and not self.allow_empty:
+                continue
+            elif num_bbox == 0:
+                is_empty = True
+
+            gt_bbox = np.zeros((num_bbox, 4), dtype=np.float32)
+            gt_class = np.zeros((num_bbox, 1), dtype=np.int32)
+            gt_score = np.ones((num_bbox, 1), dtype=np.float32)
+            is_crowd = np.zeros((num_bbox, 1), dtype=np.int32)
+            difficult = np.zeros((num_bbox, 1), dtype=np.int32)
+            gt_poly = [None] * num_bbox
+
+            has_segmentation = False
+            for i, box in reversed(list(enumerate(bboxes))):
+                catid = box['category_id']
+                gt_class[i][0] = catid2clsid[catid]
+                gt_bbox[i, :] = box['clean_bbox']
+                is_crowd[i][0] = box['iscrowd']
+                if 'segmentation' in box and box['iscrowd'] == 1:
+                    gt_poly[i] = [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]
+                elif 'segmentation' in box and box['segmentation']:
+                    if not np.array(
+                            box['segmentation'],
+                            dtype=object).size > 0 and not self.allow_empty:
+                        gt_poly.pop(i)
+                        is_crowd = np.delete(is_crowd, i)
+                        gt_class = np.delete(gt_class, i)
+                        gt_bbox = np.delete(gt_bbox, i)
+                    else:
+                        gt_poly[i] = box['segmentation']
+                    has_segmentation = True
+            if has_segmentation and not any(gt_poly) and not self.allow_empty:
+                continue
+
+            im_info = {
+                'im_id': np.array([img_id]).astype('int32'),
+                'image_shape': np.array([im_h, im_w]).astype('int32'),
+            }
+            label_info = {
+                'is_crowd': is_crowd,
+                'gt_class': gt_class,
+                'gt_bbox': gt_bbox,
+                'gt_score': gt_score,
+                'gt_poly': gt_poly,
+                'difficult': difficult
+            }
+
+            if is_empty:
+                neg_file_list.append({
+                    'image': im_fname,
+                    **
+                    im_info,
+                    **
+                    label_info
+                })
+            else:
+                self.file_list.append({
+                    'image': im_fname,
+                    **
+                    im_info,
+                    **
+                    label_info
+                })
+            ct += 1
+
+            self.num_max_boxes = max(self.num_max_boxes, len(instances))
+
+        if not ct:
+            logging.error(
+                "No coco record found in %s' % (ann_file)", exit=True)
+        self.pos_num = len(self.file_list)
+        if self.allow_empty and neg_file_list:
+            self.file_list += self._sample_empty(neg_file_list)
+        logging.info(
+            "{} samples in file {}, including {} positive samples and {} negative samples.".
+            format(
+                len(self.file_list), ann_file, self.pos_num,
+                len(self.file_list) - self.pos_num))
+        self.num_samples = len(self.file_list)
+
+        self._epoch = 0

python/fastdeploy/vision/evaluation/utils/coco_metrics.py

@@ -0,0 +1,89 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import sys
+from collections import OrderedDict
+from .coco_utils import get_infer_results, cocoapi_eval
+
+
+class COCOMetric(object):
+    def __init__(self, coco_gt, **kwargs):
+        self.clsid2catid = {
+            i: cat['id']
+            for i, cat in enumerate(coco_gt.loadCats(coco_gt.getCatIds()))
+        }
+        self.coco_gt = coco_gt
+        self.classwise = kwargs.get('classwise', False)
+        self.bias = 0
+        self.reset()
+
+    def reset(self):
+        # only bbox and mask evaluation support currently
+        self.details = {
+            'gt': copy.deepcopy(self.coco_gt.dataset),
+            'bbox': [],
+            'mask': []
+        }
+        self.eval_stats = {}
+
+    def update(self, im_id, outputs):
+        outs = {}
+        # outputs Tensor -> numpy.ndarray
+        for k, v in outputs.items():
+            outs[k] = v
+
+        outs['im_id'] = im_id
+        infer_results = get_infer_results(
+            outs, self.clsid2catid, bias=self.bias)
+        self.details['bbox'] += infer_results[
+            'bbox'] if 'bbox' in infer_results else []
+        self.details['mask'] += infer_results[
+            'mask'] if 'mask' in infer_results else []
+
+    def accumulate(self):
+        if len(self.details['bbox']) > 0:
+            bbox_stats = cocoapi_eval(
+                copy.deepcopy(self.details['bbox']),
+                'bbox',
+                coco_gt=self.coco_gt,
+                classwise=self.classwise)
+            self.eval_stats['bbox'] = bbox_stats
+            sys.stdout.flush()
+
+        if len(self.details['mask']) > 0:
+            seg_stats = cocoapi_eval(
+                copy.deepcopy(self.details['mask']),
+                'segm',
+                coco_gt=self.coco_gt,
+                classwise=self.classwise)
+            self.eval_stats['mask'] = seg_stats
+            sys.stdout.flush()
+
+    def log(self):
+        pass
+
+    def get(self):
+        if 'bbox' not in self.eval_stats:
+            return {'bbox_mmap': 0.}
+        if 'mask' in self.eval_stats:
+            return OrderedDict(
+                zip(['bbox_mmap', 'segm_mmap'],
+                    [self.eval_stats['bbox'][0], self.eval_stats['mask'][0]]))
+        else:
+            return {'bbox_mmap': self.eval_stats['bbox'][0]}

python/fastdeploy/vision/evaluation/utils/coco_utils.py

@@ -0,0 +1,218 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+import numpy as np
+from .map_utils import draw_pr_curve
+from .json_results import get_det_res, get_det_poly_res, get_seg_res, get_solov2_segm_res
+from . import fd_logging as logging
+import copy
+
+
+def loadRes(coco_obj, anns):
+    """
+    Load result file and return a result api object.
+    :param   resFile (str)     : file name of result file
+    :return: res (obj)         : result api object
+    """
+
+    # This function has the same functionality as pycocotools.COCO.loadRes,
+    # except that the input anns is list of results rather than a json file.
+    # Refer to
+    # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/coco.py#L305,
+
+    # matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
+    # or matplotlib.backends is imported for the first time
+    # pycocotools import matplotlib
+    import matplotlib
+    matplotlib.use('Agg')
+    from pycocotools.coco import COCO
+    import pycocotools.mask as maskUtils
+    import time
+    res = COCO()
+    res.dataset['images'] = [img for img in coco_obj.dataset['images']]
+
+    tic = time.time()
+    assert type(anns) == list, 'results in not an array of objects'
+    annsImgIds = [ann['image_id'] for ann in anns]
+    assert set(annsImgIds) == (set(annsImgIds) & set(coco_obj.getImgIds())), \
+        'Results do not correspond to current coco set'
+    if 'caption' in anns[0]:
+        imgIds = set([img['id'] for img in res.dataset['images']]) & set(
+            [ann['image_id'] for ann in anns])
+        res.dataset['images'] = [
+            img for img in res.dataset['images'] if img['id'] in imgIds
+        ]
+        for id, ann in enumerate(anns):
+            ann['id'] = id + 1
+    elif 'bbox' in anns[0] and not anns[0]['bbox'] == []:
+        res.dataset['categories'] = copy.deepcopy(coco_obj.dataset[
+            'categories'])
+        for id, ann in enumerate(anns):
+            bb = ann['bbox']
+            x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]]
+            if not 'segmentation' in ann:
+                ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
+            ann['area'] = bb[2] * bb[3]
+            ann['id'] = id + 1
+            ann['iscrowd'] = 0
+    elif 'segmentation' in anns[0]:
+        res.dataset['categories'] = copy.deepcopy(coco_obj.dataset[
+            'categories'])
+        for id, ann in enumerate(anns):
+            # now only support compressed RLE format as segmentation results
+            ann['area'] = maskUtils.area(ann['segmentation'])
+            if not 'bbox' in ann:
+                ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
+            ann['id'] = id + 1
+            ann['iscrowd'] = 0
+    elif 'keypoints' in anns[0]:
+        res.dataset['categories'] = copy.deepcopy(coco_obj.dataset[
+            'categories'])
+        for id, ann in enumerate(anns):
+            s = ann['keypoints']
+            x = s[0::3]
+            y = s[1::3]
+            x0, x1, y0, y1 = np.min(x), np.max(x), np.min(y), np.max(y)
+            ann['area'] = (x1 - x0) * (y1 - y0)
+            ann['id'] = id + 1
+            ann['bbox'] = [x0, y0, x1 - x0, y1 - y0]
+
+    res.dataset['annotations'] = anns
+    res.createIndex()
+    return res
+
+
+def get_infer_results(outs, catid, bias=0):
+    """
+    Get result at the stage of inference.
+    The output format is dictionary containing bbox or mask result.
+
+    For example, bbox result is a list and each element contains
+    image_id, category_id, bbox and score.
+    """
+    if outs is None or len(outs) == 0:
+        raise ValueError(
+            'The number of valid detection result if zero. Please use reasonable model and check input data.'
+        )
+
+    im_id = outs['im_id']
+
+    infer_res = {}
+    if 'bbox' in outs:
+        if len(outs['bbox']) > 0 and len(outs['bbox'][0]) > 6:
+            infer_res['bbox'] = get_det_poly_res(
+                outs['bbox'], outs['bbox_num'], im_id, catid, bias=bias)
+        else:
+            infer_res['bbox'] = get_det_res(
+                outs['bbox'], outs['bbox_num'], im_id, catid, bias=bias)
+
+    if 'mask' in outs:
+        # mask post process
+        infer_res['mask'] = get_seg_res(outs['mask'], outs['bbox'],
+                                        outs['bbox_num'], im_id, catid)
+
+    if 'segm' in outs:
+        infer_res['segm'] = get_solov2_segm_res(outs, im_id, catid)
+
+    return infer_res
+
+
+def cocoapi_eval(anns,
+                 style,
+                 coco_gt=None,
+                 anno_file=None,
+                 max_dets=(100, 300, 1000),
+                 classwise=False):
+    """
+    Args:
+        anns: Evaluation result.
+        style (str): COCOeval style, can be `bbox` , `segm` and `proposal`.
+        coco_gt (str): Whether to load COCOAPI through anno_file,
+                 eg: coco_gt = COCO(anno_file)
+        anno_file (str): COCO annotations file.
+        max_dets (tuple): COCO evaluation maxDets.
+        classwise (bool): Whether per-category AP and draw P-R Curve or not.
+    """
+    assert coco_gt is not None or anno_file is not None
+    from pycocotools.coco import COCO
+    from pycocotools.cocoeval import COCOeval
+
+    if coco_gt is None:
+        coco_gt = COCO(anno_file)
+    logging.info("Start evaluate...")
+    coco_dt = loadRes(coco_gt, anns)
+    if style == 'proposal':
+        coco_eval = COCOeval(coco_gt, coco_dt, 'bbox')
+        coco_eval.params.useCats = 0
+        coco_eval.params.maxDets = list(max_dets)
+    else:
+        coco_eval = COCOeval(coco_gt, coco_dt, style)
+    coco_eval.evaluate()
+    coco_eval.accumulate()
+    coco_eval.summarize()
+    if classwise:
+        # Compute per-category AP and PR curve
+        try:
+            from terminaltables import AsciiTable
+        except Exception as e:
+            logging.error(
+                'terminaltables not found, plaese install terminaltables. '
+                'for example: `pip install terminaltables`.')
+            raise e
+        precisions = coco_eval.eval['precision']
+        cat_ids = coco_gt.getCatIds()
+        # precision: (iou, recall, cls, area range, max dets)
+        assert len(cat_ids) == precisions.shape[2]
+        results_per_category = []
+        for idx, catId in enumerate(cat_ids):
+            # area range index 0: all area ranges
+            # max dets index -1: typically 100 per image
+            nm = coco_gt.loadCats(catId)[0]
+            precision = precisions[:, :, idx, 0, -1]
+            precision = precision[precision > -1]
+            if precision.size:
+                ap = np.mean(precision)
+            else:
+                ap = float('nan')
+            results_per_category.append(
+                (str(nm["name"]), '{:0.3f}'.format(float(ap))))
+            pr_array = precisions[0, :, idx, 0, 2]
+            recall_array = np.arange(0.0, 1.01, 0.01)
+            draw_pr_curve(
+                pr_array,
+                recall_array,
+                out_dir=style + '_pr_curve',
+                file_name='{}_precision_recall_curve.jpg'.format(nm["name"]))
+
+        num_columns = min(6, len(results_per_category) * 2)
+
+        import itertools
+        results_flatten = list(itertools.chain(*results_per_category))
+        headers = ['category', 'AP'] * (num_columns // 2)
+        results_2d = itertools.zip_longest(
+            * [results_flatten[i::num_columns] for i in range(num_columns)])
+        table_data = [headers]
+        table_data += [result for result in results_2d]
+        table = AsciiTable(table_data)
+        logging.info('Per-category of {} AP: \n{}'.format(style, table.table))
+        logging.info("per-category PR curve has output to {} folder.".format(
+            style + '_pr_curve'))
+    # flush coco evaluation result
+    sys.stdout.flush()
+    return coco_eval.stats

python/fastdeploy/vision/evaluation/utils/fd_logging.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import os
+import sys
+import colorama
+from colorama import init
+
+init(autoreset=True)
+levels = {0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG'}
+
+
+def log(level=2, message="", use_color=False):
+    current_time = time.time()
+    time_array = time.localtime(current_time)
+    current_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    if use_color:
+        print("\033[1;31;40m{} [{}]\t{}\033[0m".format(current_time, levels[
+            level], message).encode("utf-8").decode("latin1"))
+    else:
+        print("{} [{}]\t{}".format(current_time, levels[level], message)
+              .encode("utf-8").decode("latin1"))
+    sys.stdout.flush()
+
+
+def debug(message="", use_color=False):
+    log(level=3, message=message, use_color=use_color)
+
+
+def info(message="", use_color=False):
+    log(level=2, message=message, use_color=use_color)
+
+
+def warning(message="", use_color=True):
+    log(level=1, message=message, use_color=use_color)
+
+
+def error(message="", use_color=True, exit=True):
+    log(level=0, message=message, use_color=use_color)
+    if exit:
+        sys.exit(-1)

python/fastdeploy/vision/evaluation/utils/json_results.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+import numpy as np
+
+
+def get_det_res(bboxes, bbox_nums, image_id, label_to_cat_id_map, bias=0):
+    det_res = []
+    for i in range(bbox_nums):
+        cur_image_id = int(image_id)
+        dt = bboxes[i]
+        num_id, score, xmin, ymin, xmax, ymax = dt
+        if int(num_id) < 0:
+            continue
+        category_id = label_to_cat_id_map[int(num_id)]
+        w = xmax - xmin + bias
+        h = ymax - ymin + bias
+        bbox = [xmin, ymin, w, h]
+        dt_res = {
+            'image_id': cur_image_id,
+            'category_id': category_id,
+            'bbox': bbox,
+            'score': score
+        }
+        det_res.append(dt_res)
+    return det_res
+
+
+def get_det_poly_res(bboxes, bbox_nums, image_id, label_to_cat_id_map, bias=0):
+    det_res = []
+    k = 0
+    for i in range(len(bbox_nums)):
+        cur_image_id = int(image_id[i][0])
+        det_nums = bbox_nums[i]
+        for j in range(det_nums):
+            dt = bboxes[k]
+            k = k + 1
+            num_id, score, x1, y1, x2, y2, x3, y3, x4, y4 = dt.tolist()
+            if int(num_id) < 0:
+                continue
+            category_id = label_to_cat_id_map[int(num_id)]
+            rbox = [x1, y1, x2, y2, x3, y3, x4, y4]
+            dt_res = {
+                'image_id': cur_image_id,
+                'category_id': category_id,
+                'bbox': rbox,
+                'score': score
+            }
+            det_res.append(dt_res)
+    return det_res
+
+
+def strip_mask(mask):
+    row = mask[0, 0, :]
+    col = mask[0, :, 0]
+    im_h = len(col) - np.count_nonzero(col == -1)
+    im_w = len(row) - np.count_nonzero(row == -1)
+    return mask[:, :im_h, :im_w]
+
+
+def get_seg_res(masks, bboxes, mask_nums, image_id, label_to_cat_id_map):
+    import pycocotools.mask as mask_util
+    seg_res = []
+    k = 0
+    for i in range(len(mask_nums)):
+        cur_image_id = int(image_id[i][0])
+        det_nums = mask_nums[i]
+        mask_i = masks[k:k + det_nums]
+        mask_i = strip_mask(mask_i)
+        for j in range(det_nums):
+            mask = mask_i[j].astype(np.uint8)
+            score = float(bboxes[k][1])
+            label = int(bboxes[k][0])
+            k = k + 1
+            if label == -1:
+                continue
+            cat_id = label_to_cat_id_map[label]
+            rle = mask_util.encode(
+                np.array(
+                    mask[:, :, None], order="F", dtype="uint8"))[0]
+            if six.PY3:
+                if 'counts' in rle:
+                    rle['counts'] = rle['counts'].decode("utf8")
+            sg_res = {
+                'image_id': cur_image_id,
+                'category_id': cat_id,
+                'segmentation': rle,
+                'score': score
+            }
+            seg_res.append(sg_res)
+    return seg_res
+
+
+def get_solov2_segm_res(results, image_id, num_id_to_cat_id_map):
+    import pycocotools.mask as mask_util
+    segm_res = []
+    # for each batch
+    segms = results['segm'].astype(np.uint8)
+    clsid_labels = results['cate_label']
+    clsid_scores = results['cate_score']
+    lengths = segms.shape[0]
+    im_id = int(image_id[0][0])
+    if lengths == 0 or segms is None:
+        return None
+    # for each sample
+    for i in range(lengths - 1):
+        clsid = int(clsid_labels[i])
+        catid = num_id_to_cat_id_map[clsid]
+        score = float(clsid_scores[i])
+        mask = segms[i]
+        segm = mask_util.encode(np.array(mask[:, :, np.newaxis], order='F'))[0]
+        segm['counts'] = segm['counts'].decode('utf8')
+        coco_res = {
+            'image_id': im_id,
+            'category_id': catid,
+            'segmentation': segm,
+            'score': score
+        }
+        segm_res.append(coco_res)
+    return segm_res
+
+
+def get_keypoint_res(results, im_id):
+    anns = []
+    preds = results['keypoint']
+    for idx in range(im_id.shape[0]):
+        image_id = im_id[idx].item()
+        kpts, scores = preds[idx]
+        for kpt, score in zip(kpts, scores):
+            kpt = kpt.flatten()
+            ann = {
+                'image_id': image_id,
+                'category_id': 1,  # XXX hard code
+                'keypoints': kpt.tolist(),
+                'score': float(score)
+            }
+            x = kpt[0::3]
+            y = kpt[1::3]
+            x0, x1, y0, y1 = np.min(x).item(), np.max(x).item(), np.min(
+                y).item(), np.max(y).item()
+            ann['area'] = (x1 - x0) * (y1 - y0)
+            ann['bbox'] = [x0, y0, x1 - x0, y1 - y0]
+            anns.append(ann)
+    return anns

python/fastdeploy/vision/evaluation/utils/map_utils.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+
+
+def draw_pr_curve(precision,
+                  recall,
+                  iou=0.5,
+                  out_dir='pr_curve',
+                  file_name='precision_recall_curve.jpg'):
+    if not os.path.exists(out_dir):
+        os.makedirs(out_dir)
+    output_path = os.path.join(out_dir, file_name)
+    try:
+        import matplotlib.pyplot as plt
+    except Exception as e:
+        # logger.error('Matplotlib not found, plaese install matplotlib.'
+        #              'for example: `pip install matplotlib`.')
+        raise e
+    plt.cla()
+    plt.figure('P-R Curve')
+    plt.title('Precision/Recall Curve(IoU={})'.format(iou))
+    plt.xlabel('Recall')
+    plt.ylabel('Precision')
+    plt.grid(True)
+    plt.plot(recall, precision)
+    plt.savefig(output_path)

python/fastdeploy/vision/evaluation/utils/seg_metrics.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+
+def f1_score(intersect_area, pred_area, label_area):
+    class_f1_sco = []
+    for i in range(len(intersect_area)):
+        if pred_area[i] + label_area[i] == 0:
+            f1_sco = 0
+        elif pred_area[i] == 0:
+            f1_sco = 0
+        else:
+            prec = intersect_area[i] / pred_area[i]
+            rec = intersect_area[i] / label_area[i]
+            f1_sco = 2 * prec * rec / (prec + rec)
+        class_f1_sco.append(f1_sco)
+    return np.array(class_f1_sco)
+
+
+def calculate_area(pred, label, num_classes, ignore_index=255):
+    """
+    Calculate intersect, prediction and label area
+
+    Args:
+        pred (np.ndarray): The prediction by model.
+        label (np.ndarray): The ground truth of image.
+        num_classes (int): The unique number of target classes.
+        ignore_index (int): Specifies a target value that is ignored. Default: 255.
+
+    Returns:
+        Numpy Array: The intersection area of prediction and the ground on all class.
+        Numpy Array: The prediction area on all class.
+        Numpy Array: The ground truth area on all class
+    """
+    if not pred.shape == label.shape:
+        raise ValueError('Shape of `pred` and `label should be equal, '
+                         'but there are {} and {}.'.format(pred.shape,
+                                                           label.shape))
+
+    mask = label != ignore_index
+    pred = pred + 1
+    label = label + 1
+    pred = pred * mask
+    label = label * mask
+    pred = np.eye(num_classes + 1)[pred]
+    label = np.eye(num_classes + 1)[label]
+    pred = pred[:, 1:]
+    label = label[:, 1:]
+
+    pred_area = []
+    label_area = []
+    intersect_area = []
+
+    for i in range(num_classes):
+        pred_i = pred[:, :, i]
+        label_i = label[:, :, i]
+        pred_area_i = np.sum(pred_i)
+        label_area_i = np.sum(label_i)
+        intersect_area_i = np.sum(pred_i * label_i)
+        pred_area.append(pred_area_i)
+        label_area.append(label_area_i)
+        intersect_area.append(intersect_area_i)
+    return np.array(intersect_area), np.array(pred_area), np.array(label_area)
+
+
+def mean_iou(intersect_area, pred_area, label_area):
+    """
+    Calculate iou.
+
+    Args:
+        intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes.
+        pred_area (np.ndarray): The prediction area on all classes.
+        label_area (np.ndarray): The ground truth area on all classes.
+
+    Returns:
+        np.ndarray: iou on all classes.
+        float: mean iou of all classes.
+    """
+    union = pred_area + label_area - intersect_area
+    class_iou = []
+    for i in range(len(intersect_area)):
+        if union[i] == 0:
+            iou = 0
+        else:
+            iou = intersect_area[i] / union[i]
+        class_iou.append(iou)
+    miou = np.mean(class_iou)
+    return np.array(class_iou), miou
+
+
+def accuracy(intersect_area, pred_area):
+    """
+    Calculate accuracy
+
+    Args:
+        intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes..
+        pred_area (np.ndarray): The prediction area on all classes.
+
+    Returns:
+        np.ndarray: accuracy on all classes.
+        float: mean accuracy.
+    """
+    class_acc = []
+    for i in range(len(intersect_area)):
+        if pred_area[i] == 0:
+            acc = 0
+        else:
+            acc = intersect_area[i] / pred_area[i]
+        class_acc.append(acc)
+    macc = np.sum(intersect_area) / np.sum(pred_area)
+    return np.array(class_acc), macc
+
+
+def kappa(intersect_area, pred_area, label_area):
+    """
+    Calculate kappa coefficient
+
+    Args:
+        intersect_area (np.ndarray): The intersection area of prediction and ground truth on all classes..
+        pred_area (np.ndarray): The prediction area on all classes.
+        label_area (np.ndarray): The ground truth area on all classes.
+
+    Returns:
+        float: kappa coefficient.
+    """
+    total_area = np.sum(label_area)
+    po = np.sum(intersect_area) / total_area
+    pe = np.sum(pred_area * label_area) / (total_area * total_area)
+    kappa = (po - pe) / (1 - pe)
+    return kappa

python/fastdeploy/vision/evaluation/utils/util.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import platform
+import multiprocessing as mp
+
+
+def is_pic(img_name):
+    valid_suffix = ['JPEG', 'jpeg', 'JPG', 'jpg', 'BMP', 'bmp', 'PNG', 'png']
+    suffix = img_name.split('.')[-1]
+    if suffix not in valid_suffix:
+        return False
+    return True
+
+
+def get_num_workers(num_workers):
+    if not platform.system() == 'Linux':
+        # Dataloader with multi-process model is not supported
+        # on MacOS and Windows currently.
+        return 0
+    if num_workers == 'auto':
+        num_workers = mp.cpu_count() // 2 if mp.cpu_count() // 2 < 2 else 2
+    return num_workers