From cb5c585dd61480f4a5fc7348e087a7336d082406 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Tue, 12 Jul 2022 08:36:41 +0000 Subject: [PATCH 1/6] Add multi-label function for yolov5 --- fastdeploy/vision/ultralytics/__init__.py | 39 +++++++----- .../vision/ultralytics/ultralytics_pybind.cc | 3 +- fastdeploy/vision/ultralytics/yolov5.cc | 63 +++++++++++++------ fastdeploy/vision/ultralytics/yolov5.h | 5 +- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/fastdeploy/vision/ultralytics/__init__.py b/fastdeploy/vision/ultralytics/__init__.py index 3a5446e0f22..4dcd0d6d458 100644 --- a/fastdeploy/vision/ultralytics/__init__.py +++ b/fastdeploy/vision/ultralytics/__init__.py @@ -41,31 +41,35 @@ def predict(self, input_image, conf_threshold=0.25, nms_iou_threshold=0.5): # 多数是预处理相关，可通过修改如model.size = [1280, 1280]改变预处理时resize的大小（前提是模型支持） @property def size(self): - return self.model.size + return self._model.size @property def padding_value(self): - return self.model.padding_value + return self._model.padding_value @property def is_no_pad(self): - return self.model.is_no_pad + return self._model.is_no_pad @property def is_mini_pad(self): - return self.model.is_mini_pad + return self._model.is_mini_pad @property def is_scale_up(self): - return self.model.is_scale_up + return self._model.is_scale_up @property def stride(self): - return self.model.stride + return self._model.stride @property def max_wh(self): - return self.model.max_wh + return self._model.max_wh + + @property + def multi_label(self): + return self._model.multi_label @size.setter def size(self, wh): @@ -74,43 +78,50 @@ def size(self, wh): assert len(wh) == 2,\ "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( len(wh)) - self.model.size = wh + self._model.size = wh @padding_value.setter def padding_value(self, value): assert isinstance( value, list), "The value to set `padding_value` must be type of list." - self.model.padding_value = value + self._model.padding_value = value @is_no_pad.setter def is_no_pad(self, value): assert isinstance( value, bool), "The value to set `is_no_pad` must be type of bool." - self.model.is_no_pad = value + self._model.is_no_pad = value @is_mini_pad.setter def is_mini_pad(self, value): assert isinstance( value, bool), "The value to set `is_mini_pad` must be type of bool." - self.model.is_mini_pad = value + self._model.is_mini_pad = value @is_scale_up.setter def is_scale_up(self, value): assert isinstance( value, bool), "The value to set `is_scale_up` must be type of bool." - self.model.is_scale_up = value + self._model.is_scale_up = value @stride.setter def stride(self, value): assert isinstance( value, int), "The value to set `stride` must be type of int." - self.model.stride = value + self._model.stride = value @max_wh.setter def max_wh(self, value): assert isinstance( value, float), "The value to set `max_wh` must be type of float." - self.model.max_wh = value + self._model.max_wh = value + + @multi_label.setter + def multi_label(self, value): + assert isinstance( + value, + bool), "The value to set `multi_label` must be type of bool." + self._model.multi_label = value diff --git a/fastdeploy/vision/ultralytics/ultralytics_pybind.cc b/fastdeploy/vision/ultralytics/ultralytics_pybind.cc index 3b73b586fee..c6021434962 100644 --- a/fastdeploy/vision/ultralytics/ultralytics_pybind.cc +++ b/fastdeploy/vision/ultralytics/ultralytics_pybind.cc @@ -36,6 +36,7 @@ void BindUltralytics(pybind11::module& m) { .def_readwrite("is_no_pad", &vision::ultralytics::YOLOv5::is_no_pad) .def_readwrite("is_scale_up", &vision::ultralytics::YOLOv5::is_scale_up) .def_readwrite("stride", &vision::ultralytics::YOLOv5::stride) - .def_readwrite("max_wh", &vision::ultralytics::YOLOv5::max_wh); + .def_readwrite("max_wh", &vision::ultralytics::YOLOv5::max_wh) + .def_readwrite("multi_label", &vision::ultralytics::YOLOv5::multi_label); } } // namespace fastdeploy diff --git a/fastdeploy/vision/ultralytics/yolov5.cc b/fastdeploy/vision/ultralytics/yolov5.cc index 632c825e598..372f6c060a4 100644 --- a/fastdeploy/vision/ultralytics/yolov5.cc +++ b/fastdeploy/vision/ultralytics/yolov5.cc @@ -67,6 +67,7 @@ bool YOLOv5::Initialize() { is_scale_up = false; stride = 32; max_wh = 7680.0; + multi_label = true; if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; @@ -113,10 +114,14 @@ bool YOLOv5::Preprocess(Mat* mat, FDTensor* output, bool YOLOv5::Postprocess( FDTensor& infer_result, DetectionResult* result, const std::map>& im_info, - float conf_threshold, float nms_iou_threshold) { + float conf_threshold, float nms_iou_threshold, bool multi_label) { FDASSERT(infer_result.shape[0] == 1, "Only support batch =1 now."); result->Clear(); - result->Reserve(infer_result.shape[1]); + if (multi_label) { + result->Reserve(infer_result.shape[1] * (infer_result.shape[2] - 5)); + } else { + result->Reserve(infer_result.shape[1]); + } if (infer_result.dtype != FDDataType::FP32) { FDERROR << "Only support post process with float32 data." << std::endl; return false; @@ -125,22 +130,44 @@ bool YOLOv5::Postprocess( for (size_t i = 0; i < infer_result.shape[1]; ++i) { int s = i * infer_result.shape[2]; float confidence = data[s + 4]; - float* max_class_score = - std::max_element(data + s + 5, data + s + infer_result.shape[2]); - confidence *= (*max_class_score); - // filter boxes by conf_threshold - if (confidence <= conf_threshold) { - continue; + if (multi_label) { + for (size_t j = 5; j < infer_result.shape[2]; ++j) { + confidence = data[s + 4]; + float* class_score = data + s + j; + confidence *= (*class_score); + // filter boxes by conf_threshold + if (confidence <= conf_threshold) { + continue; + } + int32_t label_id = std::distance(data + s + 5, class_score); + + // convert from [x, y, w, h] to [x1, y1, x2, y2] + result->boxes.emplace_back(std::array{ + data[s] - data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); + result->label_ids.push_back(label_id); + result->scores.push_back(confidence); + } + } else { + float* max_class_score = + std::max_element(data + s + 5, data + s + infer_result.shape[2]); + confidence *= (*max_class_score); + // filter boxes by conf_threshold + if (confidence <= conf_threshold) { + continue; + } + int32_t label_id = std::distance(data + s + 5, max_class_score); + // convert from [x, y, w, h] to [x1, y1, x2, y2] + result->boxes.emplace_back(std::array{ + data[s] - data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); + result->label_ids.push_back(label_id); + result->scores.push_back(confidence); } - int32_t label_id = std::distance(data + s + 5, max_class_score); - // convert from [x, y, w, h] to [x1, y1, x2, y2] - result->boxes.emplace_back(std::array{ - data[s] - data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, - data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); - result->label_ids.push_back(label_id); - result->scores.push_back(confidence); } utils::NMS(result, nms_iou_threshold); @@ -214,7 +241,7 @@ bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, #endif if (!Postprocess(output_tensors[0], result, im_info, conf_threshold, - nms_iou_threshold)) { + nms_iou_threshold, multi_label)) { FDERROR << "Failed to post process." << std::endl; return false; } diff --git a/fastdeploy/vision/ultralytics/yolov5.h b/fastdeploy/vision/ultralytics/yolov5.h index fab44a6e837..9a8197e53bf 100644 --- a/fastdeploy/vision/ultralytics/yolov5.h +++ b/fastdeploy/vision/ultralytics/yolov5.h @@ -48,10 +48,11 @@ class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel { // im_info 为预处理记录的信息，后处理用于还原box // conf_threshold 后处理时过滤box的置信度阈值 // nms_iou_threshold 后处理时NMS设定的iou阈值 + // multi_label 后处理时box选取是否采用多标签方式 virtual bool Postprocess( FDTensor& infer_result, DetectionResult* result, const std::map>& im_info, - float conf_threshold, float nms_iou_threshold); + float conf_threshold, float nms_iou_threshold, bool multi_label); // 模型预测接口，即用户调用的接口 // im 为用户的输入数据，目前对于CV均定义为cv::Mat @@ -81,6 +82,8 @@ class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel { int stride; // for offseting the boxes by classes when using NMS float max_wh; + // for different strategies to get boxes when postprocessing + bool multi_label; }; } // namespace ultralytics } // namespace vision From 5ac4976f06b8b3381557061111684c9b21fedf80 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Tue, 12 Jul 2022 16:41:26 +0800 Subject: [PATCH 2/6] Update README.md Update doc --- docs/compile/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compile/README.md b/docs/compile/README.md index 909ac893ca3..9cf5daab4d3 100644 --- a/docs/compile/README.md +++ b/docs/compile/README.md @@ -10,7 +10,7 @@ | 选项 | 作用 | 备注 | |:---- | :--- | :--- | | ENABLE_ORT_BACKEND | 启用ONNXRuntime推理后端，默认ON | - | -| WIGH_GPU | 是否开启GPU使用，默认OFF | 当设为TRUE时，须通过CUDA_DIRECTORY指定cuda目录，如/usr/local/cuda; Mac上不支持设为ON | +| WITH_GPU | 是否开启GPU使用，默认OFF | 当设为TRUE时，须通过CUDA_DIRECTORY指定cuda目录，如/usr/local/cuda; Mac上不支持设为ON | | ENABLE_TRT_BACKEND | 启用TensorRT推理后端，默认OFF | 当设为TRUE时，需通过TRT_DIRECTORY指定tensorrt目录，如/usr/downloads/TensorRT-8.4.0.1; Mac上不支持设为ON| | ENABLE_VISION | 编译集成视觉模型模块，包括OpenCV的编译集成，默认OFF | - | | ENABLE_PADDLE_FRONTEND | 编译集成Paddle2ONNX，默认ON | - | From 4c1f986602a1b9af734d24356bc61af6bcec4551 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Thu, 14 Jul 2022 10:44:50 +0800 Subject: [PATCH 3/6] Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name --- fastdeploy/fastdeploy_runtime.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastdeploy/fastdeploy_runtime.cc b/fastdeploy/fastdeploy_runtime.cc index b053db586fa..3141e71721e 100644 --- a/fastdeploy/fastdeploy_runtime.cc +++ b/fastdeploy/fastdeploy_runtime.cc @@ -138,7 +138,7 @@ void Runtime::CreateTrtBackend() { trt_option.max_workspace_size = option.trt_max_workspace_size; trt_option.fixed_shape = option.trt_fixed_shape; trt_option.max_shape = option.trt_max_shape; - trt_option.min_shape = option.trt_max_shape; + trt_option.min_shape = option.trt_min_shape; trt_option.opt_shape = option.trt_opt_shape; trt_option.serialize_file = option.trt_serialize_file; FDASSERT(option.model_format == Frontend::PADDLE || From 6410251b8c1890d29826b78f837650060e69bf43 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Thu, 14 Jul 2022 10:52:33 +0800 Subject: [PATCH 4/6] Update runtime_option.md Update resnet model dynamic shape setting name from images to x --- docs/api/runtime_option.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/api/runtime_option.md b/docs/api/runtime_option.md index 1a7eeab2825..30bc5a29a63 100644 --- a/docs/api/runtime_option.md +++ b/docs/api/runtime_option.md @@ -77,9 +77,9 @@ option = fd.RuntimeOption() option.backend = fd.Backend.TRT # 当使用TRT后端，且为动态输入shape时 # 需配置输入shape信息 -option.trt_min_shape = {"images": [1, 3, 224, 224]} -option.trt_opt_shape = {"images": [4, 3, 224, 224]} -option.trt_max_shape = {"images": [8, 3, 224, 224]} +option.trt_min_shape = {"x": [1, 3, 224, 224]} +option.trt_opt_shape = {"x": [4, 3, 224, 224]} +option.trt_max_shape = {"x": [8, 3, 224, 224]} model = fd.vision.ppcls.Model("resnet50/inference.pdmodel", "resnet50/inference.pdiparams", @@ -117,9 +117,9 @@ model = fd.vision.ppcls.Model("resnet50/inference.pdmodel", int main() { auto option = fastdeploy::RuntimeOption(); - option.trt_min_shape["images"] = {1, 3, 224, 224}; - option.trt_opt_shape["images"] = {4, 3, 224, 224}; - option.trt_max_shape["images"] = {8, 3, 224, 224}; + option.trt_min_shape["x"] = {1, 3, 224, 224}; + option.trt_opt_shape["x"] = {4, 3, 224, 224}; + option.trt_max_shape["x"] = {8, 3, 224, 224}; auto model = fastdeploy::vision::ppcls.Model( "resnet50/inference.pdmodel", From 75945599509466cc364d4b7f44991d6de1f15c29 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Tue, 19 Jul 2022 10:03:40 +0000 Subject: [PATCH 5/6] Fix bug when inference result boxes are empty --- fastdeploy/vision/evaluation/detection.py | 178 ++++++++++++++++++++++ fastdeploy/vision/ultralytics/yolov5.cc | 5 + fastdeploy/vision/utils/sort_det_res.cc | 6 +- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 fastdeploy/vision/evaluation/detection.py diff --git a/fastdeploy/vision/evaluation/detection.py b/fastdeploy/vision/evaluation/detection.py new file mode 100644 index 00000000000..62c07ea76fd --- /dev/null +++ b/fastdeploy/vision/evaluation/detection.py @@ -0,0 +1,178 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tqdm import trange +from pathlib import Path +import cv2 +import os +import numpy as np +import glob +from .utils import box_iou, scale_coords, xywhn2xyxy, xyxy2xywhn, xywh2xyxy, calculate_padding +from .metric import ap_per_class + +# The implementation refers to +# https://github.com/ultralytics/yolov5/blob/master/val.py + +IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp' + + +def process_batch(detections, labels, iouv): + """ + Return correct predictions matrix. Both sets of boxes are in (x1, y1, x2, y2) format. + Arguments: + detections (Array[N, 6]), x1, y1, x2, y2, conf, class + labels (Array[M, 5]), class, x1, y1, x2, y2 + Returns: + correct (Array[N, 10]), for 10 IoU levels + """ + correct = np.zeros((detections.shape[0], iouv.shape[0])).astype(bool) + iou = box_iou(labels[:, 1:], detections[:, :4]) + correct_class = labels[:, 0:1] == detections[:, 5] + for i in range(len(iouv)): + x = np.where((iou >= iouv[i]) & + correct_class) # IoU > threshold and classes match + if x[0].shape[0]: + matches = np.concatenate((np.stack( + x, 1), iou[x[0], x[1]][:, None]), 1) # [label, detect, iou] + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique( + matches[:, 1], return_index=True)[1]] + matches = matches[np.unique( + matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + return correct + + +def img2label_paths(img_paths): + # Define label paths as a function of image paths + sa, sb = f'{os.sep}images{os.sep}', f'{os.sep}labels{os.sep}' # /images/, /labels/ substrings + return [ + sb.join(x.rsplit(sa, 1)).rsplit('.', 1)[0] + '.txt' for x in img_paths + ] + + +def eval_detection(model, + conf_threshold, + nms_iou_threshold, + image_file_path, + plot=False): + assert isinstance(conf_threshold, ( + float, int + )), "The conf_threshold:{} need to be int or float".format(conf_threshold) + assert isinstance(nms_iou_threshold, ( + float, + int)), "The nms_iou_threshold:{} need to be int or float".format( + nms_iou_threshold) + try: + f = [] # image files + for p in image_file_path if isinstance(image_file_path, + list) else [image_file_path]: + p = Path(p) + if p.is_dir(): # dir + f += glob.glob(str(p / '**' / '*.*'), recursive=True) + elif p.is_file(): # file + with open(p) as t: + t = t.read().strip().splitlines() + parent = str(p.parent) + os.sep + f += [ + x.replace('./', parent) if x.startswith('./') else x + for x in t + ] # local to global path + else: + raise Exception(f'{p} does not exist') + image_files = sorted( + x.replace('/', os.sep) for x in f + if x.split('.')[-1].lower() in IMG_FORMATS) + assert image_files, f'No images found' + except Exception: + raise Exception(f'Error loading data from {image_file_path}') + label_files = img2label_paths(image_files) + image_label_dict = {} + for label_file, image_file in zip(label_files, image_files): + if os.path.isfile(label_file): + with open(label_file) as f: + lb = [ + x.split() for x in f.read().strip().splitlines() if len(x) + ] + lb = np.array(lb, dtype=np.float32) + image_label_dict[image_file] = lb + stats = [] + image_num = len(image_files) + for image, i in zip(image_files, + trange( + image_num, + desc="Inference Progress")): + im = cv2.imread(image) + h0, w0 = im.shape[:2] + h = h0 + w = w0 + new_shape = 640 # resize image shape + r = new_shape / max(h0, w0) + if r != 1: + w = w * r + h = h * r + result = model.predict(im, conf_threshold, nms_iou_threshold) + max_det = 300 # max number of detection boxes + pred = [ + b + [s] + [c] + for b, s, c in zip(result.boxes, result.scores, result.label_ids) + ] + pred = np.array(pred, dtype='f') + if pred.shape[0] > max_det: + pred = pred[:300] + + old_shape = (h, w) + pad, ratio = calculate_padding(old_shape, new_shape, False) + shapes = (h0, w0), ((h / h0, w / w0), pad) + if image not in image_label_dict: + continue + labels = image_label_dict[image] + if labels.size: + labels[:, 1:] = xywhn2xyxy( + labels[:, 1:], + ratio[0] * w, + ratio[1] * h, + padw=pad[0], + padh=pad[1]) + nl = len(labels) # number of labels + if nl: + labels[:, 1:5] = xyxy2xywhn( + labels[:, 1:5], w=new_shape, h=new_shape, clip=True, eps=1E-3) + labels[:, 1:] *= np.array([new_shape, new_shape, new_shape, new_shape]) + npr = pred.shape[0] + iouv = np.linspace(0.5, 0.95, 10) # iou vector for mAP@0.5:0.95 + niou = iouv.shape[0] + correct = np.zeros((npr, niou)) + predn = np.copy(pred) + if npr == 0: + if nl: + stats.append((correct, *np.zeros((3, 0)))) + continue + if nl: + tbox = xywh2xyxy(labels[:, 1:5]) + scale_coords( + np.array([new_shape, new_shape]), tbox, shapes[0], shapes[1]) + labelsn = np.append(labels[:, 0:1], tbox, axis=1) + correct = process_batch(predn, labelsn, iouv) + stats.append((correct, pred[:, 4], pred[:, 5], labels[:, 0])) + stats = [np.concatenate(x, 0) for x in zip(*stats)] + + if len(stats) and stats[0].any(): + tp, fp, p, r, f1, ap, ap_class = ap_per_class( + *stats, plot=plot, save_dir='.') + ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 + mp, mr, map50, map = round(p.mean(), 3), round(r.mean(), 3), round( + ap50.mean(), 3), round(ap.mean(), 3) + return {"Precision": mp, "Recall": mr, "mAP@.5": map50, "mAP@.5:.95": map} diff --git a/fastdeploy/vision/ultralytics/yolov5.cc b/fastdeploy/vision/ultralytics/yolov5.cc index 372f6c060a4..99d7bff7fd6 100644 --- a/fastdeploy/vision/ultralytics/yolov5.cc +++ b/fastdeploy/vision/ultralytics/yolov5.cc @@ -169,6 +169,11 @@ bool YOLOv5::Postprocess( result->scores.push_back(confidence); } } + + if (result->boxes.size() == 0) { + return true; + } + utils::NMS(result, nms_iou_threshold); // scale the boxes to the origin image shape diff --git a/fastdeploy/vision/utils/sort_det_res.cc b/fastdeploy/vision/utils/sort_det_res.cc index e4a0db97614..93dbb69694b 100644 --- a/fastdeploy/vision/utils/sort_det_res.cc +++ b/fastdeploy/vision/utils/sort_det_res.cc @@ -68,7 +68,11 @@ void MergeSort(DetectionResult* result, size_t low, size_t high) { void SortDetectionResult(DetectionResult* result) { size_t low = 0; - size_t high = result->scores.size() - 1; + size_t high = result->scores.size(); + if (high == 0) { + return; + } + high = high - 1; MergeSort(result, low, high); } From 9332cf2f5be46f27a59447714a9020433fd0e7d3 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Tue, 19 Jul 2022 10:42:34 +0000 Subject: [PATCH 6/6] Delete detection.py --- fastdeploy/vision/evaluation/detection.py | 178 ---------------------- 1 file changed, 178 deletions(-) delete mode 100644 fastdeploy/vision/evaluation/detection.py diff --git a/fastdeploy/vision/evaluation/detection.py b/fastdeploy/vision/evaluation/detection.py deleted file mode 100644 index 62c07ea76fd..00000000000 --- a/fastdeploy/vision/evaluation/detection.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tqdm import trange -from pathlib import Path -import cv2 -import os -import numpy as np -import glob -from .utils import box_iou, scale_coords, xywhn2xyxy, xyxy2xywhn, xywh2xyxy, calculate_padding -from .metric import ap_per_class - -# The implementation refers to -# https://github.com/ultralytics/yolov5/blob/master/val.py - -IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp' - - -def process_batch(detections, labels, iouv): - """ - Return correct predictions matrix. Both sets of boxes are in (x1, y1, x2, y2) format. - Arguments: - detections (Array[N, 6]), x1, y1, x2, y2, conf, class - labels (Array[M, 5]), class, x1, y1, x2, y2 - Returns: - correct (Array[N, 10]), for 10 IoU levels - """ - correct = np.zeros((detections.shape[0], iouv.shape[0])).astype(bool) - iou = box_iou(labels[:, 1:], detections[:, :4]) - correct_class = labels[:, 0:1] == detections[:, 5] - for i in range(len(iouv)): - x = np.where((iou >= iouv[i]) & - correct_class) # IoU > threshold and classes match - if x[0].shape[0]: - matches = np.concatenate((np.stack( - x, 1), iou[x[0], x[1]][:, None]), 1) # [label, detect, iou] - if x[0].shape[0] > 1: - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique( - matches[:, 1], return_index=True)[1]] - matches = matches[np.unique( - matches[:, 0], return_index=True)[1]] - correct[matches[:, 1].astype(int), i] = True - return correct - - -def img2label_paths(img_paths): - # Define label paths as a function of image paths - sa, sb = f'{os.sep}images{os.sep}', f'{os.sep}labels{os.sep}' # /images/, /labels/ substrings - return [ - sb.join(x.rsplit(sa, 1)).rsplit('.', 1)[0] + '.txt' for x in img_paths - ] - - -def eval_detection(model, - conf_threshold, - nms_iou_threshold, - image_file_path, - plot=False): - assert isinstance(conf_threshold, ( - float, int - )), "The conf_threshold:{} need to be int or float".format(conf_threshold) - assert isinstance(nms_iou_threshold, ( - float, - int)), "The nms_iou_threshold:{} need to be int or float".format( - nms_iou_threshold) - try: - f = [] # image files - for p in image_file_path if isinstance(image_file_path, - list) else [image_file_path]: - p = Path(p) - if p.is_dir(): # dir - f += glob.glob(str(p / '**' / '*.*'), recursive=True) - elif p.is_file(): # file - with open(p) as t: - t = t.read().strip().splitlines() - parent = str(p.parent) + os.sep - f += [ - x.replace('./', parent) if x.startswith('./') else x - for x in t - ] # local to global path - else: - raise Exception(f'{p} does not exist') - image_files = sorted( - x.replace('/', os.sep) for x in f - if x.split('.')[-1].lower() in IMG_FORMATS) - assert image_files, f'No images found' - except Exception: - raise Exception(f'Error loading data from {image_file_path}') - label_files = img2label_paths(image_files) - image_label_dict = {} - for label_file, image_file in zip(label_files, image_files): - if os.path.isfile(label_file): - with open(label_file) as f: - lb = [ - x.split() for x in f.read().strip().splitlines() if len(x) - ] - lb = np.array(lb, dtype=np.float32) - image_label_dict[image_file] = lb - stats = [] - image_num = len(image_files) - for image, i in zip(image_files, - trange( - image_num, - desc="Inference Progress")): - im = cv2.imread(image) - h0, w0 = im.shape[:2] - h = h0 - w = w0 - new_shape = 640 # resize image shape - r = new_shape / max(h0, w0) - if r != 1: - w = w * r - h = h * r - result = model.predict(im, conf_threshold, nms_iou_threshold) - max_det = 300 # max number of detection boxes - pred = [ - b + [s] + [c] - for b, s, c in zip(result.boxes, result.scores, result.label_ids) - ] - pred = np.array(pred, dtype='f') - if pred.shape[0] > max_det: - pred = pred[:300] - - old_shape = (h, w) - pad, ratio = calculate_padding(old_shape, new_shape, False) - shapes = (h0, w0), ((h / h0, w / w0), pad) - if image not in image_label_dict: - continue - labels = image_label_dict[image] - if labels.size: - labels[:, 1:] = xywhn2xyxy( - labels[:, 1:], - ratio[0] * w, - ratio[1] * h, - padw=pad[0], - padh=pad[1]) - nl = len(labels) # number of labels - if nl: - labels[:, 1:5] = xyxy2xywhn( - labels[:, 1:5], w=new_shape, h=new_shape, clip=True, eps=1E-3) - labels[:, 1:] *= np.array([new_shape, new_shape, new_shape, new_shape]) - npr = pred.shape[0] - iouv = np.linspace(0.5, 0.95, 10) # iou vector for mAP@0.5:0.95 - niou = iouv.shape[0] - correct = np.zeros((npr, niou)) - predn = np.copy(pred) - if npr == 0: - if nl: - stats.append((correct, *np.zeros((3, 0)))) - continue - if nl: - tbox = xywh2xyxy(labels[:, 1:5]) - scale_coords( - np.array([new_shape, new_shape]), tbox, shapes[0], shapes[1]) - labelsn = np.append(labels[:, 0:1], tbox, axis=1) - correct = process_batch(predn, labelsn, iouv) - stats.append((correct, pred[:, 4], pred[:, 5], labels[:, 0])) - stats = [np.concatenate(x, 0) for x in zip(*stats)] - - if len(stats) and stats[0].any(): - tp, fp, p, r, f1, ap, ap_class = ap_per_class( - *stats, plot=plot, save_dir='.') - ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 - mp, mr, map50, map = round(p.mean(), 3), round(r.mean(), 3), round( - ap50.mean(), 3), round(ap.mean(), 3) - return {"Precision": mp, "Recall": mr, "mAP@.5": map50, "mAP@.5:.95": map}