diff --git a/pycocotools/pycocotools/cocoeval.py b/pycocotools/pycocotools/cocoeval.py index a7ae8834..44196cad 100644 --- a/pycocotools/pycocotools/cocoeval.py +++ b/pycocotools/pycocotools/cocoeval.py @@ -285,6 +285,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): dtm = np.zeros((T, D)) gtIg = np.array([g['_ignore'] for g in gt]) dtIg = np.zeros((T, D)) + dtIoU = np.zeros((T, D)) if not len(ious) == 0: for tind, t in enumerate(p.iouThrs): for dind, d in enumerate(dt): @@ -311,6 +312,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): dtIg[tind, dind] = gtIg[m] dtm[tind, dind] = gt[m]['id'] gtm[tind, m] = d['id'] + dtIoU[tind, dind] = iou # set unmatched detections outside of area range to ignore a = np.array([d['area'] < aRng[0] or d['area'] > aRng[1] for d in dt]).reshape((1, len(dt))) @@ -329,6 +331,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): 'dtScores': [d['score'] for d in dt], 'gtIgnore': gtIg, 'dtIgnore': dtIg, + 'dtIoUs': dtIoU, } def accumulate(self, p=None): @@ -356,6 +359,11 @@ def accumulate(self, p=None): (T, R, K, A, M)) # -1 for the precision of absent categories recall = -np.ones((T, K, A, M)) scores = -np.ones((T, R, K, A, M)) + olrp_loc = -np.ones((K, A, M)) + olrp_fp = -np.ones((K, A, M)) + olrp_fn = -np.ones((K, A, M)) + olrp = -np.ones((K, A, M)) + lrp_opt_thr = -np.ones((K, A, M)) # create dictionary for future indexing _pe = self._paramsEval @@ -399,6 +407,9 @@ def accumulate(self, p=None): dtIg = np.concatenate( [e['dtIgnore'][:, 0:maxDet] for e in E], axis=1)[:, inds] + dtIoU = np.concatenate( + [e['dtIoUs'][:, 0:maxDet] for e in E], axis=1)[:, inds] + gtIg = np.concatenate([e['gtIgnore'] for e in E]) npig = np.count_nonzero(gtIg == 0) if npig == 0: @@ -407,6 +418,7 @@ def accumulate(self, p=None): fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg)) + dtIoU = np.multiply(dtIoU, tps) tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): @@ -442,6 +454,40 @@ def accumulate(self, p=None): pass precision[t, :, k, a, m] = np.array(q) scores[t, :, k, a, m] = np.array(ss) + + # oLRP and Opt.Thr. Computation + tp_num = np.cumsum(tps[0, :]) + fp_num = np.cumsum(fps[0, :]) + fn_num = npig - tp_num + # If there is detection + if tp_num.shape[0] > 0: + # There is some TPs + if tp_num[-1] > 0: + total_loc = tp_num - np.cumsum(dtIoU[0, :]) + lrps = (total_loc / (1 - _pe.iouThrs[0]) + fp_num + + fn_num) / (tp_num + fp_num + fn_num) + opt_pos_idx = np.argmin(lrps) + olrp[k, a, m] = lrps[opt_pos_idx] + olrp_loc[k, a, m] = total_loc[opt_pos_idx] / \ + tp_num[opt_pos_idx] + olrp_fp[k, a, m] = fp_num[opt_pos_idx] / \ + (tp_num[opt_pos_idx] + fp_num[opt_pos_idx]) + olrp_fn[k, a, m] = fn_num[opt_pos_idx] / npig + lrp_opt_thr[k, a, m] = dtScoresSorted[opt_pos_idx] + # There is No TP + else: + olrp_loc[k, a, m] = np.nan + olrp_fp[k, a, m] = np.nan + olrp_fn[k, a, m] = 1. + olrp[k, a, m] = 1. + lrp_opt_thr[k, a, m] = np.nan + # No detection + else: + olrp_loc[k, a, m] = np.nan + olrp_fp[k, a, m] = np.nan + olrp_fn[k, a, m] = 1. + olrp[k, a, m] = 1. + lrp_opt_thr[k, a, m] = np.nan self.eval = { 'params': p, 'counts': [T, R, K, A, M], @@ -449,6 +495,11 @@ def accumulate(self, p=None): 'precision': precision, 'recall': recall, 'scores': scores, + 'olrp_loc': olrp_loc, + 'olrp_fp': olrp_fp, + 'olrp_fn': olrp_fn, + 'olrp': olrp, + 'lrp_opt_thr': lrp_opt_thr, } toc = time.time() print('DONE (t={:0.2f}s).'.format(toc - tic)) @@ -459,7 +510,11 @@ def summarize(self): Note this functin can *only* be applied on the default parameter setting ''' - def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100): + def _summarize(ap=1, + iouThr=None, + areaRng='all', + maxDets=100, + lrp_type=None): p = self.params iStr = '{:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' # noqa: E501 titleStr = 'Average Precision' if ap == 1 else 'Average Recall' @@ -479,13 +534,45 @@ def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100): t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:, :, :, aind, mind] - else: + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + elif ap == 0: # dimension of recall: [TxKxAxM] s = self.eval['recall'] if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:, :, aind, mind] + else: + # # dimension of LRP: [KxAxM] + # Person 0, Broccoli 50 + if lrp_type == 'oLRP': + s = self.eval['olrp'][:, aind, mind] + titleStr = 'Optimal LRP' + typeStr = ' ' + if lrp_type == 'oLRP_Localisation': + s = self.eval['olrp_loc'][:, aind, mind] + titleStr = 'Optimal LRP Loc' + typeStr = ' ' + if lrp_type == 'oLRP_false_positive': + s = self.eval['olrp_fp'][:, aind, mind] + titleStr = 'Optimal LRP FP' + typeStr = ' ' + if lrp_type == 'oLRP_false_negative': + s = self.eval['olrp_fn'][:, aind, mind] + titleStr = 'Optimal LRP FN' + typeStr = ' ' + if lrp_type == 'oLRP_thresholds': + s = self.eval['lrp_opt_thr'][:, aind, mind].squeeze(axis=1) + titleStr = '# Class-specific LRP-Optimal Thresholds # \n' + typeStr = ' ' + # Floor by using 3 decimal digits + print(titleStr, np.round(s - 0.5 * 10**(-3), 3)) + return s + idx = (~np.isnan(s)) + s = s[idx] if len(s[s > -1]) == 0: mean_s = -1 else: @@ -496,7 +583,7 @@ def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100): return mean_s def _summarizeDets(): - stats = np.zeros((12, )) + stats = np.zeros((16, )) stats[0] = _summarize(1) stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) stats[2] = _summarize(1, @@ -523,10 +610,35 @@ def _summarizeDets(): stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) + stats[12] = _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=self.params.maxDets[2], + lrp_type='oLRP') + stats[13] = _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=self.params.maxDets[2], + lrp_type='oLRP_Localisation') + stats[14] = _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=self.params.maxDets[2], + lrp_type='oLRP_false_positive') + stats[15] = _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=self.params.maxDets[2], + lrp_type='oLRP_false_negative') + _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=self.params.maxDets[2], + lrp_type='oLRP_thresholds') return stats def _summarizeKps(): - stats = np.zeros((10, )) + stats = np.zeros((14, )) stats[0] = _summarize(1, maxDets=20) stats[1] = _summarize(1, maxDets=20, iouThr=.5) stats[2] = _summarize(1, maxDets=20, iouThr=.75) @@ -537,6 +649,31 @@ def _summarizeKps(): stats[7] = _summarize(0, maxDets=20, iouThr=.75) stats[8] = _summarize(0, maxDets=20, areaRng='medium') stats[9] = _summarize(0, maxDets=20, areaRng='large') + stats[10] = _summarize(-1, + maxDets=20, + iouThr=.5, + areaRng='all', + lrp_type='oLRP') + stats[11] = _summarize(-1, + maxDets=20, + iouThr=.5, + areaRng='all', + lrp_type='oLRP_Localisation') + stats[12] = _summarize(-1, + maxDets=20, + iouThr=.5, + areaRng='all', + lrp_type='oLRP_false_positive') + stats[13] = _summarize(-1, + maxDets=20, + iouThr=.5, + areaRng='all', + lrp_type='oLRP_false_negative') + _summarize(-1, + iouThr=.5, + areaRng='all', + maxDets=20, + lrp_type='oLRP_thresholds') return stats if not self.eval: