Skip to content

LRP Error on COCOapi #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kemaloksuz wants to merge 4 commits into
base: master
Choose a base branch
from
Open

LRP Error on COCOapi #22

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
682c158
LRP added to COCO dataset
Nov 18, 2020
7778c80
pep8 formatting
Nov 18, 2020
77d86f6
pep8 formatting completed
Nov 18, 2020
95b8c59
yapf format
Nov 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 141 additions & 4 deletions pycocotools/pycocotools/cocoeval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet):
dtm = np.zeros((T, D))
gtIg = np.array([g['_ignore'] for g in gt])
dtIg = np.zeros((T, D))
dtIoU = np.zeros((T, D))
if not len(ious) == 0:
for tind, t in enumerate(p.iouThrs):
for dind, d in enumerate(dt):
Expand All @@ -311,6 +312,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet):
dtIg[tind, dind] = gtIg[m]
dtm[tind, dind] = gt[m]['id']
gtm[tind, m] = d['id']
dtIoU[tind, dind] = iou
# set unmatched detections outside of area range to ignore
a = np.array([d['area'] < aRng[0] or d['area'] > aRng[1]
for d in dt]).reshape((1, len(dt)))
Expand All @@ -329,6 +331,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet):
'dtScores': [d['score'] for d in dt],
'gtIgnore': gtIg,
'dtIgnore': dtIg,
'dtIoUs': dtIoU,
}

def accumulate(self, p=None):
Expand Down Expand Up @@ -356,6 +359,11 @@ def accumulate(self, p=None):
(T, R, K, A, M)) # -1 for the precision of absent categories
recall = -np.ones((T, K, A, M))
scores = -np.ones((T, R, K, A, M))
olrp_loc = -np.ones((K, A, M))
olrp_fp = -np.ones((K, A, M))
olrp_fn = -np.ones((K, A, M))
olrp = -np.ones((K, A, M))
lrp_opt_thr = -np.ones((K, A, M))

# create dictionary for future indexing
_pe = self._paramsEval
Expand Down Expand Up @@ -399,6 +407,9 @@ def accumulate(self, p=None):
dtIg = np.concatenate(
[e['dtIgnore'][:, 0:maxDet] for e in E], axis=1)[:,
inds]
dtIoU = np.concatenate(
[e['dtIoUs'][:, 0:maxDet] for e in E], axis=1)[:, inds]

gtIg = np.concatenate([e['gtIgnore'] for e in E])
npig = np.count_nonzero(gtIg == 0)
if npig == 0:
Expand All @@ -407,6 +418,7 @@ def accumulate(self, p=None):
fps = np.logical_and(np.logical_not(dtm),
np.logical_not(dtIg))

dtIoU = np.multiply(dtIoU, tps)
tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)
fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float)
for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
Expand Down Expand Up @@ -442,13 +454,52 @@ def accumulate(self, p=None):
pass
precision[t, :, k, a, m] = np.array(q)
scores[t, :, k, a, m] = np.array(ss)

# oLRP and Opt.Thr. Computation
tp_num = np.cumsum(tps[0, :])
fp_num = np.cumsum(fps[0, :])
fn_num = npig - tp_num
# If there is detection
if tp_num.shape[0] > 0:
# There is some TPs
if tp_num[-1] > 0:
total_loc = tp_num - np.cumsum(dtIoU[0, :])
lrps = (total_loc / (1 - _pe.iouThrs[0]) + fp_num +
fn_num) / (tp_num + fp_num + fn_num)
opt_pos_idx = np.argmin(lrps)
olrp[k, a, m] = lrps[opt_pos_idx]
olrp_loc[k, a, m] = total_loc[opt_pos_idx] / \
tp_num[opt_pos_idx]
olrp_fp[k, a, m] = fp_num[opt_pos_idx] / \
(tp_num[opt_pos_idx] + fp_num[opt_pos_idx])
olrp_fn[k, a, m] = fn_num[opt_pos_idx] / npig
lrp_opt_thr[k, a, m] = dtScoresSorted[opt_pos_idx]
# There is No TP
else:
olrp_loc[k, a, m] = np.nan
olrp_fp[k, a, m] = np.nan
olrp_fn[k, a, m] = 1.
olrp[k, a, m] = 1.
lrp_opt_thr[k, a, m] = np.nan
# No detection
else:
olrp_loc[k, a, m] = np.nan
olrp_fp[k, a, m] = np.nan
olrp_fn[k, a, m] = 1.
olrp[k, a, m] = 1.
lrp_opt_thr[k, a, m] = np.nan
self.eval = {
'params': p,
'counts': [T, R, K, A, M],
'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'precision': precision,
'recall': recall,
'scores': scores,
'olrp_loc': olrp_loc,
'olrp_fp': olrp_fp,
'olrp_fn': olrp_fn,
'olrp': olrp,
'lrp_opt_thr': lrp_opt_thr,
}
toc = time.time()
print('DONE (t={:0.2f}s).'.format(toc - tic))
Expand All @@ -459,7 +510,11 @@ def summarize(self):
Note this functin can *only* be applied on the default parameter
setting
'''
def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):
def _summarize(ap=1,
iouThr=None,
areaRng='all',
maxDets=100,
lrp_type=None):
p = self.params
iStr = '{:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' # noqa: E501
titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
Expand All @@ -479,13 +534,45 @@ def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:, :, :, aind, mind]
else:
if len(s[s > -1]) == 0:
mean_s = -1
else:
mean_s = np.mean(s[s > -1])
elif ap == 0:
# dimension of recall: [TxKxAxM]
s = self.eval['recall']
if iouThr is not None:
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:, :, aind, mind]
else:
# # dimension of LRP: [KxAxM]
# Person 0, Broccoli 50
if lrp_type == 'oLRP':
s = self.eval['olrp'][:, aind, mind]
titleStr = 'Optimal LRP'
typeStr = ' '
if lrp_type == 'oLRP_Localisation':
s = self.eval['olrp_loc'][:, aind, mind]
titleStr = 'Optimal LRP Loc'
typeStr = ' '
if lrp_type == 'oLRP_false_positive':
s = self.eval['olrp_fp'][:, aind, mind]
titleStr = 'Optimal LRP FP'
typeStr = ' '
if lrp_type == 'oLRP_false_negative':
s = self.eval['olrp_fn'][:, aind, mind]
titleStr = 'Optimal LRP FN'
typeStr = ' '
if lrp_type == 'oLRP_thresholds':
s = self.eval['lrp_opt_thr'][:, aind, mind].squeeze(axis=1)
titleStr = '# Class-specific LRP-Optimal Thresholds # \n'
typeStr = ' '
# Floor by using 3 decimal digits
print(titleStr, np.round(s - 0.5 * 10**(-3), 3))
return s
idx = (~np.isnan(s))
s = s[idx]
if len(s[s > -1]) == 0:
mean_s = -1
else:
Expand All @@ -496,7 +583,7 @@ def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):
return mean_s

def _summarizeDets():
stats = np.zeros((12, ))
stats = np.zeros((16, ))
stats[0] = _summarize(1)
stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])
stats[2] = _summarize(1,
Expand All @@ -523,10 +610,35 @@ def _summarizeDets():
stats[11] = _summarize(0,
areaRng='large',
maxDets=self.params.maxDets[2])
stats[12] = _summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=self.params.maxDets[2],
lrp_type='oLRP')
stats[13] = _summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=self.params.maxDets[2],
lrp_type='oLRP_Localisation')
stats[14] = _summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=self.params.maxDets[2],
lrp_type='oLRP_false_positive')
stats[15] = _summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=self.params.maxDets[2],
lrp_type='oLRP_false_negative')
_summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=self.params.maxDets[2],
lrp_type='oLRP_thresholds')
return stats

def _summarizeKps():
stats = np.zeros((10, ))
stats = np.zeros((14, ))
stats[0] = _summarize(1, maxDets=20)
stats[1] = _summarize(1, maxDets=20, iouThr=.5)
stats[2] = _summarize(1, maxDets=20, iouThr=.75)
Expand All @@ -537,6 +649,31 @@ def _summarizeKps():
stats[7] = _summarize(0, maxDets=20, iouThr=.75)
stats[8] = _summarize(0, maxDets=20, areaRng='medium')
stats[9] = _summarize(0, maxDets=20, areaRng='large')
stats[10] = _summarize(-1,
maxDets=20,
iouThr=.5,
areaRng='all',
lrp_type='oLRP')
stats[11] = _summarize(-1,
maxDets=20,
iouThr=.5,
areaRng='all',
lrp_type='oLRP_Localisation')
stats[12] = _summarize(-1,
maxDets=20,
iouThr=.5,
areaRng='all',
lrp_type='oLRP_false_positive')
stats[13] = _summarize(-1,
maxDets=20,
iouThr=.5,
areaRng='all',
lrp_type='oLRP_false_negative')
_summarize(-1,
iouThr=.5,
areaRng='all',
maxDets=20,
lrp_type='oLRP_thresholds')
return stats

if not self.eval:
Expand Down