diff --git a/README.md b/README.md
index 7a11a8e..dfc46cf 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
- 
+ 
@@ -52,6 +52,7 @@ We are happy to announce an important update to the OpenLane family, featuring t
## News
+- **`2023/11/01`** Devkit `v2.1.0` and `v1.1.0` released.
- **`2023/08/28`** Dataset `subset_B` released.
- **`2023/07/21`** Dataset `v2.0` and Devkit `v2.0.0` released.
- **`2023/07/05`** The [test server of OpenLane Topology](https://eval.ai/web/challenges/challenge-page/1925/overview) is re-opened.
diff --git a/openlanev2/centerline/evaluation/evaluate.py b/openlanev2/centerline/evaluation/evaluate.py
index c1f1a98..f24099e 100644
--- a/openlanev2/centerline/evaluation/evaluate.py
+++ b/openlanev2/centerline/evaluation/evaluate.py
@@ -105,7 +105,7 @@ def _tpfp(gts, preds, confidences, distance_matrix, distance_threshold):
tp = np.zeros((num_preds), dtype=np.float32)
fp = np.zeros((num_preds), dtype=np.float32)
- idx_match_gt = np.ones((num_preds)) * np.nan
+ idx_match_gt = np.ones((num_preds), dtype=int) * np.nan
if num_gts == 0:
fp[...] = 1
@@ -434,29 +434,30 @@ def _mAP_topology_lclc(gts, preds, distance_thresholds):
"""
acc = []
- for r in range(10):
- for distance_threshold in distance_thresholds:
- for token in gts.keys():
- preds_topology_lclc_unmatched = preds[token]['topology_lclc']
+ for distance_threshold in distance_thresholds:
+ for token in gts.keys():
+ preds_topology_lclc_unmatched = preds[token]['topology_lclc']
- idx_match_gt = preds[token][f'lane_centerline_{distance_threshold}_idx_match_gt']
- confidence = preds[token][f'lane_centerline_{distance_threshold}_confidence']
- confidence_thresholds = preds[token][f'lane_centerline_{distance_threshold}_confidence_thresholds']
- gt_pred = {m: i for i, (m, c) in enumerate(zip(idx_match_gt, confidence)) if c >= confidence_thresholds[r] and not np.isnan(m)}
+ idx_match_gt = preds[token][f'lane_centerline_{distance_threshold}_idx_match_gt']
+ gt_pred = {m: i for i, m in enumerate(idx_match_gt) if not np.isnan(m)}
- gts_topology_lclc = gts[token]['topology_lclc']
- if 0 in gts_topology_lclc.shape:
- continue
+ gts_topology_lclc = gts[token]['topology_lclc']
+ if 0 in gts_topology_lclc.shape:
+ continue
- preds_topology_lclc = np.ones_like(gts_topology_lclc, dtype=gts_topology_lclc.dtype) * np.nan
- for i in range(preds_topology_lclc.shape[0]):
- for j in range(preds_topology_lclc.shape[1]):
- if i in gt_pred and j in gt_pred:
- preds_topology_lclc[i][j] = preds_topology_lclc_unmatched[gt_pred[i]][gt_pred[j]]
- preds_topology_lclc[np.isnan(preds_topology_lclc)] = 1 - gts_topology_lclc[np.isnan(preds_topology_lclc)]
+ gt_indices = np.array(list(gt_pred.keys())).astype(int)
+ pred_indices = np.array(list(gt_pred.values())).astype(int)
+ preds_topology_lclc = np.ones_like(gts_topology_lclc, dtype=gts_topology_lclc.dtype) * np.nan
+ xs = gt_indices[:, None].repeat(len(gt_indices), 1)
+ ys = gt_indices[None, :].repeat(len(gt_indices), 0)
+ preds_topology_lclc[xs, ys] = preds_topology_lclc_unmatched[pred_indices][:, pred_indices]
+ preds_topology_lclc[np.isnan(preds_topology_lclc)] = (
+ 1 - gts_topology_lclc[np.isnan(preds_topology_lclc)]) * (0.5 + np.finfo(np.float32).eps)
- acc.append(_AP_directerd(gts=gts_topology_lclc, preds=preds_topology_lclc))
+ acc.append(_AP_directerd(gts=gts_topology_lclc, preds=preds_topology_lclc))
+ if len(acc) == 0:
+ return np.float32(0)
return np.hstack(acc).mean()
def _mAP_topology_lcte(gts, preds, distance_thresholds):
@@ -479,41 +480,37 @@ def _mAP_topology_lcte(gts, preds, distance_thresholds):
"""
acc = []
- for r in range(10):
- for distance_threshold_lane_centerline in distance_thresholds['lane_centerline']:
- for distance_threshold_traffic_element in distance_thresholds['traffic_element']:
- for token in gts.keys():
- preds_topology_lcte_unmatched = preds[token]['topology_lcte']
-
- idx_match_gt_lane_centerline = preds[token][f'lane_centerline_{distance_threshold_lane_centerline}_idx_match_gt']
- confidence_lane_centerline = preds[token][f'lane_centerline_{distance_threshold_lane_centerline}_confidence']
- confidence_thresholds_lane_centerline = preds[token][f'lane_centerline_{distance_threshold_lane_centerline}_confidence_thresholds']
- gt_pred_lane_centerline = {
- m: i for i, (m, c) in enumerate(zip(idx_match_gt_lane_centerline, confidence_lane_centerline)) \
- if c >= confidence_thresholds_lane_centerline[r] and not np.isnan(m)
- }
-
- idx_match_gt_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_idx_match_gt']
- confidence_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_confidence']
- confidence_thresholds_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_confidence_thresholds']
- gt_pred_traffic_element = {
- m: i for i, (m, c) in enumerate(zip(idx_match_gt_traffic_element, confidence_traffic_element)) \
- if c >= confidence_thresholds_traffic_element[r] and not np.isnan(m)
- }
-
- gts_topology_lcte = gts[token]['topology_lcte']
- if 0 in gts_topology_lcte.shape:
- continue
-
- preds_topology_lcte = np.ones_like(gts_topology_lcte, dtype=gts_topology_lcte.dtype) * np.nan
- for i in range(preds_topology_lcte.shape[0]):
- for j in range(preds_topology_lcte.shape[1]):
- if i in gt_pred_lane_centerline and j in gt_pred_traffic_element:
- preds_topology_lcte[i][j] = preds_topology_lcte_unmatched[gt_pred_lane_centerline[i]][gt_pred_traffic_element[j]]
- preds_topology_lcte[np.isnan(preds_topology_lcte)] = 1 - gts_topology_lcte[np.isnan(preds_topology_lcte)]
-
- acc.append(_AP_undirecterd(gts=gts_topology_lcte, preds=preds_topology_lcte))
-
+ for distance_threshold_lane_centerline in distance_thresholds['lane_centerline']:
+ for distance_threshold_traffic_element in distance_thresholds['traffic_element']:
+ for token in gts.keys():
+ preds_topology_lcte_unmatched = preds[token]['topology_lcte']
+
+ idx_match_gt_lane_centerline = preds[token][f'lane_centerline_{distance_threshold_lane_centerline}_idx_match_gt']
+ gt_pred_lane_centerline = {m: i for i, m in enumerate(idx_match_gt_lane_centerline) if not np.isnan(m)}
+
+ idx_match_gt_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_idx_match_gt']
+ gt_pred_traffic_element = {m: i for i, m in enumerate(idx_match_gt_traffic_element) if not np.isnan(m)}
+
+ gts_topology_lcte = gts[token]['topology_lcte']
+ if 0 in gts_topology_lcte.shape:
+ continue
+
+ gt_indices_lc = np.array(list(gt_pred_lane_centerline.keys())).astype(int)
+ pred_indices_lc = np.array(list(gt_pred_lane_centerline.values())).astype(int)
+ gt_indices_te = np.array(list(gt_pred_traffic_element.keys())).astype(int)
+ pred_indices_te = np.array(list(gt_pred_traffic_element.values())).astype(int)
+
+ preds_topology_lcte = np.ones_like(gts_topology_lcte, dtype=gts_topology_lcte.dtype) * np.nan
+ xs = gt_indices_lc[:, None].repeat(len(gt_indices_te), 1)
+ ys = gt_indices_te[None, :].repeat(len(gt_indices_lc), 0)
+ preds_topology_lcte[xs, ys] = preds_topology_lcte_unmatched[pred_indices_lc][:, pred_indices_te]
+ preds_topology_lcte[np.isnan(preds_topology_lcte)] = (
+ 1 - gts_topology_lcte[np.isnan(preds_topology_lcte)]) * (0.5 + np.finfo(np.float32).eps)
+
+ acc.append(_AP_undirecterd(gts=gts_topology_lcte, preds=preds_topology_lcte))
+
+ if len(acc) == 0:
+ return np.float32(0)
return np.hstack(acc).mean()
def evaluate(ground_truth, predictions, verbose=True):
diff --git a/openlanev2/lanesegment/evaluation/evaluate.py b/openlanev2/lanesegment/evaluation/evaluate.py
index 8b47ada..2f41547 100644
--- a/openlanev2/lanesegment/evaluation/evaluate.py
+++ b/openlanev2/lanesegment/evaluation/evaluate.py
@@ -104,7 +104,7 @@ def _tpfp(gts, preds, confidences, distance_matrix, distance_threshold):
tp = np.zeros((num_preds), dtype=np.float32)
fp = np.zeros((num_preds), dtype=np.float32)
- idx_match_gt = np.ones((num_preds)) * np.nan
+ idx_match_gt = np.ones((num_preds), dtype=int) * np.nan
if num_gts == 0:
fp[...] = 1
@@ -433,29 +433,30 @@ def _mAP_topology_lsls(gts, preds, distance_thresholds):
"""
acc = []
- for r in range(10):
- for distance_threshold in distance_thresholds:
- for token in gts.keys():
- preds_topology_lsls_unmatched = preds[token]['topology_lsls']
+ for distance_threshold in distance_thresholds:
+ for token in gts.keys():
+ preds_topology_lsls_unmatched = preds[token]['topology_lsls']
- idx_match_gt = preds[token][f'lane_segment_{distance_threshold}_idx_match_gt']
- confidence = preds[token][f'lane_segment_{distance_threshold}_confidence']
- confidence_thresholds = preds[token][f'lane_segment_{distance_threshold}_confidence_thresholds']
- gt_pred = {m: i for i, (m, c) in enumerate(zip(idx_match_gt, confidence)) if c >= confidence_thresholds[r] and not np.isnan(m)}
+ idx_match_gt = preds[token][f'lane_segment_{distance_threshold}_idx_match_gt']
+ gt_pred = {m: i for i, m in enumerate(idx_match_gt) if not np.isnan(m)}
- gts_topology_lsls = gts[token]['topology_lsls']
- if 0 in gts_topology_lsls.shape:
- continue
+ gts_topology_lsls = gts[token]['topology_lsls']
+ if 0 in gts_topology_lsls.shape:
+ continue
- preds_topology_lsls = np.ones_like(gts_topology_lsls, dtype=gts_topology_lsls.dtype) * np.nan
- for i in range(preds_topology_lsls.shape[0]):
- for j in range(preds_topology_lsls.shape[1]):
- if i in gt_pred and j in gt_pred:
- preds_topology_lsls[i][j] = preds_topology_lsls_unmatched[gt_pred[i]][gt_pred[j]]
- preds_topology_lsls[np.isnan(preds_topology_lsls)] = 1 - gts_topology_lsls[np.isnan(preds_topology_lsls)]
+ gt_indices = np.array(list(gt_pred.keys())).astype(int)
+ pred_indices = np.array(list(gt_pred.values())).astype(int)
+ preds_topology_lsls = np.ones_like(gts_topology_lsls, dtype=gts_topology_lsls.dtype) * np.nan
+ xs = gt_indices[:, None].repeat(len(gt_indices), 1)
+ ys = gt_indices[None, :].repeat(len(gt_indices), 0)
+ preds_topology_lsls[xs, ys] = preds_topology_lsls_unmatched[pred_indices][:, pred_indices]
+ preds_topology_lsls[np.isnan(preds_topology_lsls)] = (
+ 1 - gts_topology_lsls[np.isnan(preds_topology_lsls)]) * (0.5 + np.finfo(np.float32).eps)
- acc.append(_AP_directerd(gts=gts_topology_lsls, preds=preds_topology_lsls))
+ acc.append(_AP_directerd(gts=gts_topology_lsls, preds=preds_topology_lsls))
+ if len(acc) == 0:
+ return np.float32(0)
return np.hstack(acc).mean()
def _mAP_topology_lste(gts, preds, distance_thresholds):
@@ -478,76 +479,38 @@ def _mAP_topology_lste(gts, preds, distance_thresholds):
"""
acc = []
- for r in range(10):
- for distance_threshold_lane_segment in distance_thresholds['lane_segment']:
- for distance_threshold_traffic_element in distance_thresholds['traffic_element']:
- for token in gts.keys():
- preds_topology_lste_unmatched = preds[token]['topology_lste']
-
- idx_match_gt_lane_segment = preds[token][f'lane_segment_{distance_threshold_lane_segment}_idx_match_gt']
- confidence_lane_segment = preds[token][f'lane_segment_{distance_threshold_lane_segment}_confidence']
- confidence_thresholds_lane_segment = preds[token][f'lane_segment_{distance_threshold_lane_segment}_confidence_thresholds']
- gt_pred_lane_segment = {
- m: i for i, (m, c) in enumerate(zip(idx_match_gt_lane_segment, confidence_lane_segment)) \
- if c >= confidence_thresholds_lane_segment[r] and not np.isnan(m)
- }
-
- idx_match_gt_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_idx_match_gt']
- confidence_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_confidence']
- confidence_thresholds_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_confidence_thresholds']
- gt_pred_traffic_element = {
- m: i for i, (m, c) in enumerate(zip(idx_match_gt_traffic_element, confidence_traffic_element)) \
- if c >= confidence_thresholds_traffic_element[r] and not np.isnan(m)
- }
-
- gts_topology_lste = gts[token]['topology_lste']
- if 0 in gts_topology_lste.shape:
- continue
-
- preds_topology_lste = np.ones_like(gts_topology_lste, dtype=gts_topology_lste.dtype) * np.nan
- for i in range(preds_topology_lste.shape[0]):
- for j in range(preds_topology_lste.shape[1]):
- if i in gt_pred_lane_segment and j in gt_pred_traffic_element:
- preds_topology_lste[i][j] = preds_topology_lste_unmatched[gt_pred_lane_segment[i]][gt_pred_traffic_element[j]]
- preds_topology_lste[np.isnan(preds_topology_lste)] = 1 - gts_topology_lste[np.isnan(preds_topology_lste)]
-
- acc.append(_AP_undirecterd(gts=gts_topology_lste, preds=preds_topology_lste))
-
- return np.hstack(acc).mean()
+ for distance_threshold_lane_segment in distance_thresholds['lane_segment']:
+ for distance_threshold_traffic_element in distance_thresholds['traffic_element']:
+ for token in gts.keys():
+ preds_topology_lste_unmatched = preds[token]['topology_lste']
+
+ idx_match_gt_lane_segment = preds[token][f'lane_segment_{distance_threshold_lane_segment}_idx_match_gt']
+ gt_pred_lane_segment = {m: i for i, m in enumerate(idx_match_gt_lane_segment) if not np.isnan(m)}
+
+ idx_match_gt_traffic_element = preds[token][f'traffic_element_{distance_threshold_traffic_element}_idx_match_gt']
+ gt_pred_traffic_element = {m: i for i, m in enumerate(idx_match_gt_traffic_element) if not np.isnan(m)}
+
+ gts_topology_lste = gts[token]['topology_lste']
+ if 0 in gts_topology_lste.shape:
+ continue
-# def _matched_list(gts, preds, distance_matrices, distance_threshold, object_type, filter):
-# gts_tp = {}
-# preds_tp = {}
-# for token in gts.keys():
-# gt = [gt for gt in gts[token][object_type] if filter(gt)]
-# pred = [pred for pred in preds[token][object_type] if filter(pred)]
-# confidence = [pred['confidence'] for pred in preds[token][object_type] if filter(pred)]
-# filtered_distance_matrix = distance_matrices[token].copy()
-# filtered_distance_matrix = filtered_distance_matrix[[filter(gt) for gt in gts[token][object_type]], :]
-# filtered_distance_matrix = filtered_distance_matrix[:, [filter(pred) for pred in preds[token][object_type]]]
-# tp, fp, idx_match_gt = _tpfp(
-# gts=gt,
-# preds=pred,
-# confidences=confidence,
-# distance_matrix=filtered_distance_matrix,
-# distance_threshold=distance_threshold,
-# )
-
-# gt_tp = []
-# pred_tp = []
-# order = []
-# for idx, flag in enumerate(idx_match_gt):
-# if flag != np.nan:
-# gt_tp.append(gt[flag.astype(np.int32)])
-# pred_tp.append(pred[idx])
-# order.append(flag.astype(np.int32))
-# gt_tp = [gt_tp[i] for i in order]
-# pred_tp = [pred_tp[i] for i in order]
-
-# gts_tp[token] = {object_type: gt_tp}
-# preds_tp[token] = {object_type: pred_tp}
-
-# return gts_tp, preds_tp
+ gt_indices_ls = np.array(list(gt_pred_lane_segment.keys())).astype(int)
+ pred_indices_ls = np.array(list(gt_pred_lane_segment.values())).astype(int)
+ gt_indices_te = np.array(list(gt_pred_traffic_element.keys())).astype(int)
+ pred_indices_te = np.array(list(gt_pred_traffic_element.values())).astype(int)
+
+ preds_topology_lste = np.ones_like(gts_topology_lste, dtype=gts_topology_lste.dtype) * np.nan
+ xs = gt_indices_ls[:, None].repeat(len(gt_indices_te), 1)
+ ys = gt_indices_te[None, :].repeat(len(gt_indices_ls), 0)
+ preds_topology_lste[xs, ys] = preds_topology_lste_unmatched[pred_indices_ls][:, pred_indices_te]
+ preds_topology_lste[np.isnan(preds_topology_lste)] = (
+ 1 - gts_topology_lste[np.isnan(preds_topology_lste)]) * (0.5 + np.finfo(np.float32).eps)
+
+ acc.append(_AP_undirecterd(gts=gts_topology_lste, preds=preds_topology_lste))
+
+ if len(acc) == 0:
+ return np.float32(0)
+ return np.hstack(acc).mean()
def evaluate(ground_truth, predictions, verbose=True):
r"""
@@ -706,10 +669,4 @@ def evaluate(ground_truth, predictions, verbose=True):
np.sqrt(metrics['OpenLane-V2 UniScore']['TOP_lt']),
]).mean()
- """
- Error
- """
-
- # gts_matched, preds_matched = _matched_list(gts=gts, preds=preds, distance_matrices=distance_matrices['laneseg'], distance_threshold=THRESHOLDS_LANESEG[1], object_type='lane_segment', filter=lambda _: True)
-
return metrics
diff --git a/setup.py b/setup.py
index 6f73040..3da4e7b 100644
--- a/setup.py
+++ b/setup.py
@@ -46,7 +46,7 @@
setup(
name='openlanev2',
- version='2.0.0',
+ version='2.1.0',
author='The OpenLane-V2 Dataset Authors',
author_email='wanghuijie@pjlab.org.cn',
description='The official devkit of the OpenLane-V2 dataset.',