File size: 8,354 Bytes
16dc4f2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
"""
Copied from MMAction2
https://github.com/open-mmlab/mmaction2/blob/master/mmaction/core/evaluation/eval_detection.py
"""
import json
import numpy as np
from sklearn.metrics import precision_recall_curve
def load_jsonl(filename):
with open(filename, "r") as f:
return [json.loads(l.strip("\n")) for l in f.readlines()]
def compute_temporal_iou_batch_paired(pred_windows, gt_windows):
""" compute intersection-over-union along temporal axis for each pair of windows in pred_windows and gt_windows.
Args:
pred_windows: np.ndarray, (N, 2), [st (float), ed (float)] * N
gt_windows: np.ndarray, (N, 2), [st (float), ed (float)] * N
Returns:
iou (float): np.ndarray, (N, )
References:
for np.divide with zeros, see https://stackoverflow.com/a/37977222
"""
intersection = np.maximum(
0, np.minimum(pred_windows[:, 1], gt_windows[:, 1]) - np.maximum(pred_windows[:, 0], gt_windows[:, 0])
)
union = np.maximum(pred_windows[:, 1], gt_windows[:, 1]) \
- np.minimum(pred_windows[:, 0], gt_windows[:, 0]) # not the correct union though
return np.divide(intersection, union, out=np.zeros_like(intersection), where=union != 0)
def compute_temporal_iou_batch_cross(spans1, spans2):
"""
Args:
spans1: (N, 2) np.ndarray, each row defines a span [st, ed]
spans2: (M, 2) np.ndarray, ...
Returns:
iou: (N, M) np.ndarray
union: (N, M) np.ndarray
>>> spans1 = np.array([[0, 0.2, 0.9], [0.5, 1.0, 0.2]])
>>> spans2 = np.array([[0, 0.3], [0., 1.0]])
>>> compute_temporal_iou_batch_cross(spans1, spans2)
(tensor([[0.6667, 0.2000],
[0.0000, 0.5000]]),
tensor([[0.3000, 1.0000],
[0.8000, 1.0000]]))
"""
areas1 = spans1[:, 1] - spans1[:, 0] # (N, )
areas2 = spans2[:, 1] - spans2[:, 0] # (M, )
left = np.maximum(spans1[:, None, 0], spans2[None, :, 0]) # (N, M)
right = np.minimum(spans1[:, None, 1], spans2[None, :, 1]) # (N, M)
inter = np.clip(right - left, 0, None) # (N, M)
union = areas1[:, None] + areas2[None, :] - inter # (N, M)
iou = inter / union
return iou, union
def interpolated_precision_recall(precision, recall):
"""Interpolated AP - VOCdevkit from VOC 2011.
Args:
precision (np.ndarray): The precision of different thresholds.
recall (np.ndarray): The recall of different thresholds.
Returns：
float: Average precision score.
"""
mprecision = np.hstack([[0], precision, [0]])
mrecall = np.hstack([[0], recall, [1]])
for i in range(len(mprecision) - 1)[::-1]:
mprecision[i] = max(mprecision[i], mprecision[i + 1])
idx = np.where(mrecall[1::] != mrecall[0:-1])[0] + 1
ap = np.sum((mrecall[idx] - mrecall[idx - 1]) * mprecision[idx])
return ap
def compute_average_precision_detection(ground_truth,
prediction,
tiou_thresholds=np.linspace(
0.5, 0.95, 10)):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as true
positive. This code is greatly inspired by Pascal VOC devkit.
Args:
ground_truth (list[dict]): List containing the ground truth instances
(dictionaries). Required keys are 'video-id', 't-start' and
't-end'.
prediction (list[dict]): List containing the prediction instances
(dictionaries). Required keys are: 'video-id', 't-start', 't-end'
and 'score'.
tiou_thresholds (np.ndarray): A 1darray indicates the temporal
intersection over union threshold, which is optional.
Default: ``np.linspace(0.5, 0.95, 10)``.
Returns:
Float: ap, Average precision score.
"""
num_thresholds = len(tiou_thresholds)
num_gts = len(ground_truth)
num_preds = len(prediction)
ap = np.zeros(num_thresholds)
if len(prediction) == 0:
return ap
num_positive = float(num_gts)
lock_gt = np.ones((num_thresholds, num_gts)) * -1
# Sort predictions by decreasing score order.
prediction.sort(key=lambda x: -x['score'])
# Initialize true positive and false positive vectors.
tp = np.zeros((num_thresholds, num_preds))
fp = np.zeros((num_thresholds, num_preds))
# Adaptation to query faster
ground_truth_by_videoid = {}
for i, item in enumerate(ground_truth):
item['index'] = i
ground_truth_by_videoid.setdefault(item['video-id'], []).append(item)
# Assigning true positive to truly grount truth instances.
for idx, pred in enumerate(prediction):
if pred['video-id'] in ground_truth_by_videoid:
gts = ground_truth_by_videoid[pred['video-id']]
else:
fp[:, idx] = 1
continue
_pred = np.array([[pred['t-start'], pred['t-end']], ])
_gt = np.array([[gt['t-start'], gt['t-end']] for gt in gts])
tiou_arr = compute_temporal_iou_batch_cross(_pred, _gt)[0]
tiou_arr = tiou_arr.reshape(-1)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for t_idx, tiou_threshold in enumerate(tiou_thresholds):
for j_idx in tiou_sorted_idx:
if tiou_arr[j_idx] < tiou_threshold:
fp[t_idx, idx] = 1
break
if lock_gt[t_idx, gts[j_idx]['index']] >= 0:
continue
# Assign as true positive after the filters above.
tp[t_idx, idx] = 1
lock_gt[t_idx, gts[j_idx]['index']] = idx
break
if fp[t_idx, idx] == 0 and tp[t_idx, idx] == 0:
fp[t_idx, idx] = 1
tp_cumsum = np.cumsum(tp, axis=1).astype(float)
fp_cumsum = np.cumsum(fp, axis=1).astype(float)
recall_cumsum = tp_cumsum / num_positive
precision_cumsum = tp_cumsum / (tp_cumsum + fp_cumsum)
for t_idx in range(len(tiou_thresholds)):
ap[t_idx] = interpolated_precision_recall(precision_cumsum[t_idx, :],
recall_cumsum[t_idx, :])
return ap
def get_ap(y_true, y_predict, interpolate=True, point_11=False):
"""
Average precision in different formats: (non-) interpolated and/or 11-point approximated
point_11=True and interpolate=True corresponds to the 11-point interpolated AP used in
the PASCAL VOC challenge up to the 2008 edition and has been verfied against the vlfeat implementation
The exact average precision (interpolate=False, point_11=False) corresponds to the one of vl_feat
:param y_true: list/ numpy vector of true labels in {0,1} for each element
:param y_predict: predicted score for each element
:param interpolate: Use interpolation?
:param point_11: Use 11-point approximation to average precision?
:return: average precision
ref: https://github.com/gyglim/video2gif_dataset/blob/master/v2g_evaluation/__init__.py
"""
# Check inputs
assert len(y_true) == len(y_predict), "Prediction and ground truth need to be of the same length"
if len(set(y_true)) == 1:
if y_true[0] == 0:
return 0 # True labels are all zeros
# raise ValueError('True labels cannot all be zero')
else:
return 1
else:
assert sorted(set(y_true)) == [0, 1], "Ground truth can only contain elements {0,1}"
# Compute precision and recall
precision, recall, _ = precision_recall_curve(y_true, y_predict)
recall = recall.astype(np.float32)
if interpolate: # Compute the interpolated precision
for i in range(1, len(precision)):
precision[i] = max(precision[i - 1], precision[i])
if point_11: # Compute the 11-point approximated AP
precision_11 = [precision[np.where(recall >= t)[0][-1]] for t in np.arange(0, 1.01, 0.1)]
return np.mean(precision_11)
else: # Compute the AP using precision at every additionally recalled sample
indices = np.where(np.diff(recall))
return np.mean(precision[indices])
|