Spaces:
Configuration error
Configuration error
small update to support unrealistic perfect classification
Browse files
ece.py
CHANGED
@@ -104,7 +104,7 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
|
|
104 |
def discretize_into_bins(P, bins):
|
105 |
|
106 |
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
107 |
-
contains_leftmost = bool(bins[0]
|
108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
109 |
|
110 |
oneDbins = np.digitize(
|
@@ -112,7 +112,6 @@ def discretize_into_bins(P, bins):
|
|
112 |
) # since bins contains extra righmost (& leftmost bins)
|
113 |
if contains_leftmost:
|
114 |
oneDbins -= 1
|
115 |
-
# oneDbins = np.digitize(P, bins) - 1 # since bins contains extra righmost (& leftmost bins)
|
116 |
|
117 |
# Fix to scipy.binned_dd_statistic:
|
118 |
# Tie-breaking to the left for rightmost bin
|
@@ -245,8 +244,8 @@ class ECE(evaluate.EvaluationModule):
|
|
245 |
# Homepage of the module for documentation
|
246 |
homepage="https://huggingface.co/spaces/jordyvl/ece",
|
247 |
# Additional links to the codebase or references
|
248 |
-
codebase_urls=["
|
249 |
-
reference_urls=["
|
250 |
)
|
251 |
|
252 |
def init_kwargs(
|
@@ -341,32 +340,27 @@ def test_equalmass_binning():
|
|
341 |
scheme="equal-mass",
|
342 |
bin_range=None,
|
343 |
proxy="upper-edge",
|
344 |
-
# proxy="center",
|
345 |
p=1,
|
346 |
detail=True,
|
347 |
)
|
348 |
bins = create_bins(
|
349 |
n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
|
350 |
)
|
351 |
-
|
352 |
test_ECE(**kwargs)
|
353 |
|
354 |
-
|
|
|
|
|
355 |
res = ECE()._compute(
|
356 |
-
references=
|
357 |
-
predictions=
|
358 |
detail=True,
|
359 |
)
|
360 |
print(f"ECE: {res['ECE']}\n {res}")
|
361 |
-
"""
|
362 |
-
# need to provide lens
|
363 |
-
|
364 |
-
import pdb
|
365 |
-
|
366 |
-
pdb.set_trace() # breakpoint 94583f9f //
|
367 |
|
368 |
|
369 |
if __name__ == "__main__":
|
|
|
370 |
test_equalmass_binning()
|
371 |
test_deterministic()
|
372 |
test_ECE()
|
|
|
104 |
def discretize_into_bins(P, bins):
|
105 |
|
106 |
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
107 |
+
contains_leftmost = bool(bins[0] <= 0) # beyond [before] bin_range[0]
|
108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
109 |
|
110 |
oneDbins = np.digitize(
|
|
|
112 |
) # since bins contains extra righmost (& leftmost bins)
|
113 |
if contains_leftmost:
|
114 |
oneDbins -= 1
|
|
|
115 |
|
116 |
# Fix to scipy.binned_dd_statistic:
|
117 |
# Tie-breaking to the left for rightmost bin
|
|
|
244 |
# Homepage of the module for documentation
|
245 |
homepage="https://huggingface.co/spaces/jordyvl/ece",
|
246 |
# Additional links to the codebase or references
|
247 |
+
codebase_urls=[""],
|
248 |
+
reference_urls=[""],
|
249 |
)
|
250 |
|
251 |
def init_kwargs(
|
|
|
340 |
scheme="equal-mass",
|
341 |
bin_range=None,
|
342 |
proxy="upper-edge",
|
|
|
343 |
p=1,
|
344 |
detail=True,
|
345 |
)
|
346 |
bins = create_bins(
|
347 |
n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
|
348 |
)
|
|
|
349 |
test_ECE(**kwargs)
|
350 |
|
351 |
+
|
352 |
+
def test_perfect_predictions(K=3):
|
353 |
+
references = [0, 1, 2]
|
354 |
res = ECE()._compute(
|
355 |
+
references=references,
|
356 |
+
predictions=np.eye(K)[references],
|
357 |
detail=True,
|
358 |
)
|
359 |
print(f"ECE: {res['ECE']}\n {res}")
|
|
|
|
|
|
|
|
|
|
|
|
|
360 |
|
361 |
|
362 |
if __name__ == "__main__":
|
363 |
+
test_perfect_predictions()
|
364 |
test_equalmass_binning()
|
365 |
test_deterministic()
|
366 |
test_ECE()
|