Spaces:
Runtime error
Runtime error
updates for equal mass binning
Browse files
ece.py
CHANGED
@@ -103,8 +103,8 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
|
|
103 |
|
104 |
def discretize_into_bins(P, bins):
|
105 |
|
106 |
-
contains_rightmost = bool(bins[-1] > 1)
|
107 |
-
contains_leftmost = bool(bins[0] == 0)
|
108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
109 |
|
110 |
oneDbins = np.digitize(
|
@@ -143,7 +143,7 @@ def discretize_into_bins(P, bins):
|
|
143 |
def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
144 |
bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
|
145 |
|
146 |
-
# indexed as in julia!
|
147 |
result = np.empty([len(bins)], float)
|
148 |
result.fill(np.nan) # cannot assume each bin will have observations
|
149 |
|
@@ -160,8 +160,10 @@ def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
|
160 |
def bin_calibrated_accuracy(bins, proxy="upper-edge"):
|
161 |
assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
|
162 |
|
|
|
|
|
163 |
if proxy == "upper-edge":
|
164 |
-
return bins[1:]
|
165 |
|
166 |
if proxy == "center":
|
167 |
return bins[:-1] + np.diff(bins) / 2
|
@@ -175,7 +177,7 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
|
|
175 |
Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
|
176 |
"""
|
177 |
|
178 |
-
n_bins = len(bins) - 1
|
179 |
bin_range = [min(bins), max(bins)]
|
180 |
|
181 |
# average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
|
@@ -185,9 +187,6 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
|
|
185 |
bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
|
186 |
anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
|
187 |
|
188 |
-
import pdb; pdb.set_trace() # breakpoint 83c9148b //
|
189 |
-
|
190 |
-
|
191 |
# Expected calibration error
|
192 |
if p < np.inf: # L^p-CE
|
193 |
CE = np.average(
|
|
|
103 |
|
104 |
def discretize_into_bins(P, bins):
|
105 |
|
106 |
+
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
107 |
+
contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
|
108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
109 |
|
110 |
oneDbins = np.digitize(
|
|
|
143 |
def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
144 |
bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
|
145 |
|
146 |
+
# indexed as in julia!
|
147 |
result = np.empty([len(bins)], float)
|
148 |
result.fill(np.nan) # cannot assume each bin will have observations
|
149 |
|
|
|
160 |
def bin_calibrated_accuracy(bins, proxy="upper-edge"):
|
161 |
assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
|
162 |
|
163 |
+
contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
|
164 |
+
|
165 |
if proxy == "upper-edge":
|
166 |
+
return bins[1:] if contains_leftmost else bins
|
167 |
|
168 |
if proxy == "center":
|
169 |
return bins[:-1] + np.diff(bins) / 2
|
|
|
177 |
Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
|
178 |
"""
|
179 |
|
180 |
+
n_bins = len(bins) - 1 # true number of bins
|
181 |
bin_range = [min(bins), max(bins)]
|
182 |
|
183 |
# average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
|
|
|
187 |
bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
|
188 |
anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
|
189 |
|
|
|
|
|
|
|
190 |
# Expected calibration error
|
191 |
if p < np.inf: # L^p-CE
|
192 |
CE = np.average(
|