jordyvl commited on
Commit
8ec938f
1 Parent(s): 0a06e4b

updates for equal mass binning

Browse files
Files changed (1) hide show
  1. ece.py +7 -8
ece.py CHANGED
@@ -103,8 +103,8 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
103
 
104
  def discretize_into_bins(P, bins):
105
 
106
- contains_rightmost = bool(bins[-1] > 1) #outlier bins
107
- contains_leftmost = bool(bins[0] == 0) #beyond [before] bin_range[0]
108
  # bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
109
 
110
  oneDbins = np.digitize(
@@ -143,7 +143,7 @@ def discretize_into_bins(P, bins):
143
  def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
144
  bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
145
 
146
- # indexed as in julia!
147
  result = np.empty([len(bins)], float)
148
  result.fill(np.nan) # cannot assume each bin will have observations
149
 
@@ -160,8 +160,10 @@ def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
160
  def bin_calibrated_accuracy(bins, proxy="upper-edge"):
161
  assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
162
 
 
 
163
  if proxy == "upper-edge":
164
- return bins[1:]
165
 
166
  if proxy == "center":
167
  return bins[:-1] + np.diff(bins) / 2
@@ -175,7 +177,7 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
175
  Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
176
  """
177
 
178
- n_bins = len(bins) - 1 #true number of bins
179
  bin_range = [min(bins), max(bins)]
180
 
181
  # average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
@@ -185,9 +187,6 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
185
  bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
186
  anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
187
 
188
- import pdb; pdb.set_trace() # breakpoint 83c9148b //
189
-
190
-
191
  # Expected calibration error
192
  if p < np.inf: # L^p-CE
193
  CE = np.average(
 
103
 
104
  def discretize_into_bins(P, bins):
105
 
106
+ contains_rightmost = bool(bins[-1] > 1) # outlier bins
107
+ contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
108
  # bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
109
 
110
  oneDbins = np.digitize(
 
143
  def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
144
  bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
145
 
146
+ # indexed as in julia!
147
  result = np.empty([len(bins)], float)
148
  result.fill(np.nan) # cannot assume each bin will have observations
149
 
 
160
  def bin_calibrated_accuracy(bins, proxy="upper-edge"):
161
  assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
162
 
163
+ contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
164
+
165
  if proxy == "upper-edge":
166
+ return bins[1:] if contains_leftmost else bins
167
 
168
  if proxy == "center":
169
  return bins[:-1] + np.diff(bins) / 2
 
177
  Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
178
  """
179
 
180
+ n_bins = len(bins) - 1 # true number of bins
181
  bin_range = [min(bins), max(bins)]
182
 
183
  # average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
 
187
  bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
188
  anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
189
 
 
 
 
190
  # Expected calibration error
191
  if p < np.inf: # L^p-CE
192
  CE = np.average(