update app

ece.py

@@ -209,29 +209,17 @@ class ECE(evaluate.EvaluationModule):
     4. apply L^p norm distance and weights
     """
 
-    def __init__(self, n_bins=10, bin_range=None, scheme="equal-range", proxy="upper-edge", p=1, **kwargs):
-        import pdb; pdb.set_trace()  # breakpoint bf1c56c7 //
-
-        super(evaluate.EvaluationModule, self).__init__(**kwargs)
-        self.n_bins = n_bins
-        self.bin_range = bin_range
-        self.scheme = scheme
-        self.proxy = proxy
-        self.p = p
-
     def _info(self):
         # TODO: Specifies the evaluate.EvaluationModuleInfo object
         return evaluate.EvaluationModuleInfo(
-            # This is the description that will appear on the modules page.
             module_type="metric",
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
-            # This defines the format of each prediction and reference
             features=datasets.Features(
                 {
-                    "predictions": datasets.Value("float32"),
-                    "references": datasets.Value("int64"),
+                    "predictions": datasets.Array2D(datasets.Value("float32")),  # or Sequence
+                    "references": datasets.Sequence(datasets.Value("int64")),
                 }
             ),
             # Homepage of the module for documentation
@@ -241,42 +229,43 @@ class ECE(evaluate.EvaluationModule):
             reference_urls=["http://path.to.reference.url/new_module"],
         )
 
-    def _download_and_prepare(self, dl_manager):
-        """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
-        pass
-
-    def _compute(self, predictions, references):
-        """Returns the scores"""
-        ECE = top_1_CE(references, predictions, **self.__dict__)
-        return {
-            "ECE": ECE,
-        }
-
+    def init_kwargs(
+        self, n_bins=10, bin_range=None, scheme="equal-range", proxy="upper-edge", p=1, **kwargs
+    ):
+        # super(evaluate.EvaluationModule, self).__init__(**kwargs)
+        self.n_bins = n_bins
+        self.bin_range = bin_range
+        self.scheme = scheme
+        self.proxy = proxy
+        self.p = p
 
-def test_ECE():
-    N = 10  # N evaluation instances {(x_i,y_i)}_{i=1}^N
-    K = 5  # K class problem
+    def _compute(self, predictions, references, **kwargs):
 
-    def random_mc_instance(concentration=1, onehot=False):
-        reference = np.argmax(
-            np.random.dirichlet(([concentration for _ in range(K)])), -1
-        )  # class targets
-        prediction = np.random.dirichlet(([concentration for _ in range(K)]))  # probabilities
-        if onehot:
-            reference = np.eye(K)[np.argmax(reference, -1)]
-        return reference, prediction
+        # convert to numpy arrays
+        references = np.array(references, dtype=np.int64)
+        predictions = np.array(predictions, dtype=np.float32)
 
-    references, predictions = list(zip(*[random_mc_instance() for i in range(N)]))
-    references = np.array(references, dtype=np.int64)
-    predictions = np.array(predictions, dtype=np.float32)
-    res = ECE()._compute(predictions, references)
-    print(f"ECE: {res['ECE']}")
+        assert (
+            predictions.shape[0] == references.shape[0]
+        ), "Need to pass similar predictions and references"
 
+        # Assert that arrays are 2D
+        if len(predictions.shape) != 2:
+            raise ValueError("Expected `predictions` to be a 2D vector (N x K)")
 
-if __name__ == "__main__":
-    test_ECE()
+        if len(references.shape) != 1:
+            # could check if wrongly passed as onehot
+            if (references.shape[-1] == predictions.shape[1]) and (
+                np.sum(references) == predictions.shape[0]
+            ):
+                references = np.argmax(references, -1)
+            else:
+                raise ValueError("Expected `references` to be a 1D vector (N,)")
 
-    evaluate.load("jordyvl/ece")
-    import pdb; pdb.set_trace()  # breakpoint 058892ee //
+        self.init_kwargs(**kwargs)
 
+        """Returns the scores"""
+        ECE = top_1_CE(references, predictions, **self.__dict__)
+        return {
+            "ECE": ECE,
+        }

tests.py

@@ -1,3 +1,5 @@
+import numpy as np
+
 test_cases = [
     {
         "predictions": [0, 0],
@@ -14,4 +16,24 @@ test_cases = [
         "references": [1, 1],
         "result": {"metric_score": 0.5}
     }
-]
+]
+
+
+def test_ECE():
+    N = 10  # N evaluation instances {(x_i,y_i)}_{i=1}^N
+    K = 5  # K class problem
+
+    def random_mc_instance(concentration=1, onehot=False):
+        reference = np.argmax(
+            np.random.dirichlet(([concentration for _ in range(K)])), -1
+        )  # class targets
+        prediction = np.random.dirichlet(([concentration for _ in range(K)]))  # probabilities
+        if onehot:
+            reference = np.eye(K)[np.argmax(reference, -1)]
+        return reference, prediction
+
+    references, predictions = list(zip(*[random_mc_instance() for i in range(N)]))
+    references = np.array(references, dtype=np.int64)
+    predictions = np.array(predictions, dtype=np.float32)
+    res = ECE()._compute(predictions, references)
+    print(f"ECE: {res['ECE']}")