Prova

metrica_tesi.py

@@ -38,14 +38,17 @@ _KWARGS_DESCRIPTION = """
 Calculates how good are predictions given some references, using certain scores
 Args:
     predictions: list of predictions to score. Each prediction
-        should be a string with tokens separated by spaces.
-        Special tokens must be included.
+        should be an input id.
     references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.  
-        Special tokens must be included.
-    n: number of last tokens to be considered for the calculation.
+        reference should be an input id.
+    actions_seen: number of actions token seen before generating the predicted action token.
+    max_actions_seen: the number of scores to calculate. For example, with max_actions_seen = 5,
+        it will calculate score for prediction with actions_seen = 0, 1, 2, 3, 4, 5.
 Returns:
-    score: accuracy score calculated on the last n action tokens of every pair prediction-reference.
+    score_k: accuracy score calculated on predictions with n = k. The number of scores
+        calculated in this way depends on the value of max_actions_seen. For example,
+        with max_actions_seen = 5, we will have score_0, score_1, ..., score_5.
+    support_k: the number of predictions that support the corresponding score_k.
 Examples:
     Examples should be written in doctest format, and should illustrate how
     to use the function.
@@ -60,10 +63,6 @@ Examples:
 BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
 
 
-def get_last_n_tokens(string_of_tokens, n):
-    return string_of_tokens.split(" ")[-n:]
-
-
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class MetricaTesi(evaluate.Metric):
     """TODO: Short description of my evaluation module."""
@@ -79,8 +78,9 @@ class MetricaTesi(evaluate.Metric):
             # This defines the format of each prediction and reference
             features=datasets.Features(
                 {
-                    "predictions": datasets.Value("string", id="sequence"),
-                    "references": datasets.Value("string", id="sequence"),
+                    "predictions": datasets.Value("int32"),
+                    "references": datasets.Value("int32"),
+                    "n": datasets.Value("int32"),
                 }
             ),
             # Homepage of the module for documentation
@@ -95,14 +95,17 @@ class MetricaTesi(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references, n=2):
+    def _compute(self, predictions, references, actions_seen, max_actions_seen=20):
         """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        score = 0
-        for pair in zip(map(get_last_n_tokens, predictions, repeat(n)), map(get_last_n_tokens, references, repeat(n))):
-            if len(pair[0]) == 2:
-                score += sum(pred_token == ref_token for pred_token, ref_token in zip(*pair)) / 2
-        score /= len(predictions)
-        return {
-            "score": score,
-        }
+        results = dict()
+        for i in range(max_actions_seen):
+            score = 0.0
+            support = sum(n == i for n in actions_seen)
+            if support != 0:
+                for prediction, reference, n in zip(predictions, references, actions_seen):
+                    if prediction == reference:
+                        score += 1
+                score /= support
+            results[f"support_{i}"] = support
+            results[f"score_{i}"] = score
+        return results