def _update_ranking_metrics(self, observation, labels):
    text_cands = observation.get('text_candidates', None)
    if (text_cands is None):
        return
    label_set = set((normalize_answer(l) for l in labels))
    cnts = {k: 0 for k in self.eval_pr}
    cnt = 0
    for c in text_cands:
        cnt += 1
        if (normalize_answer(c) in label_set):
            for k in self.eval_pr:
                if (cnt <= k):
                    cnts[k] += 1
    for k in self.eval_pr:
        self.add(f'hits@{k}', AverageMetric((cnts[k] > 0)))
def _update_ranking_metrics(self, observation, labels):
    text_cands = observation.get('text_candidates', None)
    if (text_cands is None):
        return
    label_set = set((normalize_answer(l) for l in labels))
    cnts = {k: 0 for k in self.eval_pr}
    cnt = 0
    for c in text_cands:
        cnt += 1
        if (normalize_answer(c) in label_set):
            for k in self.eval_pr:
                if (cnt <= k):
                    cnts[k] += 1
    for k in self.eval_pr:
        self.add(f'hits@{k}', AverageMetric((cnts[k] > 0)))