


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase       =							re.compile(R'\b(a|an|the)\b', re.UNICODE)

lowerCAmelCase       =							None
def 			_a (      ):


		"""simple docstring"""




		lowercase__            =      argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''     )
		parser.add_argument('''data_file'''						,				metavar='''data.json'''						,				help='''Input data JSON file.'''     )
		parser.add_argument('''pred_file'''						,				metavar='''pred.json'''						,				help='''Model predictions.'''     )
		parser.add_argument(
		    '''--out-file'''						,				'''-o'''						,				metavar='''eval.json'''						,				help='''Write accuracy metrics to file (default is stdout).'''     )
		parser.add_argument(
		    '''--na-prob-file'''						,				'''-n'''						,				metavar='''na_prob.json'''						,				help='''Model estimates of probability of no answer.'''     )
		parser.add_argument(
		    '''--na-prob-thresh'''						,				'''-t'''						,				type=SCREAMING_SNAKE_CASE						,				default=1.0						,				help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''						,				)
		parser.add_argument(
		    '''--out-image-dir'''						,				'''-p'''						,				metavar='''out_images'''						,				default=SCREAMING_SNAKE_CASE						,				help='''Save precision-recall curves to directory.'''     )
		parser.add_argument('''--verbose'''						,				'''-v'''						,				action='''store_true'''     )
		if len(sys.argv     ) == 1:
				parser.print_help()
				sys.exit(1     )
		return parser.parse_args()
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								lowercase__            =      bool(qa['''answers''']['''text''']     )
		return qid_to_has_ans
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		def remove_articles(SCREAMING_SNAKE_CASE     ):
				return ARTICLES_REGEX.sub(''' '''						,				SCREAMING_SNAKE_CASE     )

		def white_space_fix(SCREAMING_SNAKE_CASE     ):
				return " ".join(text.split()     )

		def remove_punc(SCREAMING_SNAKE_CASE     ):
				lowercase__            =      set(string.punctuation     )
				return "".join(ch for ch in text if ch not in exclude     )

		def lower(SCREAMING_SNAKE_CASE     ):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE     )     )     )     )
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		if not s:
				return []
		return normalize_answer(SCREAMING_SNAKE_CASE     ).split()
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		return int(normalize_answer(SCREAMING_SNAKE_CASE     ) == normalize_answer(SCREAMING_SNAKE_CASE     )     )
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      get_tokens(SCREAMING_SNAKE_CASE     )
		lowercase__            =      get_tokens(SCREAMING_SNAKE_CASE     )
		lowercase__            =      collections.Counter(SCREAMING_SNAKE_CASE     ) & collections.Counter(SCREAMING_SNAKE_CASE     )
		lowercase__            =      sum(common.values()     )
		if len(SCREAMING_SNAKE_CASE     ) == 0 or len(SCREAMING_SNAKE_CASE     ) == 0:
				# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
				return int(gold_toks == pred_toks     )
		if num_same == 0:
				return 0
		lowercase__            =      1.0 * num_same / len(SCREAMING_SNAKE_CASE     )
		lowercase__            =      1.0 * num_same / len(SCREAMING_SNAKE_CASE     )
		lowercase__            =      (2 * precision * recall) / (precision + recall)
		return fa
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      {}
		lowercase__            =      {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								lowercase__            =      qa['''id''']
								lowercase__            =      [t for t in qa['''answers''']['''text'''] if normalize_answer(SCREAMING_SNAKE_CASE     )]
								if not gold_answers:
										# For unanswerable questions, only correct answer is empty string
										lowercase__            =      ['''''']
								if qid not in preds:
										print(f'Missing prediction for {qid}'     )
										continue
								lowercase__            =      preds[qid]
								# Take max over all gold answers
								lowercase__            =      max(compute_exact(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ) for a in gold_answers     )
								lowercase__            =      max(compute_fa(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ) for a in gold_answers     )
		return exact_scores, fa_scores
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      {}
		for qid, s in scores.items():
				lowercase__            =      na_probs[qid] > na_prob_thresh
				if pred_na:
						lowercase__            =      float(not qid_to_has_ans[qid]     )
				else:
						lowercase__            =      s
		return new_scores
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE=None     ):


		"""simple docstring"""




		if not qid_list:
				lowercase__            =      len(SCREAMING_SNAKE_CASE     )
				return collections.OrderedDict(
				    [
				        ('''exact''', 100.0 * sum(exact_scores.values()     ) / total),
				        ('''f1''', 100.0 * sum(fa_scores.values()     ) / total),
				        ('''total''', total),
				    ]     )
		else:
				lowercase__            =      len(SCREAMING_SNAKE_CASE     )
				return collections.OrderedDict(
				    [
				        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list     ) / total),
				        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list     ) / total),
				        ('''total''', total),
				    ]     )
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		for k in new_eval:
				lowercase__            =      new_eval[k]
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		plt.step(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				color='''b'''						,				alpha=0.2						,				where='''post'''     )
		plt.fill_between(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				step='''post'''						,				alpha=0.2						,				color='''b'''     )
		plt.xlabel('''Recall'''     )
		plt.ylabel('''Precision'''     )
		plt.xlim([0.0, 1.05]     )
		plt.ylim([0.0, 1.05]     )
		plt.title(SCREAMING_SNAKE_CASE     )
		plt.savefig(SCREAMING_SNAKE_CASE     )
		plt.clf()
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE=None						,				SCREAMING_SNAKE_CASE=None     ):


		"""simple docstring"""




		lowercase__            =      sorted(SCREAMING_SNAKE_CASE						,				key=lambda SCREAMING_SNAKE_CASE     : na_probs[k]     )
		lowercase__            =      0.0
		lowercase__            =      1.0
		lowercase__            =      0.0
		lowercase__            =      [1.0]
		lowercase__            =      [0.0]
		lowercase__            =      0.0
		for i, qid in enumerate(SCREAMING_SNAKE_CASE     ):
				if qid_to_has_ans[qid]:
						true_pos += scores[qid]
				lowercase__            =      true_pos / float(i + 1     )
				lowercase__            =      true_pos / float(SCREAMING_SNAKE_CASE     )
				if i == len(SCREAMING_SNAKE_CASE     ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
						# i.e., if we can put a threshold after this point
						avg_prec += cur_p * (cur_r - recalls[-1])
						precisions.append(SCREAMING_SNAKE_CASE     )
						recalls.append(SCREAMING_SNAKE_CASE     )
		if out_image:
				plot_pr_curve(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		return {"ap": 100.0 * avg_prec}
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		if out_image_dir and not os.path.exists(SCREAMING_SNAKE_CASE     ):
				os.makedirs(SCREAMING_SNAKE_CASE     )
		lowercase__            =      sum(1 for v in qid_to_has_ans.values() if v     )
		if num_true_pos == 0:
				return
		lowercase__            =      make_precision_recall_eval(
		    SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				out_image=os.path.join(SCREAMING_SNAKE_CASE						,				'''pr_exact.png'''     )						,				title='''Precision-Recall curve for Exact Match score'''						,				)
		lowercase__            =      make_precision_recall_eval(
		    SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				out_image=os.path.join(SCREAMING_SNAKE_CASE						,				'''pr_f1.png'''     )						,				title='''Precision-Recall curve for F1 score'''						,				)
		lowercase__            =      {k: float(SCREAMING_SNAKE_CASE     ) for k, v in qid_to_has_ans.items()}
		lowercase__            =      make_precision_recall_eval(
		    SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				out_image=os.path.join(SCREAMING_SNAKE_CASE						,				'''pr_oracle.png'''     )						,				title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''						,				)
		merge_eval(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				'''pr_exact'''     )
		merge_eval(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				'''pr_f1'''     )
		merge_eval(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				'''pr_oracle'''     )
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		if not qid_list:
				return
		lowercase__            =      [na_probs[k] for k in qid_list]
		lowercase__            =      np.ones_like(SCREAMING_SNAKE_CASE     ) / float(len(SCREAMING_SNAKE_CASE     )     )
		plt.hist(SCREAMING_SNAKE_CASE						,				weights=SCREAMING_SNAKE_CASE						,				bins=20						,				range=(0.0, 1.0)     )
		plt.xlabel('''Model probability of no-answer'''     )
		plt.ylabel('''Proportion of dataset'''     )
		plt.title(f'Histogram of no-answer probability: {name}'     )
		plt.savefig(os.path.join(SCREAMING_SNAKE_CASE						,				f'na_prob_hist_{name}.png'     )     )
		plt.clf()
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]     )
		lowercase__            =      num_no_ans
		lowercase__            =      cur_score
		lowercase__            =      0.0
		lowercase__            =      sorted(SCREAMING_SNAKE_CASE						,				key=lambda SCREAMING_SNAKE_CASE     : na_probs[k]     )
		for i, qid in enumerate(SCREAMING_SNAKE_CASE     ):
				if qid not in scores:
						continue
				if qid_to_has_ans[qid]:
						lowercase__            =      scores[qid]
				else:
						if preds[qid]:
								lowercase__            =      -1
						else:
								lowercase__            =      0
				cur_score += diff
				if cur_score > best_score:
						lowercase__            =      cur_score
						lowercase__            =      na_probs[qid]
		return 100.0 * best_score / len(SCREAMING_SNAKE_CASE     ), best_thresh
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__     ,		lowercase__            =      find_best_thresh(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		lowercase__     ,		lowercase__            =      find_best_thresh(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		lowercase__            =      best_exact
		lowercase__            =      exact_thresh
		lowercase__            =      best_fa
		lowercase__            =      fa_thresh
def 			_a (      ):


		"""simple docstring"""




		with open(OPTS.data_file     ) as f:
				lowercase__            =      json.load(SCREAMING_SNAKE_CASE     )
				lowercase__            =      dataset_json['''data''']
		with open(OPTS.pred_file     ) as f:
				lowercase__            =      json.load(SCREAMING_SNAKE_CASE     )
		if OPTS.na_prob_file:
				with open(OPTS.na_prob_file     ) as f:
						lowercase__            =      json.load(SCREAMING_SNAKE_CASE     )
		else:
				lowercase__            =      {k: 0.0 for k in preds}
		lowercase__            =      make_qid_to_has_ans(SCREAMING_SNAKE_CASE     )  # maps qid to True/False
		lowercase__            =      [k for k, v in qid_to_has_ans.items() if v]
		lowercase__            =      [k for k, v in qid_to_has_ans.items() if not v]
		lowercase__     ,		lowercase__            =      get_raw_scores(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		lowercase__            =      apply_no_ans_threshold(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				OPTS.na_prob_thresh     )
		lowercase__            =      apply_no_ans_threshold(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				OPTS.na_prob_thresh     )
		lowercase__            =      make_eval_dict(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		if has_ans_qids:
				lowercase__            =      make_eval_dict(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				qid_list=SCREAMING_SNAKE_CASE     )
				merge_eval(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				'''HasAns'''     )
		if no_ans_qids:
				lowercase__            =      make_eval_dict(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				qid_list=SCREAMING_SNAKE_CASE     )
				merge_eval(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				'''NoAns'''     )
		if OPTS.na_prob_file:
				find_all_best_thresh(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		if OPTS.na_prob_file and OPTS.out_image_dir:
				run_precision_recall_analysis(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				OPTS.out_image_dir     )
				histogram_na_prob(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				OPTS.out_image_dir						,				'''hasAns'''     )
				histogram_na_prob(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				OPTS.out_image_dir						,				'''noAns'''     )
		if OPTS.out_file:
				with open(OPTS.out_file						,				'''w'''     ) as f:
						json.dump(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		else:
				print(json.dumps(SCREAMING_SNAKE_CASE						,				indent=2     )     )


if __name__ == "__main__":
			lowerCAmelCase       =							parse_args()
			if OPTS.out_image_dir:
						import matplotlib

						matplotlib.use('Agg')
						import matplotlib.pyplot as plt
			main()






from __future__ import annotations

from collections.abc import Iterator


class        _a							:

							def __init__(							self:	List[str]    ,						UpperCamelCase_:	int							)			->   None:





									"""simple docstring"""







									lowercase__            =      value
									lowercase__            =      None
									lowercase__            =      None


class        _a							:

							def __init__(							self:	Union[str, Any]    ,						UpperCamelCase_:	Node							)			->   None:





									"""simple docstring"""







									lowercase__            =      tree

							def       lowerCamelCase_ (							self:	Any    ,						UpperCamelCase_:	Node | None							)			->   int:





									"""simple docstring"""







									if node is None:
											return 0
									return node.value + (
									    self.depth_first_search(node.left							) + self.depth_first_search(node.right							)
									)


							def __iter__(							self:	List[str]							)			->   Iterator[int]:





									"""simple docstring"""







									yield self.depth_first_search(self.tree							)


if __name__ == "__main__":
			import doctest

			doctest.testmod()


