


import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_torch_available
from transformers.testing_utils import require_torch, torch_device


if is_torch_available():
    from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments







@require_torch
class      lowerCAmelCase__       (				unittest.TestCase):




     '''simple docstring'''


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Dict:
            for model_result in results.values():
                   for batch_size, sequence_length in zip(model_result["bs"] ,						model_result["ss"]):
                          _A      :       Optional[int]				    = model_result["result"][batch_size][sequence_length]
                          self.assertIsNotNone(__lowerCamelCase)


     def  _lowerCamelCase (						self)			->				int:
            _A      :       Optional[int]				    = "sshleifer/tiny-gpt2"
            _A      :       int				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       List[str]				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       Optional[Any]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				Dict:
            _A      :       int				    = "sgugger/tiny-distilbert-classification"
            _A      :       str				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						only_pretrain_model=__lowerCamelCase ,						)
            _A      :       Dict				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       Union[str, Any]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				Optional[Any]:
            _A      :       Tuple				    = "sshleifer/tiny-gpt2"
            _A      :       Optional[Any]				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						torchscript=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Union[str, Any]				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       List[str]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     @unittest.skipIf(torch_device == "cpu" ,						"Cant do half precision")
     def  _lowerCamelCase (						self)			->				int:
            _A      :       Any				    = "sshleifer/tiny-gpt2"
            _A      :       Tuple				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						fpaa=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Any				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       Union[str, Any]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				Any:
            _A      :       Union[str, Any]				    = "sshleifer/tiny-gpt2"
            _A      :       Any				    = AutoConfig.from_pretrained(__lowerCamelCase)
            # set architectures equal to `None`
            _A      :       Dict				    = None
            _A      :       Any				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Union[str, Any]				    = PyTorchBenchmark(__lowerCamelCase ,						configs=[config])
            _A      :       int				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				int:
            _A      :       List[Any]				    = "sshleifer/tiny-gpt2"
            _A      :       int				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Optional[Any]				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       int				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_train_result)
            self.check_results_dict_not_empty(results.memory_train_result)


     @unittest.skipIf(torch_device == "cpu" ,						"Can't do half precision")
     def  _lowerCamelCase (						self)			->				Optional[Any]:
            _A      :       Any				    = "sshleifer/tiny-gpt2"
            _A      :       Tuple				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						fpaa=__lowerCamelCase ,						multi_process=__lowerCamelCase ,						)
            _A      :       List[Any]				    = PyTorchBenchmark(__lowerCamelCase)
            _A      :       Tuple				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_train_result)
            self.check_results_dict_not_empty(results.memory_train_result)


     def  _lowerCamelCase (						self)			->				str:
            _A      :       List[str]				    = "sshleifer/tiny-gpt2"
            _A      :       Union[str, Any]				    = AutoConfig.from_pretrained(__lowerCamelCase)
            _A      :       Any				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Optional[Any]				    = PyTorchBenchmark(__lowerCamelCase ,						configs=[config])
            _A      :       Tuple				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				int:
            _A      :       Tuple				    = "sshleifer/tinier_bart"
            _A      :       Optional[Any]				    = AutoConfig.from_pretrained(__lowerCamelCase)
            _A      :       Optional[int]				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Dict				    = PyTorchBenchmark(__lowerCamelCase ,						configs=[config])
            _A      :       Optional[int]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_inference_result)
            self.check_results_dict_not_empty(results.memory_inference_result)


     def  _lowerCamelCase (						self)			->				str:
            _A      :       List[Any]				    = "sshleifer/tiny-gpt2"
            _A      :       Optional[Any]				    = AutoConfig.from_pretrained(__lowerCamelCase)
            _A      :       Optional[Any]				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       List[str]				    = PyTorchBenchmark(__lowerCamelCase ,						configs=[config])
            _A      :       Dict				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_train_result)
            self.check_results_dict_not_empty(results.memory_train_result)


     def  _lowerCamelCase (						self)			->				int:
            _A      :       int				    = "sshleifer/tinier_bart"
            _A      :       str				    = AutoConfig.from_pretrained(__lowerCamelCase)
            _A      :       Tuple				    = PyTorchBenchmarkArguments(
                models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						multi_process=__lowerCamelCase ,						)
            _A      :       Tuple				    = PyTorchBenchmark(__lowerCamelCase ,						configs=[config])
            _A      :       Union[str, Any]				    = benchmark.run()
            self.check_results_dict_not_empty(results.time_train_result)
            self.check_results_dict_not_empty(results.memory_train_result)


     def  _lowerCamelCase (						self)			->				Dict:
            _A      :       List[str]				    = "sshleifer/tiny-gpt2"
            with tempfile.TemporaryDirectory() as tmp_dir:
                   _A      :       Optional[Any]				    = PyTorchBenchmarkArguments(
                       models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						save_to_csv=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						inference_time_csv_file=os.path.join(__lowerCamelCase ,						"inf_time.csv") ,						train_memory_csv_file=os.path.join(__lowerCamelCase ,						"train_mem.csv") ,						inference_memory_csv_file=os.path.join(__lowerCamelCase ,						"inf_mem.csv") ,						train_time_csv_file=os.path.join(__lowerCamelCase ,						"train_time.csv") ,						env_info_csv_file=os.path.join(__lowerCamelCase ,						"env.csv") ,						multi_process=__lowerCamelCase ,						)
                   _A      :       Tuple				    = PyTorchBenchmark(__lowerCamelCase)
                   benchmark.run()
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"inf_time.csv")).exists())
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"train_time.csv")).exists())
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"inf_mem.csv")).exists())
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"train_mem.csv")).exists())
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"env.csv")).exists())


     def  _lowerCamelCase (						self)			->				int:
            _A      :       Dict				    = "sshleifer/tiny-gpt2"

            def _check_summary_is_not_empty(__lowerCamelCase):
                   self.assertTrue(hasattr(__lowerCamelCase ,						"sequential"))
                   self.assertTrue(hasattr(__lowerCamelCase ,						"cumulative"))
                   self.assertTrue(hasattr(__lowerCamelCase ,						"current"))
                   self.assertTrue(hasattr(__lowerCamelCase ,						"total"))

            with tempfile.TemporaryDirectory() as tmp_dir:
                   _A      :       Union[str, Any]				    = PyTorchBenchmarkArguments(
                       models=[MODEL_ID] ,						training=__lowerCamelCase ,						inference=__lowerCamelCase ,						sequence_lengths=[8] ,						batch_sizes=[1] ,						log_filename=os.path.join(__lowerCamelCase ,						"log.txt") ,						log_print=__lowerCamelCase ,						trace_memory_line_by_line=__lowerCamelCase ,						multi_process=__lowerCamelCase ,						)
                   _A      :       Optional[int]				    = PyTorchBenchmark(__lowerCamelCase)
                   _A      :       Dict				    = benchmark.run()
                   _check_summary_is_not_empty(result.inference_summary)
                   _check_summary_is_not_empty(result.train_summary)
                   self.assertTrue(Path(os.path.join(__lowerCamelCase ,						"log.txt")).exists())


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


SCREAMING_SNAKE_CASE__   :			Union[str, Any]												=    re.compile(r"\b(a|an|the)\b", re.UNICODE)

SCREAMING_SNAKE_CASE__   :			int												=    None




def  __magic_name__ (   )			->   str:
				__lowerCamelCase    					=	argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''							)
				parser.add_argument('''data_file'''		,  metavar='''data.json'''		,  help='''Input data JSON file.'''							)
				parser.add_argument('''pred_file'''		,  metavar='''pred.json'''		,  help='''Model predictions.'''							)
				parser.add_argument(
				    '''--out-file'''		,  '''-o'''		,  metavar='''eval.json'''		,  help='''Write accuracy metrics to file (default is stdout).'''							)
				parser.add_argument(
				    '''--na-prob-file'''		,  '''-n'''		,  metavar='''na_prob.json'''		,  help='''Model estimates of probability of no answer.'''							)
				parser.add_argument(
				    '''--na-prob-thresh'''		,  '''-t'''		,  type=__lowerCAmelCase		,  default=1.0		,  help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''		,  )
				parser.add_argument(
				    '''--out-image-dir'''		,  '''-p'''		,  metavar='''out_images'''		,  default=__lowerCAmelCase		,  help='''Save precision-recall curves to directory.'''							)
				parser.add_argument('''--verbose'''		,  '''-v'''		,  action='''store_true'''							)
				if len(sys.argv							) == 1:
								parser.print_help()
								sys.exit(1							)
				return parser.parse_args()




def  __magic_name__ (   __lowerCAmelCase	:						List[str]							)			->   Union[str, Any]:
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	bool(qa['''answers''']['''text''']							)
				return qid_to_has_ans




def  __magic_name__ (   __lowerCAmelCase	:						Dict							)			->   Optional[Any]:

				def remove_articles(__lowerCAmelCase	:						Optional[int]							):
								return ARTICLES_REGEX.sub(''' '''		,  __lowerCAmelCase							)

				def white_space_fix(__lowerCAmelCase	:						Optional[int]							):
								return " ".join(text.split()							)

				def remove_punc(__lowerCAmelCase	:						Union[str, Any]							):
								__lowerCamelCase    					=	set(string.punctuation							)
								return "".join(ch for ch in text if ch not in exclude							)

				def lower(__lowerCAmelCase	:						Dict							):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase							)							)							)							)




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]							)			->   Optional[int]:
				if not s:
								return []
				return normalize_answer(__lowerCAmelCase							).split()




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple							)			->   int:
				return int(normalize_answer(__lowerCAmelCase							) == normalize_answer(__lowerCAmelCase							)							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Tuple							)			->   str:
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	collections.Counter(__lowerCAmelCase							) & collections.Counter(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(common.values()							)
				if len(__lowerCAmelCase							) == 0 or len(__lowerCAmelCase							) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks							)
				if num_same == 0:
								return 0
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	(2 * precision * recall) / (precision + recall)
				return fa




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						List[str]							)			->   Optional[Any]:
				__lowerCamelCase    					=	{}
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	qa['''id''']
																__lowerCamelCase    					=	[t for t in qa['''answers''']['''text'''] if normalize_answer(__lowerCAmelCase							)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				__lowerCamelCase    					=	['''''']
																if qid not in preds:
																				print(f'''Missing prediction for {qid}'''							)
																				continue
																__lowerCamelCase    					=	preds[qid]
																# Take max over all gold answers
																__lowerCamelCase    					=	max(compute_exact(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
																__lowerCamelCase    					=	max(compute_fa(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
				return exact_scores, fa_scores




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   List[str]:
				__lowerCamelCase    					=	{}
				for qid, s in scores.items():
								__lowerCamelCase    					=	na_probs[qid] > na_prob_thresh
								if pred_na:
												__lowerCamelCase    					=	float(not qid_to_has_ans[qid]							)
								else:
												__lowerCamelCase    					=	s
				return new_scores




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Optional[Any]=None							)			->   Union[str, Any]:
				if not qid_list:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores.values()							) / total),
								        ('''f1''', 100.0 * sum(fa_scores.values()							) / total),
								        ('''total''', total),
								    ]							)
				else:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list							) / total),
								        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list							) / total),
								        ('''total''', total),
								    ]							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				for k in new_eval:
								__lowerCamelCase    					=	new_eval[k]




def  __magic_name__ (   __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[Any]:
				plt.step(__lowerCAmelCase		,  __lowerCAmelCase		,  color='''b'''		,  alpha=0.2		,  where='''post'''							)
				plt.fill_between(__lowerCAmelCase		,  __lowerCAmelCase		,  step='''post'''		,  alpha=0.2		,  color='''b'''							)
				plt.xlabel('''Recall'''							)
				plt.ylabel('''Precision'''							)
				plt.xlim([0.0, 1.05]							)
				plt.ylim([0.0, 1.05]							)
				plt.title(__lowerCAmelCase							)
				plt.savefig(__lowerCAmelCase							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Optional[Any]=None		,  __lowerCAmelCase	:						Tuple=None							)			->   int:
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	1.0
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	[1.0]
				__lowerCamelCase    					=	[0.0]
				__lowerCamelCase    					=	0.0
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								__lowerCamelCase    					=	true_pos / float(i + 1							)
								__lowerCamelCase    					=	true_pos / float(__lowerCAmelCase							)
								if i == len(__lowerCAmelCase							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__lowerCAmelCase							)
												recalls.append(__lowerCAmelCase							)
				if out_image:
								plot_pr_curve(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				return {"ap": 100.0 * avg_prec}




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						List[Any]							)			->   List[Any]:
				if out_image_dir and not os.path.exists(__lowerCAmelCase							):
								os.makedirs(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(1 for v in qid_to_has_ans.values() if v							)
				if num_true_pos == 0:
								return
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_exact.png'''							)		,  title='''Precision-Recall curve for Exact Match score'''		,  )
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_f1.png'''							)		,  title='''Precision-Recall curve for F1 score'''		,  )
				__lowerCamelCase    					=	{k: float(__lowerCAmelCase							) for k, v in qid_to_has_ans.items()}
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_oracle.png'''							)		,  title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''		,  )
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_exact'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_f1'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_oracle'''							)




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						int							)			->   Optional[Any]:
				if not qid_list:
								return
				__lowerCamelCase    					=	[na_probs[k] for k in qid_list]
				__lowerCamelCase    					=	np.ones_like(__lowerCAmelCase							) / float(len(__lowerCAmelCase							)							)
				plt.hist(__lowerCAmelCase		,  weights=__lowerCAmelCase		,  bins=20		,  range=(0.0, 1.0)							)
				plt.xlabel('''Model probability of no-answer'''							)
				plt.ylabel('''Proportion of dataset'''							)
				plt.title(f'''Histogram of no-answer probability: {name}'''							)
				plt.savefig(os.path.join(__lowerCAmelCase		,  f'''na_prob_hist_{name}.png'''							)							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[int]:
				__lowerCamelCase    					=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
				__lowerCamelCase    					=	num_no_ans
				__lowerCamelCase    					=	cur_score
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												__lowerCamelCase    					=	scores[qid]
								else:
												if preds[qid]:
																__lowerCamelCase    					=	-1
												else:
																__lowerCamelCase    					=	0
								cur_score += diff
								if cur_score > best_score:
												__lowerCamelCase    					=	cur_score
												__lowerCamelCase    					=	na_probs[qid]
				return 100.0 * best_score / len(__lowerCAmelCase							), best_thresh




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Union[str, Any]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	best_exact
				__lowerCamelCase    					=	exact_thresh
				__lowerCamelCase    					=	best_fa
				__lowerCamelCase    					=	fa_thresh




def  __magic_name__ (   )			->   Optional[int]:
				with open(OPTS.data_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
								__lowerCamelCase    					=	dataset_json['''data''']
				with open(OPTS.pred_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file							) as f:
												__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				else:
								__lowerCamelCase    					=	{k: 0.0 for k in preds}
				__lowerCamelCase    					=	make_qid_to_has_ans(__lowerCAmelCase							)  # maps qid to True/False
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if v]
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if not v]
				__lowerCamelCase					,    __lowerCamelCase    					=	get_raw_scores(__lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase							)
				if has_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''HasAns'''							)
				if no_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''NoAns'''							)
				if OPTS.na_prob_file:
								find_all_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''hasAns'''							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''noAns'''							)
				if OPTS.out_file:
								with open(OPTS.out_file		,  '''w'''							) as f:
												json.dump(__lowerCAmelCase		,  __lowerCAmelCase							)
				else:
								print(json.dumps(__lowerCAmelCase		,  indent=2							)							)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE__   :			Any												=    parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use("Agg")
										import matplotlib.pyplot as plt
					main()



