
import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
					from transformers.trainer import DEFAULT_CALLBACKS

					from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel

class    lowerCAmelCase__     (	__lowercase			):






		def __init__(		self		:							int  )     ->							Optional[int]:
						__lowerCamelCase    					=	[]






		def  __A       (		self		:							Any							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							str							,			**SCREAMING_SNAKE_CASE__		:							List[Any]  )     ->							Tuple:
						self.events.append('''on_init_end'''  )






		def  __A       (		self		:							Tuple							,			SCREAMING_SNAKE_CASE__		:							Optional[int]							,			SCREAMING_SNAKE_CASE__		:							Optional[Any]							,			SCREAMING_SNAKE_CASE__		:							Optional[int]							,			**SCREAMING_SNAKE_CASE__		:							Optional[Any]  )     ->							Any:
						self.events.append('''on_train_begin'''  )






		def  __A       (		self		:							str							,			SCREAMING_SNAKE_CASE__		:							int							,			SCREAMING_SNAKE_CASE__		:							Any							,			SCREAMING_SNAKE_CASE__		:							Dict							,			**SCREAMING_SNAKE_CASE__		:							int  )     ->							Any:
						self.events.append('''on_train_end'''  )






		def  __A       (		self		:							Union[str, Any]							,			SCREAMING_SNAKE_CASE__		:							Dict							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			**SCREAMING_SNAKE_CASE__		:							List[Any]  )     ->							int:
						self.events.append('''on_epoch_begin'''  )






		def  __A       (		self		:							Optional[Any]							,			SCREAMING_SNAKE_CASE__		:							int							,			SCREAMING_SNAKE_CASE__		:							Any							,			SCREAMING_SNAKE_CASE__		:							Optional[Any]							,			**SCREAMING_SNAKE_CASE__		:							Tuple  )     ->							Optional[Any]:
						self.events.append('''on_epoch_end'''  )






		def  __A       (		self		:							str							,			SCREAMING_SNAKE_CASE__		:							Tuple							,			SCREAMING_SNAKE_CASE__		:							Optional[Any]							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]							,			**SCREAMING_SNAKE_CASE__		:							str  )     ->							List[Any]:
						self.events.append('''on_step_begin'''  )






		def  __A       (		self		:							Any							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							int							,			SCREAMING_SNAKE_CASE__		:							Any							,			**SCREAMING_SNAKE_CASE__		:							Optional[int]  )     ->							Tuple:
						self.events.append('''on_step_end'''  )






		def  __A       (		self		:							int							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							int							,			**SCREAMING_SNAKE_CASE__		:							List[Any]  )     ->							Union[str, Any]:
						self.events.append('''on_evaluate'''  )






		def  __A       (		self		:							Union[str, Any]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							str							,			**SCREAMING_SNAKE_CASE__		:							str  )     ->							str:
						self.events.append('''on_predict'''  )






		def  __A       (		self		:							Optional[int]							,			SCREAMING_SNAKE_CASE__		:							int							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							Optional[int]							,			**SCREAMING_SNAKE_CASE__		:							Optional[Any]  )     ->							Tuple:
						self.events.append('''on_save'''  )






		def  __A       (		self		:							Optional[int]							,			SCREAMING_SNAKE_CASE__		:							Dict							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]							,			**SCREAMING_SNAKE_CASE__		:							List[str]  )     ->							List[str]:
						self.events.append('''on_log'''  )







		def  __A       (		self		:							int							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							List[Any]							,			**SCREAMING_SNAKE_CASE__		:							Optional[int]  )     ->							str:
						self.events.append('''on_prediction_step'''  )





@require_torch
class    lowerCAmelCase__     (	unittest.TestCase			):






		def  __A       (		self		:							Union[str, Any]  )     ->							Optional[Any]:
						__lowerCamelCase    					=	tempfile.mkdtemp()






		def  __A       (		self		:							int  )     ->							List[str]:
						shutil.rmtree(self.output_dir  )






		def  __A       (		self		:							Tuple							,			SCREAMING_SNAKE_CASE__		:							List[Any]=0							,			SCREAMING_SNAKE_CASE__		:							Any=0							,			SCREAMING_SNAKE_CASE__		:							List[str]=64							,			SCREAMING_SNAKE_CASE__		:							Optional[int]=64							,			SCREAMING_SNAKE_CASE__		:							Optional[int]=None							,			SCREAMING_SNAKE_CASE__		:							Optional[int]=False							,			**SCREAMING_SNAKE_CASE__		:							Optional[Any]  )     ->							Union[str, Any]:
						# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
						# its set to False since the tests later on depend on its value.
						__lowerCamelCase    					=	RegressionDataset(length=SCREAMING_SNAKE_CASE__  )
						__lowerCamelCase    					=	RegressionDataset(length=SCREAMING_SNAKE_CASE__  )
						__lowerCamelCase    					=	RegressionModelConfig(a=SCREAMING_SNAKE_CASE__							,			b=SCREAMING_SNAKE_CASE__  )
						__lowerCamelCase    					=	RegressionPreTrainedModel(SCREAMING_SNAKE_CASE__  )

						__lowerCamelCase    					=	TrainingArguments(self.output_dir							,			disable_tqdm=SCREAMING_SNAKE_CASE__							,			report_to=[]							,			**SCREAMING_SNAKE_CASE__  )
						return Trainer(
						    SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			train_dataset=SCREAMING_SNAKE_CASE__							,			eval_dataset=SCREAMING_SNAKE_CASE__							,			callbacks=SCREAMING_SNAKE_CASE__							,			)






		def  __A       (		self		:							List[Any]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							str  )     ->							Optional[int]:
						self.assertEqual(len(SCREAMING_SNAKE_CASE__  )							,			len(SCREAMING_SNAKE_CASE__  )  )

						# Order doesn't matter
						__lowerCamelCase    					=	sorted(SCREAMING_SNAKE_CASE__							,			key=lambda SCREAMING_SNAKE_CASE__  : cb.__name__ if isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ) else cb.__class__.__name__  )
						__lowerCamelCase    					=	sorted(SCREAMING_SNAKE_CASE__							,			key=lambda SCREAMING_SNAKE_CASE__  : cb.__name__ if isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ) else cb.__class__.__name__  )

						for cba, cba in zip(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ):
										if isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ) and isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ):
														self.assertEqual(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  )
										elif isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ) and not isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ):
														self.assertEqual(SCREAMING_SNAKE_CASE__							,			cba.__class__  )
										elif not isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ) and isinstance(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  ):
														self.assertEqual(cba.__class__							,			SCREAMING_SNAKE_CASE__  )
										else:
														self.assertEqual(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  )






		def  __A       (		self		:							Union[str, Any]							,			SCREAMING_SNAKE_CASE__		:							Optional[int]  )     ->							Tuple:
						__lowerCamelCase    					=	['''on_init_end''', '''on_train_begin''']
						__lowerCamelCase    					=	0
						__lowerCamelCase    					=	len(trainer.get_eval_dataloader()  )
						__lowerCamelCase    					=	['''on_prediction_step'''] * len(trainer.get_eval_dataloader()  ) + ['''on_log''', '''on_evaluate''']
						for _ in range(trainer.state.num_train_epochs  ):
										expected_events.append('''on_epoch_begin'''  )
										for _ in range(SCREAMING_SNAKE_CASE__  ):
														step += 1
														expected_events += ["on_step_begin", "on_step_end"]
														if step % trainer.args.logging_steps == 0:
																		expected_events.append('''on_log'''  )
														if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
																		expected_events += evaluation_events.copy()
														if step % trainer.args.save_steps == 0:
																		expected_events.append('''on_save'''  )
										expected_events.append('''on_epoch_end'''  )
										if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
														expected_events += evaluation_events.copy()
						expected_events += ["on_log", "on_train_end"]
						return expected_events






		def  __A       (		self		:							Union[str, Any]  )     ->							int:
						__lowerCamelCase    					=	self.get_trainer()
						__lowerCamelCase    					=	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						# Callbacks passed at init are added to the default callbacks
						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]  )
						expected_callbacks.append(SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
						__lowerCamelCase    					=	self.get_trainer(disable_tqdm=SCREAMING_SNAKE_CASE__  )
						__lowerCamelCase    					=	DEFAULT_CALLBACKS.copy() + [PrinterCallback]
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )






		def  __A       (		self		:							List[Any]  )     ->							str:
						__lowerCamelCase    					=	DEFAULT_CALLBACKS.copy() + [ProgressCallback]
						__lowerCamelCase    					=	self.get_trainer()

						# We can add, pop, or remove by class name
						trainer.remove_callback(SCREAMING_SNAKE_CASE__  )
						expected_callbacks.remove(SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						__lowerCamelCase    					=	self.get_trainer()
						__lowerCamelCase    					=	trainer.pop_callback(SCREAMING_SNAKE_CASE__  )
						self.assertEqual(cb.__class__							,			SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						trainer.add_callback(SCREAMING_SNAKE_CASE__  )
						expected_callbacks.insert(0							,			SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						# We can also add, pop, or remove by instance
						__lowerCamelCase    					=	self.get_trainer()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[0]
						trainer.remove_callback(SCREAMING_SNAKE_CASE__  )
						expected_callbacks.remove(SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						__lowerCamelCase    					=	self.get_trainer()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[0]
						__lowerCamelCase    					=	trainer.pop_callback(SCREAMING_SNAKE_CASE__  )
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )

						trainer.add_callback(SCREAMING_SNAKE_CASE__  )
						expected_callbacks.insert(0							,			SCREAMING_SNAKE_CASE__  )
						self.check_callbacks_equality(trainer.callback_handler.callbacks							,			SCREAMING_SNAKE_CASE__  )







		def  __A       (		self		:							Union[str, Any]  )     ->							Any:
						import warnings

						# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
						warnings.simplefilter(action='''ignore'''							,			category=SCREAMING_SNAKE_CASE__  )

						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]  )
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						# Independent log/save/eval
						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]							,			logging_steps=5  )
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]							,			save_steps=5  )
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]							,			eval_steps=5							,			evaluation_strategy='''steps'''  )
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						__lowerCamelCase    					=	self.get_trainer(callbacks=[MyTestTrainerCallback]							,			evaluation_strategy='''epoch'''  )
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						# A bit of everything
						__lowerCamelCase    					=	self.get_trainer(
						    callbacks=[MyTestTrainerCallback]							,			logging_steps=3							,			save_steps=10							,			eval_steps=5							,			evaluation_strategy='''steps'''							,			)
						trainer.train()
						__lowerCamelCase    					=	trainer.callback_handler.callbacks[-2].events
						self.assertEqual(SCREAMING_SNAKE_CASE__							,			self.get_expected_events(SCREAMING_SNAKE_CASE__  )  )

						# warning should be emitted for duplicated callbacks
						with patch('''transformers.trainer_callback.logger.warning'''  ) as warn_mock:
										__lowerCamelCase    					=	self.get_trainer(
										    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]							,			)
										assert str(SCREAMING_SNAKE_CASE__  ) in warn_mock.call_args[0][0]





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


SCREAMING_SNAKE_CASE__   :			Union[str, Any]												=    re.compile(r"\b(a|an|the)\b", re.UNICODE)

SCREAMING_SNAKE_CASE__   :			int												=    None




def  __magic_name__ (   )			->   str:
				__lowerCamelCase    					=	argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''							)
				parser.add_argument('''data_file'''		,  metavar='''data.json'''		,  help='''Input data JSON file.'''							)
				parser.add_argument('''pred_file'''		,  metavar='''pred.json'''		,  help='''Model predictions.'''							)
				parser.add_argument(
				    '''--out-file'''		,  '''-o'''		,  metavar='''eval.json'''		,  help='''Write accuracy metrics to file (default is stdout).'''							)
				parser.add_argument(
				    '''--na-prob-file'''		,  '''-n'''		,  metavar='''na_prob.json'''		,  help='''Model estimates of probability of no answer.'''							)
				parser.add_argument(
				    '''--na-prob-thresh'''		,  '''-t'''		,  type=__lowerCAmelCase		,  default=1.0		,  help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''		,  )
				parser.add_argument(
				    '''--out-image-dir'''		,  '''-p'''		,  metavar='''out_images'''		,  default=__lowerCAmelCase		,  help='''Save precision-recall curves to directory.'''							)
				parser.add_argument('''--verbose'''		,  '''-v'''		,  action='''store_true'''							)
				if len(sys.argv							) == 1:
								parser.print_help()
								sys.exit(1							)
				return parser.parse_args()




def  __magic_name__ (   __lowerCAmelCase	:						List[str]							)			->   Union[str, Any]:
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	bool(qa['''answers''']['''text''']							)
				return qid_to_has_ans




def  __magic_name__ (   __lowerCAmelCase	:						Dict							)			->   Optional[Any]:

				def remove_articles(__lowerCAmelCase	:						Optional[int]							):
								return ARTICLES_REGEX.sub(''' '''		,  __lowerCAmelCase							)

				def white_space_fix(__lowerCAmelCase	:						Optional[int]							):
								return " ".join(text.split()							)

				def remove_punc(__lowerCAmelCase	:						Union[str, Any]							):
								__lowerCamelCase    					=	set(string.punctuation							)
								return "".join(ch for ch in text if ch not in exclude							)

				def lower(__lowerCAmelCase	:						Dict							):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase							)							)							)							)




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]							)			->   Optional[int]:
				if not s:
								return []
				return normalize_answer(__lowerCAmelCase							).split()




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple							)			->   int:
				return int(normalize_answer(__lowerCAmelCase							) == normalize_answer(__lowerCAmelCase							)							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Tuple							)			->   str:
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	collections.Counter(__lowerCAmelCase							) & collections.Counter(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(common.values()							)
				if len(__lowerCAmelCase							) == 0 or len(__lowerCAmelCase							) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks							)
				if num_same == 0:
								return 0
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	(2 * precision * recall) / (precision + recall)
				return fa




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						List[str]							)			->   Optional[Any]:
				__lowerCamelCase    					=	{}
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	qa['''id''']
																__lowerCamelCase    					=	[t for t in qa['''answers''']['''text'''] if normalize_answer(__lowerCAmelCase							)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				__lowerCamelCase    					=	['''''']
																if qid not in preds:
																				print(f'''Missing prediction for {qid}'''							)
																				continue
																__lowerCamelCase    					=	preds[qid]
																# Take max over all gold answers
																__lowerCamelCase    					=	max(compute_exact(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
																__lowerCamelCase    					=	max(compute_fa(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
				return exact_scores, fa_scores




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   List[str]:
				__lowerCamelCase    					=	{}
				for qid, s in scores.items():
								__lowerCamelCase    					=	na_probs[qid] > na_prob_thresh
								if pred_na:
												__lowerCamelCase    					=	float(not qid_to_has_ans[qid]							)
								else:
												__lowerCamelCase    					=	s
				return new_scores




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Optional[Any]=None							)			->   Union[str, Any]:
				if not qid_list:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores.values()							) / total),
								        ('''f1''', 100.0 * sum(fa_scores.values()							) / total),
								        ('''total''', total),
								    ]							)
				else:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list							) / total),
								        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list							) / total),
								        ('''total''', total),
								    ]							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				for k in new_eval:
								__lowerCamelCase    					=	new_eval[k]




def  __magic_name__ (   __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[Any]:
				plt.step(__lowerCAmelCase		,  __lowerCAmelCase		,  color='''b'''		,  alpha=0.2		,  where='''post'''							)
				plt.fill_between(__lowerCAmelCase		,  __lowerCAmelCase		,  step='''post'''		,  alpha=0.2		,  color='''b'''							)
				plt.xlabel('''Recall'''							)
				plt.ylabel('''Precision'''							)
				plt.xlim([0.0, 1.05]							)
				plt.ylim([0.0, 1.05]							)
				plt.title(__lowerCAmelCase							)
				plt.savefig(__lowerCAmelCase							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Optional[Any]=None		,  __lowerCAmelCase	:						Tuple=None							)			->   int:
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	1.0
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	[1.0]
				__lowerCamelCase    					=	[0.0]
				__lowerCamelCase    					=	0.0
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								__lowerCamelCase    					=	true_pos / float(i + 1							)
								__lowerCamelCase    					=	true_pos / float(__lowerCAmelCase							)
								if i == len(__lowerCAmelCase							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__lowerCAmelCase							)
												recalls.append(__lowerCAmelCase							)
				if out_image:
								plot_pr_curve(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				return {"ap": 100.0 * avg_prec}




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						List[Any]							)			->   List[Any]:
				if out_image_dir and not os.path.exists(__lowerCAmelCase							):
								os.makedirs(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(1 for v in qid_to_has_ans.values() if v							)
				if num_true_pos == 0:
								return
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_exact.png'''							)		,  title='''Precision-Recall curve for Exact Match score'''		,  )
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_f1.png'''							)		,  title='''Precision-Recall curve for F1 score'''		,  )
				__lowerCamelCase    					=	{k: float(__lowerCAmelCase							) for k, v in qid_to_has_ans.items()}
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_oracle.png'''							)		,  title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''		,  )
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_exact'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_f1'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_oracle'''							)




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						int							)			->   Optional[Any]:
				if not qid_list:
								return
				__lowerCamelCase    					=	[na_probs[k] for k in qid_list]
				__lowerCamelCase    					=	np.ones_like(__lowerCAmelCase							) / float(len(__lowerCAmelCase							)							)
				plt.hist(__lowerCAmelCase		,  weights=__lowerCAmelCase		,  bins=20		,  range=(0.0, 1.0)							)
				plt.xlabel('''Model probability of no-answer'''							)
				plt.ylabel('''Proportion of dataset'''							)
				plt.title(f'''Histogram of no-answer probability: {name}'''							)
				plt.savefig(os.path.join(__lowerCAmelCase		,  f'''na_prob_hist_{name}.png'''							)							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[int]:
				__lowerCamelCase    					=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
				__lowerCamelCase    					=	num_no_ans
				__lowerCamelCase    					=	cur_score
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												__lowerCamelCase    					=	scores[qid]
								else:
												if preds[qid]:
																__lowerCamelCase    					=	-1
												else:
																__lowerCamelCase    					=	0
								cur_score += diff
								if cur_score > best_score:
												__lowerCamelCase    					=	cur_score
												__lowerCamelCase    					=	na_probs[qid]
				return 100.0 * best_score / len(__lowerCAmelCase							), best_thresh




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Union[str, Any]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	best_exact
				__lowerCamelCase    					=	exact_thresh
				__lowerCamelCase    					=	best_fa
				__lowerCamelCase    					=	fa_thresh




def  __magic_name__ (   )			->   Optional[int]:
				with open(OPTS.data_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
								__lowerCamelCase    					=	dataset_json['''data''']
				with open(OPTS.pred_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file							) as f:
												__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				else:
								__lowerCamelCase    					=	{k: 0.0 for k in preds}
				__lowerCamelCase    					=	make_qid_to_has_ans(__lowerCAmelCase							)  # maps qid to True/False
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if v]
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if not v]
				__lowerCamelCase					,    __lowerCamelCase    					=	get_raw_scores(__lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase							)
				if has_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''HasAns'''							)
				if no_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''NoAns'''							)
				if OPTS.na_prob_file:
								find_all_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''hasAns'''							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''noAns'''							)
				if OPTS.out_file:
								with open(OPTS.out_file		,  '''w'''							) as f:
												json.dump(__lowerCAmelCase		,  __lowerCAmelCase							)
				else:
								print(json.dumps(__lowerCAmelCase		,  indent=2							)							)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE__   :			Any												=    parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use("Agg")
										import matplotlib.pyplot as plt
					main()



