


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a			:			List[str]												=					re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

a			:			Tuple												=					None


def 		lowercase_	(  ):


			'''simple docstring'''







			__lowercase			       = argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''	)
			parser.add_argument('''data_file'''							,      metavar='''data.json'''							,      help='''Input data JSON file.'''	)
			parser.add_argument('''pred_file'''							,      metavar='''pred.json'''							,      help='''Model predictions.'''	)
			parser.add_argument(
			    '''--out-file'''							,      '''-o'''							,      metavar='''eval.json'''							,      help='''Write accuracy metrics to file (default is stdout).'''	)
			parser.add_argument(
			    '''--na-prob-file'''							,      '''-n'''							,      metavar='''na_prob.json'''							,      help='''Model estimates of probability of no answer.'''	)
			parser.add_argument(
			    '''--na-prob-thresh'''							,      '''-t'''							,      type=_UpperCamelCase							,      default=1.0							,      help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''							,      )
			parser.add_argument(
			    '''--out-image-dir'''							,      '''-p'''							,      metavar='''out_images'''							,      default=_UpperCamelCase							,      help='''Save precision-recall curves to directory.'''	)
			parser.add_argument('''--verbose'''							,      '''-v'''							,      action='''store_true'''	)
			if len(sys.argv	) == 1:
						parser.print_help()
						sys.exit(1	)
			return parser.parse_args()


def 		lowercase_	(  _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												__lowercase			       = bool(qa['''answers''']['''text''']	)
			return qid_to_has_ans


def 		lowercase_	(  _UpperCamelCase	):


			'''simple docstring'''







			def remove_articles(_UpperCamelCase	):
						return ARTICLES_REGEX.sub(''' '''							,      _UpperCamelCase	)

			def white_space_fix(_UpperCamelCase	):
						return " ".join(text.split()	)

			def remove_punc(_UpperCamelCase	):
						__lowercase			       = set(string.punctuation	)
						return "".join(ch for ch in text if ch not in exclude	)

			def lower(_UpperCamelCase	):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(_UpperCamelCase	)	)	)	)


def 		lowercase_	(  _UpperCamelCase	):


			'''simple docstring'''







			if not s:
						return []
			return normalize_answer(_UpperCamelCase	).split()


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			return int(normalize_answer(_UpperCamelCase	) == normalize_answer(_UpperCamelCase	)	)


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = get_tokens(_UpperCamelCase	)
			__lowercase			       = get_tokens(_UpperCamelCase	)
			__lowercase			       = collections.Counter(_UpperCamelCase	) & collections.Counter(_UpperCamelCase	)
			__lowercase			       = sum(common.values()	)
			if len(_UpperCamelCase	) == 0 or len(_UpperCamelCase	) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks	)
			if num_same == 0:
						return 0
			__lowercase			       = 1.0 * num_same / len(_UpperCamelCase	)
			__lowercase			       = 1.0 * num_same / len(_UpperCamelCase	)
			__lowercase			       = (2 * precision * recall) / (precision + recall)
			return fa


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = {}
			__lowercase			       = {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												__lowercase			       = qa['''id''']
												__lowercase			       = [t for t in qa['''answers''']['''text'''] if normalize_answer(_UpperCamelCase	)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															__lowercase			       = ['''''']
												if qid not in preds:
															print(F'Missing prediction for {qid}'	)
															continue
												__lowercase			       = preds[qid]
												# Take max over all gold answers
												__lowercase			       = max(compute_exact(_UpperCamelCase							,      _UpperCamelCase	) for a in gold_answers	)
												__lowercase			       = max(compute_fa(_UpperCamelCase							,      _UpperCamelCase	) for a in gold_answers	)
			return exact_scores, fa_scores


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = {}
			for qid, s in scores.items():
						__lowercase			       = na_probs[qid] > na_prob_thresh
						if pred_na:
									__lowercase			       = float(not qid_to_has_ans[qid]	)
						else:
									__lowercase			       = s
			return new_scores


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase=None	):


			'''simple docstring'''







			if not qid_list:
						__lowercase			       = len(_UpperCamelCase	)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores.values()	) / total),
						        ('''f1''', 100.0 * sum(fa_scores.values()	) / total),
						        ('''total''', total),
						    ]	)
			else:
						__lowercase			       = len(_UpperCamelCase	)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list	) / total),
						        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list	) / total),
						        ('''total''', total),
						    ]	)


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			for k in new_eval:
						__lowercase			       = new_eval[k]


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			plt.step(_UpperCamelCase							,      _UpperCamelCase							,      color='''b'''							,      alpha=0.2							,      where='''post'''	)
			plt.fill_between(_UpperCamelCase							,      _UpperCamelCase							,      step='''post'''							,      alpha=0.2							,      color='''b'''	)
			plt.xlabel('''Recall'''	)
			plt.ylabel('''Precision'''	)
			plt.xlim([0.0, 1.05]	)
			plt.ylim([0.0, 1.05]	)
			plt.title(_UpperCamelCase	)
			plt.savefig(_UpperCamelCase	)
			plt.clf()


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase=None							,      _UpperCamelCase=None	):


			'''simple docstring'''







			__lowercase			       = sorted(_UpperCamelCase							,      key=lambda _UpperCamelCase	: na_probs[k]	)
			__lowercase			       = 0.0
			__lowercase			       = 1.0
			__lowercase			       = 0.0
			__lowercase			       = [1.0]
			__lowercase			       = [0.0]
			__lowercase			       = 0.0
			for i, qid in enumerate(_UpperCamelCase	):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						__lowercase			       = true_pos / float(i + 1	)
						__lowercase			       = true_pos / float(_UpperCamelCase	)
						if i == len(_UpperCamelCase	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(_UpperCamelCase	)
									recalls.append(_UpperCamelCase	)
			if out_image:
						plot_pr_curve(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	)
			return {"ap": 100.0 * avg_prec}


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			if out_image_dir and not os.path.exists(_UpperCamelCase	):
						os.makedirs(_UpperCamelCase	)
			__lowercase			       = sum(1 for v in qid_to_has_ans.values() if v	)
			if num_true_pos == 0:
						return
			__lowercase			       = make_precision_recall_eval(
			    _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      out_image=os.path.join(_UpperCamelCase							,      '''pr_exact.png'''	)							,      title='''Precision-Recall curve for Exact Match score'''							,      )
			__lowercase			       = make_precision_recall_eval(
			    _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      out_image=os.path.join(_UpperCamelCase							,      '''pr_f1.png'''	)							,      title='''Precision-Recall curve for F1 score'''							,      )
			__lowercase			       = {k: float(_UpperCamelCase	) for k, v in qid_to_has_ans.items()}
			__lowercase			       = make_precision_recall_eval(
			    _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      out_image=os.path.join(_UpperCamelCase							,      '''pr_oracle.png'''	)							,      title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''							,      )
			merge_eval(_UpperCamelCase							,      _UpperCamelCase							,      '''pr_exact'''	)
			merge_eval(_UpperCamelCase							,      _UpperCamelCase							,      '''pr_f1'''	)
			merge_eval(_UpperCamelCase							,      _UpperCamelCase							,      '''pr_oracle'''	)


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			if not qid_list:
						return
			__lowercase			       = [na_probs[k] for k in qid_list]
			__lowercase			       = np.ones_like(_UpperCamelCase	) / float(len(_UpperCamelCase	)	)
			plt.hist(_UpperCamelCase							,      weights=_UpperCamelCase							,      bins=20							,      range=(0.0, 1.0)	)
			plt.xlabel('''Model probability of no-answer'''	)
			plt.ylabel('''Proportion of dataset'''	)
			plt.title(F'Histogram of no-answer probability: {name}'	)
			plt.savefig(os.path.join(_UpperCamelCase							,      F'na_prob_hist_{name}.png'	)	)
			plt.clf()


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
			__lowercase			       = num_no_ans
			__lowercase			       = cur_score
			__lowercase			       = 0.0
			__lowercase			       = sorted(_UpperCamelCase							,      key=lambda _UpperCamelCase	: na_probs[k]	)
			for i, qid in enumerate(_UpperCamelCase	):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									__lowercase			       = scores[qid]
						else:
									if preds[qid]:
												__lowercase			       = -1
									else:
												__lowercase			       = 0
						cur_score += diff
						if cur_score > best_score:
									__lowercase			       = cur_score
									__lowercase			       = na_probs[qid]
			return 100.0 * best_score / len(_UpperCamelCase	), best_thresh


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase       ,			__lowercase			       = find_best_thresh(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	)
			__lowercase       ,			__lowercase			       = find_best_thresh(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	)
			__lowercase			       = best_exact
			__lowercase			       = exact_thresh
			__lowercase			       = best_fa
			__lowercase			       = fa_thresh


def 		lowercase_	(  ):


			'''simple docstring'''







			with open(OPTS.data_file	) as f:
						__lowercase			       = json.load(_UpperCamelCase	)
						__lowercase			       = dataset_json['''data''']
			with open(OPTS.pred_file	) as f:
						__lowercase			       = json.load(_UpperCamelCase	)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file	) as f:
									__lowercase			       = json.load(_UpperCamelCase	)
			else:
						__lowercase			       = {k: 0.0 for k in preds}
			__lowercase			       = make_qid_to_has_ans(_UpperCamelCase	)  # maps qid to True/False
			__lowercase			       = [k for k, v in qid_to_has_ans.items() if v]
			__lowercase			       = [k for k, v in qid_to_has_ans.items() if not v]
			__lowercase       ,			__lowercase			       = get_raw_scores(_UpperCamelCase							,      _UpperCamelCase	)
			__lowercase			       = apply_no_ans_threshold(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      OPTS.na_prob_thresh	)
			__lowercase			       = apply_no_ans_threshold(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      OPTS.na_prob_thresh	)
			__lowercase			       = make_eval_dict(_UpperCamelCase							,      _UpperCamelCase	)
			if has_ans_qids:
						__lowercase			       = make_eval_dict(_UpperCamelCase							,      _UpperCamelCase							,      qid_list=_UpperCamelCase	)
						merge_eval(_UpperCamelCase							,      _UpperCamelCase							,      '''HasAns'''	)
			if no_ans_qids:
						__lowercase			       = make_eval_dict(_UpperCamelCase							,      _UpperCamelCase							,      qid_list=_UpperCamelCase	)
						merge_eval(_UpperCamelCase							,      _UpperCamelCase							,      '''NoAns'''	)
			if OPTS.na_prob_file:
						find_all_best_thresh(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      OPTS.out_image_dir	)
						histogram_na_prob(_UpperCamelCase							,      _UpperCamelCase							,      OPTS.out_image_dir							,      '''hasAns'''	)
						histogram_na_prob(_UpperCamelCase							,      _UpperCamelCase							,      OPTS.out_image_dir							,      '''noAns'''	)
			if OPTS.out_file:
						with open(OPTS.out_file							,      '''w'''	) as f:
									json.dump(_UpperCamelCase							,      _UpperCamelCase	)
			else:
						print(json.dumps(_UpperCamelCase							,      indent=2	)	)


if __name__ == "__main__":
	a			:			str												=					parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use('''Agg''')
		import matplotlib.pyplot as plt
	main()






import doctest
from collections import deque

import numpy as np
class 						lowerCamelCase_							:



					'''simple docstring'''







					def __init__(					self    ) ->      None:







								'''simple docstring'''

								__lowercase			       = [2, 1, 2, -1]
								__lowercase			       = [1, 2, 3, 4]






					def A   (					self    ) ->      list[float]:







								'''simple docstring'''

								__lowercase			       = len(self.first_signal    )
								__lowercase			       = len(self.second_signal    )

								__lowercase			       = max(snake_case_			,							snake_case_    )

								# create a zero matrix of max_length x max_length
								__lowercase			       = [[0] * max_length for i in range(snake_case_    )]

								# fills the smaller signal with zeros to make both signals of same length
								if length_first_signal < length_second_signal:
											self.first_signal += [0] * (max_length - length_first_signal)
								elif length_first_signal > length_second_signal:
											self.second_signal += [0] * (max_length - length_second_signal)
								for i in range(snake_case_    ):
											__lowercase			       = deque(self.second_signal    )
											rotated_signal.rotate(snake_case_    )
											for j, item in enumerate(snake_case_    ):
														matrix[i][j] += item

        # multiply the matrix with the first signal
								__lowercase			       = np.matmul(np.transpose(snake_case_    )			,							np.transpose(self.first_signal    )    )

								# rounding-off to two decimal places
								return [round(snake_case_			,							2    ) for i in final_signal]


if __name__ == "__main__":
	doctest.testmod()


