import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


A_:   List[str]   =      re.compile(R'\b(a|an|the)\b', re.UNICODE)

A_:   Tuple   =      None


def        __lowerCAmelCase      (	):



							"""simple docstring"""


							_lowercase					=						argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""       )
							parser.add_argument("""data_file"""			,metavar="""data.json"""			,help="""Input data JSON file."""       )
							parser.add_argument("""pred_file"""			,metavar="""pred.json"""			,help="""Model predictions."""       )
							parser.add_argument(
							    """--out-file"""			,"""-o"""			,metavar="""eval.json"""			,help="""Write accuracy metrics to file (default is stdout)."""       )
							parser.add_argument(
							    """--na-prob-file"""			,"""-n"""			,metavar="""na_prob.json"""			,help="""Model estimates of probability of no answer."""       )
							parser.add_argument(
							    """--na-prob-thresh"""			,"""-t"""			,type=_UpperCamelCase			,default=1.0			,help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""			,)
							parser.add_argument(
							    """--out-image-dir"""			,"""-p"""			,metavar="""out_images"""			,default=_UpperCamelCase			,help="""Save precision-recall curves to directory."""       )
							parser.add_argument("""--verbose"""			,"""-v"""			,action="""store_true"""       )
							if len(sys.argv       ) == 1:
														parser.print_help()
														sys.exit(1       )
							return parser.parse_args()


def        __lowerCAmelCase      (	_A       ):



							"""simple docstring"""


							_lowercase					=						{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_lowercase					=						bool(qa["""answers"""]["""text"""]       )
							return qid_to_has_ans


def        __lowerCAmelCase      (	_A       ):



							"""simple docstring"""


							def remove_articles(_A       ):
														return ARTICLES_REGEX.sub(""" """			,_UpperCamelCase       )

							def white_space_fix(_A       ):
														return " ".join(text.split()       )

							def remove_punc(_A       ):
														_lowercase					=						set(string.punctuation       )
														return "".join(ch for ch in text if ch not in exclude       )

							def lower(_A       ):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(_UpperCamelCase       )       )       )       )


def        __lowerCAmelCase      (	_A       ):



							"""simple docstring"""


							if not s:
														return []
							return normalize_answer(_UpperCamelCase       ).split()


def        __lowerCAmelCase      (	_A			,_A       ):



							"""simple docstring"""


							return int(normalize_answer(_UpperCamelCase       ) == normalize_answer(_UpperCamelCase       )       )


def        __lowerCAmelCase      (	_A			,_A       ):



							"""simple docstring"""


							_lowercase					=						get_tokens(_UpperCamelCase       )
							_lowercase					=						get_tokens(_UpperCamelCase       )
							_lowercase					=						collections.Counter(_UpperCamelCase       ) & collections.Counter(_UpperCamelCase       )
							_lowercase					=						sum(common.values()       )
							if len(_UpperCamelCase       ) == 0 or len(_UpperCamelCase       ) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks       )
							if num_same == 0:
														return 0
							_lowercase					=						1.0 * num_same / len(_UpperCamelCase       )
							_lowercase					=						1.0 * num_same / len(_UpperCamelCase       )
							_lowercase					=						(2 * precision * recall) / (precision + recall)
							return fa


def        __lowerCAmelCase      (	_A			,_A       ):



							"""simple docstring"""


							_lowercase					=						{}
							_lowercase					=						{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_lowercase					=						qa["""id"""]
																												_lowercase					=						[t for t in qa["""answers"""]["""text"""] if normalize_answer(_UpperCamelCase       )]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			_lowercase					=						[""""""]
																												if qid not in preds:
																																			print(f'''Missing prediction for {qid}'''       )
																																			continue
																												_lowercase					=						preds[qid]
																												# Take max over all gold answers
																												_lowercase					=						max(compute_exact(_UpperCamelCase			,_UpperCamelCase       ) for a in gold_answers       )
																												_lowercase					=						max(compute_fa(_UpperCamelCase			,_UpperCamelCase       ) for a in gold_answers       )
							return exact_scores, fa_scores


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							_lowercase					=						{}
							for qid, s in scores.items():
														_lowercase					=						na_probs[qid] > na_prob_thresh
														if pred_na:
																					_lowercase					=						float(not qid_to_has_ans[qid]       )
														else:
																					_lowercase					=						s
							return new_scores


def        __lowerCAmelCase      (	_A			,_A			,_A=None       ):



							"""simple docstring"""


							if not qid_list:
														_lowercase					=						len(_UpperCamelCase       )
														return collections.OrderedDict(
														    [
														        ("""exact""", 1_0_0.0 * sum(exact_scores.values()       ) / total),
														        ("""f1""", 1_0_0.0 * sum(fa_scores.values()       ) / total),
														        ("""total""", total),
														    ]       )
							else:
														_lowercase					=						len(_UpperCamelCase       )
														return collections.OrderedDict(
														    [
														        ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list       ) / total),
														        ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list       ) / total),
														        ("""total""", total),
														    ]       )


def        __lowerCAmelCase      (	_A			,_A			,_A       ):



							"""simple docstring"""


							for k in new_eval:
														_lowercase					=						new_eval[k]


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							plt.step(_UpperCamelCase			,_UpperCamelCase			,color="""b"""			,alpha=0.2			,where="""post"""       )
							plt.fill_between(_UpperCamelCase			,_UpperCamelCase			,step="""post"""			,alpha=0.2			,color="""b"""       )
							plt.xlabel("""Recall"""       )
							plt.ylabel("""Precision"""       )
							plt.xlim([0.0, 1.0_5]       )
							plt.ylim([0.0, 1.0_5]       )
							plt.title(_UpperCamelCase       )
							plt.savefig(_UpperCamelCase       )
							plt.clf()


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A			,_A=None			,_A=None       ):



							"""simple docstring"""


							_lowercase					=						sorted(_UpperCamelCase			,key=lambda _A       : na_probs[k]       )
							_lowercase					=						0.0
							_lowercase					=						1.0
							_lowercase					=						0.0
							_lowercase					=						[1.0]
							_lowercase					=						[0.0]
							_lowercase					=						0.0
							for i, qid in enumerate(_UpperCamelCase       ):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														_lowercase					=						true_pos / float(i + 1       )
														_lowercase					=						true_pos / float(_UpperCamelCase       )
														if i == len(_UpperCamelCase       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(_UpperCamelCase       )
																					recalls.append(_UpperCamelCase       )
							if out_image:
														plot_pr_curve(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase       )
							return {"ap": 1_0_0.0 * avg_prec}


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							if out_image_dir and not os.path.exists(_UpperCamelCase       ):
														os.makedirs(_UpperCamelCase       )
							_lowercase					=						sum(1 for v in qid_to_has_ans.values() if v       )
							if num_true_pos == 0:
														return
							_lowercase					=						make_precision_recall_eval(
							    _UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,out_image=os.path.join(_UpperCamelCase			,"""pr_exact.png"""       )			,title="""Precision-Recall curve for Exact Match score"""			,)
							_lowercase					=						make_precision_recall_eval(
							    _UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,out_image=os.path.join(_UpperCamelCase			,"""pr_f1.png"""       )			,title="""Precision-Recall curve for F1 score"""			,)
							_lowercase					=						{k: float(_UpperCamelCase       ) for k, v in qid_to_has_ans.items()}
							_lowercase					=						make_precision_recall_eval(
							    _UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,out_image=os.path.join(_UpperCamelCase			,"""pr_oracle.png"""       )			,title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""			,)
							merge_eval(_UpperCamelCase			,_UpperCamelCase			,"""pr_exact"""       )
							merge_eval(_UpperCamelCase			,_UpperCamelCase			,"""pr_f1"""       )
							merge_eval(_UpperCamelCase			,_UpperCamelCase			,"""pr_oracle"""       )


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							if not qid_list:
														return
							_lowercase					=						[na_probs[k] for k in qid_list]
							_lowercase					=						np.ones_like(_UpperCamelCase       ) / float(len(_UpperCamelCase       )       )
							plt.hist(_UpperCamelCase			,weights=_UpperCamelCase			,bins=20			,range=(0.0, 1.0)       )
							plt.xlabel("""Model probability of no-answer"""       )
							plt.ylabel("""Proportion of dataset"""       )
							plt.title(f'''Histogram of no-answer probability: {name}'''       )
							plt.savefig(os.path.join(_UpperCamelCase			,f'''na_prob_hist_{name}.png'''       )       )
							plt.clf()


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							_lowercase					=						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
							_lowercase					=						num_no_ans
							_lowercase					=						cur_score
							_lowercase					=						0.0
							_lowercase					=						sorted(_UpperCamelCase			,key=lambda _A       : na_probs[k]       )
							for i, qid in enumerate(_UpperCamelCase       ):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					_lowercase					=						scores[qid]
														else:
																					if preds[qid]:
																												_lowercase					=						-1
																					else:
																												_lowercase					=						0
														cur_score += diff
														if cur_score > best_score:
																					_lowercase					=						cur_score
																					_lowercase					=						na_probs[qid]
							return 1_0_0.0 * best_score / len(_UpperCamelCase       ), best_thresh


def        __lowerCAmelCase      (	_A			,_A			,_A			,_A			,_A			,_A       ):



							"""simple docstring"""


							_lowercase					,							_lowercase					=						find_best_thresh(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase       )
							_lowercase					,							_lowercase					=						find_best_thresh(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase       )
							_lowercase					=						best_exact
							_lowercase					=						exact_thresh
							_lowercase					=						best_fa
							_lowercase					=						fa_thresh


def        __lowerCAmelCase      (	):



							"""simple docstring"""


							with open(OPTS.data_file       ) as f:
														_lowercase					=						json.load(_UpperCamelCase       )
														_lowercase					=						dataset_json["""data"""]
							with open(OPTS.pred_file       ) as f:
														_lowercase					=						json.load(_UpperCamelCase       )
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file       ) as f:
																					_lowercase					=						json.load(_UpperCamelCase       )
							else:
														_lowercase					=						{k: 0.0 for k in preds}
							_lowercase					=						make_qid_to_has_ans(_UpperCamelCase       )  # maps qid to True/False
							_lowercase					=						[k for k, v in qid_to_has_ans.items() if v]
							_lowercase					=						[k for k, v in qid_to_has_ans.items() if not v]
							_lowercase					,							_lowercase					=						get_raw_scores(_UpperCamelCase			,_UpperCamelCase       )
							_lowercase					=						apply_no_ans_threshold(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,OPTS.na_prob_thresh       )
							_lowercase					=						apply_no_ans_threshold(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,OPTS.na_prob_thresh       )
							_lowercase					=						make_eval_dict(_UpperCamelCase			,_UpperCamelCase       )
							if has_ans_qids:
														_lowercase					=						make_eval_dict(_UpperCamelCase			,_UpperCamelCase			,qid_list=_UpperCamelCase       )
														merge_eval(_UpperCamelCase			,_UpperCamelCase			,"""HasAns"""       )
							if no_ans_qids:
														_lowercase					=						make_eval_dict(_UpperCamelCase			,_UpperCamelCase			,qid_list=_UpperCamelCase       )
														merge_eval(_UpperCamelCase			,_UpperCamelCase			,"""NoAns"""       )
							if OPTS.na_prob_file:
														find_all_best_thresh(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase       )
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,_UpperCamelCase			,OPTS.out_image_dir       )
														histogram_na_prob(_UpperCamelCase			,_UpperCamelCase			,OPTS.out_image_dir			,"""hasAns"""       )
														histogram_na_prob(_UpperCamelCase			,_UpperCamelCase			,OPTS.out_image_dir			,"""noAns"""       )
							if OPTS.out_file:
														with open(OPTS.out_file			,"""w"""       ) as f:
																					json.dump(_UpperCamelCase			,_UpperCamelCase       )
							else:
														print(json.dumps(_UpperCamelCase			,indent=2       )       )


if __name__ == "__main__":
				A_:   str   =      parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('Agg')
								import matplotlib.pyplot as plt
				main()






import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate
@dataclass(frozen=lowerCAmelCase__					)
class 						lowerCamelCase_							(					lowerCAmelCase__					):



					'''simple docstring'''







					__UpperCAmelCase							  =					field(default="automatic-speech-recognition"  ,  metadata={"include_in_asdict_even_if_is_default": True}					)
					__UpperCAmelCase							  =					Features({"audio": Audio()}					)
					__UpperCAmelCase							  =					Features({"transcription": Value("string"					)}					)
					__UpperCAmelCase							  =					"audio"
					__UpperCAmelCase							  =					"transcription"






					def A   (					self			,							snake_case_    ) ->      Any:







								'''simple docstring'''

								if self.audio_column not in features:
											raise ValueError(F'Column {self.audio_column} is not present in features.'    )
								if not isinstance(features[self.audio_column]			,							snake_case_    ):
											raise ValueError(F'Column {self.audio_column} is not an Audio type.'    )
								__lowercase			       = copy.deepcopy(self    )
								__lowercase			       = self.input_schema.copy()
								__lowercase			       = features[self.audio_column]
								__lowercase			       = input_schema
								return task_template






					@property
					def A   (					self    ) ->      Dict[str, str]:







								'''simple docstring'''

								return {self.audio_column: "audio", self.transcription_column: "transcription"}


