


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


A       :		List[str]    =   re.compile(r'\b(a|an|the)\b', re.UNICODE)

A       :		Union[str, Any]    =   None






def  UpperCamelCase					(	)   ->  Optional[Any]:
		"""simple docstring"""




		lowercase__     			=		argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""     )
		parser.add_argument("""data_file"""    ,    metavar="""data.json"""    ,    help="""Input data JSON file."""     )
		parser.add_argument("""pred_file"""    ,    metavar="""pred.json"""    ,    help="""Model predictions."""     )
		parser.add_argument(
		    """--out-file"""    ,    """-o"""    ,    metavar="""eval.json"""    ,    help="""Write accuracy metrics to file (default is stdout)."""     )
		parser.add_argument(
		    """--na-prob-file"""    ,    """-n"""    ,    metavar="""na_prob.json"""    ,    help="""Model estimates of probability of no answer."""     )
		parser.add_argument(
		    """--na-prob-thresh"""    ,    """-t"""    ,    type=_lowerCamelCase    ,    default=1.0    ,    help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""    ,    )
		parser.add_argument(
		    """--out-image-dir"""    ,    """-p"""    ,    metavar="""out_images"""    ,    default=_lowerCamelCase    ,    help="""Save precision-recall curves to directory."""     )
		parser.add_argument("""--verbose"""    ,    """-v"""    ,    action="""store_true"""     )
		if len(sys.argv     ) == 1:
				parser.print_help()
				sys.exit(1     )
		return parser.parse_args()






def  UpperCamelCase					(	__magic_name__     :							List[Any]     )   ->  List[str]:
		"""simple docstring"""




		lowercase__     			=		{}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								lowercase__     			=		bool(qa["""answers"""]["""text"""]     )
		return qid_to_has_ans






def  UpperCamelCase					(	__magic_name__     :							Union[str, Any]     )   ->  Optional[Any]:
		"""simple docstring"""




		def remove_articles(__magic_name__     :							List[str]     ):
				return ARTICLES_REGEX.sub(""" """    ,    _lowerCamelCase     )

		def white_space_fix(__magic_name__     :							int     ):
				return " ".join(text.split()     )

		def remove_punc(__magic_name__     :							Tuple     ):
				lowercase__     			=		set(string.punctuation     )
				return "".join(ch for ch in text if ch not in exclude     )

		def lower(__magic_name__     :							Any     ):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(_lowerCamelCase     )     )     )     )






def  UpperCamelCase					(	__magic_name__     :							str     )   ->  int:
		"""simple docstring"""




		if not s:
				return []
		return normalize_answer(_lowerCamelCase     ).split()






def  UpperCamelCase					(	__magic_name__     :							Optional[int]    ,    __magic_name__     :							Any     )   ->  int:
		"""simple docstring"""




		return int(normalize_answer(_lowerCamelCase     ) == normalize_answer(_lowerCamelCase     )     )






def  UpperCamelCase					(	__magic_name__     :							Union[str, Any]    ,    __magic_name__     :							Optional[Any]     )   ->  str:
		"""simple docstring"""




		lowercase__     			=		get_tokens(_lowerCamelCase     )
		lowercase__     			=		get_tokens(_lowerCamelCase     )
		lowercase__     			=		collections.Counter(_lowerCamelCase     ) & collections.Counter(_lowerCamelCase     )
		lowercase__     			=		sum(common.values()     )
		if len(_lowerCamelCase     ) == 0 or len(_lowerCamelCase     ) == 0:
				# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
				return int(gold_toks == pred_toks     )
		if num_same == 0:
				return 0
		lowercase__     			=		1.0 * num_same / len(_lowerCamelCase     )
		lowercase__     			=		1.0 * num_same / len(_lowerCamelCase     )
		lowercase__     			=		(2 * precision * recall) / (precision + recall)
		return fa






def  UpperCamelCase					(	__magic_name__     :							Tuple    ,    __magic_name__     :							Any     )   ->  Dict:
		"""simple docstring"""




		lowercase__     			=		{}
		lowercase__     			=		{}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								lowercase__     			=		qa["id"]
								lowercase__     			=		[t for t in qa["answers"]["text"] if normalize_answer(_lowerCamelCase     )]
								if not gold_answers:
										# For unanswerable questions, only correct answer is empty string
										lowercase__     			=		[""]
								if qid not in preds:
										print(f'''Missing prediction for {qid}'''     )
										continue
								lowercase__     			=		preds[qid]
								# Take max over all gold answers
								lowercase__     			=		max(compute_exact(_lowerCamelCase    ,    _lowerCamelCase     ) for a in gold_answers     )
								lowercase__     			=		max(compute_fa(_lowerCamelCase    ,    _lowerCamelCase     ) for a in gold_answers     )
		return exact_scores, fa_scores






def  UpperCamelCase					(	__magic_name__     :							Union[str, Any]    ,    __magic_name__     :							Dict    ,    __magic_name__     :							Any    ,    __magic_name__     :							Any     )   ->  Optional[Any]:
		"""simple docstring"""




		lowercase__     			=		{}
		for qid, s in scores.items():
				lowercase__     			=		na_probs[qid] > na_prob_thresh
				if pred_na:
						lowercase__     			=		float(not qid_to_has_ans[qid]     )
				else:
						lowercase__     			=		s
		return new_scores






def  UpperCamelCase					(	__magic_name__     :							Dict    ,    __magic_name__     :							Union[str, Any]    ,    __magic_name__     :							Union[str, Any]=None     )   ->  Tuple:
		"""simple docstring"""




		if not qid_list:
				lowercase__     			=		len(_lowerCamelCase     )
				return collections.OrderedDict(
				    [
				        ("""exact""", 1_0_0.0 * sum(exact_scores.values()     ) / total),
				        ("""f1""", 1_0_0.0 * sum(fa_scores.values()     ) / total),
				        ("""total""", total),
				    ]     )
		else:
				lowercase__     			=		len(_lowerCamelCase     )
				return collections.OrderedDict(
				    [
				        ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list     ) / total),
				        ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list     ) / total),
				        ("""total""", total),
				    ]     )






def  UpperCamelCase					(	__magic_name__     :							Dict    ,    __magic_name__     :							Any    ,    __magic_name__     :							Any     )   ->  List[Any]:
		"""simple docstring"""




		for k in new_eval:
				lowercase__     			=		new_eval[k]






def  UpperCamelCase					(	__magic_name__     :							Optional[Any]    ,    __magic_name__     :							Optional[int]    ,    __magic_name__     :							Tuple    ,    __magic_name__     :							List[Any]     )   ->  Dict:
		"""simple docstring"""




		plt.step(_lowerCamelCase    ,    _lowerCamelCase    ,    color="""b"""    ,    alpha=0.2    ,    where="""post"""     )
		plt.fill_between(_lowerCamelCase    ,    _lowerCamelCase    ,    step="""post"""    ,    alpha=0.2    ,    color="""b"""     )
		plt.xlabel("""Recall"""     )
		plt.ylabel("""Precision"""     )
		plt.xlim([0.0, 1.0_5]     )
		plt.ylim([0.0, 1.0_5]     )
		plt.title(_lowerCamelCase     )
		plt.savefig(_lowerCamelCase     )
		plt.clf()






def  UpperCamelCase					(	__magic_name__     :							Any    ,    __magic_name__     :							List[Any]    ,    __magic_name__     :							int    ,    __magic_name__     :							List[Any]    ,    __magic_name__     :							Dict=None    ,    __magic_name__     :							Optional[int]=None     )   ->  Tuple:
		"""simple docstring"""




		lowercase__     			=		sorted(_lowerCamelCase    ,    key=lambda __magic_name__     : na_probs[k]     )
		lowercase__     			=		0.0
		lowercase__     			=		1.0
		lowercase__     			=		0.0
		lowercase__     			=		[1.0]
		lowercase__     			=		[0.0]
		lowercase__     			=		0.0
		for i, qid in enumerate(_lowerCamelCase     ):
				if qid_to_has_ans[qid]:
						true_pos += scores[qid]
				lowercase__     			=		true_pos / float(i + 1     )
				lowercase__     			=		true_pos / float(_lowerCamelCase     )
				if i == len(_lowerCamelCase     ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
						# i.e., if we can put a threshold after this point
						avg_prec += cur_p * (cur_r - recalls[-1])
						precisions.append(_lowerCamelCase     )
						recalls.append(_lowerCamelCase     )
		if out_image:
				plot_pr_curve(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase     )
		return {"ap": 1_0_0.0 * avg_prec}






def  UpperCamelCase					(	__magic_name__     :							str    ,    __magic_name__     :							Optional[Any]    ,    __magic_name__     :							int    ,    __magic_name__     :							Tuple    ,    __magic_name__     :							Union[str, Any]    ,    __magic_name__     :							Optional[Any]     )   ->  Optional[int]:
		"""simple docstring"""




		if out_image_dir and not os.path.exists(_lowerCamelCase     ):
				os.makedirs(_lowerCamelCase     )
		lowercase__     			=		sum(1 for v in qid_to_has_ans.values() if v     )
		if num_true_pos == 0:
				return
		lowercase__     			=		make_precision_recall_eval(
		    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    out_image=os.path.join(_lowerCamelCase    ,    """pr_exact.png"""     )    ,    title="""Precision-Recall curve for Exact Match score"""    ,    )
		lowercase__     			=		make_precision_recall_eval(
		    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    out_image=os.path.join(_lowerCamelCase    ,    """pr_f1.png"""     )    ,    title="""Precision-Recall curve for F1 score"""    ,    )
		lowercase__     			=		{k: float(_lowerCamelCase     ) for k, v in qid_to_has_ans.items()}
		lowercase__     			=		make_precision_recall_eval(
		    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    out_image=os.path.join(_lowerCamelCase    ,    """pr_oracle.png"""     )    ,    title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""    ,    )
		merge_eval(_lowerCamelCase    ,    _lowerCamelCase    ,    """pr_exact"""     )
		merge_eval(_lowerCamelCase    ,    _lowerCamelCase    ,    """pr_f1"""     )
		merge_eval(_lowerCamelCase    ,    _lowerCamelCase    ,    """pr_oracle"""     )






def  UpperCamelCase					(	__magic_name__     :							Tuple    ,    __magic_name__     :							List[Any]    ,    __magic_name__     :							Optional[Any]    ,    __magic_name__     :							List[Any]     )   ->  List[str]:
		"""simple docstring"""




		if not qid_list:
				return
		lowercase__     			=		[na_probs[k] for k in qid_list]
		lowercase__     			=		np.ones_like(_lowerCamelCase     ) / float(len(_lowerCamelCase     )     )
		plt.hist(_lowerCamelCase    ,    weights=_lowerCamelCase    ,    bins=20    ,    range=(0.0, 1.0)     )
		plt.xlabel("""Model probability of no-answer"""     )
		plt.ylabel("""Proportion of dataset"""     )
		plt.title(f'''Histogram of no-answer probability: {name}'''     )
		plt.savefig(os.path.join(_lowerCamelCase    ,    f'''na_prob_hist_{name}.png'''     )     )
		plt.clf()






def  UpperCamelCase					(	__magic_name__     :							List[str]    ,    __magic_name__     :							Any    ,    __magic_name__     :							Any    ,    __magic_name__     :							Optional[int]     )   ->  Dict:
		"""simple docstring"""




		lowercase__     			=		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]     )
		lowercase__     			=		num_no_ans
		lowercase__     			=		cur_score
		lowercase__     			=		0.0
		lowercase__     			=		sorted(_lowerCamelCase    ,    key=lambda __magic_name__     : na_probs[k]     )
		for i, qid in enumerate(_lowerCamelCase     ):
				if qid not in scores:
						continue
				if qid_to_has_ans[qid]:
						lowercase__     			=		scores[qid]
				else:
						if preds[qid]:
								lowercase__     			=		-1
						else:
								lowercase__     			=		0
				cur_score += diff
				if cur_score > best_score:
						lowercase__     			=		cur_score
						lowercase__     			=		na_probs[qid]
		return 1_0_0.0 * best_score / len(_lowerCamelCase     ), best_thresh






def  UpperCamelCase					(	__magic_name__     :							Optional[int]    ,    __magic_name__     :							str    ,    __magic_name__     :							List[str]    ,    __magic_name__     :							int    ,    __magic_name__     :							Optional[Any]    ,    __magic_name__     :							Optional[Any]     )   ->  Dict:
		"""simple docstring"""




		lowercase__     			=		find_best_thresh(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase     )
		lowercase__     			=		find_best_thresh(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase     )
		lowercase__     			=		best_exact
		lowercase__     			=		exact_thresh
		lowercase__     			=		best_fa
		lowercase__     			=		fa_thresh






def  UpperCamelCase					(	)   ->  Dict:
		"""simple docstring"""




		with open(OPTS.data_file     ) as f:
				lowercase__     			=		json.load(_lowerCamelCase     )
				lowercase__     			=		dataset_json["data"]
		with open(OPTS.pred_file     ) as f:
				lowercase__     			=		json.load(_lowerCamelCase     )
		if OPTS.na_prob_file:
				with open(OPTS.na_prob_file     ) as f:
						lowercase__     			=		json.load(_lowerCamelCase     )
		else:
				lowercase__     			=		{k: 0.0 for k in preds}
		lowercase__     			=		make_qid_to_has_ans(_lowerCamelCase     )  # maps qid to True/False
		lowercase__     			=		[k for k, v in qid_to_has_ans.items() if v]
		lowercase__     			=		[k for k, v in qid_to_has_ans.items() if not v]
		lowercase__     			=		get_raw_scores(_lowerCamelCase    ,    _lowerCamelCase     )
		lowercase__     			=		apply_no_ans_threshold(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    OPTS.na_prob_thresh     )
		lowercase__     			=		apply_no_ans_threshold(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    OPTS.na_prob_thresh     )
		lowercase__     			=		make_eval_dict(_lowerCamelCase    ,    _lowerCamelCase     )
		if has_ans_qids:
				lowercase__     			=		make_eval_dict(_lowerCamelCase    ,    _lowerCamelCase    ,    qid_list=_lowerCamelCase     )
				merge_eval(_lowerCamelCase    ,    _lowerCamelCase    ,    """HasAns"""     )
		if no_ans_qids:
				lowercase__     			=		make_eval_dict(_lowerCamelCase    ,    _lowerCamelCase    ,    qid_list=_lowerCamelCase     )
				merge_eval(_lowerCamelCase    ,    _lowerCamelCase    ,    """NoAns"""     )
		if OPTS.na_prob_file:
				find_all_best_thresh(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase     )
		if OPTS.na_prob_file and OPTS.out_image_dir:
				run_precision_recall_analysis(_lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    _lowerCamelCase    ,    OPTS.out_image_dir     )
				histogram_na_prob(_lowerCamelCase    ,    _lowerCamelCase    ,    OPTS.out_image_dir    ,    """hasAns"""     )
				histogram_na_prob(_lowerCamelCase    ,    _lowerCamelCase    ,    OPTS.out_image_dir    ,    """noAns"""     )
		if OPTS.out_file:
				with open(OPTS.out_file    ,    """w"""     ) as f:
						json.dump(_lowerCamelCase    ,    _lowerCamelCase     )
		else:
				print(json.dumps(_lowerCamelCase    ,    indent=2     )     )


if __name__ == "__main__":
	A       :		List[str]    =   parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use('Agg')
		import matplotlib.pyplot as plt
	main()


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCamelCase  							=						{
    '''configuration_groupvit''': [
        '''GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''GroupViTConfig''',
        '''GroupViTOnnxConfig''',
        '''GroupViTTextConfig''',
        '''GroupViTVisionConfig''',
    ],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase  							=						[
          '''GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''GroupViTModel''',
          '''GroupViTPreTrainedModel''',
          '''GroupViTTextModel''',
          '''GroupViTVisionModel''',
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase  							=						[
          '''TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''TFGroupViTModel''',
          '''TFGroupViTPreTrainedModel''',
          '''TFGroupViTTextModel''',
          '''TFGroupViTVisionModel''',
      ]

if TYPE_CHECKING:
      from .configuration_groupvit import (
          GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
          GroupViTConfig,
          GroupViTOnnxConfig,
          GroupViTTextConfig,
          GroupViTVisionConfig,
      )

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_groupvit import (
                GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                GroupViTModel,
                GroupViTPreTrainedModel,
                GroupViTTextModel,
                GroupViTVisionModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_groupvit import (
                TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFGroupViTModel,
                TFGroupViTPreTrainedModel,
                TFGroupViTTextModel,
                TFGroupViTVisionModel,
            )

else:
      import sys

      UpperCamelCase  							=						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



