
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase__		:     List[Any]     			=      re.compile(R'\b(a|an|the)\b', re.UNICODE)

UpperCAmelCase__		:     Any     			=      None






def   lowerCamelCase__							(					)      ->   Union[str, Any]:
   _A:  Optional[Any]					=       argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''      )
   parser.add_argument('''data_file'''		,		metavar='''data.json'''		,		help='''Input data JSON file.'''      )
   parser.add_argument('''pred_file'''		,		metavar='''pred.json'''		,		help='''Model predictions.'''      )
   parser.add_argument(
       '''--out-file'''		,		'''-o'''		,		metavar='''eval.json'''		,		help='''Write accuracy metrics to file (default is stdout).'''      )
   parser.add_argument(
       '''--na-prob-file'''		,		'''-n'''		,		metavar='''na_prob.json'''		,		help='''Model estimates of probability of no answer.'''      )
   parser.add_argument(
       '''--na-prob-thresh'''		,		'''-t'''		,		type=a		,		default=1.0		,		help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''		,		)
   parser.add_argument(
       '''--out-image-dir'''		,		'''-p'''		,		metavar='''out_images'''		,		default=a		,		help='''Save precision-recall curves to directory.'''      )
   parser.add_argument('''--verbose'''		,		'''-v'''		,		action='''store_true'''      )
   if len(sys.argv      ) == 1:
      parser.print_help()
      sys.exit(1      )
   return parser.parse_args()






def   lowerCamelCase__							(					a      )      ->   str:
   _A:  Union[str, Any]					=       {}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            _A:  Optional[Any]					=       bool(qa['''answers''']['''text''']      )
   return qid_to_has_ans






def   lowerCamelCase__							(					a      )      ->   Dict:

   def remove_articles(a      ):
      return ARTICLES_REGEX.sub(''' '''		,		a      )

   def white_space_fix(a      ):
      return " ".join(text.split()      )

   def remove_punc(a      ):
      _A:  int					=       set(string.punctuation      )
      return "".join(ch for ch in text if ch not in exclude      )

   def lower(a      ):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(a      )      )      )      )






def   lowerCamelCase__							(					a      )      ->   Optional[Any]:
   if not s:
      return []
   return normalize_answer(a      ).split()






def   lowerCamelCase__							(					a		,		a      )      ->   Union[str, Any]:
   return int(normalize_answer(a      ) == normalize_answer(a      )      )






def   lowerCamelCase__							(					a		,		a      )      ->   Union[str, Any]:
   _A:  Union[str, Any]					=       get_tokens(a      )
   _A:  Tuple					=       get_tokens(a      )
   _A:  Union[str, Any]					=       collections.Counter(a      ) & collections.Counter(a      )
   _A:  Union[str, Any]					=       sum(common.values()      )
   if len(a      ) == 0 or len(a      ) == 0:
      # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
      return int(gold_toks == pred_toks      )
   if num_same == 0:
      return 0
   _A:  str					=       1.0 * num_same / len(a      )
   _A:  str					=       1.0 * num_same / len(a      )
   _A:  Optional[Any]					=       (2 * precision * recall) / (precision + recall)
   return fa






def   lowerCamelCase__							(					a		,		a      )      ->   List[str]:
   _A:  List[str]					=       {}
   _A:  List[str]					=       {}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            _A:  int					=       qa['''id''']
            _A:  Tuple					=       [t for t in qa['''answers''']['''text'''] if normalize_answer(a      )]
            if not gold_answers:
               # For unanswerable questions, only correct answer is empty string
               _A:  Union[str, Any]					=       ['''''']
            if qid not in preds:
               print(f"""Missing prediction for {qid}"""      )
               continue
            _A:  Any					=       preds[qid]
            # Take max over all gold answers
            _A:  List[Any]					=       max(compute_exact(a		,		a      ) for a in gold_answers      )
            _A:  int					=       max(compute_fa(a		,		a      ) for a in gold_answers      )
   return exact_scores, fa_scores






def   lowerCamelCase__							(					a		,		a		,		a		,		a      )      ->   Tuple:
   _A:  List[str]					=       {}
   for qid, s in scores.items():
      _A:  List[str]					=       na_probs[qid] > na_prob_thresh
      if pred_na:
         _A:  Optional[int]					=       float(not qid_to_has_ans[qid]      )
      else:
         _A:  Optional[Any]					=       s
   return new_scores






def   lowerCamelCase__							(					a		,		a		,		a=None      )      ->   Optional[int]:
   if not qid_list:
      _A:  int					=       len(a      )
      return collections.OrderedDict(
          [
              ('''exact''', 100.0 * sum(exact_scores.values()      ) / total),
              ('''f1''', 100.0 * sum(fa_scores.values()      ) / total),
              ('''total''', total),
          ]      )
   else:
      _A:  int					=       len(a      )
      return collections.OrderedDict(
          [
              ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list      ) / total),
              ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list      ) / total),
              ('''total''', total),
          ]      )






def   lowerCamelCase__							(					a		,		a		,		a      )      ->   str:
   for k in new_eval:
      _A:  int					=       new_eval[k]






def   lowerCamelCase__							(					a		,		a		,		a		,		a      )      ->   int:
   plt.step(a		,		a		,		color='''b'''		,		alpha=0.2		,		where='''post'''      )
   plt.fill_between(a		,		a		,		step='''post'''		,		alpha=0.2		,		color='''b'''      )
   plt.xlabel('''Recall'''      )
   plt.ylabel('''Precision'''      )
   plt.xlim([0.0, 1.05]      )
   plt.ylim([0.0, 1.05]      )
   plt.title(a      )
   plt.savefig(a      )
   plt.clf()






def   lowerCamelCase__							(					a		,		a		,		a		,		a		,		a=None		,		a=None      )      ->   List[str]:
   _A:  int					=       sorted(a		,		key=lambda a      : na_probs[k]      )
   _A:  Optional[int]					=       0.0
   _A:  Optional[int]					=       1.0
   _A:  Union[str, Any]					=       0.0
   _A:  List[Any]					=       [1.0]
   _A:  List[str]					=       [0.0]
   _A:  Optional[int]					=       0.0
   for i, qid in enumerate(a      ):
      if qid_to_has_ans[qid]:
         true_pos += scores[qid]
      _A:  Tuple					=       true_pos / float(i + 1      )
      _A:  Tuple					=       true_pos / float(a      )
      if i == len(a      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
         # i.e., if we can put a threshold after this point
         avg_prec += cur_p * (cur_r - recalls[-1])
         precisions.append(a      )
         recalls.append(a      )
   if out_image:
      plot_pr_curve(a		,		a		,		a		,		a      )
   return {"ap": 100.0 * avg_prec}






def   lowerCamelCase__							(					a		,		a		,		a		,		a		,		a		,		a      )      ->   Any:
   if out_image_dir and not os.path.exists(a      ):
      os.makedirs(a      )
   _A:  Optional[Any]					=       sum(1 for v in qid_to_has_ans.values() if v      )
   if num_true_pos == 0:
      return
   _A:  Optional[Any]					=       make_precision_recall_eval(
       a		,		a		,		a		,		a		,		out_image=os.path.join(a		,		'''pr_exact.png'''      )		,		title='''Precision-Recall curve for Exact Match score'''		,		)
   _A:  Optional[Any]					=       make_precision_recall_eval(
       a		,		a		,		a		,		a		,		out_image=os.path.join(a		,		'''pr_f1.png'''      )		,		title='''Precision-Recall curve for F1 score'''		,		)
   _A:  Optional[Any]					=       {k: float(a      ) for k, v in qid_to_has_ans.items()}
   _A:  int					=       make_precision_recall_eval(
       a		,		a		,		a		,		a		,		out_image=os.path.join(a		,		'''pr_oracle.png'''      )		,		title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''		,		)
   merge_eval(a		,		a		,		'''pr_exact'''      )
   merge_eval(a		,		a		,		'''pr_f1'''      )
   merge_eval(a		,		a		,		'''pr_oracle'''      )






def   lowerCamelCase__							(					a		,		a		,		a		,		a      )      ->   Optional[Any]:
   if not qid_list:
      return
   _A:  List[Any]					=       [na_probs[k] for k in qid_list]
   _A:  int					=       np.ones_like(a      ) / float(len(a      )      )
   plt.hist(a		,		weights=a		,		bins=20		,		range=(0.0, 1.0)      )
   plt.xlabel('''Model probability of no-answer'''      )
   plt.ylabel('''Proportion of dataset'''      )
   plt.title(f"""Histogram of no-answer probability: {name}"""      )
   plt.savefig(os.path.join(a		,		f"""na_prob_hist_{name}.png"""      )      )
   plt.clf()






def   lowerCamelCase__							(					a		,		a		,		a		,		a      )      ->   Tuple:
   _A:  Union[str, Any]					=       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
   _A:  Optional[int]					=       num_no_ans
   _A:  Tuple					=       cur_score
   _A:  Dict					=       0.0
   _A:  Tuple					=       sorted(a		,		key=lambda a      : na_probs[k]      )
   for i, qid in enumerate(a      ):
      if qid not in scores:
         continue
      if qid_to_has_ans[qid]:
         _A:  List[str]					=       scores[qid]
      else:
         if preds[qid]:
            _A:  int					=       -1
         else:
            _A:  Optional[int]					=       0
      cur_score += diff
      if cur_score > best_score:
         _A:  Union[str, Any]					=       cur_score
         _A:  Optional[Any]					=       na_probs[qid]
   return 100.0 * best_score / len(a      ), best_thresh






def   lowerCamelCase__							(					a		,		a		,		a		,		a		,		a		,		a      )      ->   Optional[int]:
   _A			,  _A:  Dict					=       find_best_thresh(a		,		a		,		a		,		a      )
   _A			,  _A:  Dict					=       find_best_thresh(a		,		a		,		a		,		a      )
   _A:  Optional[int]					=       best_exact
   _A:  List[Any]					=       exact_thresh
   _A:  List[Any]					=       best_fa
   _A:  Optional[int]					=       fa_thresh






def   lowerCamelCase__							(					)      ->   List[Any]:
   with open(OPTS.data_file      ) as f:
      _A:  List[Any]					=       json.load(a      )
      _A:  Tuple					=       dataset_json['''data''']
   with open(OPTS.pred_file      ) as f:
      _A:  int					=       json.load(a      )
   if OPTS.na_prob_file:
      with open(OPTS.na_prob_file      ) as f:
         _A:  int					=       json.load(a      )
   else:
      _A:  Optional[int]					=       {k: 0.0 for k in preds}
   _A:  Optional[int]					=       make_qid_to_has_ans(a      )  # maps qid to True/False
   _A:  Optional[Any]					=       [k for k, v in qid_to_has_ans.items() if v]
   _A:  List[Any]					=       [k for k, v in qid_to_has_ans.items() if not v]
   _A			,  _A:  Union[str, Any]					=       get_raw_scores(a		,		a      )
   _A:  Any					=       apply_no_ans_threshold(a		,		a		,		a		,		OPTS.na_prob_thresh      )
   _A:  Tuple					=       apply_no_ans_threshold(a		,		a		,		a		,		OPTS.na_prob_thresh      )
   _A:  Dict					=       make_eval_dict(a		,		a      )
   if has_ans_qids:
      _A:  int					=       make_eval_dict(a		,		a		,		qid_list=a      )
      merge_eval(a		,		a		,		'''HasAns'''      )
   if no_ans_qids:
      _A:  Dict					=       make_eval_dict(a		,		a		,		qid_list=a      )
      merge_eval(a		,		a		,		'''NoAns'''      )
   if OPTS.na_prob_file:
      find_all_best_thresh(a		,		a		,		a		,		a		,		a		,		a      )
   if OPTS.na_prob_file and OPTS.out_image_dir:
      run_precision_recall_analysis(a		,		a		,		a		,		a		,		a		,		OPTS.out_image_dir      )
      histogram_na_prob(a		,		a		,		OPTS.out_image_dir		,		'''hasAns'''      )
      histogram_na_prob(a		,		a		,		OPTS.out_image_dir		,		'''noAns'''      )
   if OPTS.out_file:
      with open(OPTS.out_file		,		'''w'''      ) as f:
         json.dump(a		,		a      )
   else:
      print(json.dumps(a		,		indent=2      )      )


if __name__ == "__main__":
   UpperCAmelCase__		:     List[str]     			=      parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use('Agg')
      import matplotlib.pyplot as plt
   main()





def   lowerCamelCase__							(					a		,		a      )      ->   str:
   if a < 0 or b < 0:
      raise ValueError('''the value of both inputs must be positive'''      )

   _A:  Union[str, Any]					=       str(bin(a      )      )[2:]  # remove the leading "0b"
   _A:  Union[str, Any]					=       str(bin(a      )      )[2:]  # remove the leading "0b"

   _A:  Optional[int]					=       max(len(a      )		,		len(a      )      )

   return "0b" + "".join(
       str(int(char_a != char_b      )      )
       for char_a, char_b in zip(a_binary.zfill(a      )		,		b_binary.zfill(a      )      )      )


if __name__ == "__main__":
   import doctest

   doctest.testmod()



