






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a		:       List[str]			     = re.compile(r"\b(a|an|the)\b", re.UNICODE)

a		:       List[str]			     = None






def 				lowerCamelCase__      (    ):
  __UpperCAmelCase							:	Optional[Any]           =  argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""		)
  parser.add_argument("""data_file"""  ,		metavar="""data.json"""  ,		help="""Input data JSON file."""		)
  parser.add_argument("""pred_file"""  ,		metavar="""pred.json"""  ,		help="""Model predictions."""		)
  parser.add_argument(
      """--out-file"""  ,		"""-o"""  ,		metavar="""eval.json"""  ,		help="""Write accuracy metrics to file (default is stdout)."""		)
  parser.add_argument(
      """--na-prob-file"""  ,		"""-n"""  ,		metavar="""na_prob.json"""  ,		help="""Model estimates of probability of no answer."""		)
  parser.add_argument(
      """--na-prob-thresh"""  ,		"""-t"""  ,		type=__lowerCamelCase  ,		default=1.0  ,		help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,		)
  parser.add_argument(
      """--out-image-dir"""  ,		"""-p"""  ,		metavar="""out_images"""  ,		default=__lowerCamelCase  ,		help="""Save precision-recall curves to directory."""		)
  parser.add_argument("""--verbose"""  ,		"""-v"""  ,		action="""store_true"""		)
  if len(sys.argv		) == 1:
    parser.print_help()
    sys.exit(1		)
  return parser.parse_args()






def 				lowerCamelCase__      (    __lowerCamelCase			:     Tuple		):
  __UpperCAmelCase							:	Tuple           =  {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __UpperCAmelCase							:	Dict           =  bool(qa["""answers"""]["""text"""]		)
  return qid_to_has_ans






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[Any]		):

  def remove_articles(__lowerCamelCase			:     List[str]		):
    return ARTICLES_REGEX.sub(""" """  ,		__lowerCamelCase		)

  def white_space_fix(__lowerCamelCase			:     str		):
    return " ".join(text.split()		)

  def remove_punc(__lowerCamelCase			:     Optional[int]		):
    __UpperCAmelCase							:	Optional[Any]           =  set(string.punctuation		)
    return "".join(ch for ch in text if ch not in exclude		)

  def lower(__lowerCamelCase			:     List[str]		):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(__lowerCamelCase		)		)		)		)






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[str]		):
  if not s:
    return []
  return normalize_answer(__lowerCamelCase		).split()






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[str]  ,		__lowerCamelCase			:     Dict		):
  return int(normalize_answer(__lowerCamelCase		) == normalize_answer(__lowerCamelCase		)		)






def 				lowerCamelCase__      (    __lowerCamelCase			:     Dict  ,		__lowerCamelCase			:     Dict		):
  __UpperCAmelCase							:	List[Any]           =  get_tokens(__lowerCamelCase		)
  __UpperCAmelCase							:	Dict           =  get_tokens(__lowerCamelCase		)
  __UpperCAmelCase							:	Union[str, Any]           =  collections.Counter(__lowerCamelCase		) & collections.Counter(__lowerCamelCase		)
  __UpperCAmelCase							:	List[Any]           =  sum(common.values()		)
  if len(__lowerCamelCase		) == 0 or len(__lowerCamelCase		) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks		)
  if num_same == 0:
    return 0
  __UpperCAmelCase							:	Dict           =  1.0 * num_same / len(__lowerCamelCase		)
  __UpperCAmelCase							:	List[str]           =  1.0 * num_same / len(__lowerCamelCase		)
  __UpperCAmelCase							:	Optional[int]           =  (2 * precision * recall) / (precision + recall)
  return fa






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[Any]  ,		__lowerCamelCase			:     str		):
  __UpperCAmelCase							:	int           =  {}
  __UpperCAmelCase							:	int           =  {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __UpperCAmelCase							:	Optional[int]           =  qa["""id"""]
        __UpperCAmelCase							:	Dict           =  [t for t in qa["""answers"""]["""text"""] if normalize_answer(__lowerCamelCase		)]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          __UpperCAmelCase							:	Dict           =  [""""""]
        if qid not in preds:
          print(f"""Missing prediction for {qid}"""		)
          continue
        __UpperCAmelCase							:	int           =  preds[qid]
        # Take max over all gold answers
        __UpperCAmelCase							:	Optional[int]           =  max(compute_exact(__lowerCamelCase  ,		__lowerCamelCase		) for a in gold_answers		)
        __UpperCAmelCase							:	Union[str, Any]           =  max(compute_fa(__lowerCamelCase  ,		__lowerCamelCase		) for a in gold_answers		)
  return exact_scores, fa_scores






def 				lowerCamelCase__      (    __lowerCamelCase			:     Tuple  ,		__lowerCamelCase			:     Dict  ,		__lowerCamelCase			:     int  ,		__lowerCamelCase			:     Tuple		):
  __UpperCAmelCase							:	Any           =  {}
  for qid, s in scores.items():
    __UpperCAmelCase							:	List[Any]           =  na_probs[qid] > na_prob_thresh
    if pred_na:
      __UpperCAmelCase							:	Dict           =  float(not qid_to_has_ans[qid]		)
    else:
      __UpperCAmelCase							:	List[str]           =  s
  return new_scores






def 				lowerCamelCase__      (    __lowerCamelCase			:     Any  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     int=None		):
  if not qid_list:
    __UpperCAmelCase							:	int           =  len(__lowerCamelCase		)
    return collections.OrderedDict(
        [
            ("""exact""", 1_0_0.0 * sum(exact_scores.values()		) / total),
            ("""f1""", 1_0_0.0 * sum(fa_scores.values()		) / total),
            ("""total""", total),
        ]		)
  else:
    __UpperCAmelCase							:	Dict           =  len(__lowerCamelCase		)
    return collections.OrderedDict(
        [
            ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list		) / total),
            ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list		) / total),
            ("""total""", total),
        ]		)






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[str]  ,		__lowerCamelCase			:     int  ,		__lowerCamelCase			:     Optional[int]		):
  for k in new_eval:
    __UpperCAmelCase							:	Union[str, Any]           =  new_eval[k]






def 				lowerCamelCase__      (    __lowerCamelCase			:     str  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     Optional[int]  ,		__lowerCamelCase			:     Union[str, Any]		):
  plt.step(__lowerCamelCase  ,		__lowerCamelCase  ,		color="""b"""  ,		alpha=0.2  ,		where="""post"""		)
  plt.fill_between(__lowerCamelCase  ,		__lowerCamelCase  ,		step="""post"""  ,		alpha=0.2  ,		color="""b"""		)
  plt.xlabel("""Recall"""		)
  plt.ylabel("""Precision"""		)
  plt.xlim([0.0, 1.0_5]		)
  plt.ylim([0.0, 1.0_5]		)
  plt.title(__lowerCamelCase		)
  plt.savefig(__lowerCamelCase		)
  plt.clf()






def 				lowerCamelCase__      (    __lowerCamelCase			:     int  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     int  ,		__lowerCamelCase			:     List[str]  ,		__lowerCamelCase			:     str=None  ,		__lowerCamelCase			:     Dict=None		):
  __UpperCAmelCase							:	int           =  sorted(__lowerCamelCase  ,		key=lambda __lowerCamelCase		: na_probs[k]		)
  __UpperCAmelCase							:	Optional[int]           =  0.0
  __UpperCAmelCase							:	Union[str, Any]           =  1.0
  __UpperCAmelCase							:	Dict           =  0.0
  __UpperCAmelCase							:	Optional[int]           =  [1.0]
  __UpperCAmelCase							:	Optional[Any]           =  [0.0]
  __UpperCAmelCase							:	str           =  0.0
  for i, qid in enumerate(__lowerCamelCase		):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    __UpperCAmelCase							:	List[Any]           =  true_pos / float(i + 1		)
    __UpperCAmelCase							:	str           =  true_pos / float(__lowerCamelCase		)
    if i == len(__lowerCamelCase		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(__lowerCamelCase		)
      recalls.append(__lowerCamelCase		)
  if out_image:
    plot_pr_curve(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase		)
  return {"ap": 1_0_0.0 * avg_prec}






def 				lowerCamelCase__      (    __lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     Tuple  ,		__lowerCamelCase			:     Optional[int]  ,		__lowerCamelCase			:     Optional[Any]  ,		__lowerCamelCase			:     List[str]		):
  if out_image_dir and not os.path.exists(__lowerCamelCase		):
    os.makedirs(__lowerCamelCase		)
  __UpperCAmelCase							:	List[str]           =  sum(1 for v in qid_to_has_ans.values() if v		)
  if num_true_pos == 0:
    return
  __UpperCAmelCase							:	int           =  make_precision_recall_eval(
      __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		out_image=os.path.join(__lowerCamelCase  ,		"""pr_exact.png"""		)  ,		title="""Precision-Recall curve for Exact Match score"""  ,		)
  __UpperCAmelCase							:	str           =  make_precision_recall_eval(
      __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		out_image=os.path.join(__lowerCamelCase  ,		"""pr_f1.png"""		)  ,		title="""Precision-Recall curve for F1 score"""  ,		)
  __UpperCAmelCase							:	List[str]           =  {k: float(__lowerCamelCase		) for k, v in qid_to_has_ans.items()}
  __UpperCAmelCase							:	Tuple           =  make_precision_recall_eval(
      __lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		out_image=os.path.join(__lowerCamelCase  ,		"""pr_oracle.png"""		)  ,		title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,		)
  merge_eval(__lowerCamelCase  ,		__lowerCamelCase  ,		"""pr_exact"""		)
  merge_eval(__lowerCamelCase  ,		__lowerCamelCase  ,		"""pr_f1"""		)
  merge_eval(__lowerCamelCase  ,		__lowerCamelCase  ,		"""pr_oracle"""		)






def 				lowerCamelCase__      (    __lowerCamelCase			:     int  ,		__lowerCamelCase			:     str  ,		__lowerCamelCase			:     Optional[int]  ,		__lowerCamelCase			:     Tuple		):
  if not qid_list:
    return
  __UpperCAmelCase							:	str           =  [na_probs[k] for k in qid_list]
  __UpperCAmelCase							:	List[Any]           =  np.ones_like(__lowerCamelCase		) / float(len(__lowerCamelCase		)		)
  plt.hist(__lowerCamelCase  ,		weights=__lowerCamelCase  ,		bins=20  ,		range=(0.0, 1.0)		)
  plt.xlabel("""Model probability of no-answer"""		)
  plt.ylabel("""Proportion of dataset"""		)
  plt.title(f"""Histogram of no-answer probability: {name}"""		)
  plt.savefig(os.path.join(__lowerCamelCase  ,		f"""na_prob_hist_{name}.png"""		)		)
  plt.clf()






def 				lowerCamelCase__      (    __lowerCamelCase			:     Dict  ,		__lowerCamelCase			:     Optional[int]  ,		__lowerCamelCase			:     Dict  ,		__lowerCamelCase			:     Any		):
  __UpperCAmelCase							:	Optional[Any]           =  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
  __UpperCAmelCase							:	Union[str, Any]           =  num_no_ans
  __UpperCAmelCase							:	str           =  cur_score
  __UpperCAmelCase							:	Any           =  0.0
  __UpperCAmelCase							:	Optional[Any]           =  sorted(__lowerCamelCase  ,		key=lambda __lowerCamelCase		: na_probs[k]		)
  for i, qid in enumerate(__lowerCamelCase		):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      __UpperCAmelCase							:	Tuple           =  scores[qid]
    else:
      if preds[qid]:
        __UpperCAmelCase							:	Dict           =  -1
      else:
        __UpperCAmelCase							:	List[str]           =  0
    cur_score += diff
    if cur_score > best_score:
      __UpperCAmelCase							:	Union[str, Any]           =  cur_score
      __UpperCAmelCase							:	str           =  na_probs[qid]
  return 1_0_0.0 * best_score / len(__lowerCamelCase		), best_thresh






def 				lowerCamelCase__      (    __lowerCamelCase			:     Any  ,		__lowerCamelCase			:     Dict  ,		__lowerCamelCase			:     Any  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     Union[str, Any]  ,		__lowerCamelCase			:     Any		):
  __UpperCAmelCase      , __UpperCAmelCase							:	List[str]           =  find_best_thresh(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase		)
  __UpperCAmelCase      , __UpperCAmelCase							:	str           =  find_best_thresh(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase		)
  __UpperCAmelCase							:	Any           =  best_exact
  __UpperCAmelCase							:	Any           =  exact_thresh
  __UpperCAmelCase							:	Optional[int]           =  best_fa
  __UpperCAmelCase							:	List[Any]           =  fa_thresh






def 				lowerCamelCase__      (    ):
  with open(OPTS.data_file		) as f:
    __UpperCAmelCase							:	List[str]           =  json.load(__lowerCamelCase		)
    __UpperCAmelCase							:	int           =  dataset_json["""data"""]
  with open(OPTS.pred_file		) as f:
    __UpperCAmelCase							:	str           =  json.load(__lowerCamelCase		)
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file		) as f:
      __UpperCAmelCase							:	Dict           =  json.load(__lowerCamelCase		)
  else:
    __UpperCAmelCase							:	Optional[int]           =  {k: 0.0 for k in preds}
  __UpperCAmelCase							:	List[Any]           =  make_qid_to_has_ans(__lowerCamelCase		)  # maps qid to True/False
  __UpperCAmelCase							:	List[str]           =  [k for k, v in qid_to_has_ans.items() if v]
  __UpperCAmelCase							:	Union[str, Any]           =  [k for k, v in qid_to_has_ans.items() if not v]
  __UpperCAmelCase      , __UpperCAmelCase							:	List[Any]           =  get_raw_scores(__lowerCamelCase  ,		__lowerCamelCase		)
  __UpperCAmelCase							:	str           =  apply_no_ans_threshold(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		OPTS.na_prob_thresh		)
  __UpperCAmelCase							:	Tuple           =  apply_no_ans_threshold(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		OPTS.na_prob_thresh		)
  __UpperCAmelCase							:	List[str]           =  make_eval_dict(__lowerCamelCase  ,		__lowerCamelCase		)
  if has_ans_qids:
    __UpperCAmelCase							:	List[Any]           =  make_eval_dict(__lowerCamelCase  ,		__lowerCamelCase  ,		qid_list=__lowerCamelCase		)
    merge_eval(__lowerCamelCase  ,		__lowerCamelCase  ,		"""HasAns"""		)
  if no_ans_qids:
    __UpperCAmelCase							:	Any           =  make_eval_dict(__lowerCamelCase  ,		__lowerCamelCase  ,		qid_list=__lowerCamelCase		)
    merge_eval(__lowerCamelCase  ,		__lowerCamelCase  ,		"""NoAns"""		)
  if OPTS.na_prob_file:
    find_all_best_thresh(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase		)
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		OPTS.out_image_dir		)
    histogram_na_prob(__lowerCamelCase  ,		__lowerCamelCase  ,		OPTS.out_image_dir  ,		"""hasAns"""		)
    histogram_na_prob(__lowerCamelCase  ,		__lowerCamelCase  ,		OPTS.out_image_dir  ,		"""noAns"""		)
  if OPTS.out_file:
    with open(OPTS.out_file  ,		"""w"""		) as f:
      json.dump(__lowerCamelCase  ,		__lowerCamelCase		)
  else:
    print(json.dumps(__lowerCamelCase  ,		indent=2		)		)


if __name__ == "__main__":
       a		:       Optional[int]			     = parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use("Agg")
              import matplotlib.pyplot as plt
       main()










def 				lowerCamelCase__      (    __lowerCamelCase			:     Tuple  ,		__lowerCamelCase			:     Union[str, Any]		):
  __UpperCAmelCase							:	Tuple           =  [1]
  for i in range(2  ,		__lowerCamelCase		):
    factorials.append(factorials[-1] * i		)
  assert 0 <= k < factorials[-1] * n, "k out of bounds"

  __UpperCAmelCase							:	Optional[Any]           =  []
  __UpperCAmelCase							:	str           =  list(range(__lowerCamelCase		)		)

  # Find permutation
  while factorials:
    __UpperCAmelCase							:	Any           =  factorials.pop()
    __UpperCAmelCase      , __UpperCAmelCase							:	Optional[Any]           =  divmod(__lowerCamelCase  ,		__lowerCamelCase		)
    permutation.append(elements[number]		)
    elements.remove(elements[number]		)
  permutation.append(elements[0]		)

  return permutation


if __name__ == "__main__":
       import doctest

       doctest.testmod()


