

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCAmelCase			:						Union[str, Any]  					=     re.compile(R'\b(a|an|the)\b', re.UNICODE)

_lowerCAmelCase			:						Optional[int]  					=     None







def a_		(							)				-> Union[str, Any]:







 """simple docstring"""

 lowerCamelCase       =     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'	)
 parser.add_argument('data_file' ,       metavar='data.json' ,       help='Input data JSON file.'	)
 parser.add_argument('pred_file' ,       metavar='pred.json' ,       help='Model predictions.'	)
 parser.add_argument(
     '--out-file' ,       '-o' ,       metavar='eval.json' ,       help='Write accuracy metrics to file (default is stdout).'	)
 parser.add_argument(
     '--na-prob-file' ,       '-n' ,       metavar='na_prob.json' ,       help='Model estimates of probability of no answer.'	)
 parser.add_argument(
     '--na-prob-thresh' ,       '-t' ,       type=lowercase__ ,       default=1.0 ,       help='Predict "" if no-answer probability exceeds this (default = 1.0).' ,       )
 parser.add_argument(
     '--out-image-dir' ,       '-p' ,       metavar='out_images' ,       default=lowercase__ ,       help='Save precision-recall curves to directory.'	)
 parser.add_argument('--verbose' ,       '-v' ,       action='store_true'	)
 if len(sys.argv	) == 1:
  parser.print_help()
  sys.exit(1	)
 return parser.parse_args()







def a_		(							UpperCamelCase_ :				str	)				-> Optional[int]:







 """simple docstring"""

 lowerCamelCase       =     {}
 for article in dataset:
  for p in article["paragraphs"]:
   for qa in p["qas"]:
    lowerCamelCase       =     bool(qa['answers']['text']	)
 return qid_to_has_ans







def a_		(							UpperCamelCase_ :				Tuple	)				-> List[Any]:







 """simple docstring"""

 def remove_articles(UpperCamelCase_ :				Tuple	):
  return ARTICLES_REGEX.sub(' ' ,       lowercase__	)

 def white_space_fix(UpperCamelCase_ :				str	):
  return " ".join(text.split()	)

 def remove_punc(UpperCamelCase_ :				str	):
  lowerCamelCase       =     set(string.punctuation	)
  return "".join(ch for ch in text if ch not in exclude	)

 def lower(UpperCamelCase_ :				Dict	):
  return text.lower()

 return white_space_fix(remove_articles(remove_punc(lower(lowercase__	)	)	)	)







def a_		(							UpperCamelCase_ :				Union[str, Any]	)				-> List[str]:







 """simple docstring"""

 if not s:
  return []
 return normalize_answer(lowercase__	).split()







def a_		(							UpperCamelCase_ :				str ,       UpperCamelCase_ :				Any	)				-> int:







 """simple docstring"""

 return int(normalize_answer(lowercase__	) == normalize_answer(lowercase__	)	)







def a_		(							UpperCamelCase_ :				Dict ,       UpperCamelCase_ :				str	)				-> List[str]:







 """simple docstring"""

 lowerCamelCase       =     get_tokens(lowercase__	)
 lowerCamelCase       =     get_tokens(lowercase__	)
 lowerCamelCase       =     collections.Counter(lowercase__	) & collections.Counter(lowercase__	)
 lowerCamelCase       =     sum(common.values()	)
 if len(lowercase__	) == 0 or len(lowercase__	) == 0:
  # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
  return int(gold_toks == pred_toks	)
 if num_same == 0:
  return 0
 lowerCamelCase       =     1.0 * num_same / len(lowercase__	)
 lowerCamelCase       =     1.0 * num_same / len(lowercase__	)
 lowerCamelCase       =     (2 * precision * recall) / (precision + recall)
 return fa







def a_		(							UpperCamelCase_ :				int ,       UpperCamelCase_ :				Union[str, Any]	)				-> Optional[Any]:







 """simple docstring"""

 lowerCamelCase       =     {}
 lowerCamelCase       =     {}
 for article in dataset:
  for p in article["paragraphs"]:
   for qa in p["qas"]:
    lowerCamelCase       =     qa['id']
    lowerCamelCase       =     [t for t in qa['answers']['text'] if normalize_answer(lowercase__	)]
    if not gold_answers:
     # For unanswerable questions, only correct answer is empty string
     lowerCamelCase       =     ['']
    if qid not in preds:
     print(f'''Missing prediction for {qid}'''	)
     continue
    lowerCamelCase       =     preds[qid]
    # Take max over all gold answers
    lowerCamelCase       =     max(compute_exact(lowercase__ ,       lowercase__	) for a in gold_answers	)
    lowerCamelCase       =     max(compute_fa(lowercase__ ,       lowercase__	) for a in gold_answers	)
 return exact_scores, fa_scores







def a_		(							UpperCamelCase_ :				str ,       UpperCamelCase_ :				Tuple ,       UpperCamelCase_ :				Union[str, Any] ,       UpperCamelCase_ :				Optional[Any]	)				-> Any:







 """simple docstring"""

 lowerCamelCase       =     {}
 for qid, s in scores.items():
  lowerCamelCase       =     na_probs[qid] > na_prob_thresh
  if pred_na:
   lowerCamelCase       =     float(not qid_to_has_ans[qid]	)
  else:
   lowerCamelCase       =     s
 return new_scores







def a_		(							UpperCamelCase_ :				Any ,       UpperCamelCase_ :				Tuple ,       UpperCamelCase_ :				List[Any]=None	)				-> List[Any]:







 """simple docstring"""

 if not qid_list:
  lowerCamelCase       =     len(lowercase__	)
  return collections.OrderedDict(
      [
          ('exact', 1_0_0.0 * sum(exact_scores.values()	) / total),
          ('f1', 1_0_0.0 * sum(fa_scores.values()	) / total),
          ('total', total),
      ]	)
 else:
  lowerCamelCase       =     len(lowercase__	)
  return collections.OrderedDict(
      [
          ('exact', 1_0_0.0 * sum(exact_scores[k] for k in qid_list	) / total),
          ('f1', 1_0_0.0 * sum(fa_scores[k] for k in qid_list	) / total),
          ('total', total),
      ]	)







def a_		(							UpperCamelCase_ :				Dict ,       UpperCamelCase_ :				List[Any] ,       UpperCamelCase_ :				Tuple	)				-> Any:







 """simple docstring"""

 for k in new_eval:
  lowerCamelCase       =     new_eval[k]







def a_		(							UpperCamelCase_ :				List[str] ,       UpperCamelCase_ :				Optional[int] ,       UpperCamelCase_ :				List[str] ,       UpperCamelCase_ :				Any	)				-> str:







 """simple docstring"""

 plt.step(lowercase__ ,       lowercase__ ,       color='b' ,       alpha=0.2 ,       where='post'	)
 plt.fill_between(lowercase__ ,       lowercase__ ,       step='post' ,       alpha=0.2 ,       color='b'	)
 plt.xlabel('Recall'	)
 plt.ylabel('Precision'	)
 plt.xlim([0.0, 1.05]	)
 plt.ylim([0.0, 1.05]	)
 plt.title(lowercase__	)
 plt.savefig(lowercase__	)
 plt.clf()







def a_		(							UpperCamelCase_ :				Optional[Any] ,       UpperCamelCase_ :				int ,       UpperCamelCase_ :				Union[str, Any] ,       UpperCamelCase_ :				int ,       UpperCamelCase_ :				Optional[Any]=None ,       UpperCamelCase_ :				Optional[int]=None	)				-> str:







 """simple docstring"""

 lowerCamelCase       =     sorted(lowercase__ ,       key=lambda UpperCamelCase_	: na_probs[k]	)
 lowerCamelCase       =     0.0
 lowerCamelCase       =     1.0
 lowerCamelCase       =     0.0
 lowerCamelCase       =     [1.0]
 lowerCamelCase       =     [0.0]
 lowerCamelCase       =     0.0
 for i, qid in enumerate(lowercase__	):
  if qid_to_has_ans[qid]:
   true_pos += scores[qid]
  lowerCamelCase       =     true_pos / float(i + 1	)
  lowerCamelCase       =     true_pos / float(lowercase__	)
  if i == len(lowercase__	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
   # i.e., if we can put a threshold after this point
   avg_prec += cur_p * (cur_r - recalls[-1])
   precisions.append(lowercase__	)
   recalls.append(lowercase__	)
 if out_image:
  plot_pr_curve(lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__	)
 return {"ap": 1_0_0.0 * avg_prec}







def a_		(							UpperCamelCase_ :				Tuple ,       UpperCamelCase_ :				Union[str, Any] ,       UpperCamelCase_ :				int ,       UpperCamelCase_ :				Optional[int] ,       UpperCamelCase_ :				str ,       UpperCamelCase_ :				Dict	)				-> List[str]:







 """simple docstring"""

 if out_image_dir and not os.path.exists(lowercase__	):
  os.makedirs(lowercase__	)
 lowerCamelCase       =     sum(1 for v in qid_to_has_ans.values() if v	)
 if num_true_pos == 0:
  return
 lowerCamelCase       =     make_precision_recall_eval(
     lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       out_image=os.path.join(lowercase__ ,       'pr_exact.png'	) ,       title='Precision-Recall curve for Exact Match score' ,       )
 lowerCamelCase       =     make_precision_recall_eval(
     lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       out_image=os.path.join(lowercase__ ,       'pr_f1.png'	) ,       title='Precision-Recall curve for F1 score' ,       )
 lowerCamelCase       =     {k: float(lowercase__	) for k, v in qid_to_has_ans.items()}
 lowerCamelCase       =     make_precision_recall_eval(
     lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       out_image=os.path.join(lowercase__ ,       'pr_oracle.png'	) ,       title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)' ,       )
 merge_eval(lowercase__ ,       lowercase__ ,       'pr_exact'	)
 merge_eval(lowercase__ ,       lowercase__ ,       'pr_f1'	)
 merge_eval(lowercase__ ,       lowercase__ ,       'pr_oracle'	)







def a_		(							UpperCamelCase_ :				Optional[int] ,       UpperCamelCase_ :				Tuple ,       UpperCamelCase_ :				List[Any] ,       UpperCamelCase_ :				List[str]	)				-> Union[str, Any]:







 """simple docstring"""

 if not qid_list:
  return
 lowerCamelCase       =     [na_probs[k] for k in qid_list]
 lowerCamelCase       =     np.ones_like(lowercase__	) / float(len(lowercase__	)	)
 plt.hist(lowercase__ ,       weights=lowercase__ ,       bins=2_0 ,       range=(0.0, 1.0)	)
 plt.xlabel('Model probability of no-answer'	)
 plt.ylabel('Proportion of dataset'	)
 plt.title(f'''Histogram of no-answer probability: {name}'''	)
 plt.savefig(os.path.join(lowercase__ ,       f'''na_prob_hist_{name}.png'''	)	)
 plt.clf()







def a_		(							UpperCamelCase_ :				List[Any] ,       UpperCamelCase_ :				Optional[int] ,       UpperCamelCase_ :				Optional[Any] ,       UpperCamelCase_ :				List[Any]	)				-> int:







 """simple docstring"""

 lowerCamelCase       =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
 lowerCamelCase       =     num_no_ans
 lowerCamelCase       =     cur_score
 lowerCamelCase       =     0.0
 lowerCamelCase       =     sorted(lowercase__ ,       key=lambda UpperCamelCase_	: na_probs[k]	)
 for i, qid in enumerate(lowercase__	):
  if qid not in scores:
   continue
  if qid_to_has_ans[qid]:
   lowerCamelCase       =     scores[qid]
  else:
   if preds[qid]:
    lowerCamelCase       =     -1
   else:
    lowerCamelCase       =     0
  cur_score += diff
  if cur_score > best_score:
   lowerCamelCase       =     cur_score
   lowerCamelCase       =     na_probs[qid]
 return 1_0_0.0 * best_score / len(lowercase__	), best_thresh







def a_		(							UpperCamelCase_ :				Dict ,       UpperCamelCase_ :				str ,       UpperCamelCase_ :				int ,       UpperCamelCase_ :				Dict ,       UpperCamelCase_ :				List[str] ,       UpperCamelCase_ :				Tuple	)				-> Optional[int]:







 """simple docstring"""

 lowerCamelCase       =     find_best_thresh(lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__	)
 lowerCamelCase       =     find_best_thresh(lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__	)
 lowerCamelCase       =     best_exact
 lowerCamelCase       =     exact_thresh
 lowerCamelCase       =     best_fa
 lowerCamelCase       =     fa_thresh







def a_		(							)				-> Tuple:







 """simple docstring"""

 with open(OPTS.data_file	) as f:
  lowerCamelCase       =     json.load(lowercase__	)
  lowerCamelCase       =     dataset_json['data']
 with open(OPTS.pred_file	) as f:
  lowerCamelCase       =     json.load(lowercase__	)
 if OPTS.na_prob_file:
  with open(OPTS.na_prob_file	) as f:
   lowerCamelCase       =     json.load(lowercase__	)
 else:
  lowerCamelCase       =     {k: 0.0 for k in preds}
 lowerCamelCase       =     make_qid_to_has_ans(lowercase__	)  # maps qid to True/False
 lowerCamelCase       =     [k for k, v in qid_to_has_ans.items() if v]
 lowerCamelCase       =     [k for k, v in qid_to_has_ans.items() if not v]
 lowerCamelCase       =     get_raw_scores(lowercase__ ,       lowercase__	)
 lowerCamelCase       =     apply_no_ans_threshold(lowercase__ ,       lowercase__ ,       lowercase__ ,       OPTS.na_prob_thresh	)
 lowerCamelCase       =     apply_no_ans_threshold(lowercase__ ,       lowercase__ ,       lowercase__ ,       OPTS.na_prob_thresh	)
 lowerCamelCase       =     make_eval_dict(lowercase__ ,       lowercase__	)
 if has_ans_qids:
  lowerCamelCase       =     make_eval_dict(lowercase__ ,       lowercase__ ,       qid_list=lowercase__	)
  merge_eval(lowercase__ ,       lowercase__ ,       'HasAns'	)
 if no_ans_qids:
  lowerCamelCase       =     make_eval_dict(lowercase__ ,       lowercase__ ,       qid_list=lowercase__	)
  merge_eval(lowercase__ ,       lowercase__ ,       'NoAns'	)
 if OPTS.na_prob_file:
  find_all_best_thresh(lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__	)
 if OPTS.na_prob_file and OPTS.out_image_dir:
  run_precision_recall_analysis(lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       lowercase__ ,       OPTS.out_image_dir	)
  histogram_na_prob(lowercase__ ,       lowercase__ ,       OPTS.out_image_dir ,       'hasAns'	)
  histogram_na_prob(lowercase__ ,       lowercase__ ,       OPTS.out_image_dir ,       'noAns'	)
 if OPTS.out_file:
  with open(OPTS.out_file ,       'w'	) as f:
   json.dump(lowercase__ ,       lowercase__	)
 else:
  print(json.dumps(lowercase__ ,       indent=2	)	)


if __name__ == "__main__":
     _lowerCAmelCase			:						int  					=     parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('Agg')
          import matplotlib.pyplot as plt
     main()










"""simple docstring"""






import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer






class lowerCAmelCase__      (	tf.keras.layers.Layer		):






			'''simple docstring'''



			def __init__(    self    , lowercase    , lowercase    , lowercase = None    , lowercase = None    ):
							super().__init__()
							_lowerCamelCase	:     Optional[Any]       =       pad_token_id
							_lowerCamelCase	:     Dict       =       max_length
							_lowerCamelCase	:     Tuple       =       vocab
							_lowerCamelCase	:     List[str]       =       merges
							_lowerCamelCase	:     int       =       BytePairTokenizer(lowercase    , lowercase    , sequence_length=lowercase    )
			@classmethod
			def    A_       (    cls    , lowercase    , *lowercase    , **lowercase    ):
							_lowerCamelCase	:     str       =       [' '.join(lowercase    ) for m in tokenizer.bpe_ranks.keys()]
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.get_vocab()
							return cls(lowercase    , lowercase    , *lowercase    , **lowercase    )
			@classmethod
			def    A_       (    cls    , lowercase    , *lowercase    , **lowercase    ):
							_lowerCamelCase	:     Optional[int]       =       GPTaTokenizer.from_pretrained(lowercase    , *lowercase    , **lowercase    )
							return cls.from_tokenizer(lowercase    , *lowercase    , **lowercase    )
			@classmethod
			def    A_       (    cls    , lowercase    ):
							return cls(**lowercase    )
			def    A_       (    self    ):
							return {
							    "vocab": self.vocab,
							    "merges": self.merges,
							    "max_length": self.max_length,
							    "pad_token_id": self.pad_token_id,
							}






			def    A_       (    self    , lowercase    , lowercase = None    ):
							_lowerCamelCase	:     int       =       self.tf_tokenizer(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       tf.ones_like(lowercase    )

							if self.pad_token_id is not None:
											# pad the tokens up to max length
											_lowerCamelCase	:     str       =       max_length if max_length is not None else self.max_length

											if max_length is not None:
															_lowerCamelCase,					_lowerCamelCase	:     Union[str, Any]       =       pad_model_inputs(
															    lowercase    , max_seq_length=lowercase    , pad_value=self.pad_token_id    )

							return {"attention_mask": attention_mask, "input_ids": input_ids}