


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


snake_case				:    int       				=     re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

snake_case				:    Optional[Any]       				=     None
def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Optional[int] 							=       argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''						)
      parser.add_argument('''data_file''' ,  metavar='''data.json''' ,  help='''Input data JSON file.'''						)
      parser.add_argument('''pred_file''' ,  metavar='''pred.json''' ,  help='''Model predictions.'''						)
      parser.add_argument(
          '''--out-file''' ,  '''-o''' ,  metavar='''eval.json''' ,  help='''Write accuracy metrics to file (default is stdout).'''						)
      parser.add_argument(
          '''--na-prob-file''' ,  '''-n''' ,  metavar='''na_prob.json''' ,  help='''Model estimates of probability of no answer.'''						)
      parser.add_argument(
          '''--na-prob-thresh''' ,  '''-t''' ,  type=UpperCAmelCase_ ,  default=1.0 ,  help='''Predict "" if no-answer probability exceeds this (default = 1.0).''' ,  )
      parser.add_argument(
          '''--out-image-dir''' ,  '''-p''' ,  metavar='''out_images''' ,  default=UpperCAmelCase_ ,  help='''Save precision-recall curves to directory.'''						)
      parser.add_argument('''--verbose''' ,  '''-v''' ,  action='''store_true'''						)
      if len(sys.argv						) == 1:
            parser.print_help()
            sys.exit(1						)
      return parser.parse_args()
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any						):




      """simple docstring"""





      a						:Tuple 							=       {}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a						:List[str] 							=       bool(qa['''answers''']['''text''']						)
      return qid_to_has_ans
def 				__lowerCamelCase						( UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      def remove_articles(UpperCAmelCase_						:		Optional[int]						):
            return ARTICLES_REGEX.sub(''' ''' ,  UpperCAmelCase_						)

      def white_space_fix(UpperCAmelCase_						:		Tuple						):
            return " ".join(text.split()						)

      def remove_punc(UpperCAmelCase_						:		List[str]						):
            a						:List[Any] 							=       set(string.punctuation						)
            return "".join(ch for ch in text if ch not in exclude						)

      def lower(UpperCAmelCase_						:		int						):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase_						)						)						)						)
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any						):




      """simple docstring"""





      if not s:
            return []
      return normalize_answer(UpperCAmelCase_						).split()
def 				__lowerCamelCase						( UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		Optional[Any]						):




      """simple docstring"""





      return int(normalize_answer(UpperCAmelCase_						) == normalize_answer(UpperCAmelCase_						)						)
def 				__lowerCamelCase						( UpperCAmelCase_						:		List[Any] ,  UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      a						:Dict 							=       get_tokens(UpperCAmelCase_						)
      a						:Dict 							=       get_tokens(UpperCAmelCase_						)
      a						:Dict 							=       collections.Counter(UpperCAmelCase_						) & collections.Counter(UpperCAmelCase_						)
      a						:List[str] 							=       sum(common.values()						)
      if len(UpperCAmelCase_						) == 0 or len(UpperCAmelCase_						) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks						)
      if num_same == 0:
            return 0
      a						:Dict 							=       1.0 * num_same / len(UpperCAmelCase_						)
      a						:int 							=       1.0 * num_same / len(UpperCAmelCase_						)
      a						:Tuple 							=       (2 * precision * recall) / (precision + recall)
      return fa
def 				__lowerCamelCase						( UpperCAmelCase_						:		Dict ,  UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      a						:Union[str, Any] 							=       {}
      a						:int 							=       {}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a						:str 							=       qa['''id''']
                        a						:Any 							=       [t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCAmelCase_						)]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              a						:Optional[Any] 							=       ['''''']
                        if qid not in preds:
                              print(F'''Missing prediction for {qid}'''						)
                              continue
                        a						:List[Any] 							=       preds[qid]
                        # Take max over all gold answers
                        a						:Any 							=       max(compute_exact(UpperCAmelCase_ ,  UpperCAmelCase_						) for a in gold_answers						)
                        a						:Optional[int] 							=       max(compute_fa(UpperCAmelCase_ ,  UpperCAmelCase_						) for a in gold_answers						)
      return exact_scores, fa_scores
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		str ,  UpperCAmelCase_						:		Tuple ,  UpperCAmelCase_						:		Optional[Any]						):




      """simple docstring"""





      a						:Dict 							=       {}
      for qid, s in scores.items():
            a						:Any 							=       na_probs[qid] > na_prob_thresh
            if pred_na:
                  a						:int 							=       float(not qid_to_has_ans[qid]						)
            else:
                  a						:Union[str, Any] 							=       s
      return new_scores
def 				__lowerCamelCase						( UpperCAmelCase_						:		int ,  UpperCAmelCase_						:		Union[str, Any] ,  UpperCAmelCase_						:		Union[str, Any]=None						):




      """simple docstring"""





      if not qid_list:
            a						:Optional[Any] 							=       len(UpperCAmelCase_						)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores.values()						) / total),
                    ('''f1''', 100.0 * sum(fa_scores.values()						) / total),
                    ('''total''', total),
                ]						)
      else:
            a						:Optional[int] 							=       len(UpperCAmelCase_						)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list						) / total),
                    ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list						) / total),
                    ('''total''', total),
                ]						)
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      for k in new_eval:
            a						:List[Any] 							=       new_eval[k]
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		int ,  UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		List[Any]						):




      """simple docstring"""





      plt.step(UpperCAmelCase_ ,  UpperCAmelCase_ ,  color='''b''' ,  alpha=0.2 ,  where='''post'''						)
      plt.fill_between(UpperCAmelCase_ ,  UpperCAmelCase_ ,  step='''post''' ,  alpha=0.2 ,  color='''b'''						)
      plt.xlabel('''Recall'''						)
      plt.ylabel('''Precision'''						)
      plt.xlim([0.0, 1.05]						)
      plt.ylim([0.0, 1.05]						)
      plt.title(UpperCAmelCase_						)
      plt.savefig(UpperCAmelCase_						)
      plt.clf()
def 				__lowerCamelCase						( UpperCAmelCase_						:		Optional[Any] ,  UpperCAmelCase_						:		List[str] ,  UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		List[str] ,  UpperCAmelCase_						:		Tuple=None ,  UpperCAmelCase_						:		str=None						):




      """simple docstring"""





      a						:Optional[Any] 							=       sorted(UpperCAmelCase_ ,  key=lambda UpperCAmelCase_						: na_probs[k]						)
      a						:List[str] 							=       0.0
      a						:str 							=       1.0
      a						:Any 							=       0.0
      a						:Optional[Any] 							=       [1.0]
      a						:Any 							=       [0.0]
      a						:Tuple 							=       0.0
      for i, qid in enumerate(UpperCAmelCase_						):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            a						:List[str] 							=       true_pos / float(i + 1						)
            a						:str 							=       true_pos / float(UpperCAmelCase_						)
            if i == len(UpperCAmelCase_						) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(UpperCAmelCase_						)
                  recalls.append(UpperCAmelCase_						)
      if out_image:
            plot_pr_curve(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_						)
      return {"ap": 100.0 * avg_prec}
def 				__lowerCamelCase						( UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		str ,  UpperCAmelCase_						:		str ,  UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		List[Any] ,  UpperCAmelCase_						:		Any						):




      """simple docstring"""





      if out_image_dir and not os.path.exists(UpperCAmelCase_						):
            os.makedirs(UpperCAmelCase_						)
      a						:Optional[int] 							=       sum(1 for v in qid_to_has_ans.values() if v						)
      if num_true_pos == 0:
            return
      a						:Union[str, Any] 							=       make_precision_recall_eval(
          UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  out_image=os.path.join(UpperCAmelCase_ ,  '''pr_exact.png'''						) ,  title='''Precision-Recall curve for Exact Match score''' ,  )
      a						:Union[str, Any] 							=       make_precision_recall_eval(
          UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  out_image=os.path.join(UpperCAmelCase_ ,  '''pr_f1.png'''						) ,  title='''Precision-Recall curve for F1 score''' ,  )
      a						:Union[str, Any] 							=       {k: float(UpperCAmelCase_						) for k, v in qid_to_has_ans.items()}
      a						:int 							=       make_precision_recall_eval(
          UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  out_image=os.path.join(UpperCAmelCase_ ,  '''pr_oracle.png'''						) ,  title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)''' ,  )
      merge_eval(UpperCAmelCase_ ,  UpperCAmelCase_ ,  '''pr_exact'''						)
      merge_eval(UpperCAmelCase_ ,  UpperCAmelCase_ ,  '''pr_f1'''						)
      merge_eval(UpperCAmelCase_ ,  UpperCAmelCase_ ,  '''pr_oracle'''						)
def 				__lowerCamelCase						( UpperCAmelCase_						:		List[str] ,  UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		List[str] ,  UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      if not qid_list:
            return
      a						:List[Any] 							=       [na_probs[k] for k in qid_list]
      a						:List[str] 							=       np.ones_like(UpperCAmelCase_						) / float(len(UpperCAmelCase_						)						)
      plt.hist(UpperCAmelCase_ ,  weights=UpperCAmelCase_ ,  bins=20 ,  range=(0.0, 1.0)						)
      plt.xlabel('''Model probability of no-answer'''						)
      plt.ylabel('''Proportion of dataset'''						)
      plt.title(F'''Histogram of no-answer probability: {name}'''						)
      plt.savefig(os.path.join(UpperCAmelCase_ ,  F'''na_prob_hist_{name}.png'''						)						)
      plt.clf()
def 				__lowerCamelCase						( UpperCAmelCase_						:		List[str] ,  UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		str ,  UpperCAmelCase_						:		Dict						):




      """simple docstring"""





      a						:str 							=       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]						)
      a						:List[Any] 							=       num_no_ans
      a						:str 							=       cur_score
      a						:List[Any] 							=       0.0
      a						:Tuple 							=       sorted(UpperCAmelCase_ ,  key=lambda UpperCAmelCase_						: na_probs[k]						)
      for i, qid in enumerate(UpperCAmelCase_						):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  a						:Dict 							=       scores[qid]
            else:
                  if preds[qid]:
                        a						:Optional[int] 							=       -1
                  else:
                        a						:Optional[Any] 							=       0
            cur_score += diff
            if cur_score > best_score:
                  a						:Tuple 							=       cur_score
                  a						:Dict 							=       na_probs[qid]
      return 100.0 * best_score / len(UpperCAmelCase_						), best_thresh
def 				__lowerCamelCase						( UpperCAmelCase_						:		Union[str, Any] ,  UpperCAmelCase_						:		Optional[Any] ,  UpperCAmelCase_						:		Tuple ,  UpperCAmelCase_						:		Dict ,  UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		str						):




      """simple docstring"""





      a							,				a						:List[Any] 							=       find_best_thresh(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_						)
      a							,				a						:Optional[int] 							=       find_best_thresh(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_						)
      a						:Optional[Any] 							=       best_exact
      a						:int 							=       exact_thresh
      a						:Any 							=       best_fa
      a						:int 							=       fa_thresh
def 				__lowerCamelCase						( ):




      """simple docstring"""





      with open(OPTS.data_file						) as f:
            a						:Dict 							=       json.load(UpperCAmelCase_						)
            a						:List[str] 							=       dataset_json['''data''']
      with open(OPTS.pred_file						) as f:
            a						:Dict 							=       json.load(UpperCAmelCase_						)
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file						) as f:
                  a						:Dict 							=       json.load(UpperCAmelCase_						)
      else:
            a						:Any 							=       {k: 0.0 for k in preds}
      a						:int 							=       make_qid_to_has_ans(UpperCAmelCase_						)  # maps qid to True/False
      a						:Optional[Any] 							=       [k for k, v in qid_to_has_ans.items() if v]
      a						:Dict 							=       [k for k, v in qid_to_has_ans.items() if not v]
      a							,				a						:List[Any] 							=       get_raw_scores(UpperCAmelCase_ ,  UpperCAmelCase_						)
      a						:Any 							=       apply_no_ans_threshold(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  OPTS.na_prob_thresh						)
      a						:Any 							=       apply_no_ans_threshold(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  OPTS.na_prob_thresh						)
      a						:List[Any] 							=       make_eval_dict(UpperCAmelCase_ ,  UpperCAmelCase_						)
      if has_ans_qids:
            a						:Tuple 							=       make_eval_dict(UpperCAmelCase_ ,  UpperCAmelCase_ ,  qid_list=UpperCAmelCase_						)
            merge_eval(UpperCAmelCase_ ,  UpperCAmelCase_ ,  '''HasAns'''						)
      if no_ans_qids:
            a						:Optional[int] 							=       make_eval_dict(UpperCAmelCase_ ,  UpperCAmelCase_ ,  qid_list=UpperCAmelCase_						)
            merge_eval(UpperCAmelCase_ ,  UpperCAmelCase_ ,  '''NoAns'''						)
      if OPTS.na_prob_file:
            find_all_best_thresh(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_						)
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  UpperCAmelCase_ ,  OPTS.out_image_dir						)
            histogram_na_prob(UpperCAmelCase_ ,  UpperCAmelCase_ ,  OPTS.out_image_dir ,  '''hasAns'''						)
            histogram_na_prob(UpperCAmelCase_ ,  UpperCAmelCase_ ,  OPTS.out_image_dir ,  '''noAns'''						)
      if OPTS.out_file:
            with open(OPTS.out_file ,  '''w'''						) as f:
                  json.dump(UpperCAmelCase_ ,  UpperCAmelCase_						)
      else:
            print(json.dumps(UpperCAmelCase_ ,  indent=2						)						)


if __name__ == "__main__":
     snake_case				:    Optional[Any]       				=     parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('''Agg''')
          import matplotlib.pyplot as plt
     main()









def 				__lowerCamelCase						( UpperCAmelCase_						:		int = 1000						):




      """simple docstring"""





      a							,				a						:int 							=       1, 1
      a						:Any 							=       2
      while True:
            a						:Optional[int] 							=       0
            a						:str 							=       fa + fa
            a							,				a						:List[Any] 							=       fa, f
            index += 1
            for _ in str(UpperCAmelCase_						):
                  i += 1
            if i == n:
                  break
      return index


if __name__ == "__main__":
     print(solution(int(str(input()).strip())))





