


'''simple docstring'''

from abc import ABC, abstractmethod
from typing import List, Optional





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self     ):







   """simple docstring"""
   # test for the above condition
   self.test()






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  0
   lowerCamelCase_      =  False
   while not completed:
    if counter == 1:
     self.reset()
    lowerCamelCase_      =  self.advance()
    if not self.does_advance(UpperCamelCase     ):
     raise Exception(
         "Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."     )

    lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  self.update(UpperCamelCase     )
    counter += 1

    if counter > 1_0000:
     raise Exception("update() does not fulfill the constraint."     )

   if self.remaining() != 0:
    raise Exception("Custom Constraint is not defined correctly."     )






  @abstractmethod
  def   snake_case		(    self     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )






  @abstractmethod
  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )






  @abstractmethod
  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )






  @abstractmethod
  def   snake_case		(    self     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )






  @abstractmethod
  def   snake_case		(    self     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )







  @abstractmethod
  def   snake_case		(    self			,  UpperCamelCase=False     ):







   """simple docstring"""
   raise NotImplementedError(
       f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   super(UpperCamelCase			,  self     ).__init__()

   if not isinstance(UpperCamelCase			,  UpperCamelCase     ) or len(UpperCamelCase     ) == 0:
    raise ValueError(f'''`token_ids` has to be a non-empty list, but is {token_ids}.'''     )
   if any((not isinstance(UpperCamelCase			,  UpperCamelCase     ) or token_id < 0) for token_id in token_ids     ):
    raise ValueError(f'''Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'''     )

   lowerCamelCase_      =  token_ids

   lowerCamelCase_      =  len(self.token_ids     )
   lowerCamelCase_      =  -1  # the index of the currently fulfilled step
   lowerCamelCase_      =  False






  def   snake_case		(    self     ):







   """simple docstring"""
   if self.completed:
    return None
   return self.token_ids[self.fulfilled_idx + 1]






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not isinstance(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(f'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

   if self.completed:
    return False

   return token_id == self.token_ids[self.fulfilled_idx + 1]






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not isinstance(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(f'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

   lowerCamelCase_      =  False
   lowerCamelCase_      =  False
   lowerCamelCase_      =  False

   if self.does_advance(UpperCamelCase     ):
    self.fulfilled_idx += 1
    lowerCamelCase_      =  True
    if self.fulfilled_idx == (self.seqlen - 1):
     lowerCamelCase_      =  True
    lowerCamelCase_      =  completed
   else:
    # failed to make progress.
    lowerCamelCase_      =  True
    self.reset()
   return stepped, completed, reset






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  False
   lowerCamelCase_      =  0






  def   snake_case		(    self     ):







   """simple docstring"""
   return self.seqlen - (self.fulfilled_idx + 1)







  def   snake_case		(    self			,  UpperCamelCase=False     ):







   """simple docstring"""
   lowerCamelCase_      =  PhrasalConstraint(self.token_ids     )

   if stateful:
    lowerCamelCase_      =  self.seqlen
    lowerCamelCase_      =  self.fulfilled_idx
    lowerCamelCase_      =  self.completed

   return new_constraint





class      snake_case    :





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase=True     ):







   """simple docstring"""
   lowerCamelCase_      =  max([len(UpperCamelCase     ) for one in nested_token_ids]     )

   lowerCamelCase_      =  {}
   for token_ids in nested_token_ids:
    lowerCamelCase_      =  root
    for tidx, token_id in enumerate(UpperCamelCase     ):
     if token_id not in level:
      lowerCamelCase_      =  {}

     lowerCamelCase_      =  level[token_id]

   if no_subsets and self.has_subsets(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(
        "Each list in `nested_token_ids` can't be a complete subset of another list, but is"
        f''' {nested_token_ids}.'''     )

   lowerCamelCase_      =  root






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.trie

   for current_token in current_seq:
    lowerCamelCase_      =  start[current_token]

   lowerCamelCase_      =  list(start.keys()     )

   return next_tokens






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.next_tokens(UpperCamelCase     )

   return len(UpperCamelCase     ) == 0






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  list(root.values()     )
   if len(UpperCamelCase     ) == 0:
    return 1
   else:
    return sum([self.count_leaves(UpperCamelCase     ) for nn in next_nodes]     )







  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.count_leaves(UpperCamelCase     )
   return len(UpperCamelCase     ) != leaf_count





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   super(UpperCamelCase			,  self     ).__init__()

   if not isinstance(UpperCamelCase			,  UpperCamelCase     ) or len(UpperCamelCase     ) == 0:
    raise ValueError(f'''`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'''     )
   if any(not isinstance(UpperCamelCase			,  UpperCamelCase     ) for token_ids in nested_token_ids     ):
    raise ValueError(f'''`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'''     )
   if any(
       any((not isinstance(UpperCamelCase			,  UpperCamelCase     ) or token_id < 0) for token_id in token_ids     )
       for token_ids in nested_token_ids     ):
    raise ValueError(
        f'''Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'''     )

   lowerCamelCase_      =  DisjunctiveTrie(UpperCamelCase     )
   lowerCamelCase_      =  nested_token_ids

   lowerCamelCase_      =  self.trie.max_height
   lowerCamelCase_      =  []
   lowerCamelCase_      =  False






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.trie.next_tokens(self.current_seq     )

   if len(UpperCamelCase     ) == 0:
    return None
   else:
    return token_list






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not isinstance(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(f'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

   lowerCamelCase_      =  self.trie.next_tokens(self.current_seq     )

   return token_id in next_tokens






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not isinstance(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(f'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

   lowerCamelCase_      =  False
   lowerCamelCase_      =  False
   lowerCamelCase_      =  False

   if self.does_advance(UpperCamelCase     ):
    self.current_seq.append(UpperCamelCase     )
    lowerCamelCase_      =  True
   else:
    lowerCamelCase_      =  True
    self.reset()

   lowerCamelCase_      =  self.trie.reached_leaf(self.current_seq     )
   lowerCamelCase_      =  completed

   return stepped, completed, reset






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  False
   lowerCamelCase_      =  []






  def   snake_case		(    self     ):







   """simple docstring"""
   if self.completed:
    # since this can be completed without reaching max height
    return 0
   else:
    return self.seqlen - len(self.current_seq     )







  def   snake_case		(    self			,  UpperCamelCase=False     ):







   """simple docstring"""
   lowerCamelCase_      =  DisjunctiveConstraint(self.token_ids     )

   if stateful:
    lowerCamelCase_      =  self.seqlen
    lowerCamelCase_      =  self.current_seq
    lowerCamelCase_      =  self.completed

   return new_constraint





class      snake_case    :





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  constraints

   # max # of steps required to fulfill a given constraint
   lowerCamelCase_      =  max([c.seqlen for c in constraints]     )
   lowerCamelCase_      =  len(UpperCamelCase     )
   lowerCamelCase_      =  False

   self.init_state()






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   lowerCamelCase_      =  None
   lowerCamelCase_      =  [constraint.copy(stateful=UpperCamelCase     ) for constraint in self.constraints]






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  0
   if self.inprogress_constraint:
    # extra points for having a constraint mid-fulfilled
    add += self.max_seqlen - self.inprogress_constraint.remaining()

   return (len(self.complete_constraints     ) * self.max_seqlen) + add






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   if self.inprogress_constraint is None:
    for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
     lowerCamelCase_      =  constraint.advance()
     if isinstance(UpperCamelCase			,  UpperCamelCase     ):
      token_list.append(UpperCamelCase     )
     elif isinstance(UpperCamelCase			,  UpperCamelCase     ):
      token_list.extend(UpperCamelCase     )
   else:
    lowerCamelCase_      =  self.inprogress_constraint.advance()
    if isinstance(UpperCamelCase			,  UpperCamelCase     ):
     token_list.append(UpperCamelCase     )
    elif isinstance(UpperCamelCase			,  UpperCamelCase     ):
     token_list.extend(UpperCamelCase     )

   if len(UpperCamelCase     ) == 0:
    return None
   else:
    return token_list






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   self.init_state()

   if token_ids is not None:
    for token in token_ids:
     # completes or steps **one** constraint
     lowerCamelCase_ ,lowerCamelCase_      =  self.add(UpperCamelCase     )

     # the entire list of constraints are fulfilled
     if self.completed:
      break






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if not isinstance(UpperCamelCase			,  UpperCamelCase     ):
    raise ValueError(f'''`token_id` should be an `int`, but is `{token_id}`.'''     )

   lowerCamelCase_ ,lowerCamelCase_      =  False, False

   if self.completed:
    lowerCamelCase_      =  True
    lowerCamelCase_      =  False
    return complete, stepped

   if self.inprogress_constraint is not None:
    # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
    # job, simply update the state

    lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  self.inprogress_constraint.update(UpperCamelCase     )
    if reset:
     # 1. If the next token breaks the progress, then we must restart.
     #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

     #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
     #     constraint, not the full list of constraints.

     self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCamelCase     )     )
     lowerCamelCase_      =  None

    if complete:
     # 2. If the next token completes the constraint, move it to completed list, set
     #     inprogress to None. If there are no pending constraints either, then this full list of constraints
     #     is complete.

     self.complete_constraints.append(self.inprogress_constraint     )
     lowerCamelCase_      =  None

     if len(self.pending_constraints     ) == 0:
      # we're done!
      lowerCamelCase_      =  True

   else:
    # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
    # of constraints?

    for cidx, pending_constraint in enumerate(self.pending_constraints     ):
     if pending_constraint.does_advance(UpperCamelCase     ):
      lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  pending_constraint.update(UpperCamelCase     )

      if not stepped:
       raise Exception(
           "`constraint.update(token_id)` is not yielding incremental progress, "
           "even though `constraint.does_advance(token_id)` is true."     )

      if complete:
       self.complete_constraints.append(UpperCamelCase     )
       lowerCamelCase_      =  None

      if not complete and stepped:
       lowerCamelCase_      =  pending_constraint

      if complete or stepped:
       # If we made any progress at all, then it's at least not a "pending constraint".

       lowerCamelCase_      =  (
           self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
       )

       if len(self.pending_constraints     ) == 0 and self.inprogress_constraint is None:
        # If there's no longer any pending after this and no inprogress either, then we must be
        # complete.

        lowerCamelCase_      =  True

       break  # prevent accidentally stepping through multiple constraints with just one token.

   return complete, stepped







  def   snake_case		(    self			,  UpperCamelCase=True     ):







   """simple docstring"""
   lowerCamelCase_      =  ConstraintListState(self.constraints     )  # we actually never though self.constraints objects
   # throughout this process. So it's at initialization state.

   if stateful:
    lowerCamelCase_      =  [
        constraint.copy(stateful=UpperCamelCase     ) for constraint in self.complete_constraints
    ]
    if self.inprogress_constraint is not None:
     lowerCamelCase_      =  self.inprogress_constraint.copy(stateful=UpperCamelCase     )
    lowerCamelCase_      =  [constraint.copy() for constraint in self.pending_constraints]

   return new_state





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


SCREAMING_SNAKE_CASE__   :			Union[str, Any]												=    re.compile(r"\b(a|an|the)\b", re.UNICODE)

SCREAMING_SNAKE_CASE__   :			int												=    None




def  __magic_name__ (   )			->   str:
				__lowerCamelCase    					=	argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''							)
				parser.add_argument('''data_file'''		,  metavar='''data.json'''		,  help='''Input data JSON file.'''							)
				parser.add_argument('''pred_file'''		,  metavar='''pred.json'''		,  help='''Model predictions.'''							)
				parser.add_argument(
				    '''--out-file'''		,  '''-o'''		,  metavar='''eval.json'''		,  help='''Write accuracy metrics to file (default is stdout).'''							)
				parser.add_argument(
				    '''--na-prob-file'''		,  '''-n'''		,  metavar='''na_prob.json'''		,  help='''Model estimates of probability of no answer.'''							)
				parser.add_argument(
				    '''--na-prob-thresh'''		,  '''-t'''		,  type=__lowerCAmelCase		,  default=1.0		,  help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''		,  )
				parser.add_argument(
				    '''--out-image-dir'''		,  '''-p'''		,  metavar='''out_images'''		,  default=__lowerCAmelCase		,  help='''Save precision-recall curves to directory.'''							)
				parser.add_argument('''--verbose'''		,  '''-v'''		,  action='''store_true'''							)
				if len(sys.argv							) == 1:
								parser.print_help()
								sys.exit(1							)
				return parser.parse_args()




def  __magic_name__ (   __lowerCAmelCase	:						List[str]							)			->   Union[str, Any]:
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	bool(qa['''answers''']['''text''']							)
				return qid_to_has_ans




def  __magic_name__ (   __lowerCAmelCase	:						Dict							)			->   Optional[Any]:

				def remove_articles(__lowerCAmelCase	:						Optional[int]							):
								return ARTICLES_REGEX.sub(''' '''		,  __lowerCAmelCase							)

				def white_space_fix(__lowerCAmelCase	:						Optional[int]							):
								return " ".join(text.split()							)

				def remove_punc(__lowerCAmelCase	:						Union[str, Any]							):
								__lowerCamelCase    					=	set(string.punctuation							)
								return "".join(ch for ch in text if ch not in exclude							)

				def lower(__lowerCAmelCase	:						Dict							):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase							)							)							)							)




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]							)			->   Optional[int]:
				if not s:
								return []
				return normalize_answer(__lowerCAmelCase							).split()




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple							)			->   int:
				return int(normalize_answer(__lowerCAmelCase							) == normalize_answer(__lowerCAmelCase							)							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Tuple							)			->   str:
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	get_tokens(__lowerCAmelCase							)
				__lowerCamelCase    					=	collections.Counter(__lowerCAmelCase							) & collections.Counter(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(common.values()							)
				if len(__lowerCAmelCase							) == 0 or len(__lowerCAmelCase							) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks							)
				if num_same == 0:
								return 0
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	1.0 * num_same / len(__lowerCAmelCase							)
				__lowerCamelCase    					=	(2 * precision * recall) / (precision + recall)
				return fa




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						List[str]							)			->   Optional[Any]:
				__lowerCamelCase    					=	{}
				__lowerCamelCase    					=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																__lowerCamelCase    					=	qa['''id''']
																__lowerCamelCase    					=	[t for t in qa['''answers''']['''text'''] if normalize_answer(__lowerCAmelCase							)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				__lowerCamelCase    					=	['''''']
																if qid not in preds:
																				print(f'''Missing prediction for {qid}'''							)
																				continue
																__lowerCamelCase    					=	preds[qid]
																# Take max over all gold answers
																__lowerCamelCase    					=	max(compute_exact(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
																__lowerCamelCase    					=	max(compute_fa(__lowerCAmelCase		,  __lowerCAmelCase							) for a in gold_answers							)
				return exact_scores, fa_scores




def  __magic_name__ (   __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   List[str]:
				__lowerCamelCase    					=	{}
				for qid, s in scores.items():
								__lowerCamelCase    					=	na_probs[qid] > na_prob_thresh
								if pred_na:
												__lowerCamelCase    					=	float(not qid_to_has_ans[qid]							)
								else:
												__lowerCamelCase    					=	s
				return new_scores




def  __magic_name__ (   __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Optional[Any]=None							)			->   Union[str, Any]:
				if not qid_list:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores.values()							) / total),
								        ('''f1''', 100.0 * sum(fa_scores.values()							) / total),
								        ('''total''', total),
								    ]							)
				else:
								__lowerCamelCase    					=	len(__lowerCAmelCase							)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list							) / total),
								        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list							) / total),
								        ('''total''', total),
								    ]							)




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						str		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				for k in new_eval:
								__lowerCamelCase    					=	new_eval[k]




def  __magic_name__ (   __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[Any]:
				plt.step(__lowerCAmelCase		,  __lowerCAmelCase		,  color='''b'''		,  alpha=0.2		,  where='''post'''							)
				plt.fill_between(__lowerCAmelCase		,  __lowerCAmelCase		,  step='''post'''		,  alpha=0.2		,  color='''b'''							)
				plt.xlabel('''Recall'''							)
				plt.ylabel('''Precision'''							)
				plt.xlim([0.0, 1.05]							)
				plt.ylim([0.0, 1.05]							)
				plt.title(__lowerCAmelCase							)
				plt.savefig(__lowerCAmelCase							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						Optional[Any]=None		,  __lowerCAmelCase	:						Tuple=None							)			->   int:
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	1.0
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	[1.0]
				__lowerCamelCase    					=	[0.0]
				__lowerCamelCase    					=	0.0
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								__lowerCamelCase    					=	true_pos / float(i + 1							)
								__lowerCamelCase    					=	true_pos / float(__lowerCAmelCase							)
								if i == len(__lowerCAmelCase							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__lowerCAmelCase							)
												recalls.append(__lowerCAmelCase							)
				if out_image:
								plot_pr_curve(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				return {"ap": 100.0 * avg_prec}




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Tuple		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						List[Any]		,  __lowerCAmelCase	:						List[Any]							)			->   List[Any]:
				if out_image_dir and not os.path.exists(__lowerCAmelCase							):
								os.makedirs(__lowerCAmelCase							)
				__lowerCamelCase    					=	sum(1 for v in qid_to_has_ans.values() if v							)
				if num_true_pos == 0:
								return
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_exact.png'''							)		,  title='''Precision-Recall curve for Exact Match score'''		,  )
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_f1.png'''							)		,  title='''Precision-Recall curve for F1 score'''		,  )
				__lowerCamelCase    					=	{k: float(__lowerCAmelCase							) for k, v in qid_to_has_ans.items()}
				__lowerCamelCase    					=	make_precision_recall_eval(
				    __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  out_image=os.path.join(__lowerCAmelCase		,  '''pr_oracle.png'''							)		,  title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''		,  )
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_exact'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_f1'''							)
				merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''pr_oracle'''							)




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Any		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						int							)			->   Optional[Any]:
				if not qid_list:
								return
				__lowerCamelCase    					=	[na_probs[k] for k in qid_list]
				__lowerCamelCase    					=	np.ones_like(__lowerCAmelCase							) / float(len(__lowerCAmelCase							)							)
				plt.hist(__lowerCAmelCase		,  weights=__lowerCAmelCase		,  bins=20		,  range=(0.0, 1.0)							)
				plt.xlabel('''Model probability of no-answer'''							)
				plt.ylabel('''Proportion of dataset'''							)
				plt.title(f'''Histogram of no-answer probability: {name}'''							)
				plt.savefig(os.path.join(__lowerCAmelCase		,  f'''na_prob_hist_{name}.png'''							)							)
				plt.clf()




def  __magic_name__ (   __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						Optional[Any]		,  __lowerCAmelCase	:						List[str]		,  __lowerCAmelCase	:						Union[str, Any]							)			->   Optional[int]:
				__lowerCamelCase    					=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
				__lowerCamelCase    					=	num_no_ans
				__lowerCamelCase    					=	cur_score
				__lowerCamelCase    					=	0.0
				__lowerCamelCase    					=	sorted(__lowerCAmelCase		,  key=lambda __lowerCAmelCase							: na_probs[k]							)
				for i, qid in enumerate(__lowerCAmelCase							):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												__lowerCamelCase    					=	scores[qid]
								else:
												if preds[qid]:
																__lowerCamelCase    					=	-1
												else:
																__lowerCamelCase    					=	0
								cur_score += diff
								if cur_score > best_score:
												__lowerCamelCase    					=	cur_score
												__lowerCamelCase    					=	na_probs[qid]
				return 100.0 * best_score / len(__lowerCAmelCase							), best_thresh




def  __magic_name__ (   __lowerCAmelCase	:						Dict		,  __lowerCAmelCase	:						Optional[int]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Union[str, Any]		,  __lowerCAmelCase	:						int		,  __lowerCAmelCase	:						Optional[Any]							)			->   int:
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase					,    __lowerCamelCase    					=	find_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	best_exact
				__lowerCamelCase    					=	exact_thresh
				__lowerCamelCase    					=	best_fa
				__lowerCamelCase    					=	fa_thresh




def  __magic_name__ (   )			->   Optional[int]:
				with open(OPTS.data_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
								__lowerCamelCase    					=	dataset_json['''data''']
				with open(OPTS.pred_file							) as f:
								__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file							) as f:
												__lowerCamelCase    					=	json.load(__lowerCAmelCase							)
				else:
								__lowerCamelCase    					=	{k: 0.0 for k in preds}
				__lowerCamelCase    					=	make_qid_to_has_ans(__lowerCAmelCase							)  # maps qid to True/False
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if v]
				__lowerCamelCase    					=	[k for k, v in qid_to_has_ans.items() if not v]
				__lowerCamelCase					,    __lowerCamelCase    					=	get_raw_scores(__lowerCAmelCase		,  __lowerCAmelCase							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	apply_no_ans_threshold(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.na_prob_thresh							)
				__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase							)
				if has_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''HasAns'''							)
				if no_ans_qids:
								__lowerCamelCase    					=	make_eval_dict(__lowerCAmelCase		,  __lowerCAmelCase		,  qid_list=__lowerCAmelCase							)
								merge_eval(__lowerCAmelCase		,  __lowerCAmelCase		,  '''NoAns'''							)
				if OPTS.na_prob_file:
								find_all_best_thresh(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase							)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''hasAns'''							)
								histogram_na_prob(__lowerCAmelCase		,  __lowerCAmelCase		,  OPTS.out_image_dir		,  '''noAns'''							)
				if OPTS.out_file:
								with open(OPTS.out_file		,  '''w'''							) as f:
												json.dump(__lowerCAmelCase		,  __lowerCAmelCase							)
				else:
								print(json.dumps(__lowerCAmelCase		,  indent=2							)							)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE__   :			Any												=    parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use("Agg")
										import matplotlib.pyplot as plt
					main()



