

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase__		      =       re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCAmelCase__		      =       None



def     lowerCAmelCase__							(		)	->					str:




				'''simple docstring'''





				A__						=	argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."					)
				parser.add_argument("data_file"		,				metavar="data.json"		,				help="Input data JSON file."					)
				parser.add_argument("pred_file"		,				metavar="pred.json"		,				help="Model predictions."					)
				parser.add_argument(
				    "--out-file"		,				"-o"		,				metavar="eval.json"		,				help="Write accuracy metrics to file (default is stdout)."					)
				parser.add_argument(
				    "--na-prob-file"		,				"-n"		,				metavar="na_prob.json"		,				help="Model estimates of probability of no answer."					)
				parser.add_argument(
				    "--na-prob-thresh"		,				"-t"		,				type=__lowerCAmelCase		,				default=1.0		,				help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."		,				)
				parser.add_argument(
				    "--out-image-dir"		,				"-p"		,				metavar="out_images"		,				default=__lowerCAmelCase		,				help="Save precision-recall curves to directory."					)
				parser.add_argument("--verbose"		,				"-v"		,				action="store_true"					)
				if len(sys.argv					) == 1:
								parser.print_help()
								sys.exit(1					)
				return parser.parse_args()



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							List[str]					)	->					Union[str, Any]:




				'''simple docstring'''





				A__						=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																A__						=	bool(qa["answers"]["text"]					)
				return qid_to_has_ans



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Dict					)	->					Optional[Any]:




				'''simple docstring'''





				def remove_articles(SCREAMING_SNAKE_CASE_:							Optional[int]					):
								return ARTICLES_REGEX.sub(" "		,				__lowerCAmelCase					)

				def white_space_fix(SCREAMING_SNAKE_CASE_:							Optional[int]					):
								return " ".join(text.split()					)

				def remove_punc(SCREAMING_SNAKE_CASE_:							Union[str, Any]					):
								A__						=	set(string.punctuation					)
								return "".join(ch for ch in text if ch not in exclude					)

				def lower(SCREAMING_SNAKE_CASE_:							Dict					):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase					)					)					)					)



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							List[Any]					)	->					Optional[int]:




				'''simple docstring'''





				if not s:
								return []
				return normalize_answer(__lowerCAmelCase					).split()



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Tuple		,				SCREAMING_SNAKE_CASE_:							Tuple					)	->					int:




				'''simple docstring'''





				return int(normalize_answer(__lowerCAmelCase					) == normalize_answer(__lowerCAmelCase					)					)



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Any		,				SCREAMING_SNAKE_CASE_:							Tuple					)	->					str:




				'''simple docstring'''





				A__						=	get_tokens(__lowerCAmelCase					)
				A__						=	get_tokens(__lowerCAmelCase					)
				A__						=	collections.Counter(__lowerCAmelCase					) & collections.Counter(__lowerCAmelCase					)
				A__						=	sum(common.values()					)
				if len(__lowerCAmelCase					) == 0 or len(__lowerCAmelCase					) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks					)
				if num_same == 0:
								return 0
				A__						=	1.0 * num_same / len(__lowerCAmelCase					)
				A__						=	1.0 * num_same / len(__lowerCAmelCase					)
				A__						=	(2 * precision * recall) / (precision + recall)
				return fa



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Dict		,				SCREAMING_SNAKE_CASE_:							List[str]					)	->					Optional[Any]:




				'''simple docstring'''





				A__						=	{}
				A__						=	{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																A__						=	qa["id"]
																A__						=	[t for t in qa["answers"]["text"] if normalize_answer(__lowerCAmelCase					)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				A__						=	[""]
																if qid not in preds:
																				print(F'Missing prediction for {qid}'					)
																				continue
																A__						=	preds[qid]
																# Take max over all gold answers
																A__						=	max(compute_exact(__lowerCAmelCase		,				__lowerCAmelCase					) for a in gold_answers					)
																A__						=	max(compute_fa(__lowerCAmelCase		,				__lowerCAmelCase					) for a in gold_answers					)
				return exact_scores, fa_scores



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Tuple		,				SCREAMING_SNAKE_CASE_:							str		,				SCREAMING_SNAKE_CASE_:							Optional[int]		,				SCREAMING_SNAKE_CASE_:							Union[str, Any]					)	->					List[str]:




				'''simple docstring'''





				A__						=	{}
				for qid, s in scores.items():
								A__						=	na_probs[qid] > na_prob_thresh
								if pred_na:
												A__						=	float(not qid_to_has_ans[qid]					)
								else:
												A__						=	s
				return new_scores



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							List[Any]		,				SCREAMING_SNAKE_CASE_:							Optional[int]		,				SCREAMING_SNAKE_CASE_:							Optional[Any]=None					)	->					Union[str, Any]:




				'''simple docstring'''





				if not qid_list:
								A__						=	len(__lowerCAmelCase					)
								return collections.OrderedDict(
								    [
								        ("exact", 100.0 * sum(exact_scores.values()					) / total),
								        ("f1", 100.0 * sum(fa_scores.values()					) / total),
								        ("total", total),
								    ]					)
				else:
								A__						=	len(__lowerCAmelCase					)
								return collections.OrderedDict(
								    [
								        ("exact", 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
								        ("f1", 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
								        ("total", total),
								    ]					)



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Any		,				SCREAMING_SNAKE_CASE_:							str		,				SCREAMING_SNAKE_CASE_:							Optional[Any]					)	->					int:




				'''simple docstring'''





				for k in new_eval:
								A__						=	new_eval[k]



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Optional[int]		,				SCREAMING_SNAKE_CASE_:							List[Any]		,				SCREAMING_SNAKE_CASE_:							Dict		,				SCREAMING_SNAKE_CASE_:							Union[str, Any]					)	->					Optional[Any]:




				'''simple docstring'''





				plt.step(__lowerCAmelCase		,				__lowerCAmelCase		,				color="b"		,				alpha=0.2		,				where="post"					)
				plt.fill_between(__lowerCAmelCase		,				__lowerCAmelCase		,				step="post"		,				alpha=0.2		,				color="b"					)
				plt.xlabel("Recall"					)
				plt.ylabel("Precision"					)
				plt.xlim([0.0, 1.05]					)
				plt.ylim([0.0, 1.05]					)
				plt.title(__lowerCAmelCase					)
				plt.savefig(__lowerCAmelCase					)
				plt.clf()



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Any		,				SCREAMING_SNAKE_CASE_:							Optional[int]		,				SCREAMING_SNAKE_CASE_:							Tuple		,				SCREAMING_SNAKE_CASE_:							Tuple		,				SCREAMING_SNAKE_CASE_:							Optional[Any]=None		,				SCREAMING_SNAKE_CASE_:							Tuple=None					)	->					int:




				'''simple docstring'''





				A__						=	sorted(__lowerCAmelCase		,				key=lambda SCREAMING_SNAKE_CASE_					: na_probs[k]					)
				A__						=	0.0
				A__						=	1.0
				A__						=	0.0
				A__						=	[1.0]
				A__						=	[0.0]
				A__						=	0.0
				for i, qid in enumerate(__lowerCAmelCase					):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								A__						=	true_pos / float(i + 1					)
								A__						=	true_pos / float(__lowerCAmelCase					)
								if i == len(__lowerCAmelCase					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__lowerCAmelCase					)
												recalls.append(__lowerCAmelCase					)
				if out_image:
								plot_pr_curve(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase					)
				return {"ap": 100.0 * avg_prec}



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Optional[Any]		,				SCREAMING_SNAKE_CASE_:							List[str]		,				SCREAMING_SNAKE_CASE_:							Tuple		,				SCREAMING_SNAKE_CASE_:							List[str]		,				SCREAMING_SNAKE_CASE_:							List[Any]		,				SCREAMING_SNAKE_CASE_:							List[Any]					)	->					List[Any]:




				'''simple docstring'''





				if out_image_dir and not os.path.exists(__lowerCAmelCase					):
								os.makedirs(__lowerCAmelCase					)
				A__						=	sum(1 for v in qid_to_has_ans.values() if v					)
				if num_true_pos == 0:
								return
				A__						=	make_precision_recall_eval(
				    __lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				out_image=os.path.join(__lowerCAmelCase		,				"pr_exact.png"					)		,				title="Precision-Recall curve for Exact Match score"		,				)
				A__						=	make_precision_recall_eval(
				    __lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				out_image=os.path.join(__lowerCAmelCase		,				"pr_f1.png"					)		,				title="Precision-Recall curve for F1 score"		,				)
				A__						=	{k: float(__lowerCAmelCase					) for k, v in qid_to_has_ans.items()}
				A__						=	make_precision_recall_eval(
				    __lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				out_image=os.path.join(__lowerCAmelCase		,				"pr_oracle.png"					)		,				title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"		,				)
				merge_eval(__lowerCAmelCase		,				__lowerCAmelCase		,				"pr_exact"					)
				merge_eval(__lowerCAmelCase		,				__lowerCAmelCase		,				"pr_f1"					)
				merge_eval(__lowerCAmelCase		,				__lowerCAmelCase		,				"pr_oracle"					)



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Optional[Any]		,				SCREAMING_SNAKE_CASE_:							Any		,				SCREAMING_SNAKE_CASE_:							Optional[Any]		,				SCREAMING_SNAKE_CASE_:							int					)	->					Optional[Any]:




				'''simple docstring'''





				if not qid_list:
								return
				A__						=	[na_probs[k] for k in qid_list]
				A__						=	np.ones_like(__lowerCAmelCase					) / float(len(__lowerCAmelCase					)					)
				plt.hist(__lowerCAmelCase		,				weights=__lowerCAmelCase		,				bins=2_0		,				range=(0.0, 1.0)					)
				plt.xlabel("Model probability of no-answer"					)
				plt.ylabel("Proportion of dataset"					)
				plt.title(F'Histogram of no-answer probability: {name}'					)
				plt.savefig(os.path.join(__lowerCAmelCase		,				F'na_prob_hist_{name}.png'					)					)
				plt.clf()



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Optional[Any]		,				SCREAMING_SNAKE_CASE_:							Optional[Any]		,				SCREAMING_SNAKE_CASE_:							List[str]		,				SCREAMING_SNAKE_CASE_:							Union[str, Any]					)	->					Optional[int]:




				'''simple docstring'''





				A__						=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
				A__						=	num_no_ans
				A__						=	cur_score
				A__						=	0.0
				A__						=	sorted(__lowerCAmelCase		,				key=lambda SCREAMING_SNAKE_CASE_					: na_probs[k]					)
				for i, qid in enumerate(__lowerCAmelCase					):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												A__						=	scores[qid]
								else:
												if preds[qid]:
																A__						=	-1
												else:
																A__						=	0
								cur_score += diff
								if cur_score > best_score:
												A__						=	cur_score
												A__						=	na_probs[qid]
				return 100.0 * best_score / len(__lowerCAmelCase					), best_thresh



def     lowerCAmelCase__							(		SCREAMING_SNAKE_CASE_:							Dict		,				SCREAMING_SNAKE_CASE_:							Optional[int]		,				SCREAMING_SNAKE_CASE_:							int		,				SCREAMING_SNAKE_CASE_:							Union[str, Any]		,				SCREAMING_SNAKE_CASE_:							int		,				SCREAMING_SNAKE_CASE_:							Optional[Any]					)	->					int:




				'''simple docstring'''





				A__			, A__						=	find_best_thresh(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase					)
				A__			, A__						=	find_best_thresh(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase					)
				A__						=	best_exact
				A__						=	exact_thresh
				A__						=	best_fa
				A__						=	fa_thresh



def     lowerCAmelCase__							(		)	->					Optional[int]:




				'''simple docstring'''





				with open(OPTS.data_file					) as f:
								A__						=	json.load(__lowerCAmelCase					)
								A__						=	dataset_json["data"]
				with open(OPTS.pred_file					) as f:
								A__						=	json.load(__lowerCAmelCase					)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file					) as f:
												A__						=	json.load(__lowerCAmelCase					)
				else:
								A__						=	{k: 0.0 for k in preds}
				A__						=	make_qid_to_has_ans(__lowerCAmelCase					)  # maps qid to True/False
				A__						=	[k for k, v in qid_to_has_ans.items() if v]
				A__						=	[k for k, v in qid_to_has_ans.items() if not v]
				A__			, A__						=	get_raw_scores(__lowerCAmelCase		,				__lowerCAmelCase					)
				A__						=	apply_no_ans_threshold(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				OPTS.na_prob_thresh					)
				A__						=	apply_no_ans_threshold(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				OPTS.na_prob_thresh					)
				A__						=	make_eval_dict(__lowerCAmelCase		,				__lowerCAmelCase					)
				if has_ans_qids:
								A__						=	make_eval_dict(__lowerCAmelCase		,				__lowerCAmelCase		,				qid_list=__lowerCAmelCase					)
								merge_eval(__lowerCAmelCase		,				__lowerCAmelCase		,				"HasAns"					)
				if no_ans_qids:
								A__						=	make_eval_dict(__lowerCAmelCase		,				__lowerCAmelCase		,				qid_list=__lowerCAmelCase					)
								merge_eval(__lowerCAmelCase		,				__lowerCAmelCase		,				"NoAns"					)
				if OPTS.na_prob_file:
								find_all_best_thresh(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase					)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				__lowerCAmelCase		,				OPTS.out_image_dir					)
								histogram_na_prob(__lowerCAmelCase		,				__lowerCAmelCase		,				OPTS.out_image_dir		,				"hasAns"					)
								histogram_na_prob(__lowerCAmelCase		,				__lowerCAmelCase		,				OPTS.out_image_dir		,				"noAns"					)
				if OPTS.out_file:
								with open(OPTS.out_file		,				"w"					) as f:
												json.dump(__lowerCAmelCase		,				__lowerCAmelCase					)
				else:
								print(json.dumps(__lowerCAmelCase		,				indent=2					)					)


if __name__ == "__main__":
	lowerCAmelCase__		      =       parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use("""Agg""")
		import matplotlib.pyplot as plt
	main()



import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE__   :			Optional[int]												=    logging.getLogger(__name__)

@dataclass
class    lowerCAmelCase__     :
		a__       :						str
		a__       :						List[str]
		a__       :						Optional[List[str]]





@dataclass
class    lowerCAmelCase__     :
		a__       :						List[int]
		a__       :						List[int]
		a__       :						Optional[List[int]]        =		None
		a__       :						Optional[List[int]]        =		None





class    lowerCAmelCase__     (	__lowercase			):
		a__       :						Optional[Any]        =		"""train"""
		a__       :						Optional[int]        =		"""dev"""
		a__       :						Dict        =		"""test"""





class    lowerCAmelCase__     :






		@staticmethod
		def  __A       (		SCREAMING_SNAKE_CASE__		:							Union[str, Any]							,			SCREAMING_SNAKE_CASE__		:							Union[Split, str]  )     ->							List[InputExample]:
						raise NotImplementedError






		@staticmethod
		def  __A       (		SCREAMING_SNAKE_CASE__		:							str  )     ->							List[str]:
						raise NotImplementedError







		@staticmethod
		def  __A       (		SCREAMING_SNAKE_CASE__		:							List[InputExample]							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							int							,			SCREAMING_SNAKE_CASE__		:							PreTrainedTokenizer							,			SCREAMING_SNAKE_CASE__		:							Optional[Any]=False							,			SCREAMING_SNAKE_CASE__		:							List[str]="[CLS]"							,			SCREAMING_SNAKE_CASE__		:							Tuple=1							,			SCREAMING_SNAKE_CASE__		:							str="[SEP]"							,			SCREAMING_SNAKE_CASE__		:							List[Any]=False							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]=False							,			SCREAMING_SNAKE_CASE__		:							Tuple=0							,			SCREAMING_SNAKE_CASE__		:							int=0							,			SCREAMING_SNAKE_CASE__		:							str=-1_00							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]=0							,			SCREAMING_SNAKE_CASE__		:							List[Any]=True							,			)     ->							List[InputFeatures]:
						__lowerCamelCase    					=	{label: i for i, label in enumerate(SCREAMING_SNAKE_CASE__  )}

						__lowerCamelCase    					=	[]
						for ex_index, example in enumerate(SCREAMING_SNAKE_CASE__  ):
										if ex_index % 1_00_00 == 0:
														logger.info('''Writing example %d of %d'''							,			SCREAMING_SNAKE_CASE__							,			len(SCREAMING_SNAKE_CASE__  )  )

										__lowerCamelCase    					=	[]
										__lowerCamelCase    					=	[]
										for word, label in zip(example.words							,			example.labels  ):
														__lowerCamelCase    					=	tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )

														# bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
														if len(SCREAMING_SNAKE_CASE__  ) > 0:
																		tokens.extend(SCREAMING_SNAKE_CASE__  )
																		# Use the real label id for the first token of the word, and padding ids for the remaining tokens
																		label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(SCREAMING_SNAKE_CASE__  ) - 1)  )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
										__lowerCamelCase    					=	tokenizer.num_special_tokens_to_add()
										if len(SCREAMING_SNAKE_CASE__  ) > max_seq_length - special_tokens_count:
														__lowerCamelCase    					=	tokens[: (max_seq_length - special_tokens_count)]
														__lowerCamelCase    					=	label_ids[: (max_seq_length - special_tokens_count)]

										# The convention in BERT is:
										# (a) For sequence pairs:
										#  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
										#  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
										# (b) For single sequences:
										#  tokens:   [CLS] the dog is hairy . [SEP]
										#  type_ids:   0   0   0   0  0     0   0
										#
										# Where "type_ids" are used to indicate whether this is the first
										# sequence or the second sequence. The embedding vectors for `type=0` and
										# `type=1` were learned during pre-training and are added to the wordpiece
										# embedding vector (and position vector). This is not *strictly* necessary
										# since the [SEP] token unambiguously separates the sequences, but it makes
										# it easier for the model to learn the concept of sequences.
										#
										# For classification tasks, the first vector (corresponding to [CLS]) is
										# used as the "sentence vector". Note that this only makes sense because
										# the entire model is fine-tuned.
										tokens += [sep_token]
										label_ids += [pad_token_label_id]
										if sep_token_extra:
														# roberta uses an extra separator b/w pairs of sentences
														tokens += [sep_token]
														label_ids += [pad_token_label_id]
										__lowerCamelCase    					=	[sequence_a_segment_id] * len(SCREAMING_SNAKE_CASE__  )

										if cls_token_at_end:
														tokens += [cls_token]
														label_ids += [pad_token_label_id]
														segment_ids += [cls_token_segment_id]
										else:
														__lowerCamelCase    					=	[cls_token] + tokens
														__lowerCamelCase    					=	[pad_token_label_id] + label_ids
														__lowerCamelCase    					=	[cls_token_segment_id] + segment_ids

										__lowerCamelCase    					=	tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )

										# The mask has 1 for real tokens and 0 for padding tokens. Only real
										# tokens are attended to.
										__lowerCamelCase    					=	[1 if mask_padding_with_zero else 0] * len(SCREAMING_SNAKE_CASE__  )

										# Zero-pad up to the sequence length.
										__lowerCamelCase    					=	max_seq_length - len(SCREAMING_SNAKE_CASE__  )
										if pad_on_left:
														__lowerCamelCase    					=	([pad_token] * padding_length) + input_ids
														__lowerCamelCase    					=	([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
														__lowerCamelCase    					=	([pad_token_segment_id] * padding_length) + segment_ids
														__lowerCamelCase    					=	([pad_token_label_id] * padding_length) + label_ids
										else:
														input_ids += [pad_token] * padding_length
														input_mask += [0 if mask_padding_with_zero else 1] * padding_length
														segment_ids += [pad_token_segment_id] * padding_length
														label_ids += [pad_token_label_id] * padding_length

										assert len(SCREAMING_SNAKE_CASE__  ) == max_seq_length
										assert len(SCREAMING_SNAKE_CASE__  ) == max_seq_length
										assert len(SCREAMING_SNAKE_CASE__  ) == max_seq_length
										assert len(SCREAMING_SNAKE_CASE__  ) == max_seq_length

										if ex_index < 5:
														logger.info('''*** Example ***'''  )
														logger.info('''guid: %s'''							,			example.guid  )
														logger.info('''tokens: %s'''							,			''' '''.join([str(SCREAMING_SNAKE_CASE__  ) for x in tokens]  )  )
														logger.info('''input_ids: %s'''							,			''' '''.join([str(SCREAMING_SNAKE_CASE__  ) for x in input_ids]  )  )
														logger.info('''input_mask: %s'''							,			''' '''.join([str(SCREAMING_SNAKE_CASE__  ) for x in input_mask]  )  )
														logger.info('''segment_ids: %s'''							,			''' '''.join([str(SCREAMING_SNAKE_CASE__  ) for x in segment_ids]  )  )
														logger.info('''label_ids: %s'''							,			''' '''.join([str(SCREAMING_SNAKE_CASE__  ) for x in label_ids]  )  )

										if "token_type_ids" not in tokenizer.model_input_names:
														__lowerCamelCase    					=	None

										features.append(
										    InputFeatures(
										        input_ids=SCREAMING_SNAKE_CASE__							,			attention_mask=SCREAMING_SNAKE_CASE__							,			token_type_ids=SCREAMING_SNAKE_CASE__							,			label_ids=SCREAMING_SNAKE_CASE__  )  )
						return features


if is_torch_available():
					import torch
					from torch import nn
					from torch.utils.data import Dataset

					class    lowerCAmelCase__     (	__lowercase			):
							a__       :						List[InputFeatures]
							a__       :						int        =		nn.CrossEntropyLoss().ignore_index






							def __init__(		self		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							TokenClassificationTask							,			SCREAMING_SNAKE_CASE__		:							str							,			SCREAMING_SNAKE_CASE__		:							PreTrainedTokenizer							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							str							,			SCREAMING_SNAKE_CASE__		:							Optional[int] = None							,			SCREAMING_SNAKE_CASE__		:							Union[str, Any]=False							,			SCREAMING_SNAKE_CASE__		:							Split = Split.train							,			)     ->							Union[str, Any]:
											# Load data features from cache or dataset file
											__lowerCamelCase    					=	os.path.join(
											    SCREAMING_SNAKE_CASE__							,			'''cached_{}_{}_{}'''.format(mode.value							,			tokenizer.__class__.__name__							,			str(SCREAMING_SNAKE_CASE__  )  )							,			)

											# Make sure only the first process in distributed training processes the dataset,
											# and the others will use the cache.
											__lowerCamelCase    					=	cached_features_file + '''.lock'''
											with FileLock(SCREAMING_SNAKE_CASE__  ):
															if os.path.exists(SCREAMING_SNAKE_CASE__  ) and not overwrite_cache:
																			logger.info(f'''Loading features from cached file {cached_features_file}'''  )
																			__lowerCamelCase    					=	torch.load(SCREAMING_SNAKE_CASE__  )
															else:
																			logger.info(f'''Creating features from dataset file at {data_dir}'''  )
																			__lowerCamelCase    					=	token_classification_task.read_examples_from_file(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  )
																			# TODO clean up all this to leverage built-in features of tokenizers
																			__lowerCamelCase    					=	token_classification_task.convert_examples_to_features(
																			    SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			cls_token_at_end=bool(model_type in ['''xlnet''']  )							,			cls_token=tokenizer.cls_token							,			cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0							,			sep_token=tokenizer.sep_token							,			sep_token_extra=SCREAMING_SNAKE_CASE__							,			pad_on_left=bool(tokenizer.padding_side == '''left'''  )							,			pad_token=tokenizer.pad_token_id							,			pad_token_segment_id=tokenizer.pad_token_type_id							,			pad_token_label_id=self.pad_token_label_id							,			)
																			logger.info(f'''Saving features into cached file {cached_features_file}'''  )
																			torch.save(self.features							,			SCREAMING_SNAKE_CASE__  )






							def __len__(		self		:							Dict  )     ->							str:
											return len(self.features  )







							def __getitem__(		self		:							Any							,			SCREAMING_SNAKE_CASE__		:							Dict  )     ->							InputFeatures:
											return self.features[i]


if is_tf_available():
					import tensorflow as tf

					class    lowerCAmelCase__     :
							a__       :						List[InputFeatures]
							a__       :						int        =		-100






							def __init__(		self		:							Optional[Any]							,			SCREAMING_SNAKE_CASE__		:							TokenClassificationTask							,			SCREAMING_SNAKE_CASE__		:							str							,			SCREAMING_SNAKE_CASE__		:							PreTrainedTokenizer							,			SCREAMING_SNAKE_CASE__		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							str							,			SCREAMING_SNAKE_CASE__		:							Optional[int] = None							,			SCREAMING_SNAKE_CASE__		:							Optional[Any]=False							,			SCREAMING_SNAKE_CASE__		:							Split = Split.train							,			)     ->							List[Any]:
											__lowerCamelCase    					=	token_classification_task.read_examples_from_file(SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__  )
											# TODO clean up all this to leverage built-in features of tokenizers
											__lowerCamelCase    					=	token_classification_task.convert_examples_to_features(
											    SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			SCREAMING_SNAKE_CASE__							,			cls_token_at_end=bool(model_type in ['''xlnet''']  )							,			cls_token=tokenizer.cls_token							,			cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0							,			sep_token=tokenizer.sep_token							,			sep_token_extra=SCREAMING_SNAKE_CASE__							,			pad_on_left=bool(tokenizer.padding_side == '''left'''  )							,			pad_token=tokenizer.pad_token_id							,			pad_token_segment_id=tokenizer.pad_token_type_id							,			pad_token_label_id=self.pad_token_label_id							,			)

											def gen():
															for ex in self.features:
																			if ex.token_type_ids is None:
																							yield (
																							    {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
																							    ex.label_ids,
																							)
																			else:
																							yield (
																							    {
																							        "input_ids": ex.input_ids,
																							        "attention_mask": ex.attention_mask,
																							        "token_type_ids": ex.token_type_ids,
																							    },
																							    ex.label_ids,
																							)

											if "token_type_ids" not in tokenizer.model_input_names:
															__lowerCamelCase    					=	tf.data.Dataset.from_generator(
															    SCREAMING_SNAKE_CASE__							,			({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa}, tf.intaa)							,			(
															        {'''input_ids''': tf.TensorShape([None]  ), '''attention_mask''': tf.TensorShape([None]  )},
															        tf.TensorShape([None]  ),
															    )							,			)
											else:
															__lowerCamelCase    					=	tf.data.Dataset.from_generator(
															    SCREAMING_SNAKE_CASE__							,			({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa, '''token_type_ids''': tf.intaa}, tf.intaa)							,			(
															        {
															            '''input_ids''': tf.TensorShape([None]  ),
															            '''attention_mask''': tf.TensorShape([None]  ),
															            '''token_type_ids''': tf.TensorShape([None]  ),
															        },
															        tf.TensorShape([None]  ),
															    )							,			)






							def  __A       (		self		:							Union[str, Any]  )     ->							Union[str, Any]:
											__lowerCamelCase    					=	self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features  )  )  )

											return self.dataset






							def __len__(		self		:							List[Any]  )     ->							Any:
											return len(self.features  )







							def __getitem__(		self		:							List[str]							,			SCREAMING_SNAKE_CASE__		:							Optional[int]  )     ->							InputFeatures:
											return self.features[i]



