



'''simple docstring'''




import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    GPTNeoXForCausalLM,
						    GPTNeoXForQuestionAnswering,
						    GPTNeoXForSequenceClassification,
						    GPTNeoXForTokenClassification,
						    GPTNeoXModel,
						)






class 						_UpperCamelCase  :


			'''simple docstring'''

			def __init__(							self     : Optional[int]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : List[Any]=1_3							,     lowerCAmelCase__     : int=7							,     lowerCAmelCase__     : Optional[int]=True							,     lowerCAmelCase__     : Dict=True							,     lowerCAmelCase__     : str=True							,     lowerCAmelCase__     : List[str]=True							,     lowerCAmelCase__     : Union[str, Any]=9_9							,     lowerCAmelCase__     : List[str]=6_4							,     lowerCAmelCase__     : Union[str, Any]=5							,     lowerCAmelCase__     : Union[str, Any]=4							,     lowerCAmelCase__     : Optional[Any]=3_7							,     lowerCAmelCase__     : List[Any]="gelu"							,     lowerCAmelCase__     : Union[str, Any]=0.1							,     lowerCAmelCase__     : Union[str, Any]=0.1							,     lowerCAmelCase__     : Optional[Any]=5_1_2							,     lowerCAmelCase__     : Tuple=1_6							,     lowerCAmelCase__     : Tuple=2							,     lowerCAmelCase__     : str=0.02							,     lowerCAmelCase__     : List[str]=3							,     lowerCAmelCase__     : List[Any]=4							,     lowerCAmelCase__     : Dict=None							,     ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		parent
					__SCREAMING_SNAKE_CASE       :		Dict           =		batch_size
					__SCREAMING_SNAKE_CASE       :		str           =		seq_length
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		is_training
					__SCREAMING_SNAKE_CASE       :		Any           =		use_input_mask
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		use_token_type_ids
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		use_labels
					__SCREAMING_SNAKE_CASE       :		Any           =		vocab_size
					__SCREAMING_SNAKE_CASE       :		Dict           =		hidden_size
					__SCREAMING_SNAKE_CASE       :		Tuple           =		num_hidden_layers
					__SCREAMING_SNAKE_CASE       :		Dict           =		num_attention_heads
					__SCREAMING_SNAKE_CASE       :		int           =		intermediate_size
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		hidden_act
					__SCREAMING_SNAKE_CASE       :		str           =		hidden_dropout_prob
					__SCREAMING_SNAKE_CASE       :		Tuple           =		attention_probs_dropout_prob
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		max_position_embeddings
					__SCREAMING_SNAKE_CASE       :		Dict           =		type_vocab_size
					__SCREAMING_SNAKE_CASE       :		Tuple           =		type_sequence_label_size
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		initializer_range
					__SCREAMING_SNAKE_CASE       :		int           =		num_labels
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		num_choices
					__SCREAMING_SNAKE_CASE       :		Dict           =		scope
					__SCREAMING_SNAKE_CASE       :		int           =		vocab_size - 1

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		str           =		ids_tensor([self.batch_size, self.seq_length]							,     self.vocab_size						)

					__SCREAMING_SNAKE_CASE       :		List[str]           =		None
					if self.use_input_mask:
							__SCREAMING_SNAKE_CASE       :		Dict           =		random_attention_mask([self.batch_size, self.seq_length]						)

					__SCREAMING_SNAKE_CASE       :		int           =		None
					if self.use_labels:
							__SCREAMING_SNAKE_CASE       :		Dict           =		ids_tensor([self.batch_size, self.seq_length]							,     self.num_labels						)

					__SCREAMING_SNAKE_CASE       :		int           =		self.get_config()

					return config, input_ids, input_mask, token_labels

			def UpperCamelCase__					(							self     : Dict						):





					"""simple docstring"""






					return GPTNeoXConfig(
					    vocab_size=self.vocab_size							,     hidden_size=self.hidden_size							,     num_hidden_layers=self.num_hidden_layers							,     num_attention_heads=self.num_attention_heads							,     intermediate_size=self.intermediate_size							,     hidden_act=self.hidden_act							,     hidden_dropout_prob=self.hidden_dropout_prob							,     attention_probs_dropout_prob=self.attention_probs_dropout_prob							,     max_position_embeddings=self.max_position_embeddings							,     type_vocab_size=self.type_vocab_size							,     is_decoder=lowerCAmelCase__							,     initializer_range=self.initializer_range							,     pad_token_id=self.pad_token_id							,     )

			def UpperCamelCase__					(							self     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		self.prepare_config_and_inputs()

					__SCREAMING_SNAKE_CASE       :		List[str]           =		True

					return config, input_ids, input_mask, token_labels

			def UpperCamelCase__					(							self     : List[str]							,     lowerCAmelCase__     : int							,     lowerCAmelCase__     : List[str]							,     lowerCAmelCase__     : str						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		GPTNeoXModel(config=lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Any           =		model(lowerCAmelCase__						)
					self.parent.assertEqual(result.last_hidden_state.shape							,     (self.batch_size, self.seq_length, self.hidden_size)						)

			def UpperCamelCase__					(							self     : int							,     lowerCAmelCase__     : Union[str, Any]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Tuple           =		True
					__SCREAMING_SNAKE_CASE       :		Dict           =		GPTNeoXModel(lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__						)
					self.parent.assertEqual(result.last_hidden_state.shape							,     (self.batch_size, self.seq_length, self.hidden_size)						)

			def UpperCamelCase__					(							self     : List[str]							,     lowerCAmelCase__     : Tuple							,     lowerCAmelCase__     : Tuple							,     lowerCAmelCase__     : List[Any]							,     lowerCAmelCase__     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		GPTNeoXForCausalLM(config=lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		Tuple           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     labels=lowerCAmelCase__						)
					self.parent.assertEqual(result.logits.shape							,     (self.batch_size, self.seq_length, self.vocab_size)						)

			def UpperCamelCase__					(							self     : Optional[int]							,     lowerCAmelCase__     : Tuple							,     lowerCAmelCase__     : Tuple							,     lowerCAmelCase__     : List[str]							,     lowerCAmelCase__     : Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		self.num_labels
					__SCREAMING_SNAKE_CASE       :		List[str]           =		GPTNeoXForQuestionAnswering(lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__						)
					self.parent.assertEqual(result.start_logits.shape							,     (self.batch_size, self.seq_length)						)
					self.parent.assertEqual(result.end_logits.shape							,     (self.batch_size, self.seq_length)						)

			def UpperCamelCase__					(							self     : List[Any]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : int							,     lowerCAmelCase__     : Dict							,     lowerCAmelCase__     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		self.num_labels
					__SCREAMING_SNAKE_CASE       :		str           =		GPTNeoXForSequenceClassification(lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		ids_tensor([self.batch_size]							,     self.type_sequence_label_size						)
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     labels=lowerCAmelCase__						)
					self.parent.assertEqual(result.logits.shape							,     (self.batch_size, self.num_labels)						)

			def UpperCamelCase__					(							self     : str							,     lowerCAmelCase__     : Any							,     lowerCAmelCase__     : Union[str, Any]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : int						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		self.num_labels
					__SCREAMING_SNAKE_CASE       :		str           =		GPTNeoXForTokenClassification(lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()
					__SCREAMING_SNAKE_CASE       :		Any           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     labels=lowerCAmelCase__						)
					self.parent.assertEqual(result.logits.shape							,     (self.batch_size, self.seq_length, self.num_labels)						)

			def UpperCamelCase__					(							self     : List[Any]							,     lowerCAmelCase__     : Optional[Any]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		True
					__SCREAMING_SNAKE_CASE       :		str           =		GPTNeoXForCausalLM(config=lowerCAmelCase__						)
					model.to(lowerCAmelCase__						)
					model.eval()

					# first forward pass
					__SCREAMING_SNAKE_CASE       :		int           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     use_cache=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Tuple           =		outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					__SCREAMING_SNAKE_CASE       :		Any           =		ids_tensor((self.batch_size, 3)							,     config.vocab_size						)
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		ids_tensor((self.batch_size, 3)							,     vocab_size=2						)

					# append to next input_ids and
					__SCREAMING_SNAKE_CASE       :		str           =		torch.cat([input_ids, next_tokens]							,     dim=-1						)
					__SCREAMING_SNAKE_CASE       :		Any           =		torch.cat([input_mask, next_mask]							,     dim=-1						)

					__SCREAMING_SNAKE_CASE       :		List[Any]           =		model(lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     output_hidden_states=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Tuple           =		output_from_no_past["""hidden_states"""][0]
					__SCREAMING_SNAKE_CASE       :		Dict           =		model(
					    lowerCAmelCase__							,     attention_mask=lowerCAmelCase__							,     past_key_values=lowerCAmelCase__							,     output_hidden_states=lowerCAmelCase__							,     )["""hidden_states"""][0]

					# select random slice
					__SCREAMING_SNAKE_CASE       :		str           =		ids_tensor((1,)							,     output_from_past.shape[-1]						).item()
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		output_from_no_past[:, -3:, random_slice_idx].detach()
					__SCREAMING_SNAKE_CASE       :		List[str]           =		output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(lowerCAmelCase__							,     lowerCAmelCase__							,     atol=1E-3						)						)



			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		self.prepare_config_and_inputs()
					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		List[Any]           =		config_and_inputs
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		{"""input_ids""": input_ids, """attention_mask""": input_mask}
					return config, inputs_dict






@require_torch
class 						_UpperCamelCase  (							lowerCamelCase__    ,					lowerCamelCase__    ,					lowerCamelCase__    ,					unittest.TestCase    ):


			'''simple docstring'''

			_A      :		int			    =		(
			    (
			        GPTNeoXModel,
			        GPTNeoXForCausalLM,
			        GPTNeoXForQuestionAnswering,
			        GPTNeoXForSequenceClassification,
			        GPTNeoXForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			_A      :		int			    =		(GPTNeoXForCausalLM,) if is_torch_available() else ()
			_A      :		Any			    =		(
			    {
			        '''feature-extraction''': GPTNeoXModel,
			        '''question-answering''': GPTNeoXForQuestionAnswering,
			        '''text-classification''': GPTNeoXForSequenceClassification,
			        '''text-generation''': GPTNeoXForCausalLM,
			        '''token-classification''': GPTNeoXForTokenClassification,
			        '''zero-shot''': GPTNeoXForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			_A      :		Dict			    =		False
			_A      :		Dict			    =		False
			_A      :		str			    =		False
			_A      :		Optional[int]			    =		False

			def UpperCamelCase__					(							self     : Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		int           =		GPTNeoXModelTester(self						)
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		ConfigTester(self							,     config_class=lowerCAmelCase__							,     hidden_size=6_4							,     num_attention_heads=8						)

			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					self.config_tester.run_common_tests()

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		int           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(lowerCAmelCase__							,     lowerCAmelCase__							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		str           =		self.model_tester.prepare_config_and_inputs_for_decoder()
					self.model_tester.create_and_check_model_as_decoder(lowerCAmelCase__							,     lowerCAmelCase__							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Tuple           =		self.model_tester.prepare_config_and_inputs_for_decoder()

					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		None

					self.model_tester.create_and_check_model_as_decoder(lowerCAmelCase__							,     lowerCAmelCase__							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Tuple           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_decoder_model_past_large_inputs(lowerCAmelCase__							,     lowerCAmelCase__							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : str						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		int           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_causal_lm(*lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[str]           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : str						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__						)

			@unittest.skip(reason="""Feed forward chunking is not implemented"""						)
			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					pass



			@parameterized.expand([("""linear""",), ("""dynamic""",)]						)
			def UpperCamelCase__					(							self     : Union[str, Any]							,     lowerCAmelCase__     : Optional[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		self.model_tester.prepare_config_and_inputs_for_common()
					__SCREAMING_SNAKE_CASE       :		List[str]           =		ids_tensor([1, 1_0]							,     config.vocab_size						)
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		ids_tensor([1, int(config.max_position_embeddings * 1.5						)]							,     config.vocab_size						)

					set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
					__SCREAMING_SNAKE_CASE       :		Any           =		GPTNeoXModel(lowerCAmelCase__						)
					original_model.to(lowerCAmelCase__						)
					original_model.eval()
					__SCREAMING_SNAKE_CASE       :		Any           =		original_model(lowerCAmelCase__						).last_hidden_state
					__SCREAMING_SNAKE_CASE       :		int           =		original_model(lowerCAmelCase__						).last_hidden_state

					set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		{"""type""": scaling_type, """factor""": 10.0}
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		GPTNeoXModel(lowerCAmelCase__						)
					scaled_model.to(lowerCAmelCase__						)
					scaled_model.eval()
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		scaled_model(lowerCAmelCase__						).last_hidden_state
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		scaled_model(lowerCAmelCase__						).last_hidden_state

					# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
					# maximum sequence length, so the outputs for the short input should match.
					if scaling_type == "dynamic":
							self.assertTrue(torch.allclose(lowerCAmelCase__							,     lowerCAmelCase__							,     atol=1E-5						)						)
					else:
							self.assertFalse(torch.allclose(lowerCAmelCase__							,     lowerCAmelCase__							,     atol=1E-5						)						)

					# The output should be different for long inputs
					self.assertFalse(torch.allclose(lowerCAmelCase__							,     lowerCAmelCase__							,     atol=1E-5						)						)






@require_torch
class 						_UpperCamelCase  (							unittest.TestCase    ):


			'''simple docstring'''

			@slow
			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		AutoTokenizer.from_pretrained("""EleutherAI/pythia-410m-deduped"""						)
					for checkpointing in [True, False]:
							__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		GPTNeoXForCausalLM.from_pretrained("""EleutherAI/pythia-410m-deduped"""						)

							if checkpointing:
									model.gradient_checkpointing_enable()
							else:
									model.gradient_checkpointing_disable()
							model.to(lowerCAmelCase__						)

							__SCREAMING_SNAKE_CASE       :		Dict           =		tokenizer("""My favorite food is"""							,     return_tensors="""pt"""						).to(lowerCAmelCase__						)
							# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
							# See: https://github.com/huggingface/transformers/pull/24193
							__SCREAMING_SNAKE_CASE       :		Tuple           =		"""My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"""

							__SCREAMING_SNAKE_CASE       :		Optional[int]           =		model.generate(**lowerCAmelCase__							,     do_sample=lowerCAmelCase__							,     max_new_tokens=2_0						)
							__SCREAMING_SNAKE_CASE       :		Dict           =		tokenizer.batch_decode(lowerCAmelCase__						)[0]

							self.assertEqual(lowerCAmelCase__							,     lowerCAmelCase__						)




'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


UpperCamelCase__       :	List[str]    = logging.get_logger(__name__)


UpperCamelCase__       :	Tuple    = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
UpperCamelCase__       :	Tuple    = {
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase__       :	List[Any]    = {
    '''allenai/led-base-16384''': 1_63_84,
}


@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 				lowerCAmelCase_   (	):
		__SCREAMING_SNAKE_CASE       :		Tuple           =		(
		    list(range(ord("""!"""					)       ,      ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					)       ,      ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					)       ,      ord("""ÿ"""					) + 1					)					)
		)
		__SCREAMING_SNAKE_CASE       :		Any           =		bs[:]
		__SCREAMING_SNAKE_CASE       :		Tuple           =		0
		for b in range(2**8					):
				if b not in bs:
						bs.append(_lowerCamelCase					)
						cs.append(2**8 + n					)
						n += 1
		__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		[chr(_lowerCamelCase					) for n in cs]
		return dict(zip(_lowerCamelCase       ,      _lowerCamelCase					)					)


def 				lowerCAmelCase_   (	_lowerCamelCase:   Optional[int]					):
		__SCREAMING_SNAKE_CASE       :		Dict           =		set()
		__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)					)
				__SCREAMING_SNAKE_CASE       :		str           =		char
		return pairs






class 						_UpperCamelCase  (							lowerCamelCase__    ):


			'''simple docstring'''

			_A      :		Union[str, Any]			    =		VOCAB_FILES_NAMES
			_A      :		Any			    =		PRETRAINED_VOCAB_FILES_MAP
			_A      :		Any			    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A      :		Optional[Any]			    =		['''input_ids''', '''attention_mask''']

			def __init__(							self     : Tuple							,     lowerCAmelCase__     : List[str]							,     lowerCAmelCase__     : Optional[Any]							,     lowerCAmelCase__     : Union[str, Any]="replace"							,     lowerCAmelCase__     : Dict="<s>"							,     lowerCAmelCase__     : List[str]="</s>"							,     lowerCAmelCase__     : Tuple="</s>"							,     lowerCAmelCase__     : Tuple="<s>"							,     lowerCAmelCase__     : Union[str, Any]="<unk>"							,     lowerCAmelCase__     : Union[str, Any]="<pad>"							,     lowerCAmelCase__     : int="<mask>"							,     lowerCAmelCase__     : str=False							,     **lowerCAmelCase__     : int							,     ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else bos_token
					__SCREAMING_SNAKE_CASE       :		List[str]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else eos_token
					__SCREAMING_SNAKE_CASE       :		List[str]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else sep_token
					__SCREAMING_SNAKE_CASE       :		Dict           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else cls_token
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else unk_token
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		AddedToken(lowerCAmelCase__							,     lstrip=lowerCAmelCase__							,     rstrip=lowerCAmelCase__						) if isinstance(lowerCAmelCase__							,     lowerCAmelCase__						) else mask_token

					super().__init__(
					    errors=lowerCAmelCase__							,     bos_token=lowerCAmelCase__							,     eos_token=lowerCAmelCase__							,     unk_token=lowerCAmelCase__							,     sep_token=lowerCAmelCase__							,     cls_token=lowerCAmelCase__							,     pad_token=lowerCAmelCase__							,     mask_token=lowerCAmelCase__							,     add_prefix_space=lowerCAmelCase__							,     **lowerCAmelCase__							,     )

					with open(lowerCAmelCase__							,     encoding="""utf-8"""						) as vocab_handle:
							__SCREAMING_SNAKE_CASE       :		str           =		json.load(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		List[str]           =		{v: k for k, v in self.encoder.items()}
					__SCREAMING_SNAKE_CASE       :		Dict           =		errors  # how to handle errors in decoding
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		bytes_to_unicode()
					__SCREAMING_SNAKE_CASE       :		Tuple           =		{v: k for k, v in self.byte_encoder.items()}
					with open(lowerCAmelCase__							,     encoding="""utf-8"""						) as merges_handle:
							__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		merges_handle.read().split("""\n"""						)[1:-1]
					__SCREAMING_SNAKE_CASE       :		int           =		[tuple(merge.split()						) for merge in bpe_merges]
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		dict(zip(lowerCAmelCase__							,     range(len(lowerCAmelCase__						)						)						)						)
					__SCREAMING_SNAKE_CASE       :		int           =		{}
					__SCREAMING_SNAKE_CASE       :		Any           =		add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__SCREAMING_SNAKE_CASE       :		str           =		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)

			@property
			# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					return len(self.encoder						)

			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					return dict(self.encoder							,     **self.added_tokens_encoder						)

			def UpperCamelCase__					(							self     : Dict							,     lowerCAmelCase__     : Any						):





					"""simple docstring"""






					if token in self.cache:
							return self.cache[token]
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		tuple(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Dict           =		get_pairs(lowerCAmelCase__						)

					if not pairs:
							return token

					while True:
							__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		min(lowerCAmelCase__							,     key=lambda lowerCAmelCase__						: self.bpe_ranks.get(lowerCAmelCase__							,     float("""inf"""						)						)						)
							if bigram not in self.bpe_ranks:
									break
							__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Dict           =		bigram
							__SCREAMING_SNAKE_CASE       :		List[Any]           =		[]
							__SCREAMING_SNAKE_CASE       :		Optional[int]           =		0
							while i < len(lowerCAmelCase__						):
									try:
											__SCREAMING_SNAKE_CASE       :		Dict           =		word.index(lowerCAmelCase__							,     lowerCAmelCase__						)
									except ValueError:
											new_word.extend(word[i:]						)
											break
									else:
											new_word.extend(word[i:j]						)
											__SCREAMING_SNAKE_CASE       :		Dict           =		j

									if word[i] == first and i < len(lowerCAmelCase__						) - 1 and word[i + 1] == second:
											new_word.append(first + second						)
											i += 2
									else:
											new_word.append(word[i]						)
											i += 1
							__SCREAMING_SNAKE_CASE       :		Tuple           =		tuple(lowerCAmelCase__						)
							__SCREAMING_SNAKE_CASE       :		Any           =		new_word
							if len(lowerCAmelCase__						) == 1:
									break
							else:
									__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		get_pairs(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		""" """.join(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		word
					return word

			def UpperCamelCase__					(							self     : Optional[Any]							,     lowerCAmelCase__     : Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		[]
					for token in re.findall(self.pat							,     lowerCAmelCase__						):
							__SCREAMING_SNAKE_CASE       :		Any           =		"""""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__						).split(""" """						)						)
					return bpe_tokens

			def UpperCamelCase__					(							self     : List[str]							,     lowerCAmelCase__     : List[str]						):





					"""simple docstring"""






					return self.encoder.get(lowerCAmelCase__							,     self.encoder.get(self.unk_token						)						)

			def UpperCamelCase__					(							self     : int							,     lowerCAmelCase__     : Optional[int]						):





					"""simple docstring"""






					return self.decoder.get(lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Optional[Any]							,     lowerCAmelCase__     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		str           =		"""""".join(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""							,     errors=self.errors						)
					return text

			def UpperCamelCase__					(							self     : List[str]							,     lowerCAmelCase__     : str							,     lowerCAmelCase__     : Optional[str] = None						):





					"""simple docstring"""






					if not os.path.isdir(lowerCAmelCase__						):
							logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
							return
					__SCREAMING_SNAKE_CASE       :		int           =		os.path.join(
					    lowerCAmelCase__							,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		os.path.join(
					    lowerCAmelCase__							,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

					with open(lowerCAmelCase__							,     """w"""							,     encoding="""utf-8"""						) as f:
							f.write(json.dumps(self.encoder							,     indent=2							,     sort_keys=lowerCAmelCase__							,     ensure_ascii=lowerCAmelCase__						) + """\n"""						)

					__SCREAMING_SNAKE_CASE       :		Tuple           =		0
					with open(lowerCAmelCase__							,     """w"""							,     encoding="""utf-8"""						) as writer:
							writer.write("""#version: 0.2\n"""						)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,     key=lambda lowerCAmelCase__						: kv[1]						):
									if index != token_index:
											logger.warning(
											    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
											    """ Please check that the tokenizer is not corrupted!"""						)
											__SCREAMING_SNAKE_CASE       :		List[Any]           =		token_index
									writer.write(""" """.join(lowerCAmelCase__						) + """\n"""						)
									index += 1

					return vocab_file, merge_file

			def UpperCamelCase__					(							self     : str							,     lowerCAmelCase__     : List[int]							,     lowerCAmelCase__     : Optional[List[int]] = None						):





					"""simple docstring"""






					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		[self.cls_token_id]
					__SCREAMING_SNAKE_CASE       :		str           =		[self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep

			def UpperCamelCase__					(							self     : Any							,     lowerCAmelCase__     : List[int]							,     lowerCAmelCase__     : Optional[List[int]] = None							,     lowerCAmelCase__     : bool = False						):





					"""simple docstring"""






					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCAmelCase__							,     token_ids_a=lowerCAmelCase__							,     already_has_special_tokens=lowerCAmelCase__						)

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCAmelCase__						)) + [1]
					return [1] + ([0] * len(lowerCAmelCase__						)) + [1, 1] + ([0] * len(lowerCAmelCase__						)) + [1]

			def UpperCamelCase__					(							self     : Any							,     lowerCAmelCase__     : List[int]							,     lowerCAmelCase__     : Optional[List[int]] = None						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		int           =		[self.sep_token_id]
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

			def UpperCamelCase__					(							self     : Optional[Any]							,     lowerCAmelCase__     : int							,     lowerCAmelCase__     : List[Any]=False							,     **lowerCAmelCase__     : Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		kwargs.pop("""add_prefix_space"""							,     self.add_prefix_space						)
					if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase__						) > 0 and not text[0].isspace()):
							__SCREAMING_SNAKE_CASE       :		int           =		""" """ + text
					return (text, kwargs)



			def UpperCamelCase__					(							self     : Optional[Any]							,     lowerCAmelCase__     : Union[Dict[str, EncodedInput], BatchEncoding]							,     lowerCAmelCase__     : Optional[int] = None							,     lowerCAmelCase__     : PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,     lowerCAmelCase__     : Optional[int] = None							,     lowerCAmelCase__     : Optional[bool] = None							,     ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		str           =		super()._pad(
					    encoded_inputs=lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding_strategy=lowerCAmelCase__							,     pad_to_multiple_of=lowerCAmelCase__							,     return_attention_mask=lowerCAmelCase__							,     )

					# Load from model defaults
					if return_attention_mask is None:
							__SCREAMING_SNAKE_CASE       :		Tuple           =		"""attention_mask""" in self.model_input_names

					if return_attention_mask and "global_attention_mask" in encoded_inputs:
							__SCREAMING_SNAKE_CASE       :		str           =		encoded_inputs[self.model_input_names[0]]
							# `global_attention_mask` need to have the same length as other (sequential) inputs.
							__SCREAMING_SNAKE_CASE       :		str           =		len(encoded_inputs["""global_attention_mask"""]						) != len(lowerCAmelCase__						)

							if needs_to_be_padded:
									__SCREAMING_SNAKE_CASE       :		Dict           =		len(lowerCAmelCase__						) - len(encoded_inputs["""global_attention_mask"""]						)

									if self.padding_side == "right":
											# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
											__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		(
											    encoded_inputs["""global_attention_mask"""] + [-1] * difference
											)
									elif self.padding_side == "left":
											__SCREAMING_SNAKE_CASE       :		Dict           =		[-1] * difference + encoded_inputs[
											    """global_attention_mask"""
											]
									else:
											raise ValueError("""Invalid padding strategy:""" + str(self.padding_side						)						)

					return encoded_inputs