



'''simple docstring'''




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class 						_UpperCamelCase  (							lowerCamelCase__    ,					unittest.TestCase    ):


			'''simple docstring'''

			_A      :		List[str]			    =		TransfoXLTokenizer
			_A      :		List[Any]			    =		False
			_A      :		Dict			    =		False

			def UpperCamelCase__					(							self     : Optional[Any]						):





					"""simple docstring"""






					super().setUp()

					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		[
					    """<unk>""",
					    """[CLS]""",
					    """[SEP]""",
					    """want""",
					    """unwanted""",
					    """wa""",
					    """un""",
					    """running""",
					    """,""",
					    """low""",
					    """l""",
					]
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["""vocab_file"""]						)
					with open(self.vocab_file							,     """w"""							,     encoding="""utf-8"""						) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

			def UpperCamelCase__					(							self     : Optional[Any]							,     **lowerCAmelCase__     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Dict           =		True
					return TransfoXLTokenizer.from_pretrained(self.tmpdirname							,     **lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Tuple							,     lowerCAmelCase__     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		"""<unk> UNwanted , running"""
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		"""<unk> unwanted, running"""
					return input_text, output_text

			def UpperCamelCase__					(							self     : Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		TransfoXLTokenizer(vocab_file=self.vocab_file							,     lower_case=lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE       :		int           =		tokenizer.tokenize("""<unk> UNwanted , running"""						)
					self.assertListEqual(lowerCAmelCase__							,     ["""<unk>""", """unwanted""", """,""", """running"""]						)

					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__						)							,     [0, 4, 8, 7]						)

			def UpperCamelCase__					(							self     : Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		TransfoXLTokenizer(lower_case=lowerCAmelCase__						)

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """						)							,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]						)

			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Any           =		TransfoXLTokenizer(lower_case=lowerCAmelCase__						)

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """						)							,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)

			def UpperCamelCase__					(							self     : Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		TransfoXLTokenizer(lower_case=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		[
					    """Hello""",
					    """(""",
					    """bracket""",
					    """)""",
					    """and""",
					    """side""",
					    """@-@""",
					    """scrolled""",
					    """[""",
					    """and""",
					    """]""",
					    """Henry""",
					    """'s""",
					    """$""",
					    """5""",
					    """@,@""",
					    """000""",
					    """with""",
					    """3""",
					    """@.@""",
					    """34""",
					    """m""",
					    """.""",
					    """What""",
					    """'s""",
					    """up""",
					    """!""",
					    """?""",
					]

					self.assertListEqual(tokenizer.tokenize(lowerCAmelCase__						)							,     lowerCAmelCase__						)

					self.assertEqual(tokenizer.convert_tokens_to_string(lowerCAmelCase__						)							,     lowerCAmelCase__						)



			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		self.get_tokenizer()
					__SCREAMING_SNAKE_CASE       :		str           =		len(lowerCAmelCase__						)

					tokenizer.add_tokens(["""new1""", """new2"""]						)
					tokenizer.move_added_token("""new1"""							,     1						)

					# Check that moved token is not copied (duplicate)
					self.assertEqual(len(lowerCAmelCase__						)							,     original_len + 2						)
					# Check that token is moved to specified id
					self.assertEqual(tokenizer.encode("""new1"""						)							,     [1]						)
					self.assertEqual(tokenizer.decode([1]						)							,     """new1"""						)




'''simple docstring'''




from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

UpperCamelCase__       :	Dict    = [
    '''python''',
    '''tqdm''',
    '''regex''',
    '''requests''',
    '''packaging''',
    '''filelock''',
    '''numpy''',
    '''tokenizers''',
    '''huggingface-hub''',
    '''safetensors''',
    '''accelerate''',
    '''pyyaml''',
]

for pkg in pkgs_to_check_at_runtime:
						if pkg in deps:
												if pkg == "tokenizers":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_tokenizers_available

																		if not is_tokenizers_available():
																								continue  # not required, check version only if installed
												elif pkg == "accelerate":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_accelerate_available

																		# Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
																		# Transformers with PyTorch
																		if not is_accelerate_available():
																								continue  # not required, check version only if installed

												require_version_core(deps[pkg])
						else:
												raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py")


def 				lowerCAmelCase_   (	_lowerCamelCase:   List[Any]       ,      _lowerCamelCase:   Optional[int]=None					):
		require_version(deps[pkg]       ,      _lowerCamelCase					)