




import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __lowerCAmelCase (       __magic_name__			,				unittest.TestCase ):

			"""simple docstring"""







			snake_case_   		=       None
			snake_case_   		=       BloomTokenizerFast
			snake_case_   		=       BloomTokenizerFast
			snake_case_   		=       True
			snake_case_   		=       False
			snake_case_   		=       '''tokenizer_file'''
			snake_case_   		=       {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}


			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							super().setUp()
							__lowerCamelCase		       =    BloomTokenizerFast.from_pretrained('bigscience/tokenizer'							)
							tokenizer.save_pretrained(self.tmpdirname							)


			def       lowercase_    (      self			,						**lowerCamelCase__							)    ->      Optional[int]:



							'''simple docstring'''
							kwargs.update(self.special_tokens_map							)
							return BloomTokenizerFast.from_pretrained(self.tmpdirname			,						**lowerCamelCase__							)


			def       lowercase_    (      self							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    self.get_rust_tokenizer()

							__lowerCamelCase		       =    ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
							__lowerCamelCase		       =    [[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

							__lowerCamelCase		       =    tokenizer.batch_encode_plus(lowerCamelCase__							)['input_ids']
							self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__							)

							__lowerCamelCase		       =    tokenizer.batch_decode(lowerCamelCase__							)
							self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__=6							)    ->      Optional[int]:



							'''simple docstring'''
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
															__lowerCamelCase		       =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__			,						**lowerCamelCase__							)
															# tokenizer_r.pad_token = None # Hotfixing padding = None
															# Simple input
															__lowerCamelCase		       =    'This is a simple input'
															__lowerCamelCase		       =    ['This is a simple input 1', 'This is a simple input 2']
															__lowerCamelCase		       =    ('This is a simple input', 'This is a pair')
															__lowerCamelCase		       =    [
															    ('This is a simple input 1', 'This is a simple input 2'),
															    ('This is a simple pair 1', 'This is a simple pair 2'),
															]

															# Simple input tests
															try:
																			tokenizer_r.encode(lowerCamelCase__			,						max_length=lowerCamelCase__							)
																			tokenizer_r.encode_plus(lowerCamelCase__			,						max_length=lowerCamelCase__							)

																			tokenizer_r.batch_encode_plus(lowerCamelCase__			,						max_length=lowerCamelCase__							)
																			tokenizer_r.encode(lowerCamelCase__			,						max_length=lowerCamelCase__							)
																			tokenizer_r.batch_encode_plus(lowerCamelCase__			,						max_length=lowerCamelCase__							)
															except ValueError:
																			self.fail('Bloom Tokenizer should be able to deal with padding'							)

															__lowerCamelCase		       =    None  # Hotfixing padding = None
															self.assertRaises(lowerCamelCase__			,						tokenizer_r.encode			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'							)

															# Simple input
															self.assertRaises(lowerCamelCase__			,						tokenizer_r.encode_plus			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'							)

															# Simple input
															self.assertRaises(
															    lowerCamelCase__			,						tokenizer_r.batch_encode_plus			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'			,						)

															# Pair input
															self.assertRaises(lowerCamelCase__			,						tokenizer_r.encode			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'							)

															# Pair input
															self.assertRaises(lowerCamelCase__			,						tokenizer_r.encode_plus			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'							)

															# Pair input
															self.assertRaises(
															    lowerCamelCase__			,						tokenizer_r.batch_encode_plus			,						lowerCamelCase__			,						max_length=lowerCamelCase__			,						padding='max_length'			,						)


			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    self.get_rust_tokenizer()
							__lowerCamelCase		       =    load_dataset('xnli'			,						'all_languages'			,						split='test'			,						streaming=lowerCamelCase__							)

							__lowerCamelCase		       =    next(iter(lowerCamelCase__							)							)['premise']  # pick up one data
							__lowerCamelCase		       =    list(sample_data.values()							)

							__lowerCamelCase		       =    list(map(tokenizer.encode			,						lowerCamelCase__							)							)
							__lowerCamelCase		       =    [tokenizer.decode(lowerCamelCase__			,						clean_up_tokenization_spaces=lowerCamelCase__							) for x in output_tokens]
							self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__							)






			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
							# any sequence length constraints. This test of the parent class will fail since it relies on the
							# maximum sequence length of the positoonal embeddings.
							self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map							)			,						1							)
							self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()							)[0]							)			,						1							)








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A        =    data_utils.TransfoXLTokenizer
__A        =    data_utils.TransfoXLCorpus
__A        =    data_utils
__A        =    data_utils







def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Optional[int] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[Any]      )     ->       Union[str, Any]:





				"""simple docstring"""


				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(UpperCamelCase__ ,     'rb'      ) as fp:
												__lowerCamelCase		       =    pickle.load(UpperCamelCase__ ,     encoding='latin1'      )
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
								print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""      )
								__lowerCamelCase		       =    corpus.vocab.__dict__
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

								__lowerCamelCase		       =    corpus.__dict__
								corpus_dict_no_vocab.pop('vocab' ,     UpperCamelCase__      )
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + CORPUS_NAME
								print(F"""Save dataset to {pytorch_dataset_dump_path}"""      )
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )

								print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""      )
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												__lowerCamelCase		       =    TransfoXLConfig()
								else:
												__lowerCamelCase		       =    TransfoXLConfig.from_json_file(UpperCamelCase__      )
								print(F"""Building PyTorch model from configuration: {config}"""      )
								__lowerCamelCase		       =    TransfoXLLMHeadModel(UpperCamelCase__      )

								__lowerCamelCase		       =    load_tf_weights_in_transfo_xl(UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__      )
								# Save pytorch-model
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								print(F"""Save PyTorch model to {os.path.abspath(UpperCamelCase__      )}"""      )
								torch.save(model.state_dict() ,     UpperCamelCase__      )
								print(F"""Save configuration file to {os.path.abspath(UpperCamelCase__      )}"""      )
								with open(UpperCamelCase__ ,     'w' ,     encoding='utf-8'      ) as f:
												f.write(config.to_json_string()      )


if __name__ == "__main__":
					__A        =    argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A        =    parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)


