




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A        =    get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class     __lowerCAmelCase (       __magic_name__			,				unittest.TestCase ):

			"""simple docstring"""







			snake_case_   		=       XLNetTokenizer
			snake_case_   		=       XLNetTokenizerFast
			snake_case_   		=       True
			snake_case_   		=       True


			def       lowercase_    (      self							)    ->      Union[str, Any]:



							'''simple docstring'''
							super().setUp()

							# We have a SentencePiece fixture for testing
							__lowerCamelCase		       =    XLNetTokenizer(lowerCamelCase__			,						keep_accents=lowerCamelCase__							)
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname							)


			def       lowercase_    (      self							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    '<s>'
							__lowerCamelCase		       =    1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__							)			,						lowerCamelCase__							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__							)			,						lowerCamelCase__							)


			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]			,						'<unk>'							)
							self.assertEqual(vocab_keys[1]			,						'<s>'							)
							self.assertEqual(vocab_keys[-1]			,						'<eod>'							)
							self.assertEqual(len(lowerCamelCase__							)			,						1_006							)


			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							self.assertEqual(self.get_tokenizer().vocab_size			,						1_000							)


			def       lowercase_    (      self							)    ->      List[str]:



							'''simple docstring'''
							__lowerCamelCase		       =    XLNetTokenizer(lowerCamelCase__			,						keep_accents=lowerCamelCase__							)

							__lowerCamelCase		       =    tokenizer.tokenize('This is a test'							)
							self.assertListEqual(lowerCamelCase__			,						['▁This', '▁is', '▁a', '▁t', 'est']							)

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__							)			,						[285, 46, 10, 170, 382]							)

							__lowerCamelCase		       =    tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
							self.assertListEqual(
							    lowerCamelCase__			,						[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        'é',
							        '.',
							    ]			,						)
							__lowerCamelCase		       =    tokenizer.convert_tokens_to_ids(lowerCamelCase__							)
							self.assertListEqual(lowerCamelCase__			,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							)

							__lowerCamelCase		       =    tokenizer.convert_ids_to_tokens(lowerCamelCase__							)
							self.assertListEqual(
							    lowerCamelCase__			,						[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '<unk>',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        '<unk>',
							        '.',
							    ]			,						)


			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    XLNetTokenizer(lowerCamelCase__			,						do_lower_case=lowerCamelCase__							)
							__lowerCamelCase		       =    tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
							self.assertListEqual(
							    lowerCamelCase__			,						[
							        SPIECE_UNDERLINE + '',
							        'i',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        'se',
							        '.',
							    ]			,						)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)			,						['▁he', 'll', 'o']							)


			def       lowercase_    (      self							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    XLNetTokenizer(lowerCamelCase__			,						do_lower_case=lowerCamelCase__							)
							__lowerCamelCase		       =    tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
							self.assertListEqual(
							    lowerCamelCase__			,						[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        'se',
							        '.',
							    ]			,						)


			@slow
			def       lowercase_    (      self							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    XLNetTokenizer.from_pretrained('xlnet-base-cased'							)

							__lowerCamelCase		       =    tokenizer.encode('sequence builders'			,						add_special_tokens=lowerCamelCase__							)
							__lowerCamelCase		       =    tokenizer.encode('multi-sequence build'			,						add_special_tokens=lowerCamelCase__							)

							__lowerCamelCase		       =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__							)
							__lowerCamelCase		       =    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__			,						lowerCamelCase__							)

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]






			@slow
			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							# fmt: off
							__lowerCamelCase		       =    {'input_ids': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowerCamelCase__			,						model_name='xlnet-base-cased'			,						revision='c841166438c31ec7ca9a106dee7bb312b73ae511'			,						)








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A        =    data_utils.TransfoXLTokenizer
__A        =    data_utils.TransfoXLCorpus
__A        =    data_utils
__A        =    data_utils







def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Optional[int] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[Any]      )     ->       Union[str, Any]:





				"""simple docstring"""


				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(UpperCamelCase__ ,     'rb'      ) as fp:
												__lowerCamelCase		       =    pickle.load(UpperCamelCase__ ,     encoding='latin1'      )
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
								print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""      )
								__lowerCamelCase		       =    corpus.vocab.__dict__
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

								__lowerCamelCase		       =    corpus.__dict__
								corpus_dict_no_vocab.pop('vocab' ,     UpperCamelCase__      )
								__lowerCamelCase		       =    pytorch_dump_folder_path + '/' + CORPUS_NAME
								print(F"""Save dataset to {pytorch_dataset_dump_path}"""      )
								torch.save(UpperCamelCase__ ,     UpperCamelCase__      )

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.abspath(UpperCamelCase__      )

								print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""      )
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												__lowerCamelCase		       =    TransfoXLConfig()
								else:
												__lowerCamelCase		       =    TransfoXLConfig.from_json_file(UpperCamelCase__      )
								print(F"""Building PyTorch model from configuration: {config}"""      )
								__lowerCamelCase		       =    TransfoXLLMHeadModel(UpperCamelCase__      )

								__lowerCamelCase		       =    load_tf_weights_in_transfo_xl(UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__      )
								# Save pytorch-model
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
								print(F"""Save PyTorch model to {os.path.abspath(UpperCamelCase__      )}"""      )
								torch.save(model.state_dict() ,     UpperCamelCase__      )
								print(F"""Save configuration file to {os.path.abspath(UpperCamelCase__      )}"""      )
								with open(UpperCamelCase__ ,     'w' ,     encoding='utf-8'      ) as f:
												f.write(config.to_json_string()      )


if __name__ == "__main__":
					__A        =    argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A        =    parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)


