




import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__A        =    logging.get_logger(__name__)


__A        =    {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}







def 			lowerCamelCase_		(       UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      Any ,     UpperCamelCase__				:      List[Any] ,     UpperCamelCase__				:      Any      )     ->       str:





				"""simple docstring"""


				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()      )}."""      )

				if tokenizer_name is None:
								__lowerCamelCase		       =    TOKENIZER_CLASSES
				else:
								__lowerCamelCase		       =    {tokenizer_name: getattr(UpperCamelCase__ ,     tokenizer_name + 'Fast'      )}

				logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""      )

				for tokenizer_name in tokenizer_names:
								__lowerCamelCase		       =    TOKENIZER_CLASSES[tokenizer_name]

								__lowerCamelCase		       =    True
								if checkpoint_name is None:
												__lowerCamelCase		       =    list(tokenizer_class.max_model_input_sizes.keys()      )
								else:
												__lowerCamelCase		       =    [checkpoint_name]

								logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""      )

								for checkpoint in checkpoint_names:
												logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""      )

												# Load tokenizer
												__lowerCamelCase		       =    tokenizer_class.from_pretrained(UpperCamelCase__ ,     force_download=UpperCamelCase__      )

												# Save fast tokenizer
												logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""      )

												# For organization names we create sub-directories
												if "/" in checkpoint:
																__lowerCamelCase       ,						__lowerCamelCase		       =    checkpoint.split('/'      )
																__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
												elif add_prefix:
																__lowerCamelCase		       =    checkpoint
																__lowerCamelCase		       =    dump_path
												else:
																__lowerCamelCase		       =    None
																__lowerCamelCase		       =    dump_path

												logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()      )[0]:
																__lowerCamelCase		       =    list(tokenizer.pretrained_vocab_files_map.values()      )[0][checkpoint]
																__lowerCamelCase		       =    file_path.split(UpperCamelCase__      )[-1][0]
																if next_char == "/":
																				__lowerCamelCase		       =    os.path.join(UpperCamelCase__ ,     UpperCamelCase__      )
																				__lowerCamelCase		       =    None

																logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

												__lowerCamelCase		       =    tokenizer.save_pretrained(
												    UpperCamelCase__ ,     legacy_format=UpperCamelCase__ ,     filename_prefix=UpperCamelCase__      )
												logger.info(F"""=> File names {file_names}"""      )

												for file_name in file_names:
																if not file_name.endswith('tokenizer.json'      ):
																				os.remove(UpperCamelCase__      )
																				logger.info(F"""=> removing {file_name}"""      )


if __name__ == "__main__":
					__A        =    argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
					)
					parser.add_argument(
					    "--tokenizer_name",
					    default=None,
					    type=str,
					    help=(
					        f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
					        "download and convert all the checkpoints from AWS."
					    ),
					)
					parser.add_argument(
					    "--checkpoint_name",
					    default=None,
					    type=str,
					    help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
					)
					parser.add_argument(
					    "--force_download",
					    action="store_true",
					    help="Re-download checkpoints.",
					)
					__A        =    parser.parse_args()

					convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)








import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
					import jax.numpy as jnp

if is_tf_available():
					import tensorflow as tf

if is_torch_available():
					import torch







class     __lowerCAmelCase (       unittest.TestCase ):

			"""simple docstring"""







			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    {
							    'task_specific_params': {
							        'summarization': {'length_penalty': 1.0, 'max_length': 128, 'min_length': 12, 'num_beams': 4},
							        'summarization_cnn': {'length_penalty': 2.0, 'max_length': 142, 'min_length': 56, 'num_beams': 4},
							        'summarization_xsum': {'length_penalty': 1.0, 'max_length': 62, 'min_length': 11, 'num_beams': 6},
							    }
							}
							__lowerCamelCase		       =    {
							    'task_specific_params.summarization.length_penalty': 1.0,
							    'task_specific_params.summarization.max_length': 128,
							    'task_specific_params.summarization.min_length': 12,
							    'task_specific_params.summarization.num_beams': 4,
							    'task_specific_params.summarization_cnn.length_penalty': 2.0,
							    'task_specific_params.summarization_cnn.max_length': 142,
							    'task_specific_params.summarization_cnn.min_length': 56,
							    'task_specific_params.summarization_cnn.num_beams': 4,
							    'task_specific_params.summarization_xsum.length_penalty': 1.0,
							    'task_specific_params.summarization_xsum.max_length': 62,
							    'task_specific_params.summarization_xsum.min_length': 11,
							    'task_specific_params.summarization_xsum.num_beams': 6,
							}

							self.assertEqual(flatten_dict(lowerCamelCase__							)			,						lowerCamelCase__							)


			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__							)			,						x.transpose()							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__			,						axes=(1, 2, 0)							)			,						x.transpose((1, 2, 0)							)							)							)


			@require_torch
			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__							)			,						transpose(lowerCamelCase__							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__			,						axes=(1, 2, 0)							)			,						transpose(lowerCamelCase__			,						axes=(1, 2, 0)							).numpy()							)							)


			@require_tf
			def       lowercase_    (      self							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__							)			,						transpose(lowerCamelCase__							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__			,						axes=(1, 2, 0)							)			,						transpose(lowerCamelCase__			,						axes=(1, 2, 0)							).numpy()							)							)


			@require_flax
			def       lowercase_    (      self							)    ->      Optional[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__							)			,						np.asarray(transpose(lowerCamelCase__							)							)							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(transpose(lowerCamelCase__			,						axes=(1, 2, 0)							)			,						np.asarray(transpose(lowerCamelCase__			,						axes=(1, 2, 0)							)							)							)							)


			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(4, 3)							)			,						np.reshape(lowerCamelCase__			,						(4, 3)							)							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(12, 5)							)			,						np.reshape(lowerCamelCase__			,						(12, 5)							)							)							)


			@require_torch
			def       lowercase_    (      self							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(4, 3)							)			,						reshape(lowerCamelCase__			,						(4, 3)							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(12, 5)							)			,						reshape(lowerCamelCase__			,						(12, 5)							).numpy()							)							)


			@require_tf
			def       lowercase_    (      self							)    ->      Optional[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(4, 3)							)			,						reshape(lowerCamelCase__			,						(4, 3)							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(12, 5)							)			,						reshape(lowerCamelCase__			,						(12, 5)							).numpy()							)							)


			@require_flax
			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(4, 3)							)			,						np.asarray(reshape(lowerCamelCase__			,						(4, 3)							)							)							)							)

							__lowerCamelCase		       =    np.random.randn(3			,						4			,						5							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(reshape(lowerCamelCase__			,						(12, 5)							)			,						np.asarray(reshape(lowerCamelCase__			,						(12, 5)							)							)							)							)


			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(1			,						3			,						4							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__							)			,						np.squeeze(lowerCamelCase__							)							)							)

							__lowerCamelCase		       =    np.random.randn(1			,						4			,						1			,						5							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__			,						axis=2							)			,						np.squeeze(lowerCamelCase__			,						axis=2							)							)							)


			@require_torch
			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(1			,						3			,						4							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__							)			,						squeeze(lowerCamelCase__							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(1			,						4			,						1			,						5							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__			,						axis=2							)			,						squeeze(lowerCamelCase__			,						axis=2							).numpy()							)							)


			@require_tf
			def       lowercase_    (      self							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(1			,						3			,						4							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__							)			,						squeeze(lowerCamelCase__							).numpy()							)							)

							__lowerCamelCase		       =    np.random.randn(1			,						4			,						1			,						5							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__			,						axis=2							)			,						squeeze(lowerCamelCase__			,						axis=2							).numpy()							)							)


			@require_flax
			def       lowercase_    (      self							)    ->      List[str]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(1			,						3			,						4							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__							)			,						np.asarray(squeeze(lowerCamelCase__							)							)							)							)

							__lowerCamelCase		       =    np.random.randn(1			,						4			,						1			,						5							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(squeeze(lowerCamelCase__			,						axis=2							)			,						np.asarray(squeeze(lowerCamelCase__			,						axis=2							)							)							)							)


			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							self.assertTrue(np.allclose(expand_dims(lowerCamelCase__			,						axis=1							)			,						np.expand_dims(lowerCamelCase__			,						axis=1							)							)							)


			@require_torch
			def       lowercase_    (      self							)    ->      Any:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    torch.tensor(lowerCamelCase__							)
							self.assertTrue(np.allclose(expand_dims(lowerCamelCase__			,						axis=1							)			,						expand_dims(lowerCamelCase__			,						axis=1							).numpy()							)							)


			@require_tf
			def       lowercase_    (      self							)    ->      Any:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    tf.constant(lowerCamelCase__							)
							self.assertTrue(np.allclose(expand_dims(lowerCamelCase__			,						axis=1							)			,						expand_dims(lowerCamelCase__			,						axis=1							).numpy()							)							)






			@require_flax
			def       lowercase_    (      self							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    np.random.randn(3			,						4							)
							__lowerCamelCase		       =    jnp.array(lowerCamelCase__							)
							self.assertTrue(np.allclose(expand_dims(lowerCamelCase__			,						axis=1							)			,						np.asarray(expand_dims(lowerCamelCase__			,						axis=1							)							)							)							)


