





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

UpperCamelCase__           =			logging.get_logger(__name__)


UpperCamelCase__           =			{name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}





def      _UpperCamelCase  (a__ :Dict ,      a__ :List[str] ,      a__ :Dict ,      a__ :Tuple							):






							"""simple docstring"""





							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()							)}."""							)

							if tokenizer_name is None:
														UpperCamelCase__					  =      TOKENIZER_CLASSES
							else:
														UpperCamelCase__					  =      {tokenizer_name: getattr(a__ ,      tokenizer_name + """Fast"""							)}

							logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""							)

							for tokenizer_name in tokenizer_names:
														UpperCamelCase__					  =      TOKENIZER_CLASSES[tokenizer_name]

														UpperCamelCase__					  =      True
														if checkpoint_name is None:
																					UpperCamelCase__					  =      list(tokenizer_class.max_model_input_sizes.keys()							)
														else:
																					UpperCamelCase__					  =      [checkpoint_name]

														logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""							)

														for checkpoint in checkpoint_names:
																					logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""							)

																					# Load tokenizer
																					UpperCamelCase__					  =      tokenizer_class.from_pretrained(a__ ,      force_download=a__							)

																					# Save fast tokenizer
																					logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""							)

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												UpperCamelCase__  ,  UpperCamelCase__					  =      checkpoint.split("""/"""							)
																												UpperCamelCase__					  =      os.path.join(a__ ,      a__							)
																					elif add_prefix:
																												UpperCamelCase__					  =      checkpoint
																												UpperCamelCase__					  =      dump_path
																					else:
																												UpperCamelCase__					  =      None
																												UpperCamelCase__					  =      dump_path

																					logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""							)

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()							)[0]:
																												UpperCamelCase__					  =      list(tokenizer.pretrained_vocab_files_map.values()							)[0][checkpoint]
																												UpperCamelCase__					  =      file_path.split(a__							)[-1][0]
																												if next_char == "/":
																																			UpperCamelCase__					  =      os.path.join(a__ ,      a__							)
																																			UpperCamelCase__					  =      None

																												logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""							)

																					UpperCamelCase__					  =      tokenizer.save_pretrained(
																					    a__ ,      legacy_format=a__ ,      filename_prefix=a__							)
																					logger.info(f"""=> File names {file_names}"""							)

																					for file_name in file_names:
																												if not file_name.endswith("""tokenizer.json"""							):
																																			os.remove(a__							)
																																			logger.info(f"""=> removing {file_name}"""							)


if __name__ == "__main__":
					UpperCamelCase__           =			argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
					)
					parser.add_argument(
					    "--tokenizer_name",
					    default=None,
					    type=str,
					    help=(
					        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
					        "download and convert all the checkpoints from AWS."
					    ),
					)
					parser.add_argument(
					    "--checkpoint_name",
					    default=None,
					    type=str,
					    help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
					)
					parser.add_argument(
					    "--force_download",
					    action="store_true",
					    help="Re-download checkpoints.",
					)
					UpperCamelCase__           =			parser.parse_args()

					convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)











import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							"""encoder-decoder"""
							snake_case  :   Optional[int]      =							True






							def __init__(    self	,			**__lowerCAmelCase	):
														super().__init__(**__lowerCAmelCase	)
														assert (
														    "encoder" in kwargs and "decoder" in kwargs
														), "Config has to be initialized with encoder and decoder config"
														UpperCamelCase__					  =      kwargs.pop("""encoder"""	)
														UpperCamelCase__					  =      encoder_config.pop("""model_type"""	)
														UpperCamelCase__					  =      kwargs.pop("""decoder"""	)
														UpperCamelCase__					  =      decoder_config.pop("""model_type"""	)

														from ..auto.configuration_auto import AutoConfig

														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      True






							@classmethod
							def 					_lowerCamelCase		(    cls	,			__lowerCAmelCase	,			__lowerCAmelCase	,			**__lowerCAmelCase	):
														logger.info("""Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"""	)
														UpperCamelCase__					  =      True
														UpperCamelCase__					  =      True

														return cls(encoder=encoder_config.to_dict()	,			decoder=decoder_config.to_dict()	,			**__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      copy.deepcopy(self.__dict__	)
														UpperCamelCase__					  =      self.encoder.to_dict()
														UpperCamelCase__					  =      self.decoder.to_dict()
														UpperCamelCase__					  =      self.__class__.model_type
														return output




