





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__UpperCAmelCase       :   Tuple				       =							logging.get_logger(__name__)


__UpperCAmelCase       :   int				       =							{name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}







def 			A__							(    SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)   ->    Dict:
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(F'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys())}.''')

  if tokenizer_name is None:
    __snake_case:						Tuple   =						TOKENIZER_CLASSES
  else:
    __snake_case:						int   =						{tokenizer_name: getattr(SCREAMING_SNAKE_CASE__					,					tokenizer_name + """Fast""")}

  logger.info(F'''Loading tokenizer classes: {tokenizer_names}''')

  for tokenizer_name in tokenizer_names:
    __snake_case:						Any   =						TOKENIZER_CLASSES[tokenizer_name]

    __snake_case:						str   =						True
    if checkpoint_name is None:
      __snake_case:						Dict   =						list(tokenizer_class.max_model_input_sizes.keys())
    else:
      __snake_case:						str   =						[checkpoint_name]

    logger.info(F'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}''')

    for checkpoint in checkpoint_names:
      logger.info(F'''Loading {tokenizer_class.__class__.__name__} {checkpoint}''')

      # Load tokenizer
      __snake_case:						int   =						tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__					,					force_download=SCREAMING_SNAKE_CASE__)

      # Save fast tokenizer
      logger.info(F'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}''')

      # For organization names we create sub-directories
      if "/" in checkpoint:
        __snake_case  ,   __snake_case:						List[str]   =						checkpoint.split("""/""")
        __snake_case:						Union[str, Any]   =						os.path.join(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)
      elif add_prefix:
        __snake_case:						Union[str, Any]   =						checkpoint
        __snake_case:						str   =						dump_path
      else:
        __snake_case:						Tuple   =						None
        __snake_case:						Tuple   =						dump_path

      logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}''')

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values())[0]:
        __snake_case:						Union[str, Any]   =						list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint]
        __snake_case:						Tuple   =						file_path.split(SCREAMING_SNAKE_CASE__)[-1][0]
        if next_char == "/":
          __snake_case:						Tuple   =						os.path.join(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)
          __snake_case:						List[Any]   =						None

        logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}''')

      __snake_case:						int   =						tokenizer.save_pretrained(
          SCREAMING_SNAKE_CASE__					,					legacy_format=SCREAMING_SNAKE_CASE__					,					filename_prefix=SCREAMING_SNAKE_CASE__)
      logger.info(F'''=> File names {file_names}''')

      for file_name in file_names:
        if not file_name.endswith("""tokenizer.json"""):
          os.remove(SCREAMING_SNAKE_CASE__)
          logger.info(F'''=> removing {file_name}''')


if __name__ == "__main__":
      __UpperCAmelCase       :   List[Any]				       =							argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
      )
      parser.add_argument(
          "--tokenizer_name",
          default=None,
          type=str,
          help=(
              f'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
              "download and convert all the checkpoints from AWS."
          ),
      )
      parser.add_argument(
          "--checkpoint_name",
          default=None,
          type=str,
          help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
      )
      parser.add_argument(
          "--force_download",
          action="store_true",
          help="Re-download checkpoints.",
      )
      __UpperCAmelCase       :   Union[str, Any]				       =							parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)













import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
__UpperCAmelCase       :   Any				       =							"src/diffusers"

# Matches is_xxx_available()
__UpperCAmelCase       :   List[str]				       =							re.compile(R"is\_([a-z_]*)_available\(\)")
# Matches from xxx import bla
__UpperCAmelCase       :   Dict				       =							re.compile(R"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")


__UpperCAmelCase       :   int				       =							"\n{0} = None\n"

__UpperCAmelCase       :   List[str]				       =							"\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n\n    @classmethod\n    def from_config(cls, *args, **kwargs):\n        requires_backends(cls, {1})\n\n    @classmethod\n    def from_pretrained(cls, *args, **kwargs):\n        requires_backends(cls, {1})\n"


__UpperCAmelCase       :   Tuple				       =							"\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n"







def 			A__							(    SCREAMING_SNAKE_CASE__)   ->    Any:
  __snake_case:						int   =						_re_backend.findall(SCREAMING_SNAKE_CASE__)
  if len(SCREAMING_SNAKE_CASE__) == 0:
    return None

  return "_and_".join(SCREAMING_SNAKE_CASE__)







def 			A__							(    )   ->    Optional[int]:
  with open(os.path.join(SCREAMING_SNAKE_CASE__					,					"""__init__.py""")					,					"""r"""					,					encoding="""utf-8"""					,					newline="""\n""") as f:
    __snake_case:						Optional[Any]   =						f.readlines()

  # Get to the point we do the actual imports for type checking
  __snake_case:						Tuple   =						0
  __snake_case:						Any   =						{}
  # Go through the end of the file
  while line_index < len(SCREAMING_SNAKE_CASE__):
    # If the line contains is_backend_available, we grab all objects associated with the `else` block
    __snake_case:						List[Any]   =						find_backend(lines[line_index])
    if backend is not None:
      while not lines[line_index].startswith("""else:"""):
        line_index += 1
      line_index += 1
      __snake_case:						Any   =						[]
      # Until we unindent, add backend objects to the list
      while line_index < len(SCREAMING_SNAKE_CASE__) and len(lines[line_index]) > 1:
        __snake_case:						List[Any]   =						lines[line_index]
        __snake_case:						str   =						_re_single_line_import.search(SCREAMING_SNAKE_CASE__)
        if single_line_import_search is not None:
          objects.extend(single_line_import_search.groups()[0].split(""", """))
        elif line.startswith(""" """ * 8):
          objects.append(line[8:-2])
        line_index += 1

      if len(SCREAMING_SNAKE_CASE__) > 0:
        __snake_case:						Optional[Any]   =						objects
    else:
      line_index += 1

  return backend_specific_objects







def 			A__							(    SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)   ->    List[Any]:
  if name.isupper():
    return DUMMY_CONSTANT.format(SCREAMING_SNAKE_CASE__)
  elif name.islower():
    return DUMMY_FUNCTION.format(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)
  else:
    return DUMMY_CLASS.format(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)







def 			A__							(    SCREAMING_SNAKE_CASE__=None)   ->    Optional[int]:
  if backend_specific_objects is None:
    __snake_case:						Union[str, Any]   =						read_init()
  # For special correspondence backend to module name as used in the function requires_modulename
  __snake_case:						Union[str, Any]   =						{}

  for backend, objects in backend_specific_objects.items():
    __snake_case:						List[Any]   =						"""[""" + """, """.join(F'''"{b}"''' for b in backend.split("""_and_""")) + """]"""
    __snake_case:						Dict   =						"""# This file is autogenerated by the command `make fix-copies`, do not edit.\n"""
    dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
    dummy_file += "\n".join([create_dummy_object(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__) for o in objects])
    __snake_case:						List[str]   =						dummy_file

  return dummy_files







def 			A__							(    SCREAMING_SNAKE_CASE__=False)   ->    int:
  __snake_case:						List[str]   =						create_dummy_files()
  # For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
  __snake_case:						Dict   =						{"""torch""": """pt"""}

  # Locate actual dummy modules and read their content.
  __snake_case:						Optional[Any]   =						os.path.join(SCREAMING_SNAKE_CASE__					,					"""utils""")
  __snake_case:						List[Any]   =						{
      backend: os.path.join(SCREAMING_SNAKE_CASE__					,					F'''dummy_{short_names.get(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)}_objects.py''')
      for backend in dummy_files.keys()
  }

  __snake_case:						int   =						{}
  for backend, file_path in dummy_file_paths.items():
    if os.path.isfile(SCREAMING_SNAKE_CASE__):
      with open(SCREAMING_SNAKE_CASE__					,					"""r"""					,					encoding="""utf-8"""					,					newline="""\n""") as f:
        __snake_case:						Any   =						f.read()
    else:
      __snake_case:						Tuple   =						""""""

  for backend in dummy_files.keys():
    if dummy_files[backend] != actual_dummies[backend]:
      if overwrite:
        print(
            F'''Updating diffusers.utils.dummy_{short_names.get(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)}_objects.py as the main '''
            """__init__ has new objects.""")
        with open(dummy_file_paths[backend]					,					"""w"""					,					encoding="""utf-8"""					,					newline="""\n""") as f:
          f.write(dummy_files[backend])
      else:
        raise ValueError(
            """The main __init__ has objects that are not present in """
            F'''diffusers.utils.dummy_{short_names.get(SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)}_objects.py. Run `make fix-copies` '''
            """to fix this.""")


if __name__ == "__main__":
      __UpperCAmelCase       :   Optional[Any]				       =							argparse.ArgumentParser()
      parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
      __UpperCAmelCase       :   List[Any]				       =							parser.parse_args()

      check_dummies(args.fix_and_overwrite)






