"""simple docstring"""


import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
    from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__     						=  get_tests_dir("""fixtures/test_sentencepiece.model""")

UpperCAmelCase__     						=  {"target_lang": "fi", "source_lang": "en"}
UpperCAmelCase__     						=  ">>zh<<"
UpperCAmelCase__     						=  "Helsinki-NLP/"

if is_torch_available():
    UpperCAmelCase__     						=  "pt"
elif is_tf_available():
    UpperCAmelCase__     						=  "tf"
else:
    UpperCAmelCase__     						=  "jax"


@require_sentencepiece
class 				a (						_SCREAMING_SNAKE_CASE		,							unittest.TestCase      ):
    _snake_case   :     str   							=							MarianTokenizer
    _snake_case   :     Any   							=							False
    _snake_case   :     Optional[Any]   							=							True


    def    lowerCAmelCase_     (    self		:     Any    ):
      super().setUp()
      _UpperCAmelCase         =					["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
      _UpperCAmelCase         =					dict(zip(__lowerCAmelCase      , range(len(__lowerCAmelCase    )    )    )    )
      _UpperCAmelCase         =					Path(self.tmpdirname    )
      save_json(__lowerCAmelCase      , save_dir / VOCAB_FILES_NAMES["""vocab"""]    )
      save_json(__lowerCAmelCase      , save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""]    )
      if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
        copyfile(__lowerCAmelCase      , save_dir / VOCAB_FILES_NAMES["""source_spm"""]    )
        copyfile(__lowerCAmelCase      , save_dir / VOCAB_FILES_NAMES["""target_spm"""]    )

      _UpperCAmelCase         =					MarianTokenizer.from_pretrained(self.tmpdirname    )
      tokenizer.save_pretrained(self.tmpdirname    )


    def    lowerCAmelCase_     (    self		:     Dict      , **__lowerCAmelCase		:     Optional[Any]    ):
      return MarianTokenizer.from_pretrained(self.tmpdirname      , **__lowerCAmelCase    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     List[Any]    ):
      return (
          "This is a test",
          "This is a test",
      )


    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase         =					"""</s>"""
      _UpperCAmelCase         =					0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase    )      , __lowerCAmelCase    )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase    )      , __lowerCAmelCase    )


    def    lowerCAmelCase_     (    self		:     Optional[int]    ):
      _UpperCAmelCase         =					list(self.get_tokenizer().get_vocab().keys()    )

      self.assertEqual(vocab_keys[0]      , """</s>"""    )
      self.assertEqual(vocab_keys[1]      , """<unk>"""    )
      self.assertEqual(vocab_keys[-1]      , """<pad>"""    )
      self.assertEqual(len(__lowerCAmelCase    )      , 9    )


    def    lowerCAmelCase_     (    self		:     int    ):
      self.assertEqual(self.get_tokenizer().vocab_size      , 9    )


    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      _UpperCAmelCase         =					MarianTokenizer.from_pretrained(f'''{ORG_NAME}opus-mt-en-de'''    )
      _UpperCAmelCase         =					en_de_tokenizer(["""I am a small frog"""]      , return_tensors=__lowerCAmelCase    )
      self.assertIsInstance(__lowerCAmelCase      , __lowerCAmelCase    )
      _UpperCAmelCase         =					[38, 121, 14, 697, 3_8848, 0]
      self.assertListEqual(__lowerCAmelCase      , batch.input_ids[0]    )

      _UpperCAmelCase         =					tempfile.mkdtemp()
      en_de_tokenizer.save_pretrained(__lowerCAmelCase    )
      _UpperCAmelCase         =					[x.name for x in Path(__lowerCAmelCase    ).glob("""*"""    )]
      self.assertIn("""source.spm"""      , __lowerCAmelCase    )
      MarianTokenizer.from_pretrained(__lowerCAmelCase    )


    def    lowerCAmelCase_     (    self		:     Tuple    ):
      _UpperCAmelCase         =					self.get_tokenizer()

      _UpperCAmelCase         =					tok(
          ["""I am a small frog""" * 1000, """I am a small frog"""]      , padding=__lowerCAmelCase      , truncation=__lowerCAmelCase      , return_tensors=__lowerCAmelCase    )
      self.assertIsInstance(__lowerCAmelCase      , __lowerCAmelCase    )
      self.assertEqual(batch.input_ids.shape      , (2, 512)    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase         =					self.get_tokenizer()
      _UpperCAmelCase         =					tok(["""I am a tiny frog""", """I am a small frog"""]      , padding=__lowerCAmelCase      , return_tensors=__lowerCAmelCase    )
      self.assertIsInstance(__lowerCAmelCase      , __lowerCAmelCase    )
      self.assertEqual(batch_smaller.input_ids.shape      , (2, 10)    )


    @slow
    def    lowerCAmelCase_     (    self		:     Any    ):
      # fmt: off
      _UpperCAmelCase         =					{"""input_ids""": [[4_3495, 462, 20, 4_2164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 3_8999, 6, 8, 464, 132, 1703, 492, 13, 4669, 3_7867, 13, 7525, 27, 1593, 988, 13, 3_3972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 1_2338, 2, 1_3958, 387, 2, 3629, 6953, 188, 2900, 2, 1_3958, 8011, 1_1501, 23, 8460, 4073, 3_4009, 20, 435, 1_1439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 3_7867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 2_6453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 1_0767, 6, 316, 304, 4239, 3, 0], [148, 1_5722, 19, 1839, 12, 1350, 13, 2_2327, 5082, 5418, 4_7567, 3_5938, 59, 318, 1_9552, 108, 2183, 54, 1_4976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 1_9088, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100], [36, 6395, 1_2570, 3_9147, 1_1597, 6, 266, 4, 4_5405, 7296, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=__lowerCAmelCase      , model_name="""Helsinki-NLP/opus-mt-en-de"""      , revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""      , decode_kwargs={"""use_source_tokenizer""": True}      , )


    def    lowerCAmelCase_     (    self		:     List[str]    ):
      _UpperCAmelCase         =					MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs"""    )

      _UpperCAmelCase         =					"""Tämä on testi"""
      _UpperCAmelCase         =					"""This is a test"""

      _UpperCAmelCase         =					[76, 7, 2047, 2]
      _UpperCAmelCase         =					[69, 12, 11, 940, 2]

      _UpperCAmelCase         =					tokenizer(__lowerCAmelCase    ).input_ids
      self.assertListEqual(__lowerCAmelCase      , __lowerCAmelCase    )

      _UpperCAmelCase         =					tokenizer(text_target=__lowerCAmelCase    ).input_ids
      self.assertListEqual(__lowerCAmelCase      , __lowerCAmelCase    )

      _UpperCAmelCase         =					tokenizer.decode(__lowerCAmelCase      , skip_special_tokens=__lowerCAmelCase    )
      self.assertEqual(__lowerCAmelCase      , __lowerCAmelCase    )




'''simple docstring'''






import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
  from cookiecutter.main import cookiecutter

  __A					:  List[Any]											=      True
except ImportError:
  __A					:  int											=      False

__A					:  str											=      logging.get_logger(__name__)  # pylint: disable=invalid-name







def 				UpperCamelCase_			(    A__					:   Namespace					):
  '''simple docstring'''





  return AddNewModelCommand(args.testing							,    args.testing_file							,    path=args.path					)






class    __snake_case       (   _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    @staticmethod
    def __lowercase   (   lowerCamelCase    :  ArgumentParser		)   ->  int:
      lowerCAmelCase_			:      Optional[int]			=   parser.add_parser("""add-new-model"""		)
      add_new_model_parser.add_argument("""--testing"""    ,						action="""store_true"""    ,						help="""If in testing mode."""		)
      add_new_model_parser.add_argument("""--testing_file"""    ,						type=lowerCamelCase    ,						help="""Configuration file on which to run."""		)
      add_new_model_parser.add_argument(
          """--path"""    ,						type=lowerCamelCase    ,						help="""Path to cookiecutter. Should only be used for testing purposes."""		)
      add_new_model_parser.set_defaults(func=lowerCamelCase		)

    def __init__(   self    :  List[str]    ,						lowerCamelCase    :  bool    ,						lowerCamelCase    :  str    ,						lowerCamelCase    :  Any=None    ,						*lowerCamelCase    :  List[str]		)   ->  Optional[Any]:
      lowerCAmelCase_			:      int			=   testing
      lowerCAmelCase_			:      Union[str, Any]			=   testing_file
      lowerCAmelCase_			:      Tuple			=   path





    def __lowercase   (   self    :  Tuple		)   ->  int:
      warnings.warn(
          """The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. """
          """It is not actively maintained anymore, so might give a result that won't pass all tests and quality """
          """checks, you should use `transformers-cli add-new-model-like` instead."""		)
      if not _has_cookiecutter:
        raise ImportError(
            """Model creation dependencies are required to use the `add_new_model` command. Install them by running """
            """the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"""		)
      # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
      lowerCAmelCase_			:      int			=   [directory for directory in os.listdir() if """cookiecutter-template-""" == directory[:22]]
      if len(lowerCamelCase		) > 0:
        raise ValueError(
            """Several directories starting with `cookiecutter-template-` in current working directory. """
            """Please clean your directory by removing all folders starting with `cookiecutter-template-` or """
            """change your working directory."""		)

      lowerCAmelCase_			:      List[Any]			=   (
          Path(lowerCamelCase		).parent.parent.parent.parent if self._path is None else Path(self._path		).parent.parent
      )
      lowerCAmelCase_			:      Dict			=   path_to_transformer_root / """templates""" / """adding_a_new_model"""

      # Execute cookiecutter
      if not self._testing:
        cookiecutter(str(lowerCamelCase		)		)
      else:
        with open(self._testing_file    ,						"""r"""		) as configuration_file:
          lowerCAmelCase_			:      Tuple			=   json.load(lowerCamelCase		)

        cookiecutter(
            str(path_to_cookiecutter if self._path is None else self._path		)    ,						no_input=lowerCamelCase    ,						extra_context=lowerCamelCase    ,						)

      lowerCAmelCase_			:      List[str]			=   [directory for directory in os.listdir() if """cookiecutter-template-""" in directory[:22]][0]

      # Retrieve configuration
      with open(directory + """/configuration.json"""    ,						"""r"""		) as configuration_file:
        lowerCAmelCase_			:      Tuple			=   json.load(lowerCamelCase		)

      lowerCAmelCase_			:      str			=   configuration["""lowercase_modelname"""]
      lowerCAmelCase_			:      List[str]			=   configuration["""generate_tensorflow_pytorch_and_flax"""]
      os.remove(F'{directory}/configuration.json'		)

      lowerCAmelCase_			:      Dict			=   """PyTorch""" in generate_tensorflow_pytorch_and_flax
      lowerCAmelCase_			:      Optional[int]			=   """TensorFlow""" in generate_tensorflow_pytorch_and_flax
      lowerCAmelCase_			:      List[str]			=   """Flax""" in generate_tensorflow_pytorch_and_flax

      lowerCAmelCase_			:      Union[str, Any]			=   F'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
      os.makedirs(lowerCamelCase    ,						exist_ok=lowerCamelCase		)
      os.makedirs(F'{path_to_transformer_root}/tests/models/{lowercase_model_name}'    ,						exist_ok=lowerCamelCase		)

      # Tests require submodules as they have parent imports
      with open(F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'    ,						"""w"""		):
        pass

      shutil.move(
          F'{directory}/__init__.py'    ,						F'{model_dir}/__init__.py'    ,						)
      shutil.move(
          F'{directory}/configuration_{lowercase_model_name}.py'    ,						F'{model_dir}/configuration_{lowercase_model_name}.py'    ,						)

      def remove_copy_lines(lowerCamelCase    :  Any		):
        with open(lowerCamelCase    ,						"""r"""		) as f:
          lowerCAmelCase_			:      List[str]			=   f.readlines()
        with open(lowerCamelCase    ,						"""w"""		) as f:
          for line in lines:
            if "# Copied from transformers." not in line:
              f.write(lowerCamelCase		)

      if output_pytorch:
        if not self._testing:
          remove_copy_lines(F'{directory}/modeling_{lowercase_model_name}.py'		)

        shutil.move(
            F'{directory}/modeling_{lowercase_model_name}.py'    ,						F'{model_dir}/modeling_{lowercase_model_name}.py'    ,						)

        shutil.move(
            F'{directory}/test_modeling_{lowercase_model_name}.py'    ,						F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'    ,						)
      else:
        os.remove(F'{directory}/modeling_{lowercase_model_name}.py'		)
        os.remove(F'{directory}/test_modeling_{lowercase_model_name}.py'		)

      if output_tensorflow:
        if not self._testing:
          remove_copy_lines(F'{directory}/modeling_tf_{lowercase_model_name}.py'		)

        shutil.move(
            F'{directory}/modeling_tf_{lowercase_model_name}.py'    ,						F'{model_dir}/modeling_tf_{lowercase_model_name}.py'    ,						)

        shutil.move(
            F'{directory}/test_modeling_tf_{lowercase_model_name}.py'    ,						F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'    ,						)
      else:
        os.remove(F'{directory}/modeling_tf_{lowercase_model_name}.py'		)
        os.remove(F'{directory}/test_modeling_tf_{lowercase_model_name}.py'		)

      if output_flax:
        if not self._testing:
          remove_copy_lines(F'{directory}/modeling_flax_{lowercase_model_name}.py'		)

        shutil.move(
            F'{directory}/modeling_flax_{lowercase_model_name}.py'    ,						F'{model_dir}/modeling_flax_{lowercase_model_name}.py'    ,						)

        shutil.move(
            F'{directory}/test_modeling_flax_{lowercase_model_name}.py'    ,						F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'    ,						)
      else:
        os.remove(F'{directory}/modeling_flax_{lowercase_model_name}.py'		)
        os.remove(F'{directory}/test_modeling_flax_{lowercase_model_name}.py'		)

      shutil.move(
          F'{directory}/{lowercase_model_name}.md'    ,						F'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'    ,						)

      shutil.move(
          F'{directory}/tokenization_{lowercase_model_name}.py'    ,						F'{model_dir}/tokenization_{lowercase_model_name}.py'    ,						)

      shutil.move(
          F'{directory}/tokenization_fast_{lowercase_model_name}.py'    ,						F'{model_dir}/tokenization_{lowercase_model_name}_fast.py'    ,						)

      from os import fdopen, remove
      from shutil import copymode, move
      from tempfile import mkstemp

      def replace(lowerCamelCase    :  str    ,						lowerCamelCase    :  str    ,						lowerCamelCase    :  List[str]		):
        # Create temp file
        lowerCAmelCase_,							lowerCAmelCase_			:      int			=   mkstemp()
        lowerCAmelCase_			:      List[Any]			=   False
        with fdopen(lowerCamelCase    ,						"""w"""		) as new_file:
          with open(lowerCamelCase		) as old_file:
            for line in old_file:
              new_file.write(lowerCamelCase		)
              if line_to_copy_below in line:
                lowerCAmelCase_			:      List[str]			=   True
                for line_to_copy in lines_to_copy:
                  new_file.write(lowerCamelCase		)

        if not line_found:
          raise ValueError(F'Line {line_to_copy_below} was not found in file.'		)

        # Copy the file permissions from the old file to the new file
        copymode(lowerCamelCase    ,						lowerCamelCase		)
        # Remove original file
        remove(lowerCamelCase		)
        # Move new file
        move(lowerCamelCase    ,						lowerCamelCase		)

      def skip_units(lowerCamelCase    :  Optional[int]		):
        return (
            ("generating PyTorch" in line and not output_pytorch)
            or ("generating TensorFlow" in line and not output_tensorflow)
            or ("generating Flax" in line and not output_flax)
        )

      def replace_in_files(lowerCamelCase    :  Any		):
        with open(lowerCamelCase		) as datafile:
          lowerCAmelCase_			:      Dict			=   []
          lowerCAmelCase_			:      List[str]			=   False
          lowerCAmelCase_			:      str			=   False
          for line in datafile:
            if "# To replace in: " in line and "##" not in line:
              lowerCAmelCase_			:      Dict			=   line.split("""\""""		)[1]
              lowerCAmelCase_			:      int			=   skip_units(lowerCamelCase		)
            elif "# Below: " in line and "##" not in line:
              lowerCAmelCase_			:      Any			=   line.split("""\""""		)[1]
              lowerCAmelCase_			:      Tuple			=   skip_units(lowerCamelCase		)
            elif "# End." in line and "##" not in line:
              if not skip_file and not skip_snippet:
                replace(lowerCamelCase    ,						lowerCamelCase    ,						lowerCamelCase		)

              lowerCAmelCase_			:      Dict			=   []
            elif "# Replace with" in line and "##" not in line:
              lowerCAmelCase_			:      int			=   []
            elif "##" not in line:
              lines_to_copy.append(lowerCamelCase		)

        remove(lowerCamelCase		)

      replace_in_files(F'{directory}/to_replace_{lowercase_model_name}.py'		)
      os.rmdir(lowerCamelCase		)



