





"""simple docstring"""






import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
		from cookiecutter.main import cookiecutter

		__UpperCAmelCase						=			True
except ImportError:
		__UpperCAmelCase						=			False

__UpperCAmelCase						=			logging.get_logger(__name__)  # pylint: disable=invalid-name






def      lowercase__ (					lowerCAmelCase__   :			Namespace			)    ->			Tuple:



			'''simple docstring'''
			return AddNewModelCommand(args.testing							,			args.testing_file							,			path=args.path			)





class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				@staticmethod
				def       UpperCAmelCase      (		a_    :			ArgumentParser    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	List[str]		 =			parser.add_parser("add-new-model"    )
							add_new_model_parser.add_argument("--testing"	, action="store_true"	, help="If in testing mode."    )
							add_new_model_parser.add_argument("--testing_file"	, type=a_	, help="Configuration file on which to run."    )
							add_new_model_parser.add_argument(
							    "--path"	, type=a_	, help="Path to cookiecutter. Should only be used for testing purposes."    )
							add_new_model_parser.set_defaults(func=a_    )
				def __init__(		self    :			List[str]	, a_    :			bool	, a_    :			str	, a_    :			Dict=None	, *a_    :			Union[str, Any]    )	->	int:



							'''simple docstring'''



							a__  :	List[str]		 =			testing
							a__  :	Optional[Any]		 =			testing_file
							a__  :	List[str]		 =			path






				def       UpperCAmelCase      (		self    :			int    )	->	List[str]:



							'''simple docstring'''



							warnings.warn(
							    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
							    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
							    "checks, you should use `transformers-cli add-new-model-like` instead."    )
							if not _has_cookiecutter:
										raise ImportError(
										    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
										    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"    )
							# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
							a__  :	List[str]		 =			[directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
							if len(a_    ) > 0:
										raise ValueError(
										    "Several directories starting with `cookiecutter-template-` in current working directory. "
										    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
										    "change your working directory."    )

							a__  :	Tuple		 =			(
							    Path(a_    ).parent.parent.parent.parent if self._path is None else Path(self._path    ).parent.parent
							)
							a__  :	Optional[int]		 =			path_to_transformer_root / "templates" / "adding_a_new_model"

							# Execute cookiecutter
							if not self._testing:
										cookiecutter(str(a_    )    )
							else:
										with open(self._testing_file	, "r"    ) as configuration_file:
													a__  :	Any		 =			json.load(a_    )

										cookiecutter(
										    str(path_to_cookiecutter if self._path is None else self._path    )	, no_input=a_	, extra_context=a_	, )

							a__  :	List[str]		 =			[directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

							# Retrieve configuration
							with open(directory + "/configuration.json"	, "r"    ) as configuration_file:
										a__  :	int		 =			json.load(a_    )

							a__  :	List[str]		 =			configuration["lowercase_modelname"]
							a__  :	List[Any]		 =			configuration["generate_tensorflow_pytorch_and_flax"]
							os.remove(F"{directory}/configuration.json"    )

							a__  :	List[str]		 =			"PyTorch" in generate_tensorflow_pytorch_and_flax
							a__  :	Any		 =			"TensorFlow" in generate_tensorflow_pytorch_and_flax
							a__  :	Optional[Any]		 =			"Flax" in generate_tensorflow_pytorch_and_flax

							a__  :	int		 =			F"{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"
							os.makedirs(a_	, exist_ok=a_    )
							os.makedirs(F"{path_to_transformer_root}/tests/models/{lowercase_model_name}"	, exist_ok=a_    )

							# Tests require submodules as they have parent imports
							with open(F"{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py"	, "w"    ):
										pass

							shutil.move(
							    F"{directory}/__init__.py"	, F"{model_dir}/__init__.py"	, )
							shutil.move(
							    F"{directory}/configuration_{lowercase_model_name}.py"	, F"{model_dir}/configuration_{lowercase_model_name}.py"	, )

							def remove_copy_lines(a_    :			Any    ):
										with open(a_	, "r"    ) as f:
													a__  :	List[str]		 =			f.readlines()
										with open(a_	, "w"    ) as f:
													for line in lines:
																if "# Copied from transformers." not in line:
																			f.write(a_    )

							if output_pytorch:
										if not self._testing:
													remove_copy_lines(F"{directory}/modeling_{lowercase_model_name}.py"    )

										shutil.move(
										    F"{directory}/modeling_{lowercase_model_name}.py"	, F"{model_dir}/modeling_{lowercase_model_name}.py"	, )

										shutil.move(
										    F"{directory}/test_modeling_{lowercase_model_name}.py"	, F"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py"	, )
							else:
										os.remove(F"{directory}/modeling_{lowercase_model_name}.py"    )
										os.remove(F"{directory}/test_modeling_{lowercase_model_name}.py"    )

							if output_tensorflow:
										if not self._testing:
													remove_copy_lines(F"{directory}/modeling_tf_{lowercase_model_name}.py"    )

										shutil.move(
										    F"{directory}/modeling_tf_{lowercase_model_name}.py"	, F"{model_dir}/modeling_tf_{lowercase_model_name}.py"	, )

										shutil.move(
										    F"{directory}/test_modeling_tf_{lowercase_model_name}.py"	, F"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py"	, )
							else:
										os.remove(F"{directory}/modeling_tf_{lowercase_model_name}.py"    )
										os.remove(F"{directory}/test_modeling_tf_{lowercase_model_name}.py"    )

							if output_flax:
										if not self._testing:
													remove_copy_lines(F"{directory}/modeling_flax_{lowercase_model_name}.py"    )

										shutil.move(
										    F"{directory}/modeling_flax_{lowercase_model_name}.py"	, F"{model_dir}/modeling_flax_{lowercase_model_name}.py"	, )

										shutil.move(
										    F"{directory}/test_modeling_flax_{lowercase_model_name}.py"	, F"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py"	, )
							else:
										os.remove(F"{directory}/modeling_flax_{lowercase_model_name}.py"    )
										os.remove(F"{directory}/test_modeling_flax_{lowercase_model_name}.py"    )

							shutil.move(
							    F"{directory}/{lowercase_model_name}.md"	, F"{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md"	, )

							shutil.move(
							    F"{directory}/tokenization_{lowercase_model_name}.py"	, F"{model_dir}/tokenization_{lowercase_model_name}.py"	, )

							shutil.move(
							    F"{directory}/tokenization_fast_{lowercase_model_name}.py"	, F"{model_dir}/tokenization_{lowercase_model_name}_fast.py"	, )

							from os import fdopen, remove
							from shutil import copymode, move
							from tempfile import mkstemp

							def replace(a_    :			str	, a_    :			str	, a_    :			List[str]    ):
										# Create temp file
										a__ ,	a__  :	Optional[Any]		 =			mkstemp()
										a__  :	str		 =			False
										with fdopen(a_	, "w"    ) as new_file:
													with open(a_    ) as old_file:
																for line in old_file:
																			new_file.write(a_    )
																			if line_to_copy_below in line:
																						a__  :	Tuple		 =			True
																						for line_to_copy in lines_to_copy:
																									new_file.write(a_    )

										if not line_found:
													raise ValueError(F"Line {line_to_copy_below} was not found in file."    )

										# Copy the file permissions from the old file to the new file
										copymode(a_	, a_    )
										# Remove original file
										remove(a_    )
										# Move new file
										move(a_	, a_    )

							def skip_units(a_    :			Any    ):
										return (
										    ("generating PyTorch" in line and not output_pytorch)
										    or ("generating TensorFlow" in line and not output_tensorflow)
										    or ("generating Flax" in line and not output_flax)
										)

							def replace_in_files(a_    :			int    ):
										with open(a_    ) as datafile:
													a__  :	List[str]		 =			[]
													a__  :	List[Any]		 =			False
													a__  :	Union[str, Any]		 =			False
													for line in datafile:
																if "# To replace in: " in line and "##" not in line:
																			a__  :	Tuple		 =			line.split("\""    )[1]
																			a__  :	Any		 =			skip_units(a_    )
																elif "# Below: " in line and "##" not in line:
																			a__  :	List[str]		 =			line.split("\""    )[1]
																			a__  :	str		 =			skip_units(a_    )
																elif "# End." in line and "##" not in line:
																			if not skip_file and not skip_snippet:
																						replace(a_	, a_	, a_    )

																			a__  :	List[str]		 =			[]
																elif "# Replace with" in line and "##" not in line:
																			a__  :	str		 =			[]
																elif "##" not in line:
																			lines_to_copy.append(a_    )

										remove(a_    )

							replace_in_files(F"{directory}/to_replace_{lowercase_model_name}.py"    )
							os.rmdir(a_    )






"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__UpperCAmelCase						=			logging.get_logger(__name__)

__UpperCAmelCase						=			{'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase						=			{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}



class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				def __init__(		self    :			str	, a_    :			Dict	, a_    :			List[str]=False	, a_    :			Any=True	, a_    :			int=False	, a_    :			Union[str, Any]="<s>"	, a_    :			Optional[int]="</s>"	, a_    :			int="<unk>"	, a_    :			List[Any]="<sep>"	, a_    :			Dict="<pad>"	, a_    :			Any="<cls>"	, a_    :			Optional[Any]="<mask>"	, a_    :			int=["<eop>", "<eod>"]	, a_    :			Optional[Dict[str, Any]] = None	, **a_    :			int	, )	->	None:



							'''simple docstring'''



							a__  :	List[str]		 =			AddedToken(a_	, lstrip=a_	, rstrip=a_    ) if isinstance(a_	, a_    ) else mask_token

							a__  :	List[str]		 =			{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=a_	, remove_space=a_	, keep_accents=a_	, bos_token=a_	, eos_token=a_	, unk_token=a_	, sep_token=a_	, pad_token=a_	, cls_token=a_	, mask_token=a_	, additional_special_tokens=a_	, sp_model_kwargs=self.sp_model_kwargs	, **a_	, )

							a__  :	Union[str, Any]		 =			3

							a__  :	Dict		 =			do_lower_case
							a__  :	Union[str, Any]		 =			remove_space
							a__  :	int		 =			keep_accents
							a__  :	str		 =			vocab_file

							a__  :	str		 =			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(a_    )

							try:
										import jieba
							except ModuleNotFoundError as error:
										raise error.__class__(
										    "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
										    "See https://pypi.org/project/jieba/ for installation."    )
							a__  :	Optional[int]		 =			jieba
							a__  :	Optional[int]		 =			str.maketrans(" \n"	, "\u2582\u2583"    )
				@property
				# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	str:



							'''simple docstring'''



							return len(self.sp_model    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	Optional[int]		 =			{self.convert_ids_to_tokens(a_    ): i for i in range(self.vocab_size    )}
							vocab.update(self.added_tokens_encoder    )
							return vocab
				def __getstate__(		self    :			Tuple    )	->	List[str]:



							'''simple docstring'''



							a__  :	Tuple		 =			self.__dict__.copy()
							a__  :	Union[str, Any]		 =			None
							return state
				def __setstate__(		self    :			Tuple	, a_    :			int    )	->	List[str]:



							'''simple docstring'''



							a__  :	Any		 =			d

							# for backward compatibility
							if not hasattr(self	, "sp_model_kwargs"    ):
										a__  :	str		 =			{}

							a__  :	List[str]		 =			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(self.vocab_file    )
				def       UpperCAmelCase      (		self    :			List[Any]	, a_    :			Optional[Any]    )	->	List[str]:



							'''simple docstring'''



							if self.remove_space:
										a__  :	Union[str, Any]		 =			" ".join(inputs.strip().split()    )
							else:
										a__  :	Optional[Any]		 =			inputs
							a__  :	List[str]		 =			outputs.replace("``"	, "\""    ).replace("''"	, "\""    )

							if not self.keep_accents:
										a__  :	Union[str, Any]		 =			unicodedata.normalize("NFKD"	, a_    )
										a__  :	Union[str, Any]		 =			"".join([c for c in outputs if not unicodedata.combining(a_    )]    )
							if self.do_lower_case:
										a__  :	List[Any]		 =			outputs.lower()

							return outputs
				def       UpperCAmelCase      (		self    :			Any	, a_    :			str    )	->	List[str]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			self.preprocess_text(a_    )
							a__  :	Dict		 =			self.sp_model.encode(a_	, out_type=a_    )
							a__  :	Optional[Any]		 =			[]
							for piece in pieces:
										if len(a_    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
													a__  :	Optional[Any]		 =			self.sp_model.EncodeAsPieces(piece[:-1].replace(a_	, ""    )    )
													if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																if len(cur_pieces[0]    ) == 1:
																			a__  :	List[str]		 =			cur_pieces[1:]
																else:
																			a__  :	List[str]		 =			cur_pieces[0][1:]
													cur_pieces.append(piece[-1]    )
													new_pieces.extend(a_    )
										else:
													new_pieces.append(a_    )

							return new_pieces
				def       UpperCAmelCase      (		self    :			int	, a_    :			Dict    )	->	int:



							'''simple docstring'''



							return self.sp_model.PieceToId(a_    )
				def       UpperCAmelCase      (		self    :			Dict	, a_    :			Tuple    )	->	List[Any]:



							'''simple docstring'''



							return self.sp_model.IdToPiece(a_    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]	, a_    :			List[Any]    )	->	str:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			"".join(a_    ).replace(a_	, " "    ).strip()
							return out_string
				def       UpperCAmelCase      (		self    :			str	, a_    :			List[int]	, a_    :			Optional[List[int]] = None    )	->	List[int]:



							'''simple docstring'''



							a__  :	List[Any]		 =			[self.sep_token_id]
							a__  :	int		 =			[self.cls_token_id]
							if token_ids_a is None:
										return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls
				def       UpperCAmelCase      (		self    :			int	, a_    :			List[int]	, a_    :			Optional[List[int]] = None	, a_    :			bool = False    )	->	List[int]:



							'''simple docstring'''



							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=a_	, token_ids_a=a_	, already_has_special_tokens=a_    )

							if token_ids_a is not None:
										return ([0] * len(a_    )) + [1] + ([0] * len(a_    )) + [1, 1]
							return ([0] * len(a_    )) + [1, 1]
				def       UpperCAmelCase      (		self    :			List[Any]	, a_    :			List[int]	, a_    :			Optional[List[int]] = None    )	->	List[int]:



							'''simple docstring'''



							a__  :	List[str]		 =			[self.sep_token_id]
							a__  :	Tuple		 =			[2]

							if token_ids_a is None:
										return len(token_ids_a + sep    ) * [0] + cls_segment_id
							return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id
				def       UpperCAmelCase      (		self    :			Dict	, a_    :			str	, a_    :			Optional[str] = None    )	->	Tuple[str]:



							'''simple docstring'''



							if not os.path.isdir(a_    ):
										logger.error(F"Vocabulary path ({save_directory}) should be a directory"    )
										return
							a__  :	Optional[int]		 =			os.path.join(
							    a_	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(a_    ) and os.path.isfile(self.vocab_file    ):
										copyfile(self.vocab_file	, a_    )
							elif not os.path.isfile(self.vocab_file    ):
										with open(a_	, "wb"    ) as fi:
													a__  :	int		 =			self.sp_model.serialized_model_proto()
													fi.write(a_    )

							return (out_vocab_file,)






				def       UpperCAmelCase      (		self    :			str	, *a_    :			Union[str, Any]	, **a_    :			Any    )	->	int:



							'''simple docstring'''



							a__  :	Optional[int]		 =			super()._decode(*a_	, **a_    )
							a__  :	Tuple		 =			text.replace(" "	, ""    ).replace("\u2582"	, " "    ).replace("\u2583"	, "\n"    )
							return text