


'''simple docstring'''
import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
		from cookiecutter.main import cookiecutter

		__snake_case							      =     True
except ImportError:
		__snake_case							      =     False

__snake_case							      =     logging.get_logger(__name__)  # pylint: disable=invalid-name


def 			a      (			__a   )  ->	Union[str, Any]:



					'''simple docstring'''






					return AddNewModelCommand(args.testing ,					args.testing_file ,					path=args.path   )







class    lowercase	(							A__					):

		"""simple docstring"""

		@staticmethod
		def     lowerCAmelCase__				(		UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= parser.add_parser('''add-new-model'''							)
							add_new_model_parser.add_argument('''--testing'''    ,   action='''store_true'''    ,   help='''If in testing mode.'''							)
							add_new_model_parser.add_argument('''--testing_file'''    ,   type=UpperCamelCase_    ,   help='''Configuration file on which to run.'''							)
							add_new_model_parser.add_argument(
							    '''--path'''    ,   type=UpperCamelCase_    ,   help='''Path to cookiecutter. Should only be used for testing purposes.'''							)
							add_new_model_parser.set_defaults(func=UpperCamelCase_							)




		def __init__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_=None    ,   *UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :int												= testing
							UpperCamelCase__   :Optional[int]												= testing_file
							UpperCamelCase__   :Dict												= path




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							warnings.warn(
							    '''The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '''
							    '''It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '''
							    '''checks, you should use `transformers-cli add-new-model-like` instead.'''							)
							if not _has_cookiecutter:
												raise ImportError(
												    '''Model creation dependencies are required to use the `add_new_model` command. Install them by running '''
												    '''the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'''							)
							# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
							UpperCamelCase__   :Optional[int]												= [directory for directory in os.listdir() if '''cookiecutter-template-''' == directory[:22]]
							if len(UpperCamelCase_							) > 0:
												raise ValueError(
												    '''Several directories starting with `cookiecutter-template-` in current working directory. '''
												    '''Please clean your directory by removing all folders starting with `cookiecutter-template-` or '''
												    '''change your working directory.'''							)

							UpperCamelCase__   :Optional[int]												= (
							    Path(UpperCamelCase_							).parent.parent.parent.parent if self._path is None else Path(self._path							).parent.parent
							)
							UpperCamelCase__   :int												= path_to_transformer_root / '''templates''' / '''adding_a_new_model'''

							# Execute cookiecutter
							if not self._testing:
												cookiecutter(str(UpperCamelCase_							)							)
							else:
												with open(self._testing_file    ,   '''r'''							) as configuration_file:
																	UpperCamelCase__   :Union[str, Any]												= json.load(UpperCamelCase_							)

												cookiecutter(
												    str(path_to_cookiecutter if self._path is None else self._path							)    ,   no_input=UpperCamelCase_    ,   extra_context=UpperCamelCase_    ,   )

							UpperCamelCase__   :List[Any]												= [directory for directory in os.listdir() if '''cookiecutter-template-''' in directory[:22]][0]

							# Retrieve configuration
							with open(directory + '''/configuration.json'''    ,   '''r'''							) as configuration_file:
												UpperCamelCase__   :Any												= json.load(UpperCamelCase_							)

							UpperCamelCase__   :Union[str, Any]												= configuration['''lowercase_modelname''']
							UpperCamelCase__   :Union[str, Any]												= configuration['''generate_tensorflow_pytorch_and_flax''']
							os.remove(F'''{directory}/configuration.json'''							)

							UpperCamelCase__   :str												= '''PyTorch''' in generate_tensorflow_pytorch_and_flax
							UpperCamelCase__   :str												= '''TensorFlow''' in generate_tensorflow_pytorch_and_flax
							UpperCamelCase__   :List[str]												= '''Flax''' in generate_tensorflow_pytorch_and_flax

							UpperCamelCase__   :List[Any]												= F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
							os.makedirs(UpperCamelCase_    ,   exist_ok=UpperCamelCase_							)
							os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''    ,   exist_ok=UpperCamelCase_							)

							# Tests require submodules as they have parent imports
							with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''    ,   '''w'''							):
												pass

							shutil.move(
							    F'''{directory}/__init__.py'''    ,   F'''{model_dir}/__init__.py'''    ,   )
							shutil.move(
							    F'''{directory}/configuration_{lowercase_model_name}.py'''    ,   F'''{model_dir}/configuration_{lowercase_model_name}.py'''    ,   )

							def remove_copy_lines(UpperCamelCase_							):
												with open(UpperCamelCase_    ,   '''r'''							) as f:
																	UpperCamelCase__   :List[str]												= f.readlines()
												with open(UpperCamelCase_    ,   '''w'''							) as f:
																	for line in lines:
																						if "# Copied from transformers." not in line:
																											f.write(UpperCamelCase_							)

							if output_pytorch:
												if not self._testing:
																	remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''							)

												shutil.move(
												    F'''{directory}/modeling_{lowercase_model_name}.py'''    ,   F'''{model_dir}/modeling_{lowercase_model_name}.py'''    ,   )

												shutil.move(
												    F'''{directory}/test_modeling_{lowercase_model_name}.py'''    ,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''    ,   )
							else:
												os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''							)
												os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''							)

							if output_tensorflow:
												if not self._testing:
																	remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)

												shutil.move(
												    F'''{directory}/modeling_tf_{lowercase_model_name}.py'''    ,   F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''    ,   )

												shutil.move(
												    F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''    ,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''    ,   )
							else:
												os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)
												os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''							)

							if output_flax:
												if not self._testing:
																	remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)

												shutil.move(
												    F'''{directory}/modeling_flax_{lowercase_model_name}.py'''    ,   F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''    ,   )

												shutil.move(
												    F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''    ,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''    ,   )
							else:
												os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)
												os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''							)

							shutil.move(
							    F'''{directory}/{lowercase_model_name}.md'''    ,   F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''    ,   )

							shutil.move(
							    F'''{directory}/tokenization_{lowercase_model_name}.py'''    ,   F'''{model_dir}/tokenization_{lowercase_model_name}.py'''    ,   )

							shutil.move(
							    F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''    ,   F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''    ,   )

							from os import fdopen, remove
							from shutil import copymode, move
							from tempfile import mkstemp

							def replace(UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):
												# Create temp file
												UpperCamelCase__					,				UpperCamelCase__   :List[str]												= mkstemp()
												UpperCamelCase__   :Optional[int]												= False
												with fdopen(UpperCamelCase_    ,   '''w'''							) as new_file:
																	with open(UpperCamelCase_							) as old_file:
																						for line in old_file:
																											new_file.write(UpperCamelCase_							)
																											if line_to_copy_below in line:
																																UpperCamelCase__   :int												= True
																																for line_to_copy in lines_to_copy:
																																					new_file.write(UpperCamelCase_							)

												if not line_found:
																	raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''							)

												# Copy the file permissions from the old file to the new file
												copymode(UpperCamelCase_    ,   UpperCamelCase_							)
												# Remove original file
												remove(UpperCamelCase_							)
												# Move new file
												move(UpperCamelCase_    ,   UpperCamelCase_							)

							def skip_units(UpperCamelCase_							):
												return (
												    ("generating PyTorch" in line and not output_pytorch)
												    or ("generating TensorFlow" in line and not output_tensorflow)
												    or ("generating Flax" in line and not output_flax)
												)

							def replace_in_files(UpperCamelCase_							):
												with open(UpperCamelCase_							) as datafile:
																	UpperCamelCase__   :Union[str, Any]												= []
																	UpperCamelCase__   :int												= False
																	UpperCamelCase__   :Dict												= False
																	for line in datafile:
																						if "# To replace in: " in line and "##" not in line:
																											UpperCamelCase__   :List[str]												= line.split('''"'''							)[1]
																											UpperCamelCase__   :int												= skip_units(UpperCamelCase_							)
																						elif "# Below: " in line and "##" not in line:
																											UpperCamelCase__   :str												= line.split('''"'''							)[1]
																											UpperCamelCase__   :Tuple												= skip_units(UpperCamelCase_							)
																						elif "# End." in line and "##" not in line:
																											if not skip_file and not skip_snippet:
																																replace(UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							)

																											UpperCamelCase__   :Union[str, Any]												= []
																						elif "# Replace with" in line and "##" not in line:
																											UpperCamelCase__   :str												= []
																						elif "##" not in line:
																											lines_to_copy.append(UpperCamelCase_							)

												remove(UpperCamelCase_							)

							replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''							)
							os.rmdir(UpperCamelCase_							)



'''simple docstring'''
from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream
class    lowercase	(							A__					):

		"""simple docstring"""

		def __init__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = False    ,   UpperCamelCase_ = False    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = None    ,   **UpperCamelCase_    ,   ):

							'''simple docstring'''


							super().__init__(
							    features=UpperCamelCase_    ,   cache_dir=UpperCamelCase_    ,   keep_in_memory=UpperCamelCase_    ,   streaming=UpperCamelCase_    ,   num_proc=UpperCamelCase_    ,   **UpperCamelCase_    ,   )
							UpperCamelCase__   :Any												= Generator(
							    cache_dir=UpperCamelCase_    ,   features=UpperCamelCase_    ,   generator=UpperCamelCase_    ,   gen_kwargs=UpperCamelCase_    ,   **UpperCamelCase_    ,   )




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							if self.streaming:
												UpperCamelCase__   :Optional[Any]												= self.builder.as_streaming_dataset(split='''train'''							)
							# Build regular (map-style) dataset
							else:
												UpperCamelCase__   :Optional[int]												= None
												UpperCamelCase__   :int												= None
												UpperCamelCase__   :Any												= None
												UpperCamelCase__   :Any												= None

												self.builder.download_and_prepare(
												    download_config=UpperCamelCase_    ,   download_mode=UpperCamelCase_    ,   verification_mode=UpperCamelCase_    ,   base_path=UpperCamelCase_    ,   num_proc=self.num_proc    ,   )
												UpperCamelCase__   :List[Any]												= self.builder.as_dataset(
												    split='''train'''    ,   verification_mode=UpperCamelCase_    ,   in_memory=self.keep_in_memory							)
							return dataset