



import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
				from cookiecutter.main import cookiecutter

				__snake_case     :Tuple			       =True
except ImportError:
				__snake_case     :Dict			       =False

__snake_case     :str			       =logging.get_logger(__name__)  # pylint: disable=invalid-name






def  lowerCamelCase_	(      lowerCAmelCase__	:	Namespace				)       ->							str:

							'''simple docstring'''






							return AddNewModelCommand(args.testing			,	args.testing_file			,	path=args.path				)



class        lowerCAmelCase__					(  _lowerCamelCase       ):


				@staticmethod
				def __UpperCamelCase						(							__UpperCamelCase      :  ArgumentParser       )		->     Tuple:
											A      						=    parser.add_parser('add-new-model'       )
											add_new_model_parser.add_argument('--testing'						, action='store_true'						, help='If in testing mode.'       )
											add_new_model_parser.add_argument('--testing_file'						, type=__UpperCamelCase						, help='Configuration file on which to run.'       )
											add_new_model_parser.add_argument(
											    '--path'						, type=__UpperCamelCase						, help='Path to cookiecutter. Should only be used for testing purposes.'       )
											add_new_model_parser.set_defaults(func=__UpperCamelCase       )


				def __init__(							self      :  Any						, __UpperCamelCase      :  bool						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Dict=None						, *__UpperCamelCase      :  List[str]       )		->     Optional[Any]:
											A      						=    testing
											A      						=    testing_file
											A      						=    path


				def __UpperCamelCase						(							self      :  List[str]       )		->     Union[str, Any]:
											warnings.warn(
											    'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
											    'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
											    'checks, you should use `transformers-cli add-new-model-like` instead.'       )
											if not _has_cookiecutter:
																		raise ImportError(
																		    'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
																		    'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'       )
											# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
											A      						=    [directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:22]]
											if len(__UpperCamelCase       ) > 0:
																		raise ValueError(
																		    'Several directories starting with `cookiecutter-template-` in current working directory. '
																		    'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
																		    'change your working directory.'       )

											A      						=    (
											    Path(__UpperCamelCase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
											)
											A      						=    path_to_transformer_root / 'templates' / 'adding_a_new_model'

											# Execute cookiecutter
											if not self._testing:
																		cookiecutter(str(__UpperCamelCase       )       )
											else:
																		with open(self._testing_file						, 'r'       ) as configuration_file:
																									A      						=    json.load(__UpperCamelCase       )

																		cookiecutter(
																		    str(path_to_cookiecutter if self._path is None else self._path       )						, no_input=__UpperCamelCase						, extra_context=__UpperCamelCase						, )

											A      						=    [directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:22]][0]

											# Retrieve configuration
											with open(directory + '/configuration.json'						, 'r'       ) as configuration_file:
																		A      						=    json.load(__UpperCamelCase       )

											A      						=    configuration['lowercase_modelname']
											A      						=    configuration['generate_tensorflow_pytorch_and_flax']
											os.remove(f'''{directory}/configuration.json'''       )

											A      						=    'PyTorch' in generate_tensorflow_pytorch_and_flax
											A      						=    'TensorFlow' in generate_tensorflow_pytorch_and_flax
											A      						=    'Flax' in generate_tensorflow_pytorch_and_flax

											A      						=    f'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
											os.makedirs(__UpperCamelCase						, exist_ok=__UpperCamelCase       )
											os.makedirs(f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''						, exist_ok=__UpperCamelCase       )

											# Tests require submodules as they have parent imports
											with open(f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''						, 'w'       ):
																		pass

											shutil.move(
											    f'''{directory}/__init__.py'''						, f'''{model_dir}/__init__.py'''						, )
											shutil.move(
											    f'''{directory}/configuration_{lowercase_model_name}.py'''						, f'''{model_dir}/configuration_{lowercase_model_name}.py'''						, )

											def remove_copy_lines(__UpperCamelCase      :  List[str]       ):
																		with open(__UpperCamelCase						, 'r'       ) as f:
																									A      						=    f.readlines()
																		with open(__UpperCamelCase						, 'w'       ) as f:
																									for line in lines:
																																if "# Copied from transformers." not in line:
																																							f.write(__UpperCamelCase       )

											if output_pytorch:
																		if not self._testing:
																									remove_copy_lines(f'''{directory}/modeling_{lowercase_model_name}.py'''       )

																		shutil.move(
																		    f'''{directory}/modeling_{lowercase_model_name}.py'''						, f'''{model_dir}/modeling_{lowercase_model_name}.py'''						, )

																		shutil.move(
																		    f'''{directory}/test_modeling_{lowercase_model_name}.py'''						, f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''						, )
											else:
																		os.remove(f'''{directory}/modeling_{lowercase_model_name}.py'''       )
																		os.remove(f'''{directory}/test_modeling_{lowercase_model_name}.py'''       )

											if output_tensorflow:
																		if not self._testing:
																									remove_copy_lines(f'''{directory}/modeling_tf_{lowercase_model_name}.py'''       )

																		shutil.move(
																		    f'''{directory}/modeling_tf_{lowercase_model_name}.py'''						, f'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''						, )

																		shutil.move(
																		    f'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''						, f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''						, )
											else:
																		os.remove(f'''{directory}/modeling_tf_{lowercase_model_name}.py'''       )
																		os.remove(f'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''       )

											if output_flax:
																		if not self._testing:
																									remove_copy_lines(f'''{directory}/modeling_flax_{lowercase_model_name}.py'''       )

																		shutil.move(
																		    f'''{directory}/modeling_flax_{lowercase_model_name}.py'''						, f'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''						, )

																		shutil.move(
																		    f'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''						, f'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''						, )
											else:
																		os.remove(f'''{directory}/modeling_flax_{lowercase_model_name}.py'''       )
																		os.remove(f'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''       )

											shutil.move(
											    f'''{directory}/{lowercase_model_name}.md'''						, f'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''						, )

											shutil.move(
											    f'''{directory}/tokenization_{lowercase_model_name}.py'''						, f'''{model_dir}/tokenization_{lowercase_model_name}.py'''						, )

											shutil.move(
											    f'''{directory}/tokenization_fast_{lowercase_model_name}.py'''						, f'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''						, )

											from os import fdopen, remove
											from shutil import copymode, move
											from tempfile import mkstemp

											def replace(__UpperCamelCase      :  str						, __UpperCamelCase      :  str						, __UpperCamelCase      :  List[str]       ):
																		# Create temp file
																		A						,   A      						=    mkstemp()
																		A      						=    False
																		with fdopen(__UpperCamelCase						, 'w'       ) as new_file:
																									with open(__UpperCamelCase       ) as old_file:
																																for line in old_file:
																																							new_file.write(__UpperCamelCase       )
																																							if line_to_copy_below in line:
																																														A      						=    True
																																														for line_to_copy in lines_to_copy:
																																																					new_file.write(__UpperCamelCase       )

																		if not line_found:
																									raise ValueError(f'''Line {line_to_copy_below} was not found in file.'''       )

																		# Copy the file permissions from the old file to the new file
																		copymode(__UpperCamelCase						, __UpperCamelCase       )
																		# Remove original file
																		remove(__UpperCamelCase       )
																		# Move new file
																		move(__UpperCamelCase						, __UpperCamelCase       )

											def skip_units(__UpperCamelCase      :  List[Any]       ):
																		return (
																		    ("generating PyTorch" in line and not output_pytorch)
																		    or ("generating TensorFlow" in line and not output_tensorflow)
																		    or ("generating Flax" in line and not output_flax)
																		)

											def replace_in_files(__UpperCamelCase      :  Tuple       ):
																		with open(__UpperCamelCase       ) as datafile:
																									A      						=    []
																									A      						=    False
																									A      						=    False
																									for line in datafile:
																																if "# To replace in: " in line and "##" not in line:
																																							A      						=    line.split('"'       )[1]
																																							A      						=    skip_units(__UpperCamelCase       )
																																elif "# Below: " in line and "##" not in line:
																																							A      						=    line.split('"'       )[1]
																																							A      						=    skip_units(__UpperCamelCase       )
																																elif "# End." in line and "##" not in line:
																																							if not skip_file and not skip_snippet:
																																														replace(__UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase       )

																																							A      						=    []
																																elif "# Replace with" in line and "##" not in line:
																																							A      						=    []
																																elif "##" not in line:
																																							lines_to_copy.append(__UpperCamelCase       )

																		remove(__UpperCamelCase       )

											replace_in_files(f'''{directory}/to_replace_{lowercase_model_name}.py'''       )
											os.rmdir(__UpperCamelCase       )




from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class      _UpperCAmelCase      (							A__      ,   A__      ,   A__  ):
     UpperCamelCase__		     =					[r'''h\.\d+\.attn\.bias''', r'''h\.\d+\.attn\.masked_bias''']




     @register_to_config
     def __init__(					self			,    a__			,    a__			,    a__ = None			,    a__ = 5_0_2_5_7			,    a__ = 1_0_2_4			,    a__ = 7_6_8			,    a__ = 1_2			,    a__ = 1_2			,    a__ = None			,    a__ = "gelu_new"			,    a__ = 0.1			,    a__ = 0.1			,    a__ = 0.1			,    a__ = 1e-5			,    a__ = 0.0_2			,    a__ = True			,    a__ = True			,    a__ = False			,    a__ = False			,    ):
         super().__init__()

         A__					=       prefix_length

         if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
             raise ValueError(
                 F"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"
                 F" `n_embd`: {n_embd} are not equal.")

         A__					=       prefix_inner_dim
         A__					=       prefix_hidden_dim

         A__					=       (
             nn.Linear(self.prefix_inner_dim			,    self.prefix_hidden_dim)
             if self.prefix_hidden_dim is not None
             else nn.Identity()
         )
         A__					=       (
             nn.Linear(self.prefix_hidden_dim			,    a__) if self.prefix_hidden_dim is not None else nn.Identity()
         )

         A__					=       GPTaConfig(
             vocab_size=a__			,    n_positions=a__			,    n_embd=a__			,    n_layer=a__			,    n_head=a__			,    n_inner=a__			,    activation_function=a__			,    resid_pdrop=a__			,    embd_pdrop=a__			,    attn_pdrop=a__			,    layer_norm_epsilon=a__			,    initializer_range=a__			,    scale_attn_weights=a__			,    use_cache=a__			,    scale_attn_by_inverse_layer_idx=a__			,    reorder_and_upcast_attn=a__			,    )
         A__					=       GPTaLMHeadModel(a__)




     def 		snake_case_		(					self			,    a__			,    a__			,    a__ = None			,    a__ = None			,    ):
         A__					=       self.transformer.transformer.wte(a__)
         A__					=       self.encode_prefix(a__)
         A__					=       self.decode_prefix(a__)
         A__					=       torch.cat((prefix_embeds, embedding_text)			,    dim=1)

         if labels is not None:
             A__					=       self.get_dummy_token(input_ids.shape[0]			,    input_ids.device)
             A__					=       torch.cat((dummy_token, input_ids)			,    dim=1)
         A__					=       self.transformer(inputs_embeds=a__			,    labels=a__			,    attention_mask=a__)
         if self.prefix_hidden_dim is not None:
             return out, hidden
         else:
             return out




     def 		snake_case_		(					self			,    a__			,    a__):
         return torch.zeros(a__			,    self.prefix_length			,    dtype=torch.intaa			,    device=a__)




     def 		snake_case_		(					self			,    a__):
         return self.encode_prefix(a__)




     @torch.no_grad()
     def 		snake_case_		(					self			,    a__			,    a__			,    a__):
         A__					=       torch.split(a__			,    1			,    dim=0)
         A__					=       []
         A__					=       []
         for feature in features:
             A__					=       self.decode_prefix(feature.to(a__))  # back to the clip feature
             # Only support beam search for now
             A__			,						A__					=       self.generate_beam(
                 input_embeds=a__			,    device=a__			,    eos_token_id=a__)
             generated_tokens.append(output_tokens[0])
             generated_seq_lengths.append(seq_lengths[0])
         A__					=       torch.stack(a__)
         A__					=       torch.stack(a__)
         return generated_tokens, generated_seq_lengths




     @torch.no_grad()
     def 		snake_case_		(					self			,    a__=None			,    a__=None			,    a__=None			,    a__ = 5			,    a__ = 6_7			,    a__ = 1.0			,    a__ = None			,    ):
         A__					=       eos_token_id
         A__					=       None
         A__					=       None
         A__					=       torch.ones(a__			,    device=a__			,    dtype=torch.int)
         A__					=       torch.zeros(a__			,    device=a__			,    dtype=torch.bool)

         if input_embeds is not None:
             A__					=       input_embeds
         else:
             A__					=       self.transformer.transformer.wte(a__)

         for i in range(a__):
             A__					=       self.transformer(inputs_embeds=a__)
             A__					=       outputs.logits
             A__					=       logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
             A__					=       logits.softmax(-1).log()

             if scores is None:
                 A__			,						A__					=       logits.topk(a__			,    -1)
                 A__					=       generated.expand(a__			,    *generated.shape[1:])
                 A__			,						A__					=       next_tokens.permute(1			,    0), scores.squeeze(0)
                 if tokens is None:
                     A__					=       next_tokens
                 else:
                     A__					=       tokens.expand(a__			,    *tokens.shape[1:])
                     A__					=       torch.cat((tokens, next_tokens)			,    dim=1)
             else:
                 A__					=       -float(np.inf)
                 A__					=       0
                 A__					=       scores[:, None] + logits
                 seq_lengths[~is_stopped] += 1
                 A__					=       scores_sum / seq_lengths[:, None]
                 A__			,						A__					=       scores_sum_average.view(-1).topk(a__			,    -1)
                 A__					=       next_tokens // scores_sum.shape[1]
                 A__					=       seq_lengths[next_tokens_source]
                 A__					=       next_tokens % scores_sum.shape[1]
                 A__					=       next_tokens.unsqueeze(1)
                 A__					=       tokens[next_tokens_source]
                 A__					=       torch.cat((tokens, next_tokens)			,    dim=1)
                 A__					=       generated[next_tokens_source]
                 A__					=       scores_sum_average * seq_lengths
                 A__					=       is_stopped[next_tokens_source]

             A__					=       self.transformer.transformer.wte(next_tokens.squeeze()).view(generated.shape[0]			,    1			,    -1)
             A__					=       torch.cat((generated, next_token_embed)			,    dim=1)
             A__					=       is_stopped + next_tokens.eq(a__).squeeze()
             if is_stopped.all():
                 break

         A__					=       scores / seq_lengths
         A__					=       scores.argsort(descending=a__)
         # tokens tensors are already padded to max_seq_length
         A__					=       [tokens[i] for i in order]
         A__					=       torch.stack(a__			,    dim=0)
         A__					=       torch.tensor([seq_lengths[i] for i in order]			,    dtype=seq_lengths.dtype)
         return output_texts, seq_lengths
