



'''simple docstring'''



import os
import re
import shutil
import sys
import tempfile
import unittest

import black


lowerCAmelCase_  :							Optional[int]             =    os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
lowerCAmelCase_  :							Dict             =    "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"
class 						lowerCamelCase_				(       unittest.TestCase      ):







						def        __lowercase     (	self :   Tuple					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Any         =      tempfile.mkdtemp()
													os.makedirs(os.path.join(self.transformer_dir					,     '''models/bert/'''					)					)
													SCREAMING_SNAKE_CASE       :   List[Any]         =      self.transformer_dir
													shutil.copy(
													    os.path.join(__UpperCamelCase					,     '''src/transformers/models/bert/modeling_bert.py'''					)					,     os.path.join(self.transformer_dir					,     '''models/bert/modeling_bert.py'''					)					,     )







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      """src/transformers"""
													shutil.rmtree(self.transformer_dir					)







						def        __lowercase     (	self :   List[Any]					,     lowerCAmelCase__ :   Union[str, Any]					,     lowerCAmelCase__ :   Any					,     lowerCAmelCase__ :   int					,     lowerCAmelCase__ :   int=None					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Any         =      comment + F"""\nclass {class_name}(nn.Module):\n""" + class_code
													if overwrite_result is not None:
																				SCREAMING_SNAKE_CASE       :   Tuple         =      comment + F"""\nclass {class_name}(nn.Module):\n""" + overwrite_result
													SCREAMING_SNAKE_CASE       :   List[str]         =      black.Mode(target_versions={black.TargetVersion.PYaa}					,     line_length=1_19					)
													SCREAMING_SNAKE_CASE       :   str         =      black.format_str(__UpperCamelCase					,     mode=__UpperCamelCase					)
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      os.path.join(self.transformer_dir					,     '''new_code.py'''					)
													with open(__UpperCamelCase					,     '''w'''					,     newline='''\n'''					) as f:
																				f.write(__UpperCamelCase					)
													if overwrite_result is None:
																				self.assertTrue(len(check_copies.is_copy_consistent(__UpperCamelCase					)					) == 0					)
													else:
																				check_copies.is_copy_consistent(f.name					,     overwrite=__UpperCamelCase					)
																				with open(__UpperCamelCase					,     '''r'''					) as f:
																											self.assertTrue(f.read()					,     __UpperCamelCase					)







						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   List[str]         =      check_copies.find_code_in_transformers('''models.bert.modeling_bert.BertLMPredictionHead'''					)
													self.assertEqual(__UpperCamelCase					,     __UpperCamelCase					)







						def        __lowercase     (	self :   List[Any]					):

													"""simple docstring"""
													# Base copy consistency
													self.check_copy_consistency(
													    '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'''					,     '''BertLMPredictionHead'''					,     REFERENCE_CODE + '''\n'''					,     )

													# With no empty line at the end
													self.check_copy_consistency(
													    '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'''					,     '''BertLMPredictionHead'''					,     __UpperCamelCase					,     )

													# Copy consistency with rename
													self.check_copy_consistency(
													    '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'''					,     '''TestModelLMPredictionHead'''					,     re.sub('''Bert'''					,     '''TestModel'''					,     __UpperCamelCase					)					,     )

													# Copy consistency with a really long name
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      """TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
													self.check_copy_consistency(
													    F"""# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"""					,     F"""{long_class_name}LMPredictionHead"""					,     re.sub('''Bert'''					,     __UpperCamelCase					,     __UpperCamelCase					)					,     )

													# Copy consistency with overwrite
													self.check_copy_consistency(
													    '''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'''					,     '''TestModelLMPredictionHead'''					,     __UpperCamelCase					,     overwrite_result=re.sub('''Bert'''					,     '''TestModel'''					,     __UpperCamelCase					)					,     )







						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Dict         =      check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

													SCREAMING_SNAKE_CASE       :   int         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
													    """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
													    """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
													    """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
													    """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
													    """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
													    """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
													    """ method has been applied to compress GPT2 into"""
													    """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
													    """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
													    """ Multilingual BERT into"""
													    """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
													    """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
													    """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
													    """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
													    """ Luong, Quoc V. Le, Christopher D. Manning."""
													)
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
													    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
													    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
													    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
													)
													SCREAMING_SNAKE_CASE       :   Any         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
													    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
													    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
													    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
													    """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
													    """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
													    """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
													    """ method has been applied to compress GPT2 into"""
													    """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
													    """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
													    """ Multilingual BERT into"""
													    """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
													    """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
													    """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
													    """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
													    """ Christopher D. Manning 发布。\n"""
													)

													SCREAMING_SNAKE_CASE       :   Any         =      check_copies.convert_to_localized_md(
													    __UpperCamelCase					,     __UpperCamelCase					,     localized_readme['''format_model_list''']					)

													self.assertFalse(__UpperCamelCase					)
													self.assertEqual(__UpperCamelCase					,     __UpperCamelCase					)

													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      check_copies.convert_to_localized_md(
													    __UpperCamelCase					,     __UpperCamelCase					,     localized_readme['''format_model_list''']					)

													# Check whether the number of models is equal to README.md after conversion.
													self.assertTrue(__UpperCamelCase					)

													SCREAMING_SNAKE_CASE       :   int         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
													    """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
													    """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
													    """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
													)
													SCREAMING_SNAKE_CASE       :   int         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
													    """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
													    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
													    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
													)
													SCREAMING_SNAKE_CASE       :   List[Any]         =      (
													    """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
													    """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
													    """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
													    """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
													)

													SCREAMING_SNAKE_CASE       :   Tuple         =      check_copies.convert_to_localized_md(
													    __UpperCamelCase					,     __UpperCamelCase					,     localized_readme['''format_model_list''']					)

													# Check if the model link is synchronized.
													self.assertEqual(__UpperCamelCase					,     __UpperCamelCase					)





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
	from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


SCREAMING_SNAKE_CASE		:     List[str]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     str         =	{
    "bigscience/bloom": "https://huggingface.co/bigscience/bloom/resolve/main/config.json",
    "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/config.json",
    "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json",
    "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json",
    "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/config.json",
    "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json",
}

class        UpperCamelCase						(				__a				):
						a__		:Any      =			'''bloom'''
						a__		:Optional[int]      =			['''past_key_values''']
						a__		:Tuple      =			{
						    '''num_hidden_layers''': '''n_layer''',
						    '''num_attention_heads''': '''n_head''',
						}





						def __init__(self			,     __UpperCamelCase=250_880			,     __UpperCamelCase=64			,     __UpperCamelCase=2			,     __UpperCamelCase=8			,     __UpperCamelCase=1E-5			,     __UpperCamelCase=0.02			,     __UpperCamelCase=True			,     __UpperCamelCase=1			,     __UpperCamelCase=2			,     __UpperCamelCase=False			,     __UpperCamelCase=0.0			,     __UpperCamelCase=0.0			,     __UpperCamelCase=1			,     __UpperCamelCase=False			,     **__UpperCamelCase			,     )				->				List[Any]:
												UpperCamelCase_		:							Dict  =		vocab_size
												# Backward compatibility with n_embed kwarg
												UpperCamelCase_		:							Optional[Any]  =		kwargs.pop("""n_embed"""			,     __UpperCamelCase						)
												UpperCamelCase_		:							List[Any]  =		hidden_size if n_embed is None else n_embed
												UpperCamelCase_		:							List[Any]  =		n_layer
												UpperCamelCase_		:							Optional[Any]  =		n_head
												UpperCamelCase_		:							List[str]  =		layer_norm_epsilon
												UpperCamelCase_		:							int  =		initializer_range
												UpperCamelCase_		:							List[str]  =		use_cache
												UpperCamelCase_		:							List[Any]  =		pretraining_tp
												UpperCamelCase_		:							Optional[int]  =		apply_residual_connection_post_layernorm
												UpperCamelCase_		:							Any  =		hidden_dropout
												UpperCamelCase_		:							Optional[Any]  =		attention_dropout

												UpperCamelCase_		:							List[str]  =		bos_token_id
												UpperCamelCase_		:							Optional[Any]  =		eos_token_id
												UpperCamelCase_		:							str  =		slow_but_exact

												super().__init__(bos_token_id=__UpperCamelCase			,     eos_token_id=__UpperCamelCase			,     **__UpperCamelCase						)


class        UpperCamelCase						(				__a				):
						a__		:Dict      =			version.parse('''1.12'''				)





						def __init__(self			,     __UpperCamelCase			,     __UpperCamelCase = "default"			,     __UpperCamelCase = None			,     __UpperCamelCase = False			,     )				->				List[str]:
												super().__init__(__UpperCamelCase			,     task=__UpperCamelCase			,     patching_specs=__UpperCamelCase			,     use_past=__UpperCamelCase						)
												if not getattr(self._config			,     """pad_token_id"""			,     __UpperCamelCase						):
																		# TODO: how to do that better?
																		UpperCamelCase_		:							Optional[Any]  =		0





						@property
						def        A_							(self						)				->				Mapping[str, Mapping[int, str]]:
												UpperCamelCase_		:							str  =		OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}						)
												if self.use_past:
																		# BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
																		self.fill_with_past_key_values_(__UpperCamelCase			,     direction="""inputs"""			,     inverted_values_shape=__UpperCamelCase						)
																		UpperCamelCase_		:							int  =		{0: """batch""", 1: """past_sequence + sequence"""}
												else:
																		UpperCamelCase_		:							Dict  =		{0: """batch""", 1: """sequence"""}

												return common_inputs





						@property
						def        A_							(self						)				->				int:
												return self._config.n_layer





						@property
						def        A_							(self						)				->				int:
												return self._config.n_head





						@property
						def        A_							(self						)				->				float:
												return 1E-3





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = -1			,     __UpperCamelCase = -1			,     __UpperCamelCase = False			,     __UpperCamelCase = None			,     )				->				Mapping[str, Any]:
												UpperCamelCase_		:							Union[str, Any]  =		super(__UpperCamelCase			,     self						).generate_dummy_inputs(
												    __UpperCamelCase			,     batch_size=__UpperCamelCase			,     seq_length=__UpperCamelCase			,     is_pair=__UpperCamelCase			,     framework=__UpperCamelCase						)

												# We need to order the input in the way they appears in the forward()
												UpperCamelCase_		:							List[Any]  =		OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}						)

												# Need to add the past_keys
												if self.use_past:
																		if not is_torch_available():
																								raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""						)
																		else:
																								import torch

																								UpperCamelCase_,UpperCamelCase_		:							List[str]  =		common_inputs["""input_ids"""].shape
																								# Not using the same length for past_key_values
																								UpperCamelCase_		:							List[str]  =		seqlen + 2
																								UpperCamelCase_		:							Tuple  =		self._config.hidden_size // self.num_attention_heads
																								UpperCamelCase_		:							List[Any]  =		(
																								    batch * self.num_attention_heads,
																								    head_dim,
																								    past_key_values_length,
																								)
																								UpperCamelCase_		:							str  =		(
																								    batch * self.num_attention_heads,
																								    past_key_values_length,
																								    head_dim,
																								)
																								UpperCamelCase_		:							Any  =		[
																								    (torch.zeros(__UpperCamelCase						), torch.zeros(__UpperCamelCase						)) for _ in range(self.num_layers						)
																								]

												UpperCamelCase_		:							int  =		common_inputs["""attention_mask"""]
												if self.use_past:
																		UpperCamelCase_		:							List[str]  =		ordered_inputs["""attention_mask"""].dtype
																		UpperCamelCase_		:							Any  =		torch.cat(
																		    [ordered_inputs["""attention_mask"""], torch.ones(__UpperCamelCase			,     __UpperCamelCase			,     dtype=__UpperCamelCase						)]			,     dim=1						)

												return ordered_inputs





						@property
						def        A_							(self						)				->				int:
												return 13






