


'''simple docstring'''



import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


__lowercase	:						Dict       =	{
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 1_28,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 50,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 10,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 10,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}





@is_staging_test
class 							__UpperCamelCase				(		unittest.TestCase					):




	@classmethod
	def       __UpperCAmelCase							(   cls							):

						'''simple docstring'''





						__a			:       int					    =  TOKEN
						HfFolder.save_token(__a							)




	@classmethod
	def       __UpperCAmelCase							(   cls							):

						'''simple docstring'''





						try:
											delete_repo(token=cls._token						,				repo_id='test-config'							)
						except HTTPError:
											pass

						try:
											delete_repo(token=cls._token						,				repo_id='valid_org/test-config-org'							)
						except HTTPError:
											pass

						try:
											delete_repo(token=cls._token						,				repo_id='test-dynamic-config'							)
						except HTTPError:
											pass




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  BertConfig(
						    vocab_size=99						,				hidden_size=32						,				num_hidden_layers=5						,				num_attention_heads=4						,				intermediate_size=37							)
						config.push_to_hub('test-config'						,				use_auth_token=self._token							)

						__a			:       Dict					    =  BertConfig.from_pretrained(f"""{USER}/test-config"""							)
						for k, v in config.to_dict().items():
											if k != "transformers_version":
																self.assertEqual(__a						,				getattr(__a						,				__a							)							)

        # Reset repo
						delete_repo(token=self._token						,				repo_id='test-config'							)

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
											config.save_pretrained(__a						,				repo_id='test-config'						,				push_to_hub=__a						,				use_auth_token=self._token							)

						__a			:       List[str]					    =  BertConfig.from_pretrained(f"""{USER}/test-config"""							)
						for k, v in config.to_dict().items():
											if k != "transformers_version":
																self.assertEqual(__a						,				getattr(__a						,				__a							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Any					    =  BertConfig(
						    vocab_size=99						,				hidden_size=32						,				num_hidden_layers=5						,				num_attention_heads=4						,				intermediate_size=37							)
						config.push_to_hub('valid_org/test-config-org'						,				use_auth_token=self._token							)

						__a			:       List[Any]					    =  BertConfig.from_pretrained('valid_org/test-config-org'							)
						for k, v in config.to_dict().items():
											if k != "transformers_version":
																self.assertEqual(__a						,				getattr(__a						,				__a							)							)

        # Reset repo
						delete_repo(token=self._token						,				repo_id='valid_org/test-config-org'							)

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
											config.save_pretrained(
											    __a						,				repo_id='valid_org/test-config-org'						,				push_to_hub=__a						,				use_auth_token=self._token							)

						__a			:       List[Any]					    =  BertConfig.from_pretrained('valid_org/test-config-org'							)
						for k, v in config.to_dict().items():
											if k != "transformers_version":
																self.assertEqual(__a						,				getattr(__a						,				__a							)							)





	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						CustomConfig.register_for_auto_class()
						__a			:       str					    =  CustomConfig(attribute=42							)

						config.push_to_hub('test-dynamic-config'						,				use_auth_token=self._token							)

						# This has added the proper auto_map field to the config
						self.assertDictEqual(config.auto_map						,				{'AutoConfig': 'custom_configuration.CustomConfig'}							)

						__a			:       Tuple					    =  AutoConfig.from_pretrained(f"""{USER}/test-dynamic-config"""						,				trust_remote_code=__a							)
						# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
						self.assertEqual(new_config.__class__.__name__						,				'CustomConfig'							)
						self.assertEqual(new_config.attribute						,				42							)





class 							__UpperCamelCase				(		unittest.TestCase					):




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Dict					    =  GPTaConfig()

						# attempt to modify each of int/float/bool/str config records and verify they were updated
						__a			:       Dict					    =  c.n_embd + 1  # int
						__a			:       List[str]					    =  c.resid_pdrop + 1.0  # float
						__a			:       Optional[int]					    =  not c.scale_attn_weights  # bool
						__a			:       Union[str, Any]					    =  c.summary_type + 'foo'  # str
						c.update_from_string(
						    f"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""							)
						self.assertEqual(__a						,				c.n_embd						,				'mismatch for key: n_embd'							)
						self.assertEqual(__a						,				c.resid_pdrop						,				'mismatch for key: resid_pdrop'							)
						self.assertEqual(__a						,				c.scale_attn_weights						,				'mismatch for key: scale_attn_weights'							)
						self.assertEqual(__a						,				c.summary_type						,				'mismatch for key: summary_type'							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[str]					    =  PretrainedConfig()
						__a			:       Dict					    =  [key for key in base_config.__dict__ if key not in config_common_kwargs]
						# If this part of the test fails, you have arguments to addin config_common_kwargs above.
						self.assertListEqual(
						    __a						,				['is_encoder_decoder', '_name_or_path', '_commit_hash', 'transformers_version']							)
						__a			:       int					    =  [key for key, value in config_common_kwargs.items() if value == getattr(__a						,				__a							)]
						if len(__a							) > 0:
											raise ValueError(
											    'The following keys are set with the default values in'
											    ' `test_configuration_common.config_common_kwargs` pick another value for them:'
											    f""" {", ".join(__a							)}."""							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						with self.assertRaises(__a							):
											# config is in subfolder, the following should not work without specifying the subfolder
											__a			:       List[Any]					    =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert-subfolder'							)

						__a			:       Optional[int]					    =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert-subfolder'						,				subfolder='bert'							)

						self.assertIsNotNone(__a							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[str]					    =  mock.Mock()
						__a			:       Dict					    =  500
						__a			:       Any					    =  {}
						__a			:       int					    =  HTTPError
						__a			:       Tuple					    =  {}

						# Download this model to make sure it's in the cache.
						__a			:       Union[str, Any]					    =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert'							)

						# Under the mock environment we get a 500 error when trying to reach the model.
						with mock.patch('requests.Session.request'						,				return_value=__a							) as mock_head:
											__a			:       Any					    =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert'							)
											# This check we did call the fake head request
											mock_head.assert_called()




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Union[str, Any]					    =  BertConfig.from_pretrained(
						    'https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json'							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       str					    =  AutoConfig.from_pretrained('bert-base-cased'							)
						__a			:       Optional[Any]					    =  ['config.4.0.0.json']

						with tempfile.TemporaryDirectory() as tmp_dir:
											configuration.save_pretrained(__a							)
											__a			:       List[str]					    =  2
											json.dump(configuration.to_dict()						,				open(os.path.join(__a						,				'config.4.0.0.json'							)						,				'w'							)							)

											# This should pick the new configuration file as the version of Transformers is > 4.0.0
											__a			:       Optional[int]					    =  AutoConfig.from_pretrained(__a							)
											self.assertEqual(new_configuration.hidden_size						,				2							)

											# Will need to be adjusted if we reach v42 and this test is still here.
											# Should pick the old configuration file as the version of Transformers is < 4.42.0
											__a			:       str					    =  ['config.42.0.0.json']
											__a			:       List[Any]					    =  768
											configuration.save_pretrained(__a							)
											shutil.move(os.path.join(__a						,				'config.4.0.0.json'							)						,				os.path.join(__a						,				'config.42.0.0.json'							)							)
											__a			:       List[str]					    =  AutoConfig.from_pretrained(__a							)
											self.assertEqual(new_configuration.hidden_size						,				768							)





	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  'hf-internal-testing/test-two-configs'

						import transformers as new_transformers

						__a			:       Optional[Any]					    =  'v4.0.0'
						__a    ,				__a			:       Any					    =  new_transformers.models.auto.AutoConfig.from_pretrained(
						    __a						,				return_unused_kwargs=__a							)
						self.assertEqual(new_configuration.hidden_size						,				2							)
						# This checks `_configuration_file` ia not kept in the kwargs by mistake.
						self.assertDictEqual(__a						,				{}							)

						# Testing an older version by monkey-patching the version in the module it's used.
						import transformers as old_transformers

						__a			:       Dict					    =  'v3.0.0'
						__a			:       str					    =  old_transformers.models.auto.AutoConfig.from_pretrained(__a							)
						self.assertEqual(old_configuration.hidden_size						,				768							)







'''simple docstring'''
import re

from filelock import FileLock


try:
		import nltk

		__snake_case							      =     True
except (ImportError, ModuleNotFoundError):
		__snake_case							      =     False

if NLTK_AVAILABLE:
		with FileLock('''.lock''') as lock:
				nltk.download('''punkt''', quiet=True)


def 			a      (			__a   )  ->	str:



					'''simple docstring'''






					re.sub('''<n>''' ,					'''''' ,					__a   )  # remove pegasus newline char
					assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
					return "\n".join(nltk.sent_tokenize(__a   )   )