





import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    MegatronBertForCausalLM,
							    MegatronBertForMaskedLM,
							    MegatronBertForMultipleChoice,
							    MegatronBertForNextSentencePrediction,
							    MegatronBertForPreTraining,
							    MegatronBertForQuestionAnswering,
							    MegatronBertForSequenceClassification,
							    MegatronBertForTokenClassification,
							    MegatronBertModel,
							)





class 				a			:






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase=13			, _lowerCAmelCase=7			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=True			, _lowerCAmelCase=99			, _lowerCAmelCase=64			, _lowerCAmelCase=32			, _lowerCAmelCase=5			, _lowerCAmelCase=4			, _lowerCAmelCase=37			, _lowerCAmelCase="gelu"			, _lowerCAmelCase=0.1			, _lowerCAmelCase=0.1			, _lowerCAmelCase=512			, _lowerCAmelCase=16			, _lowerCAmelCase=2			, _lowerCAmelCase=0.02			, _lowerCAmelCase=3			, _lowerCAmelCase=4			, _lowerCAmelCase=None			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   parent
										__SCREAMING_SNAKE_CASE:				Any           =   batch_size
										__SCREAMING_SNAKE_CASE:				Dict           =   seq_length
										__SCREAMING_SNAKE_CASE:				List[str]           =   is_training
										__SCREAMING_SNAKE_CASE:				Any           =   use_input_mask
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   use_token_type_ids
										__SCREAMING_SNAKE_CASE:				Dict           =   use_labels
										__SCREAMING_SNAKE_CASE:				Any           =   vocab_size
										__SCREAMING_SNAKE_CASE:				List[Any]           =   hidden_size
										__SCREAMING_SNAKE_CASE:				str           =   embedding_size
										__SCREAMING_SNAKE_CASE:				Any           =   num_hidden_layers
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   num_attention_heads
										__SCREAMING_SNAKE_CASE:				List[str]           =   intermediate_size
										__SCREAMING_SNAKE_CASE:				Any           =   hidden_act
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   hidden_dropout_prob
										__SCREAMING_SNAKE_CASE:				str           =   attention_probs_dropout_prob
										__SCREAMING_SNAKE_CASE:				List[Any]           =   max_position_embeddings
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   type_vocab_size
										__SCREAMING_SNAKE_CASE:				Any           =   type_sequence_label_size
										__SCREAMING_SNAKE_CASE:				Dict           =   initializer_range
										__SCREAMING_SNAKE_CASE:				List[Any]           =   num_labels
										__SCREAMING_SNAKE_CASE:				List[str]           =   num_choices
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   scope






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   ids_tensor([self.batch_size, self.seq_length]			, self.vocab_size					)

										__SCREAMING_SNAKE_CASE:				Any           =   None
										if self.use_input_mask:
																__SCREAMING_SNAKE_CASE:				Tuple           =   random_attention_mask([self.batch_size, self.seq_length]					)

										__SCREAMING_SNAKE_CASE:				str           =   None
										if self.use_token_type_ids:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   ids_tensor([self.batch_size, self.seq_length]			, self.type_vocab_size					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   None
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   None
										__SCREAMING_SNAKE_CASE:				Tuple           =   None
										if self.use_labels:
																__SCREAMING_SNAKE_CASE:				Any           =   ids_tensor([self.batch_size]			, self.type_sequence_label_size					)
																__SCREAMING_SNAKE_CASE:				int           =   ids_tensor([self.batch_size, self.seq_length]			, self.num_labels					)
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   ids_tensor([self.batch_size]			, self.num_choices					)

										__SCREAMING_SNAKE_CASE:				Any           =   self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






				def 			snake_case_    (  self					):

										"""simple docstring"""




										return MegatronBertConfig(
										    vocab_size=self.vocab_size			, hidden_size=self.hidden_size			, num_hidden_layers=self.num_hidden_layers			, num_attention_heads=self.num_attention_heads			, intermediate_size=self.intermediate_size			, embedding_size=self.embedding_size			, hidden_act=self.hidden_act			, hidden_dropout_prob=self.hidden_dropout_prob			, attention_probs_dropout_prob=self.attention_probs_dropout_prob			, max_position_embeddings=self.max_position_embeddings			, type_vocab_size=self.type_vocab_size			, is_decoder=_lowerCAmelCase			, initializer_range=self.initializer_range			, )






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   MegatronBertModel(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				str           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   model(_lowerCAmelCase			, token_type_ids=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   model(_lowerCAmelCase					)

										self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)					)
										self.parent.assertEqual(result.pooler_output.shape			, (self.batch_size, self.hidden_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   MegatronBertForMaskedLM(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				int           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   MegatronBertForCausalLM(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				List[Any]           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   MegatronBertForNextSentencePrediction(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				str           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase			, )
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, 2)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   MegatronBertForPreTraining(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Tuple           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase			, next_sentence_label=_lowerCAmelCase			, )
										self.parent.assertEqual(result.prediction_logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)
										self.parent.assertEqual(result.seq_relationship_logits.shape			, (self.batch_size, 2)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   MegatronBertForQuestionAnswering(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, start_positions=_lowerCAmelCase			, end_positions=_lowerCAmelCase			, )
										self.parent.assertEqual(result.start_logits.shape			, (self.batch_size, self.seq_length)					)
										self.parent.assertEqual(result.end_logits.shape			, (self.batch_size, self.seq_length)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   self.num_labels
										__SCREAMING_SNAKE_CASE:				str           =   MegatronBertForSequenceClassification(_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_labels)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   self.num_labels
										__SCREAMING_SNAKE_CASE:				str           =   MegatronBertForTokenClassification(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				List[str]           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.num_labels)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   self.num_choices
										__SCREAMING_SNAKE_CASE:				Any           =   MegatronBertForMultipleChoice(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Any           =   input_ids.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Dict           =   token_type_ids.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Dict           =   input_mask.unsqueeze(1					).expand(-1			, self.num_choices			, -1					).contiguous()
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   model(
										    _lowerCAmelCase			, attention_mask=_lowerCAmelCase			, token_type_ids=_lowerCAmelCase			, labels=_lowerCAmelCase			, )
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_choices)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.prepare_config_and_inputs()
										(
										    (
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,(
										    __SCREAMING_SNAKE_CASE						
										)       ,						
										):				str           =   config_and_inputs
										__SCREAMING_SNAKE_CASE:				Any           =   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 				a			(      __lowercase   ,__lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     (
				    (
				        MegatronBertModel,
				        MegatronBertForMaskedLM,
				        MegatronBertForCausalLM,
				        MegatronBertForMultipleChoice,
				        MegatronBertForNextSentencePrediction,
				        MegatronBertForPreTraining,
				        MegatronBertForQuestionAnswering,
				        MegatronBertForSequenceClassification,
				        MegatronBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     (
				    {
				        "feature-extraction": MegatronBertModel,
				        "fill-mask": MegatronBertForMaskedLM,
				        "question-answering": MegatronBertForQuestionAnswering,
				        "text-classification": MegatronBertForSequenceClassification,
				        "text-generation": MegatronBertForCausalLM,
				        "token-classification": MegatronBertForTokenClassification,
				        "zero-shot": MegatronBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     True
				# test_resize_embeddings = False
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     False






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase=False					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   super()._prepare_for_class(_lowerCAmelCase			, _lowerCAmelCase			, return_labels=_lowerCAmelCase					)

										if return_labels:
																if model_class in get_values(_lowerCAmelCase					):
																						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)			, dtype=torch.long			, device=_lowerCAmelCase					)
																						__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.zeros(
																						    self.model_tester.batch_size			, dtype=torch.long			, device=_lowerCAmelCase					)
										return inputs_dict






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   MegatronBertModelTester(self					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   ConfigTester(self			, config_class=_lowerCAmelCase			, hidden_size=37					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										self.config_tester.run_common_tests()






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_model(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_pretraining(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_question_answering(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_megatron_bert_for_token_classification(*_lowerCAmelCase					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   List[Any]					)      ->    Optional[Any]:

						"""simple docstring"""





						return torch.tensor(
						    UpperCamelCase__       ,			dtype=torch.long       ,			device=UpperCamelCase__       ,			)


lowerCAmelCase							:       Any					 =	1e-4





@require_torch
@require_sentencepiece
@require_tokenizers
class 				a			(      unittest.TestCase			):






				@slow
				@unittest.skip('''Model is not available.'''					)
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   '''nvidia/megatron-bert-uncased-345m'''
										if "MYDIR" in os.environ:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   os.path.join(os.environ['''MYDIR''']			, _lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				int           =   MegatronBertModel.from_pretrained(_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.half()
										__SCREAMING_SNAKE_CASE:				str           =   _long_tensor([[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]					)
										with torch.no_grad():
																__SCREAMING_SNAKE_CASE:				Dict           =   model(_lowerCAmelCase					)[0]
										__SCREAMING_SNAKE_CASE:				Dict           =   torch.Size((1, 9, 1024)					)
										self.assertEqual(output.shape			, _lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				str           =   [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
										for ii in range(3					):
																for jj in range(3					):
																						__SCREAMING_SNAKE_CASE:				Tuple           =   output[0, ii, jj]
																						__SCREAMING_SNAKE_CASE:				Tuple           =   expected[3 * ii + jj]
																						__SCREAMING_SNAKE_CASE:				List[Any]           =   '''ii={} jj={} a={} b={}'''.format(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)
																						self.assertTrue(math.isclose(_lowerCAmelCase			, _lowerCAmelCase			, rel_tol=_lowerCAmelCase			, abs_tol=_lowerCAmelCase					)			, msg=_lowerCAmelCase					)





import pprint

import requests

lowerCamelCase__						=				"https://zenquotes.io/api"


def 				__A()		->					list:

		"""simple docstring"""






		return requests.get(API_ENDPOINT_URL + """/today"""			).json()


def 				__A()		->					list:

		"""simple docstring"""






		return requests.get(API_ENDPOINT_URL + """/random"""			).json()


if __name__ == "__main__":
		lowerCamelCase__						=				random_quotes()
		pprint.pprint(response)



