



'''simple docstring'''

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    MegatronBertForCausalLM,
				    MegatronBertForMaskedLM,
				    MegatronBertForMultipleChoice,
				    MegatronBertForNextSentencePrediction,
				    MegatronBertForPreTraining,
				    MegatronBertForQuestionAnswering,
				    MegatronBertForSequenceClassification,
				    MegatronBertForTokenClassification,
				    MegatronBertModel,
				)

class __snake_case:


						'''simple docstring'''



						def __init__(   self      ,			A_      ,			A_=13      ,			A_=7      ,			A_=True      ,			A_=True      ,			A_=True      ,			A_=True      ,			A_=99      ,			A_=64      ,			A_=32      ,			A_=5      ,			A_=4      ,			A_=37      ,			A_="gelu"      ,			A_=0.1      ,			A_=0.1      ,			A_=512      ,			A_=16      ,			A_=2      ,			A_=0.0_2      ,			A_=3      ,			A_=4      ,			A_=None      ,			)      ->      Any:
							lowerCAmelCase       =      parent
							lowerCAmelCase       =      batch_size
							lowerCAmelCase       =      seq_length
							lowerCAmelCase       =      is_training
							lowerCAmelCase       =      use_input_mask
							lowerCAmelCase       =      use_token_type_ids
							lowerCAmelCase       =      use_labels
							lowerCAmelCase       =      vocab_size
							lowerCAmelCase       =      hidden_size
							lowerCAmelCase       =      embedding_size
							lowerCAmelCase       =      num_hidden_layers
							lowerCAmelCase       =      num_attention_heads
							lowerCAmelCase       =      intermediate_size
							lowerCAmelCase       =      hidden_act
							lowerCAmelCase       =      hidden_dropout_prob
							lowerCAmelCase       =      attention_probs_dropout_prob
							lowerCAmelCase       =      max_position_embeddings
							lowerCAmelCase       =      type_vocab_size
							lowerCAmelCase       =      type_sequence_label_size
							lowerCAmelCase       =      initializer_range
							lowerCAmelCase       =      num_labels
							lowerCAmelCase       =      num_choices
							lowerCAmelCase       =      scope


						def 							__snake_case       (   self )      ->      Optional[int]:
							lowerCAmelCase       =      ids_tensor([self.batch_size, self.seq_length]      ,			self.vocab_size )

							lowerCAmelCase       =      None
							if self.use_input_mask:
								lowerCAmelCase       =      random_attention_mask([self.batch_size, self.seq_length] )

							lowerCAmelCase       =      None
							if self.use_token_type_ids:
								lowerCAmelCase       =      ids_tensor([self.batch_size, self.seq_length]      ,			self.type_vocab_size )

							lowerCAmelCase       =      None
							lowerCAmelCase       =      None
							lowerCAmelCase       =      None
							if self.use_labels:
								lowerCAmelCase       =      ids_tensor([self.batch_size]      ,			self.type_sequence_label_size )
								lowerCAmelCase       =      ids_tensor([self.batch_size, self.seq_length]      ,			self.num_labels )
								lowerCAmelCase       =      ids_tensor([self.batch_size]      ,			self.num_choices )

							lowerCAmelCase       =      self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def 							__snake_case       (   self )      ->      Optional[Any]:
							return MegatronBertConfig(
							    vocab_size=self.vocab_size      ,			hidden_size=self.hidden_size      ,			num_hidden_layers=self.num_hidden_layers      ,			num_attention_heads=self.num_attention_heads      ,			intermediate_size=self.intermediate_size      ,			embedding_size=self.embedding_size      ,			hidden_act=self.hidden_act      ,			hidden_dropout_prob=self.hidden_dropout_prob      ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,			max_position_embeddings=self.max_position_embeddings      ,			type_vocab_size=self.type_vocab_size      ,			is_decoder=A_      ,			initializer_range=self.initializer_range      ,			)


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Any:
							lowerCAmelCase       =      MegatronBertModel(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(A_      ,			attention_mask=A_      ,			token_type_ids=A_ )
							lowerCAmelCase       =      model(A_      ,			token_type_ids=A_ )
							lowerCAmelCase       =      model(A_ )

							self.parent.assertEqual(result.last_hidden_state.shape      ,			(self.batch_size, self.seq_length, self.hidden_size) )
							self.parent.assertEqual(result.pooler_output.shape      ,			(self.batch_size, self.hidden_size) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      int:
							lowerCAmelCase       =      MegatronBertForMaskedLM(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_ )
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, self.seq_length, self.vocab_size) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Union[str, Any]:
							lowerCAmelCase       =      MegatronBertForCausalLM(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_ )
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, self.seq_length, self.vocab_size) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Any:
							lowerCAmelCase       =      MegatronBertForNextSentencePrediction(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(
							    A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_      ,			)
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, 2) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Tuple:
							lowerCAmelCase       =      MegatronBertForPreTraining(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(
							    A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_      ,			next_sentence_label=A_      ,			)
							self.parent.assertEqual(result.prediction_logits.shape      ,			(self.batch_size, self.seq_length, self.vocab_size) )
							self.parent.assertEqual(result.seq_relationship_logits.shape      ,			(self.batch_size, 2) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Tuple:
							lowerCAmelCase       =      MegatronBertForQuestionAnswering(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(
							    A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			start_positions=A_      ,			end_positions=A_      ,			)
							self.parent.assertEqual(result.start_logits.shape      ,			(self.batch_size, self.seq_length) )
							self.parent.assertEqual(result.end_logits.shape      ,			(self.batch_size, self.seq_length) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      Union[str, Any]:
							lowerCAmelCase       =      self.num_labels
							lowerCAmelCase       =      MegatronBertForSequenceClassification(A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_ )
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, self.num_labels) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      List[Any]:
							lowerCAmelCase       =      self.num_labels
							lowerCAmelCase       =      MegatronBertForTokenClassification(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      model(A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_ )
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, self.seq_length, self.num_labels) )


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_      ,			A_ )      ->      List[Any]:
							lowerCAmelCase       =      self.num_choices
							lowerCAmelCase       =      MegatronBertForMultipleChoice(config=A_ )
							model.to(A_ )
							model.eval()
							lowerCAmelCase       =      input_ids.unsqueeze(1 ).expand(-1      ,			self.num_choices      ,			-1 ).contiguous()
							lowerCAmelCase       =      token_type_ids.unsqueeze(1 ).expand(-1      ,			self.num_choices      ,			-1 ).contiguous()
							lowerCAmelCase       =      input_mask.unsqueeze(1 ).expand(-1      ,			self.num_choices      ,			-1 ).contiguous()
							lowerCAmelCase       =      model(
							    A_      ,			attention_mask=A_      ,			token_type_ids=A_      ,			labels=A_      ,			)
							self.parent.assertEqual(result.logits.shape      ,			(self.batch_size, self.num_choices) )




						def 							__snake_case       (   self )      ->      Optional[int]:
							lowerCAmelCase       =      self.prepare_config_and_inputs()
							(
							    lowerCAmelCase       
							)       =      config_and_inputs
							lowerCAmelCase       =      {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class __snake_case( __snake_case						,				__snake_case						,				unittest.TestCase   ):


						'''simple docstring'''



						UpperCAmelCase				:       Tuple 	=						(
						    (
						        MegatronBertModel,
						        MegatronBertForMaskedLM,
						        MegatronBertForCausalLM,
						        MegatronBertForMultipleChoice,
						        MegatronBertForNextSentencePrediction,
						        MegatronBertForPreTraining,
						        MegatronBertForQuestionAnswering,
						        MegatronBertForSequenceClassification,
						        MegatronBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						UpperCAmelCase				:       Optional[Any] 	=						(
						    {
						        'feature-extraction': MegatronBertModel,
						        'fill-mask': MegatronBertForMaskedLM,
						        'question-answering': MegatronBertForQuestionAnswering,
						        'text-classification': MegatronBertForSequenceClassification,
						        'text-generation': MegatronBertForCausalLM,
						        'token-classification': MegatronBertForTokenClassification,
						        'zero-shot': MegatronBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						UpperCAmelCase				:       Optional[Any] 	=						True
						# test_resize_embeddings = False
						UpperCAmelCase				:       Optional[Any] 	=						False


						def 							__snake_case       (   self      ,			A_      ,			A_      ,			A_=False )      ->      List[str]:
							lowerCAmelCase       =      super()._prepare_for_class(A_      ,			A_      ,			return_labels=A_ )

							if return_labels:
								if model_class in get_values(A_ ):
									lowerCAmelCase       =      torch.zeros(
									    (self.model_tester.batch_size, self.model_tester.seq_length)      ,			dtype=torch.long      ,			device=A_ )
									lowerCAmelCase       =      torch.zeros(
									    self.model_tester.batch_size      ,			dtype=torch.long      ,			device=A_ )
							return inputs_dict


						def 							__snake_case       (   self )      ->      Union[str, Any]:
							lowerCAmelCase       =      MegatronBertModelTester(self )
							lowerCAmelCase       =      ConfigTester(self      ,			config_class=A_      ,			hidden_size=37 )


						def 							__snake_case       (   self )      ->      Dict:
							self.config_tester.run_common_tests()


						def 							__snake_case       (   self )      ->      Union[str, Any]:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_model(*A_ )


						def 							__snake_case       (   self )      ->      List[Any]:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_masked_lm(*A_ )


						def 							__snake_case       (   self )      ->      str:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*A_ )


						def 							__snake_case       (   self )      ->      Any:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*A_ )


						def 							__snake_case       (   self )      ->      Union[str, Any]:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_pretraining(*A_ )


						def 							__snake_case       (   self )      ->      Any:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_question_answering(*A_ )


						def 							__snake_case       (   self )      ->      Any:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*A_ )




						def 							__snake_case       (   self )      ->      List[str]:
							lowerCAmelCase       =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_token_classification(*A_ )







def 						_snake_case						(   _SCREAMING_SNAKE_CASE	:		Dict				)     ->		int:







	"""simple docstring"""
	return torch.tensor(
	    _lowerCAmelCase				,						dtype=torch.long				,						device=_lowerCAmelCase				,						)


UpperCAmelCase      						=       1e-4





@require_torch
@require_sentencepiece
@require_tokenizers
class __snake_case( unittest.TestCase   ):


						'''simple docstring'''



						@slow
						@unittest.skip("""Model is not available.""" )
						def 							__snake_case       (   self )      ->      Union[str, Any]:
							lowerCAmelCase       =      "nvidia/megatron-bert-uncased-345m"
							if "MYDIR" in os.environ:
								lowerCAmelCase       =      os.path.join(os.environ["""MYDIR"""]      ,			A_ )
							lowerCAmelCase       =      MegatronBertModel.from_pretrained(A_ )
							model.to(A_ )
							model.half()
							lowerCAmelCase       =      _long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]] )
							with torch.no_grad():
								lowerCAmelCase       =      model(A_ )[0]
							lowerCAmelCase       =      torch.Size((1, 9, 1024) )
							self.assertEqual(output.shape      ,			A_ )

							lowerCAmelCase       =      [-0.6_0_4_0, -0.2_5_1_7, -0.1_0_2_5, 0.3_4_2_0, -0.6_7_5_8, -0.0_0_1_7, -0.1_0_8_9, -0.1_9_9_0, 0.5_7_2_8]
							for ii in range(3 ):
								for jj in range(3 ):
									lowerCAmelCase       =      output[0, ii, jj]
									lowerCAmelCase       =      expected[3 * ii + jj]
									lowerCAmelCase       =      "ii={} jj={} a={} b={}".format(A_      ,			A_      ,			A_      ,			A_ )
									self.assertTrue(math.isclose(A_      ,			A_      ,			rel_tol=A_      ,			abs_tol=A_ )      ,			msg=A_ )






import os



def 						A_							(	)		->					Union[str, Any]:
 with open(os.path.dirname(_lowerCAmelCase	) + "/grid.txt"	) as f:
  UpperCamelCase      :   Optional[Any]			      =  []  # noqa: E741
  for _ in range(20	):
   l.append([int(_lowerCAmelCase	) for x in f.readline().split()]	)

  UpperCamelCase      :   str			      =  0

  # right
  for i in range(20	):
   for j in range(17	):
    UpperCamelCase      :   int			      =  l[i][j] * l[i][j + 1] * l[i][j + 2] * l[i][j + 3]
    if temp > maximum:
     UpperCamelCase      :   List[Any]			      =  temp

        # down
  for i in range(17	):
   for j in range(20	):
    UpperCamelCase      :   List[str]			      =  l[i][j] * l[i + 1][j] * l[i + 2][j] * l[i + 3][j]
    if temp > maximum:
     UpperCamelCase      :   List[str]			      =  temp

        # diagonal 1
  for i in range(17	):
   for j in range(17	):
    UpperCamelCase      :   Any			      =  l[i][j] * l[i + 1][j + 1] * l[i + 2][j + 2] * l[i + 3][j + 3]
    if temp > maximum:
     UpperCamelCase      :   Tuple			      =  temp

        # diagonal 2
  for i in range(17	):
   for j in range(3				,  20	):
    UpperCamelCase      :   Tuple			      =  l[i][j] * l[i + 1][j - 1] * l[i + 2][j - 2] * l[i + 3][j - 3]
    if temp > maximum:
     UpperCamelCase      :   List[Any]			      =  temp
  return maximum


if __name__ == "__main__":
    print(solution())
