

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    MegatronBertForCausalLM,
					    MegatronBertForMaskedLM,
					    MegatronBertForMultipleChoice,
					    MegatronBertForNextSentencePrediction,
					    MegatronBertForPreTraining,
					    MegatronBertForQuestionAnswering,
					    MegatronBertForSequenceClassification,
					    MegatronBertForTokenClassification,
					    MegatronBertModel,
					)






class        A_		:
							def __init__(		self		:			Dict   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			str=1_3   ,SCREAMING_SNAKE_CASE__		:			List[str]=7   ,SCREAMING_SNAKE_CASE__		:			int=True   ,SCREAMING_SNAKE_CASE__		:			List[str]=True   ,SCREAMING_SNAKE_CASE__		:			List[Any]=True   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]=True   ,SCREAMING_SNAKE_CASE__		:			Dict=9_9   ,SCREAMING_SNAKE_CASE__		:			Tuple=6_4   ,SCREAMING_SNAKE_CASE__		:			Any=3_2   ,SCREAMING_SNAKE_CASE__		:			Optional[int]=5   ,SCREAMING_SNAKE_CASE__		:			Optional[int]=4   ,SCREAMING_SNAKE_CASE__		:			List[str]=3_7   ,SCREAMING_SNAKE_CASE__		:			Dict="gelu"   ,SCREAMING_SNAKE_CASE__		:			int=0.1   ,SCREAMING_SNAKE_CASE__		:			Any=0.1   ,SCREAMING_SNAKE_CASE__		:			List[str]=5_1_2   ,SCREAMING_SNAKE_CASE__		:			Any=1_6   ,SCREAMING_SNAKE_CASE__		:			List[Any]=2   ,SCREAMING_SNAKE_CASE__		:			str=0.02   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]=3   ,SCREAMING_SNAKE_CASE__		:			List[str]=4   ,SCREAMING_SNAKE_CASE__		:			Tuple=None   ,):
								__lowerCamelCase				:				Optional[int]					       = parent
								__lowerCamelCase				:				Any					       = batch_size
								__lowerCamelCase				:				List[str]					       = seq_length
								__lowerCamelCase				:				List[str]					       = is_training
								__lowerCamelCase				:				int					       = use_input_mask
								__lowerCamelCase				:				List[str]					       = use_token_type_ids
								__lowerCamelCase				:				List[str]					       = use_labels
								__lowerCamelCase				:				Dict					       = vocab_size
								__lowerCamelCase				:				Optional[int]					       = hidden_size
								__lowerCamelCase				:				Union[str, Any]					       = embedding_size
								__lowerCamelCase				:				Optional[int]					       = num_hidden_layers
								__lowerCamelCase				:				int					       = num_attention_heads
								__lowerCamelCase				:				Dict					       = intermediate_size
								__lowerCamelCase				:				Union[str, Any]					       = hidden_act
								__lowerCamelCase				:				Dict					       = hidden_dropout_prob
								__lowerCamelCase				:				List[Any]					       = attention_probs_dropout_prob
								__lowerCamelCase				:				Optional[int]					       = max_position_embeddings
								__lowerCamelCase				:				Tuple					       = type_vocab_size
								__lowerCamelCase				:				List[Any]					       = type_sequence_label_size
								__lowerCamelCase				:				Tuple					       = initializer_range
								__lowerCamelCase				:				int					       = num_labels
								__lowerCamelCase				:				Optional[int]					       = num_choices
								__lowerCamelCase				:				Optional[Any]					       = scope
							def     lowerCAmelCase     (		self		:			List[Any]):
								__lowerCamelCase				:				Any					       = ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size)

								__lowerCamelCase				:				Optional[Any]					       = None
								if self.use_input_mask:
									__lowerCamelCase				:				Optional[Any]					       = random_attention_mask([self.batch_size, self.seq_length])

								__lowerCamelCase				:				List[str]					       = None
								if self.use_token_type_ids:
									__lowerCamelCase				:				int					       = ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size)

								__lowerCamelCase				:				Dict					       = None
								__lowerCamelCase				:				Dict					       = None
								__lowerCamelCase				:				Tuple					       = None
								if self.use_labels:
									__lowerCamelCase				:				Optional[Any]					       = ids_tensor([self.batch_size]   ,self.type_sequence_label_size)
									__lowerCamelCase				:				Any					       = ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels)
									__lowerCamelCase				:				Union[str, Any]					       = ids_tensor([self.batch_size]   ,self.num_choices)

								__lowerCamelCase				:				int					       = self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
							def     lowerCAmelCase     (		self		:			Optional[int]):
								return MegatronBertConfig(
								    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,embedding_size=self.embedding_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,is_decoder=SCREAMING_SNAKE_CASE__   ,initializer_range=self.initializer_range   ,)
							def     lowerCAmelCase     (		self		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			str   ,SCREAMING_SNAKE_CASE__		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			List[Any]):
								__lowerCamelCase				:				str					       = MegatronBertModel(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				str					       = model(SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				str					       = model(SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				Any					       = model(SCREAMING_SNAKE_CASE__)

								self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size))
								self.parent.assertEqual(result.pooler_output.shape   ,(self.batch_size, self.hidden_size))
							def     lowerCAmelCase     (		self		:			Dict   ,SCREAMING_SNAKE_CASE__		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			str   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			List[str]):
								__lowerCamelCase				:				Any					       = MegatronBertForMaskedLM(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Optional[Any]					       = model(SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size))
							def     lowerCAmelCase     (		self		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]   ,SCREAMING_SNAKE_CASE__		:			Dict   ,SCREAMING_SNAKE_CASE__		:			Dict   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			List[Any]):
								__lowerCamelCase				:				int					       = MegatronBertForCausalLM(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				int					       = model(SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size))
							def     lowerCAmelCase     (		self		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			str   ,SCREAMING_SNAKE_CASE__		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]):
								__lowerCamelCase				:				Optional[Any]					       = MegatronBertForNextSentencePrediction(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				List[Any]					       = model(
								    SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__   ,)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, 2))
							def     lowerCAmelCase     (		self		:			Dict   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			Dict   ,SCREAMING_SNAKE_CASE__		:			Dict):
								__lowerCamelCase				:				Dict					       = MegatronBertForPreTraining(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Union[str, Any]					       = model(
								    SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__   ,next_sentence_label=SCREAMING_SNAKE_CASE__   ,)
								self.parent.assertEqual(result.prediction_logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size))
								self.parent.assertEqual(result.seq_relationship_logits.shape   ,(self.batch_size, 2))
							def     lowerCAmelCase     (		self		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			Dict   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Any):
								__lowerCamelCase				:				Optional[int]					       = MegatronBertForQuestionAnswering(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Union[str, Any]					       = model(
								    SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,start_positions=SCREAMING_SNAKE_CASE__   ,end_positions=SCREAMING_SNAKE_CASE__   ,)
								self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length))
							def     lowerCAmelCase     (		self		:			Dict   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Any):
								__lowerCamelCase				:				int					       = self.num_labels
								__lowerCamelCase				:				Tuple					       = MegatronBertForSequenceClassification(SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Union[str, Any]					       = model(SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels))
							def     lowerCAmelCase     (		self		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			Any):
								__lowerCamelCase				:				int					       = self.num_labels
								__lowerCamelCase				:				Dict					       = MegatronBertForTokenClassification(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Dict					       = model(SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels))
							def     lowerCAmelCase     (		self		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			List[Any]   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]):
								__lowerCamelCase				:				Dict					       = self.num_choices
								__lowerCamelCase				:				Optional[Any]					       = MegatronBertForMultipleChoice(config=SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.eval()
								__lowerCamelCase				:				Tuple					       = input_ids.unsqueeze(1).expand(-1   ,self.num_choices   ,-1).contiguous()
								__lowerCamelCase				:				Any					       = token_type_ids.unsqueeze(1).expand(-1   ,self.num_choices   ,-1).contiguous()
								__lowerCamelCase				:				int					       = input_mask.unsqueeze(1).expand(-1   ,self.num_choices   ,-1).contiguous()
								__lowerCamelCase				:				Union[str, Any]					       = model(
								    SCREAMING_SNAKE_CASE__   ,attention_mask=SCREAMING_SNAKE_CASE__   ,token_type_ids=SCREAMING_SNAKE_CASE__   ,labels=SCREAMING_SNAKE_CASE__   ,)
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices))


							def     lowerCAmelCase     (		self		:			List[str]):
								__lowerCamelCase				:				Optional[Any]					       = self.prepare_config_and_inputs()
								(
								    (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , (
								    __lowerCamelCase
								)       , 
								)				:				Optional[int]					       = config_and_inputs
								__lowerCamelCase				:				str					       = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict






@require_torch
class        A_		(       SCREAMING_SNAKE_CASE			,							SCREAMING_SNAKE_CASE			,							unittest.TestCase				):
							_UpperCAmelCase    :  List[str]						      =      (
							    (
							        MegatronBertModel,
							        MegatronBertForMaskedLM,
							        MegatronBertForCausalLM,
							        MegatronBertForMultipleChoice,
							        MegatronBertForNextSentencePrediction,
							        MegatronBertForPreTraining,
							        MegatronBertForQuestionAnswering,
							        MegatronBertForSequenceClassification,
							        MegatronBertForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							_UpperCAmelCase    :  Union[str, Any]						      =      (
							    {
							        '''feature-extraction''': MegatronBertModel,
							        '''fill-mask''': MegatronBertForMaskedLM,
							        '''question-answering''': MegatronBertForQuestionAnswering,
							        '''text-classification''': MegatronBertForSequenceClassification,
							        '''text-generation''': MegatronBertForCausalLM,
							        '''token-classification''': MegatronBertForTokenClassification,
							        '''zero-shot''': MegatronBertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_UpperCAmelCase    :  Union[str, Any]						      =      True
							# test_resize_embeddings = False
							_UpperCAmelCase    :  List[Any]						      =      False
							def     lowerCAmelCase     (		self		:			str   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Any   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]=False):
								__lowerCamelCase				:				Dict					       = super()._prepare_for_class(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__   ,return_labels=SCREAMING_SNAKE_CASE__)

								if return_labels:
									if model_class in get_values(SCREAMING_SNAKE_CASE__):
										__lowerCamelCase				:				Tuple					       = torch.zeros(
										    (self.model_tester.batch_size, self.model_tester.seq_length)   ,dtype=torch.long   ,device=SCREAMING_SNAKE_CASE__)
										__lowerCamelCase				:				int					       = torch.zeros(
										    self.model_tester.batch_size   ,dtype=torch.long   ,device=SCREAMING_SNAKE_CASE__)
								return inputs_dict
							def     lowerCAmelCase     (		self		:			Optional[int]):
								__lowerCamelCase				:				str					       = MegatronBertModelTester(self)
								__lowerCamelCase				:				Tuple					       = ConfigTester(self   ,config_class=SCREAMING_SNAKE_CASE__   ,hidden_size=3_7)
							def     lowerCAmelCase     (		self		:			Optional[int]):
								self.config_tester.run_common_tests()
							def     lowerCAmelCase     (		self		:			Any):
								__lowerCamelCase				:				Optional[Any]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_model(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			str):
								__lowerCamelCase				:				Optional[Any]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_masked_lm(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			int):
								__lowerCamelCase				:				List[Any]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			List[Any]):
								__lowerCamelCase				:				Optional[int]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			Optional[int]):
								__lowerCamelCase				:				Tuple					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_pretraining(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			Any):
								__lowerCamelCase				:				Union[str, Any]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_question_answering(*SCREAMING_SNAKE_CASE__)
							def     lowerCAmelCase     (		self		:			int):
								__lowerCamelCase				:				Dict					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*SCREAMING_SNAKE_CASE__)


							def     lowerCAmelCase     (		self		:			List[str]):
								__lowerCamelCase				:				List[str]					       = self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_megatron_bert_for_token_classification(*SCREAMING_SNAKE_CASE__)






def    SCREAMING_SNAKE_CASE__       (  lowerCamelCase__			)  ->     Tuple:
	return torch.tensor(
	    lowerCamelCase__	, dtype=torch.long	, device=lowerCamelCase__	, )


a		=1E-4






@require_torch
@require_sentencepiece
@require_tokenizers
class        A_		(       unittest.TestCase				):
							@slow
							@unittest.skip('Model is not available.')
							def     lowerCAmelCase     (		self		:			Tuple):
								__lowerCamelCase				:				Optional[Any]					       = 'nvidia/megatron-bert-uncased-345m'
								if "MYDIR" in os.environ:
									__lowerCamelCase				:				Optional[Any]					       = os.path.join(os.environ['MYDIR']   ,SCREAMING_SNAKE_CASE__)
								__lowerCamelCase				:				str					       = MegatronBertModel.from_pretrained(SCREAMING_SNAKE_CASE__)
								model.to(SCREAMING_SNAKE_CASE__)
								model.half()
								__lowerCamelCase				:				List[str]					       = _long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]])
								with torch.no_grad():
									__lowerCamelCase				:				str					       = model(SCREAMING_SNAKE_CASE__)[0]
								__lowerCamelCase				:				List[Any]					       = torch.Size((1, 9, 1_0_2_4))
								self.assertEqual(output.shape   ,SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				str					       = [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
								for ii in range(3):
									for jj in range(3):
										__lowerCamelCase				:				int					       = output[0, ii, jj]
										__lowerCamelCase				:				Dict					       = expected[3 * ii + jj]
										__lowerCamelCase				:				List[Any]					       = 'ii={} jj={} a={} b={}'.format(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__)
										self.assertTrue(math.isclose(SCREAMING_SNAKE_CASE__   ,SCREAMING_SNAKE_CASE__   ,rel_tol=SCREAMING_SNAKE_CASE__   ,abs_tol=SCREAMING_SNAKE_CASE__)   ,msg=SCREAMING_SNAKE_CASE__)











"""simple docstring"""




from transformers import BertTokenizerFast

from .custom_tokenization import CustomTokenizer

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : List[str]	    =   CustomTokenizer
      pass