
"""simple docstring"""





import inspect
import unittest

from transformers import SegformerConfig, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_MAPPING,
							    SegformerForImageClassification,
							    SegformerForSemanticSegmentation,
							    SegformerModel,
							)
							from transformers.models.segformer.modeling_segformer import SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from PIL import Image

							from transformers import SegformerImageProcessor
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.config_class(**self.inputs_dict)
				self.parent.assertTrue(hasattr(lowerCAmelCase__							,  """hidden_sizes"""))
				self.parent.assertTrue(hasattr(lowerCAmelCase__							,  """num_attention_heads"""))
				self.parent.assertTrue(hasattr(lowerCAmelCase__							,  """num_encoder_blocks"""))



class 						SCREAMING_SNAKE_CASE_ :




		"""simple docstring"""





		def __init__(     self							,  lowerCAmelCase__							,  lowerCAmelCase__=1_3							,  lowerCAmelCase__=6_4							,  lowerCAmelCase__=3							,  lowerCAmelCase__=4							,  lowerCAmelCase__=[2, 2, 2, 2]							,  lowerCAmelCase__=[8, 4, 2, 1]							,  lowerCAmelCase__=[1_6, 3_2, 6_4, 1_2_8]							,  lowerCAmelCase__=[1, 4, 8, 1_6]							,  lowerCAmelCase__=[1, 2, 4, 8]							,  lowerCAmelCase__=True							,  lowerCAmelCase__=True							,  lowerCAmelCase__="gelu"							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=0.02							,  lowerCAmelCase__=3							,  lowerCAmelCase__=None							,  ):
				__SCREAMING_SNAKE_CASE            = parent
				__SCREAMING_SNAKE_CASE            = batch_size
				__SCREAMING_SNAKE_CASE            = image_size
				__SCREAMING_SNAKE_CASE            = num_channels
				__SCREAMING_SNAKE_CASE            = num_encoder_blocks
				__SCREAMING_SNAKE_CASE            = sr_ratios
				__SCREAMING_SNAKE_CASE            = depths
				__SCREAMING_SNAKE_CASE            = hidden_sizes
				__SCREAMING_SNAKE_CASE            = downsampling_rates
				__SCREAMING_SNAKE_CASE            = num_attention_heads
				__SCREAMING_SNAKE_CASE            = is_training
				__SCREAMING_SNAKE_CASE            = use_labels
				__SCREAMING_SNAKE_CASE            = hidden_act
				__SCREAMING_SNAKE_CASE            = hidden_dropout_prob
				__SCREAMING_SNAKE_CASE            = attention_probs_dropout_prob
				__SCREAMING_SNAKE_CASE            = initializer_range
				__SCREAMING_SNAKE_CASE            = num_labels
				__SCREAMING_SNAKE_CASE            = scope




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

				__SCREAMING_SNAKE_CASE            = None
				if self.use_labels:
						__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size, self.image_size, self.image_size]							,  self.num_labels)

				__SCREAMING_SNAKE_CASE            = self.get_config()
				return config, pixel_values, labels




		def 	snake_case_     (     self):
				return SegformerConfig(
				    image_size=self.image_size							,  num_channels=self.num_channels							,  num_encoder_blocks=self.num_encoder_blocks							,  depths=self.depths							,  hidden_sizes=self.hidden_sizes							,  num_attention_heads=self.num_attention_heads							,  hidden_act=self.hidden_act							,  hidden_dropout_prob=self.hidden_dropout_prob							,  attention_probs_dropout_prob=self.attention_probs_dropout_prob							,  initializer_range=self.initializer_range							,  )




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = SegformerModel(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = __SCREAMING_SNAKE_CASE            = self.image_size // (self.downsampling_rates[-1] * 2)
				self.parent.assertEqual(
				    result.last_hidden_state.shape							,  (self.batch_size, self.hidden_sizes[-1], expected_height, expected_width))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = self.num_labels
				__SCREAMING_SNAKE_CASE            = SegformerForSemanticSegmentation(lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)
				self.parent.assertEqual(
				    result.logits.shape							,  (self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  labels=lowerCAmelCase__)
				self.parent.assertEqual(
				    result.logits.shape							,  (self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
				self.parent.assertGreater(result.loss							,  0.0)




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = 1
				__SCREAMING_SNAKE_CASE            = SegformerForSemanticSegmentation(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = torch.randint(0							,  1							,  (self.batch_size, self.image_size, self.image_size)).to(lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  labels=lowerCAmelCase__)
				self.parent.assertGreater(result.loss							,  0.0)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.prepare_config_and_inputs()
				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = config_and_inputs
				__SCREAMING_SNAKE_CASE            = {"""pixel_values""": pixel_values}
				return config, inputs_dict



@require_torch
class 						SCREAMING_SNAKE_CASE_ (						__a     ,				__a     ,				unittest.TestCase ):




		"""simple docstring"""





		__lowercase							:  int						   =		(
		    (
		        SegformerModel,
		        SegformerForSemanticSegmentation,
		        SegformerForImageClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowercase							:  int						   =		(
		    {
		        '''feature-extraction''': SegformerModel,
		        '''image-classification''': SegformerForImageClassification,
		        '''image-segmentation''': SegformerForSemanticSegmentation,
		    }
		    if is_torch_available()
		    else {}
		)

		__lowercase							:  Tuple						   =		True
		__lowercase							:  List[str]						   =		False
		__lowercase							:  Optional[int]						   =		False
		__lowercase							:  Tuple						   =		False




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = SegformerModelTester(self)
				__SCREAMING_SNAKE_CASE            = SegformerConfigTester(self							,  config_class=lowerCAmelCase__)




		def 	snake_case_     (     self):
				self.config_tester.run_common_tests()




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_binary_image_segmentation(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_image_segmentation(*lowerCAmelCase__)




		@unittest.skip("""SegFormer does not use inputs_embeds""")
		def 	snake_case_     (     self):
				pass




		@unittest.skip("""SegFormer does not have get_input_embeddings method and get_output_embeddings methods""")
		def 	snake_case_     (     self):
				pass




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						__SCREAMING_SNAKE_CASE            = inspect.signature(model.forward)
						# signature.parameters is an OrderedDict => so arg_names order is deterministic
						__SCREAMING_SNAKE_CASE            = [*signature.parameters.keys()]

						__SCREAMING_SNAKE_CASE            = ["""pixel_values"""]
						self.assertListEqual(arg_names[:1]							,  lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE            = True

				for model_class in self.all_model_classes:
						__SCREAMING_SNAKE_CASE            = True
						__SCREAMING_SNAKE_CASE            = False
						__SCREAMING_SNAKE_CASE            = True
						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						model.to(lowerCAmelCase__)
						model.eval()
						with torch.no_grad():
								__SCREAMING_SNAKE_CASE            = model(**self._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__))
						__SCREAMING_SNAKE_CASE            = outputs.attentions

						__SCREAMING_SNAKE_CASE            = sum(self.model_tester.depths)
						self.assertEqual(len(lowerCAmelCase__)							,  lowerCAmelCase__)

						# check that output_attentions also work using config
						del inputs_dict["output_attentions"]
						__SCREAMING_SNAKE_CASE            = True
						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						model.to(lowerCAmelCase__)
						model.eval()
						with torch.no_grad():
								__SCREAMING_SNAKE_CASE            = model(**self._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__))
						__SCREAMING_SNAKE_CASE            = outputs.attentions

						self.assertEqual(len(lowerCAmelCase__)							,  lowerCAmelCase__)

						# verify the first attentions (first block, first layer)
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // 4) ** 2
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
						self.assertListEqual(
						    list(attentions[0].shape[-3:])							,  [self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]							,  )

						# verify the last attentions (last block, last layer)
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // 3_2) ** 2
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // (3_2 * self.model_tester.sr_ratios[-1])) ** 2
						self.assertListEqual(
						    list(attentions[-1].shape[-3:])							,  [self.model_tester.num_attention_heads[-1], expected_seq_len, expected_reduced_seq_len]							,  )
						__SCREAMING_SNAKE_CASE            = len(lowerCAmelCase__)

						# Check attention is always last and order is fine
						__SCREAMING_SNAKE_CASE            = True
						__SCREAMING_SNAKE_CASE            = True
						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						model.to(lowerCAmelCase__)
						model.eval()
						with torch.no_grad():
								__SCREAMING_SNAKE_CASE            = model(**self._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__))

						self.assertEqual(out_len + 1							,  len(lowerCAmelCase__))

						__SCREAMING_SNAKE_CASE            = outputs.attentions

						self.assertEqual(len(lowerCAmelCase__)							,  lowerCAmelCase__)
						# verify the first attentions (first block, first layer)
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // 4) ** 2
						__SCREAMING_SNAKE_CASE            = (self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
						self.assertListEqual(
						    list(self_attentions[0].shape[-3:])							,  [self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]							,  )




		def 	snake_case_     (     self):
				def check_hidden_states_output(lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						model.to(lowerCAmelCase__)
						model.eval()

						with torch.no_grad():
								__SCREAMING_SNAKE_CASE            = model(**self._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__))

						__SCREAMING_SNAKE_CASE            = outputs.hidden_states

						__SCREAMING_SNAKE_CASE            = self.model_tester.num_encoder_blocks
						self.assertEqual(len(lowerCAmelCase__)							,  lowerCAmelCase__)

						# verify the first hidden states (first block)
						self.assertListEqual(
						    list(hidden_states[0].shape[-3:])							,  [
						        self.model_tester.hidden_sizes[0],
						        self.model_tester.image_size // 4,
						        self.model_tester.image_size // 4,
						    ]							,  )

				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						__SCREAMING_SNAKE_CASE            = True
						check_hidden_states_output(lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__)

						# check that output_hidden_states also work using config
						del inputs_dict["output_hidden_states"]
						__SCREAMING_SNAKE_CASE            = True

						check_hidden_states_output(lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__)




		def 	snake_case_     (     self):
				if not self.model_tester.is_training:
						return

				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs_for_common()
				__SCREAMING_SNAKE_CASE            = True

				for model_class in self.all_model_classes:
						if model_class in get_values(lowerCAmelCase__):
								continue

						__SCREAMING_SNAKE_CASE            = model_class(lowerCAmelCase__)
						model.to(lowerCAmelCase__)
						model.train()
						__SCREAMING_SNAKE_CASE            = self._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__							,  return_labels=lowerCAmelCase__)
						__SCREAMING_SNAKE_CASE            = model(**lowerCAmelCase__).loss
						loss.backward()




		@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests.""")
		def 	snake_case_     (     self):
				pass




		@slow
		def 	snake_case_     (     self):
				for model_name in SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__SCREAMING_SNAKE_CASE            = SegformerModel.from_pretrained(lowerCAmelCase__)
						self.assertIsNotNone(lowerCAmelCase__)



def 						_lowerCAmelCase				(				):
		__SCREAMING_SNAKE_CASE            = Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""				)
		return image
@require_torch
class 						SCREAMING_SNAKE_CASE_ (						unittest.TestCase ):




		"""simple docstring"""





		@slow
		def 	snake_case_     (     self):
				# only resize + normalize
				__SCREAMING_SNAKE_CASE            = SegformerImageProcessor(
				    image_scale=(5_1_2, 5_1_2)							,  keep_ratio=lowerCAmelCase__							,  align=lowerCAmelCase__							,  do_random_crop=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = SegformerForSemanticSegmentation.from_pretrained("""nvidia/segformer-b0-finetuned-ade-512-512""").to(
				    lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = prepare_img()
				__SCREAMING_SNAKE_CASE            = image_processor(images=lowerCAmelCase__							,  return_tensors="""pt""")
				__SCREAMING_SNAKE_CASE            = encoded_inputs.pixel_values.to(lowerCAmelCase__)

				with torch.no_grad():
						__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = torch.Size((1, model.config.num_labels, 1_2_8, 1_2_8))
				self.assertEqual(outputs.logits.shape							,  lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = torch.tensor(
				    [
				        [[-4.63_10, -5.52_32, -6.23_56], [-5.19_21, -6.14_44, -6.59_96], [-5.44_24, -6.27_90, -6.75_74]],
				        [[-12.13_91, -13.31_22, -13.95_54], [-12.87_32, -13.93_52, -14.35_63], [-12.94_38, -13.82_26, -14.25_13]],
				        [[-12.51_34, -13.46_86, -14.49_15], [-12.86_69, -14.43_43, -14.77_58], [-13.25_23, -14.58_19, -15.06_94]],
				    ]).to(lowerCAmelCase__)
				self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]							,  lowerCAmelCase__							,  atol=1E-4))




		@slow
		def 	snake_case_     (     self):
				# only resize + normalize
				__SCREAMING_SNAKE_CASE            = SegformerImageProcessor(
				    image_scale=(5_1_2, 5_1_2)							,  keep_ratio=lowerCAmelCase__							,  align=lowerCAmelCase__							,  do_random_crop=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = SegformerForSemanticSegmentation.from_pretrained(
				    """nvidia/segformer-b1-finetuned-cityscapes-1024-1024""").to(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = prepare_img()
				__SCREAMING_SNAKE_CASE            = image_processor(images=lowerCAmelCase__							,  return_tensors="""pt""")
				__SCREAMING_SNAKE_CASE            = encoded_inputs.pixel_values.to(lowerCAmelCase__)

				with torch.no_grad():
						__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = torch.Size((1, model.config.num_labels, 1_2_8, 1_2_8))
				self.assertEqual(outputs.logits.shape							,  lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = torch.tensor(
				    [
				        [[-13.57_48, -13.91_11, -12.65_00], [-14.35_00, -15.36_83, -14.23_28], [-14.75_32, -16.04_24, -15.60_87]],
				        [[-17.16_51, -15.87_25, -12.96_53], [-17.25_80, -17.37_18, -14.82_23], [-16.60_58, -16.87_83, -16.74_52]],
				        [[-3.64_56, -3.02_09, -1.42_03], [-3.07_97, -3.19_59, -2.00_00], [-1.87_57, -1.92_17, -1.69_97]],
				    ]).to(lowerCAmelCase__)
				self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]							,  lowerCAmelCase__							,  atol=1E-1))




		@slow
		def 	snake_case_     (     self):
				# only resize + normalize
				__SCREAMING_SNAKE_CASE            = SegformerImageProcessor(
				    image_scale=(5_1_2, 5_1_2)							,  keep_ratio=lowerCAmelCase__							,  align=lowerCAmelCase__							,  do_random_crop=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = SegformerForSemanticSegmentation.from_pretrained("""nvidia/segformer-b0-finetuned-ade-512-512""").to(
				    lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = prepare_img()
				__SCREAMING_SNAKE_CASE            = image_processor(images=lowerCAmelCase__							,  return_tensors="""pt""")
				__SCREAMING_SNAKE_CASE            = encoded_inputs.pixel_values.to(lowerCAmelCase__)

				with torch.no_grad():
						__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = outputs.logits.detach().cpu()

				__SCREAMING_SNAKE_CASE            = image_processor.post_process_semantic_segmentation(outputs=lowerCAmelCase__							,  target_sizes=[(5_0_0, 3_0_0)])
				__SCREAMING_SNAKE_CASE            = torch.Size((5_0_0, 3_0_0))
				self.assertEqual(segmentation[0].shape							,  lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = image_processor.post_process_semantic_segmentation(outputs=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = torch.Size((1_2_8, 1_2_8))
				self.assertEqual(segmentation[0].shape							,  lowerCAmelCase__)






"""simple docstring"""





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class 						SCREAMING_SNAKE_CASE_ :




		"""simple docstring"""





		def __init__(     self							,  lowerCAmelCase__							,  lowerCAmelCase__=1_3							,  lowerCAmelCase__=7							,  lowerCAmelCase__=True							,  lowerCAmelCase__=True							,  lowerCAmelCase__=True							,  lowerCAmelCase__=True							,  lowerCAmelCase__=9_9							,  lowerCAmelCase__=1_6							,  lowerCAmelCase__=3_6							,  lowerCAmelCase__=6							,  lowerCAmelCase__=6							,  lowerCAmelCase__=6							,  lowerCAmelCase__=3_7							,  lowerCAmelCase__="gelu"							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=5_1_2							,  lowerCAmelCase__=1_6							,  lowerCAmelCase__=2							,  lowerCAmelCase__=0.02							,  lowerCAmelCase__=3							,  lowerCAmelCase__=4							,  lowerCAmelCase__=None							,  ):
				__SCREAMING_SNAKE_CASE            = parent
				__SCREAMING_SNAKE_CASE            = batch_size
				__SCREAMING_SNAKE_CASE            = seq_length
				__SCREAMING_SNAKE_CASE            = is_training
				__SCREAMING_SNAKE_CASE            = use_input_mask
				__SCREAMING_SNAKE_CASE            = use_token_type_ids
				__SCREAMING_SNAKE_CASE            = use_labels
				__SCREAMING_SNAKE_CASE            = vocab_size
				__SCREAMING_SNAKE_CASE            = embedding_size
				__SCREAMING_SNAKE_CASE            = hidden_size
				__SCREAMING_SNAKE_CASE            = num_hidden_layers
				__SCREAMING_SNAKE_CASE            = num_hidden_groups
				__SCREAMING_SNAKE_CASE            = num_attention_heads
				__SCREAMING_SNAKE_CASE            = intermediate_size
				__SCREAMING_SNAKE_CASE            = hidden_act
				__SCREAMING_SNAKE_CASE            = hidden_dropout_prob
				__SCREAMING_SNAKE_CASE            = attention_probs_dropout_prob
				__SCREAMING_SNAKE_CASE            = max_position_embeddings
				__SCREAMING_SNAKE_CASE            = type_vocab_size
				__SCREAMING_SNAKE_CASE            = type_sequence_label_size
				__SCREAMING_SNAKE_CASE            = initializer_range
				__SCREAMING_SNAKE_CASE            = num_labels
				__SCREAMING_SNAKE_CASE            = num_choices
				__SCREAMING_SNAKE_CASE            = scope




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size, self.seq_length]							,  self.vocab_size)

				__SCREAMING_SNAKE_CASE            = None
				if self.use_input_mask:
						__SCREAMING_SNAKE_CASE            = random_attention_mask([self.batch_size, self.seq_length])

				__SCREAMING_SNAKE_CASE            = None
				if self.use_token_type_ids:
						__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size, self.seq_length]							,  self.type_vocab_size)

				__SCREAMING_SNAKE_CASE            = None
				__SCREAMING_SNAKE_CASE            = None
				__SCREAMING_SNAKE_CASE            = None
				if self.use_labels:
						__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size]							,  self.type_sequence_label_size)
						__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size, self.seq_length]							,  self.num_labels)
						__SCREAMING_SNAKE_CASE            = ids_tensor([self.batch_size]							,  self.num_choices)

				__SCREAMING_SNAKE_CASE            = self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




		def 	snake_case_     (     self):
				return AlbertConfig(
				    vocab_size=self.vocab_size							,  hidden_size=self.hidden_size							,  num_hidden_layers=self.num_hidden_layers							,  num_attention_heads=self.num_attention_heads							,  intermediate_size=self.intermediate_size							,  hidden_act=self.hidden_act							,  hidden_dropout_prob=self.hidden_dropout_prob							,  attention_probs_dropout_prob=self.attention_probs_dropout_prob							,  max_position_embeddings=self.max_position_embeddings							,  type_vocab_size=self.type_vocab_size							,  initializer_range=self.initializer_range							,  num_hidden_groups=self.num_hidden_groups							,  )




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = AlbertModel(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__)
				self.parent.assertEqual(result.last_hidden_state.shape							,  (self.batch_size, self.seq_length, self.hidden_size))
				self.parent.assertEqual(result.pooler_output.shape							,  (self.batch_size, self.hidden_size))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = AlbertForPreTraining(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(
				    lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  labels=lowerCAmelCase__							,  sentence_order_label=lowerCAmelCase__							,  )
				self.parent.assertEqual(result.prediction_logits.shape							,  (self.batch_size, self.seq_length, self.vocab_size))
				self.parent.assertEqual(result.sop_logits.shape							,  (self.batch_size, config.num_labels))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = AlbertForMaskedLM(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  labels=lowerCAmelCase__)
				self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.vocab_size))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = AlbertForQuestionAnswering(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(
				    lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  start_positions=lowerCAmelCase__							,  end_positions=lowerCAmelCase__							,  )
				self.parent.assertEqual(result.start_logits.shape							,  (self.batch_size, self.seq_length))
				self.parent.assertEqual(result.end_logits.shape							,  (self.batch_size, self.seq_length))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = self.num_labels
				__SCREAMING_SNAKE_CASE            = AlbertForSequenceClassification(lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  labels=lowerCAmelCase__)
				self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.num_labels))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = self.num_labels
				__SCREAMING_SNAKE_CASE            = AlbertForTokenClassification(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  labels=lowerCAmelCase__)
				self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.num_labels))




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = self.num_choices
				__SCREAMING_SNAKE_CASE            = AlbertForMultipleChoice(config=lowerCAmelCase__)
				model.to(lowerCAmelCase__)
				model.eval()
				__SCREAMING_SNAKE_CASE            = input_ids.unsqueeze(1).expand(-1							,  self.num_choices							,  -1).contiguous()
				__SCREAMING_SNAKE_CASE            = token_type_ids.unsqueeze(1).expand(-1							,  self.num_choices							,  -1).contiguous()
				__SCREAMING_SNAKE_CASE            = input_mask.unsqueeze(1).expand(-1							,  self.num_choices							,  -1).contiguous()
				__SCREAMING_SNAKE_CASE            = model(
				    lowerCAmelCase__							,  attention_mask=lowerCAmelCase__							,  token_type_ids=lowerCAmelCase__							,  labels=lowerCAmelCase__							,  )
				self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.num_choices))




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.prepare_config_and_inputs()
				(
				    (
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,(
				    __SCREAMING_SNAKE_CASE
				)       ,
				)            = config_and_inputs
				__SCREAMING_SNAKE_CASE            = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
				return config, inputs_dict



@require_torch
class 						SCREAMING_SNAKE_CASE_ (						__a     ,				__a     ,				unittest.TestCase ):




		"""simple docstring"""





		__lowercase							:  Union[str, Any]						   =		(
		    (
		        AlbertModel,
		        AlbertForPreTraining,
		        AlbertForMaskedLM,
		        AlbertForMultipleChoice,
		        AlbertForSequenceClassification,
		        AlbertForTokenClassification,
		        AlbertForQuestionAnswering,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowercase							:  Optional[int]						   =		(
		    {
		        '''feature-extraction''': AlbertModel,
		        '''fill-mask''': AlbertForMaskedLM,
		        '''question-answering''': AlbertForQuestionAnswering,
		        '''text-classification''': AlbertForSequenceClassification,
		        '''token-classification''': AlbertForTokenClassification,
		        '''zero-shot''': AlbertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__lowercase							:  List[Any]						   =		True




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__=False):
				__SCREAMING_SNAKE_CASE            = super()._prepare_for_class(lowerCAmelCase__							,  lowerCAmelCase__							,  return_labels=lowerCAmelCase__)

				if return_labels:
						if model_class in get_values(lowerCAmelCase__):
								__SCREAMING_SNAKE_CASE            = torch.zeros(
								    (self.model_tester.batch_size, self.model_tester.seq_length)							,  dtype=torch.long							,  device=lowerCAmelCase__)
								__SCREAMING_SNAKE_CASE            = torch.zeros(
								    self.model_tester.batch_size							,  dtype=torch.long							,  device=lowerCAmelCase__)
				return inputs_dict




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = AlbertModelTester(self)
				__SCREAMING_SNAKE_CASE            = ConfigTester(self							,  config_class=lowerCAmelCase__							,  hidden_size=3_7)




		def 	snake_case_     (     self):
				self.config_tester.run_common_tests()




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__)




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
						__SCREAMING_SNAKE_CASE            = type
						self.model_tester.create_and_check_model(*lowerCAmelCase__)




		@slow
		def 	snake_case_     (     self):
				for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__SCREAMING_SNAKE_CASE            = AlbertModel.from_pretrained(lowerCAmelCase__)
						self.assertIsNotNone(lowerCAmelCase__)



@require_torch
class 						SCREAMING_SNAKE_CASE_ (						unittest.TestCase ):




		"""simple docstring"""





		@slow
		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = AlbertModel.from_pretrained("""albert-base-v2""")
				__SCREAMING_SNAKE_CASE            = torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]])
				__SCREAMING_SNAKE_CASE            = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
				with torch.no_grad():
						__SCREAMING_SNAKE_CASE            = model(lowerCAmelCase__							,  attention_mask=lowerCAmelCase__)[0]
				__SCREAMING_SNAKE_CASE            = torch.Size((1, 1_1, 7_6_8))
				self.assertEqual(output.shape							,  lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = torch.tensor(
				    [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]])

				self.assertTrue(torch.allclose(output[:, 1:4, 1:4]							,  lowerCAmelCase__							,  atol=1E-4))




