





"""simple docstring"""




import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
		    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    LayoutLMvaConfig,
		    LayoutLMvaForQuestionAnswering,
		    LayoutLMvaForSequenceClassification,
		    LayoutLMvaForTokenClassification,
		    LayoutLMvaModel,
		)
		from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
		from PIL import Image

		from transformers import LayoutLMvaImageProcessor


class 			__UpperCamelCase							:






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=2 ,   lowerCAmelCase__=3 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=2 ,   lowerCAmelCase__=7 ,   lowerCAmelCase__=True ,   lowerCAmelCase__=True ,   lowerCAmelCase__=True ,   lowerCAmelCase__=True ,   lowerCAmelCase__=99 ,   lowerCAmelCase__=36 ,   lowerCAmelCase__=3 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=37 ,   lowerCAmelCase__="gelu" ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=512 ,   lowerCAmelCase__=16 ,   lowerCAmelCase__=2 ,   lowerCAmelCase__=0.02 ,   lowerCAmelCase__=6 ,   lowerCAmelCase__=6 ,   lowerCAmelCase__=3 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=None ,   lowerCAmelCase__=1000 ,   )      ->					Union[str, Any]:
								a :						Union[str, Any]      =							parent
								a :						List[Any]      =							batch_size
								a :						Tuple      =							num_channels
								a :						Tuple      =							image_size
								a :						Any      =							patch_size
								a :						List[str]      =							text_seq_length
								a :						Tuple      =							is_training
								a :						List[str]      =							use_input_mask
								a :						List[str]      =							use_token_type_ids
								a :						Union[str, Any]      =							use_labels
								a :						Optional[Any]      =							vocab_size
								a :						Optional[Any]      =							hidden_size
								a :						int      =							num_hidden_layers
								a :						List[Any]      =							num_attention_heads
								a :						int      =							intermediate_size
								a :						List[Any]      =							hidden_act
								a :						Union[str, Any]      =							hidden_dropout_prob
								a :						Tuple      =							attention_probs_dropout_prob
								a :						str      =							max_position_embeddings
								a :						Optional[int]      =							type_vocab_size
								a :						List[Any]      =							type_sequence_label_size
								a :						Optional[Any]      =							initializer_range
								a :						Union[str, Any]      =							coordinate_size
								a :						Dict      =							shape_size
								a :						str      =							num_labels
								a :						Optional[int]      =							num_choices
								a :						Tuple      =							scope
								a :						Optional[int]      =							range_bbox

								# LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
								a :						List[Any]      =							text_seq_length
								a :						Optional[Any]      =							(image_size // patch_size) ** 2 + 1
								a :						int      =							self.text_seq_length + self.image_seq_length






							def        __a  (  self )      ->					Optional[Any]:
								a :						Tuple      =							ids_tensor([self.batch_size, self.text_seq_length] ,   self.vocab_size )

								a :						List[str]      =							ids_tensor([self.batch_size, self.text_seq_length, 4] ,   self.range_bbox )
								# Ensure that bbox is legal
								for i in range(bbox.shape[0] ):
									for j in range(bbox.shape[1] ):
										if bbox[i, j, 3] < bbox[i, j, 1]:
											a :						Tuple      =							bbox[i, j, 3]
											a :						List[str]      =							bbox[i, j, 1]
											a :						Optional[Any]      =							t
										if bbox[i, j, 2] < bbox[i, j, 0]:
											a :						Any      =							bbox[i, j, 2]
											a :						Optional[int]      =							bbox[i, j, 0]
											a :						str      =							t

								a :						List[Any]      =							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

								a :						List[Any]      =							None
								if self.use_input_mask:
									a :						Any      =							random_attention_mask([self.batch_size, self.text_seq_length] )

								a :						Tuple      =							None
								if self.use_token_type_ids:
									a :						Tuple      =							ids_tensor([self.batch_size, self.text_seq_length] ,   self.type_vocab_size )

								a :						List[str]      =							None
								a :						List[str]      =							None
								if self.use_labels:
									a :						Optional[int]      =							ids_tensor([self.batch_size] ,   self.type_sequence_label_size )
									a :						Dict      =							ids_tensor([self.batch_size, self.text_seq_length] ,   self.num_labels )

								a :						Any      =							LayoutLMvaConfig(
								    vocab_size=self.vocab_size ,   hidden_size=self.hidden_size ,   num_hidden_layers=self.num_hidden_layers ,   num_attention_heads=self.num_attention_heads ,   intermediate_size=self.intermediate_size ,   hidden_act=self.hidden_act ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   max_position_embeddings=self.max_position_embeddings ,   type_vocab_size=self.type_vocab_size ,   initializer_range=self.initializer_range ,   coordinate_size=self.coordinate_size ,   shape_size=self.shape_size ,   input_size=self.image_size ,   patch_size=self.patch_size ,   )

								return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Tuple:
								a :						Optional[int]      =							LayoutLMvaModel(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()

								# text + image
								a :						Union[str, Any]      =							model(lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ )
								a :						Union[str, Any]      =							model(
								    lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   token_type_ids=lowerCAmelCase__ )
								a :						Optional[Any]      =							model(lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ ,   token_type_ids=lowerCAmelCase__ )
								a :						Dict      =							model(lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ )

								self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size) )

								# text only
								a :						Union[str, Any]      =							model(lowerCAmelCase__ )
								self.parent.assertEqual(
								    result.last_hidden_state.shape ,   (self.batch_size, self.text_seq_length, self.hidden_size) )

								# image only
								a :						List[Any]      =							model(pixel_values=lowerCAmelCase__ )
								self.parent.assertEqual(
								    result.last_hidden_state.shape ,   (self.batch_size, self.image_seq_length, self.hidden_size) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Dict:
								a :						List[str]      =							self.num_labels
								a :						Optional[Any]      =							LayoutLMvaForSequenceClassification(lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Optional[int]      =							model(
								    lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   token_type_ids=lowerCAmelCase__ ,   labels=lowerCAmelCase__ ,   )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_labels) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Optional[int]:
								a :						List[Any]      =							self.num_labels
								a :						Any      =							LayoutLMvaForTokenClassification(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Union[str, Any]      =							model(
								    lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   token_type_ids=lowerCAmelCase__ ,   labels=lowerCAmelCase__ ,   )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.text_seq_length, self.num_labels) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Tuple:
								a :						List[Any]      =							LayoutLMvaForQuestionAnswering(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Optional[Any]      =							model(
								    lowerCAmelCase__ ,   bbox=lowerCAmelCase__ ,   pixel_values=lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   token_type_ids=lowerCAmelCase__ ,   start_positions=lowerCAmelCase__ ,   end_positions=lowerCAmelCase__ ,   )
								self.parent.assertEqual(result.start_logits.shape ,   (self.batch_size, self.seq_length) )
								self.parent.assertEqual(result.end_logits.shape ,   (self.batch_size, self.seq_length) )






							def        __a  (  self )      ->					Optional[int]:
								a :						int      =							self.prepare_config_and_inputs()
								(
								    (
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			
								) :						Optional[Any]      =							config_and_inputs
								a :						List[str]      =							{
								    "input_ids": input_ids,
								    "bbox": bbox,
								    "pixel_values": pixel_values,
								    "token_type_ids": token_type_ids,
								    "attention_mask": input_mask,
								}
								return config, inputs_dict


@require_torch
class 			__UpperCamelCase							(		a__						,			a__						,			unittest.TestCase		):
							lowerCamelCase  :   Union[str, Any]    =False
							lowerCamelCase  :   Any    =False
							lowerCamelCase  :   str    =False

							lowerCamelCase  :   str    =(
							    (
							        LayoutLMvaModel,
							        LayoutLMvaForSequenceClassification,
							        LayoutLMvaForTokenClassification,
							        LayoutLMvaForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase  :   Dict    =(
							    {"""document-question-answering""": LayoutLMvaForQuestionAnswering, """feature-extraction""": LayoutLMvaModel}
							    if is_torch_available()
							    else {}
							)






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					List[str]:
								# `DocumentQuestionAnsweringPipeline` is expected to work with this model, but it combines the text and visual
								# embedding along the sequence dimension (dim 1), which causes an error during post-processing as `p_mask` has
								# the sequence dimension of the text embedding only.
								# (see the line `embedding_output = torch.cat([embedding_output, visual_embeddings], dim=1)`)
								return True






							def        __a  (  self )      ->					Optional[Any]:
								a :						Any      =							LayoutLMvaModelTester(self )
								a :						Optional[Any]      =							ConfigTester(self ,   config_class=lowerCAmelCase__ ,   hidden_size=37 )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__=False )      ->					Dict:
								a :						Tuple      =							copy.deepcopy(lowerCAmelCase__ )
								if model_class in get_values(lowerCAmelCase__ ):
									a :						Optional[Any]      =							{
									    k: v.unsqueeze(1 ).expand(-1 ,   self.model_tester.num_choices ,   -1 ).contiguous()
									    if isinstance(lowerCAmelCase__ ,   torch.Tensor ) and v.ndim > 1
									    else v
									    for k, v in inputs_dict.items()
									}
								if return_labels:
									if model_class in get_values(lowerCAmelCase__ ):
										a :						List[str]      =							torch.ones(self.model_tester.batch_size ,   dtype=torch.long ,   device=lowerCAmelCase__ )
									elif model_class in get_values(lowerCAmelCase__ ):
										a :						Optional[Any]      =							torch.zeros(
										    self.model_tester.batch_size ,   dtype=torch.long ,   device=lowerCAmelCase__ )
										a :						Tuple      =							torch.zeros(
										    self.model_tester.batch_size ,   dtype=torch.long ,   device=lowerCAmelCase__ )
									elif model_class in [
									    *get_values(lowerCAmelCase__ ),
									]:
										a :						Union[str, Any]      =							torch.zeros(
										    self.model_tester.batch_size ,   dtype=torch.long ,   device=lowerCAmelCase__ )
									elif model_class in [
									    *get_values(lowerCAmelCase__ ),
									]:
										a :						Optional[int]      =							torch.zeros(
										    (self.model_tester.batch_size, self.model_tester.text_seq_length) ,   dtype=torch.long ,   device=lowerCAmelCase__ ,   )

								return inputs_dict






							def        __a  (  self )      ->					List[str]:
								self.config_tester.run_common_tests()






							def        __a  (  self )      ->					List[Any]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					str:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									a :						Dict      =							type
									self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[Any]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__ )






							def        __a  (  self )      ->					List[Any]:
								a :						Optional[int]      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__ )






							def        __a  (  self )      ->					List[str]:
								a :						Union[str, Any]      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__ )






							@slow
							def        __a  (  self )      ->					int:
								for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									a :						Union[str, Any]      =							LayoutLMvaModel.from_pretrained(lowerCAmelCase__ )
									self.assertIsNotNone(lowerCAmelCase__ )






def  _SCREAMING_SNAKE_CASE						(      )					->Any:






	'''simple docstring'''






	a :						List[str]      =							Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"   )
	return image







@require_torch
class 			__UpperCamelCase							(		unittest.TestCase		):






							@cached_property
							def        __a  (  self )      ->					Tuple:
								return LayoutLMvaImageProcessor(apply_ocr=lowerCAmelCase__ ) if is_vision_available() else None






							@slow
							def        __a  (  self )      ->					Any:
								a :						int      =							LayoutLMvaModel.from_pretrained("microsoft/layoutlmv3-base" ).to(lowerCAmelCase__ )

								a :						List[Any]      =							self.default_image_processor
								a :						str      =							prepare_img()
								a :						List[Any]      =							image_processor(images=lowerCAmelCase__ ,   return_tensors="pt" ).pixel_values.to(lowerCAmelCase__ )

								a :						Dict      =							torch.tensor([[1, 2]] )
								a :						Any      =							torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]] ).unsqueeze(0 )

								# forward pass
								a :						int      =							model(
								    input_ids=input_ids.to(lowerCAmelCase__ ) ,   bbox=bbox.to(lowerCAmelCase__ ) ,   pixel_values=pixel_values.to(lowerCAmelCase__ ) ,   )

								# verify the logits
								a :						int      =							torch.Size((1, 199, 768) )
								self.assertEqual(outputs.last_hidden_state.shape ,   lowerCAmelCase__ )

								a :						Any      =							torch.tensor(
								    [[-0.0_529, 0.3_618, 0.1_632], [-0.1_587, -0.1_667, -0.0_400], [-0.1_557, -0.1_671, -0.0_505]] ).to(lowerCAmelCase__ )

								self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3] ,   lowerCAmelCase__ ,   atol=1E-4 ) )









"""simple docstring"""




import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
		from transformers.models.esm.modeling_esm import (
		    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    EsmEmbeddings,
		    create_position_ids_from_input_ids,
		)


class 			__UpperCamelCase							:






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=13 ,   lowerCAmelCase__=7 ,   lowerCAmelCase__=False ,   lowerCAmelCase__=True ,   lowerCAmelCase__=False ,   lowerCAmelCase__=True ,   lowerCAmelCase__=33 ,   lowerCAmelCase__=32 ,   lowerCAmelCase__=5 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=37 ,   lowerCAmelCase__="gelu" ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=512 ,   lowerCAmelCase__=16 ,   lowerCAmelCase__=2 ,   lowerCAmelCase__=0.02 ,   lowerCAmelCase__=3 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=None ,   )      ->					List[str]:
								a :						List[str]      =							parent
								a :						Optional[int]      =							batch_size
								a :						Optional[int]      =							seq_length
								a :						Tuple      =							is_training
								a :						Optional[int]      =							use_input_mask
								a :						List[Any]      =							use_token_type_ids
								a :						Tuple      =							use_labels
								a :						Union[str, Any]      =							vocab_size
								a :						str      =							hidden_size
								a :						List[Any]      =							num_hidden_layers
								a :						Dict      =							num_attention_heads
								a :						int      =							intermediate_size
								a :						str      =							hidden_act
								a :						Optional[int]      =							hidden_dropout_prob
								a :						str      =							attention_probs_dropout_prob
								a :						str      =							max_position_embeddings
								a :						List[str]      =							type_vocab_size
								a :						List[str]      =							type_sequence_label_size
								a :						Union[str, Any]      =							initializer_range
								a :						int      =							num_labels
								a :						Optional[int]      =							num_choices
								a :						Union[str, Any]      =							scope






							def        __a  (  self )      ->					Optional[Any]:
								a :						str      =							ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size )

								a :						Dict      =							None
								if self.use_input_mask:
									a :						str      =							random_attention_mask([self.batch_size, self.seq_length] )

								a :						Union[str, Any]      =							None
								a :						int      =							None
								a :						int      =							None
								if self.use_labels:
									a :						Optional[int]      =							ids_tensor([self.batch_size] ,   self.type_sequence_label_size )
									a :						Optional[int]      =							ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels )
									a :						List[Any]      =							ids_tensor([self.batch_size] ,   self.num_choices )

								a :						str      =							self.get_config()

								return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






							def        __a  (  self )      ->					Optional[int]:
								return EsmConfig(
								    vocab_size=self.vocab_size ,   hidden_size=self.hidden_size ,   pad_token_id=1 ,   num_hidden_layers=self.num_hidden_layers ,   num_attention_heads=self.num_attention_heads ,   intermediate_size=self.intermediate_size ,   hidden_act=self.hidden_act ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   max_position_embeddings=self.max_position_embeddings ,   type_vocab_size=self.type_vocab_size ,   initializer_range=self.initializer_range ,   )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					str:
								a :						Union[str, Any]      =							EsmModel(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Optional[int]      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ )
								a :						List[str]      =							model(lowerCAmelCase__ )
								a :						Optional[Any]      =							model(lowerCAmelCase__ )

								self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size) )
								self.parent.assertEqual(result.pooler_output.shape ,   (self.batch_size, self.hidden_size) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Tuple:
								a :						int      =							EsmForMaskedLM(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Union[str, Any]      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   labels=lowerCAmelCase__ )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Optional[Any]:
								a :						str      =							self.num_labels
								a :						int      =							EsmForTokenClassification(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						int      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   labels=lowerCAmelCase__ )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels) )






							def        __a  (  self )      ->					Union[str, Any]:
								a :						str      =							self.prepare_config_and_inputs()
								(
								    (
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			
								) :						Optional[int]      =							config_and_inputs
								a :						int      =							{"input_ids": input_ids, "attention_mask": input_mask}
								return config, inputs_dict


@require_torch
class 			__UpperCamelCase							(		a__						,			a__						,			unittest.TestCase		):
							lowerCamelCase  :   Optional[Any]    =False

							lowerCamelCase  :   Tuple    =(
							    (
							        EsmForMaskedLM,
							        EsmModel,
							        EsmForSequenceClassification,
							        EsmForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase  :   Tuple    =()
							lowerCamelCase  :   Optional[int]    =(
							    {
							        """feature-extraction""": EsmModel,
							        """fill-mask""": EsmForMaskedLM,
							        """text-classification""": EsmForSequenceClassification,
							        """token-classification""": EsmForTokenClassification,
							        """zero-shot""": EsmForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase  :   Any    =True






							def        __a  (  self )      ->					Tuple:
								a :						str      =							EsmModelTester(self )
								a :						Union[str, Any]      =							ConfigTester(self ,   config_class=lowerCAmelCase__ ,   hidden_size=37 )






							def        __a  (  self )      ->					Dict:
								self.config_tester.run_common_tests()






							def        __a  (  self )      ->					Union[str, Any]:
								a :						Tuple      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					List[str]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									a :						List[str]      =							type
									self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[Any]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__ )






							def        __a  (  self )      ->					Dict:
								a :						int      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__ )






							@slow
							def        __a  (  self )      ->					Optional[Any]:
								for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									a :						Tuple      =							EsmModel.from_pretrained(lowerCAmelCase__ )
									self.assertIsNotNone(lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[int]:
								a :						List[str]      =							self.model_tester.prepare_config_and_inputs()[0]
								a :						str      =							EsmEmbeddings(config=lowerCAmelCase__ )

								a :						Optional[Any]      =							torch.as_tensor([[12, 31, 13, model.padding_idx]] )
								a :						Tuple      =							torch.as_tensor(
								    [
								        [
								            0 + model.padding_idx + 1,
								            1 + model.padding_idx + 1,
								            2 + model.padding_idx + 1,
								            model.padding_idx,
								        ]
								    ] )
								a :						str      =							create_position_ids_from_input_ids(lowerCAmelCase__ ,   model.padding_idx )
								self.assertEqual(position_ids.shape ,   expected_positions.shape )
								self.assertTrue(torch.all(torch.eq(lowerCAmelCase__ ,   lowerCAmelCase__ ) ) )






							def        __a  (  self )      ->					List[Any]:
								a :						int      =							self.model_tester.prepare_config_and_inputs()[0]
								a :						Dict      =							EsmEmbeddings(config=lowerCAmelCase__ )

								a :						Any      =							torch.empty(2 ,   4 ,   30 )
								a :						str      =							[
								    0 + embeddings.padding_idx + 1,
								    1 + embeddings.padding_idx + 1,
								    2 + embeddings.padding_idx + 1,
								    3 + embeddings.padding_idx + 1,
								]
								a :						Optional[Any]      =							torch.as_tensor([expected_single_positions, expected_single_positions] )
								a :						str      =							embeddings.create_position_ids_from_inputs_embeds(lowerCAmelCase__ )
								self.assertEqual(position_ids.shape ,   expected_positions.shape )
								self.assertTrue(torch.all(torch.eq(lowerCAmelCase__ ,   lowerCAmelCase__ ) ) )






							@unittest.skip("Esm does not support embedding resizing" )
							def        __a  (  self )      ->					Any:
								pass






							@unittest.skip("Esm does not support embedding resizing" )
							def        __a  (  self )      ->					Dict:
								pass






							@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests." )
							def        __a  (  self )      ->					str:
								pass


@require_torch
class 			__UpperCamelCase							(		a__		):






							@slow
							def        __a  (  self )      ->					int:
								with torch.no_grad():
									a :						str      =							EsmForMaskedLM.from_pretrained("facebook/esm2_t6_8M_UR50D" )
									model.eval()
									a :						List[str]      =							torch.tensor([[0, 1, 2, 3, 4, 5]] )
									a :						Any      =							model(lowerCAmelCase__ )[0]

									a :						Union[str, Any]      =							33

									a :						str      =							torch.Size((1, 6, vocab_size) )
									self.assertEqual(output.shape ,   lowerCAmelCase__ )

									a :						Optional[int]      =							torch.tensor(
									    [[[8.9_215, -10.5_898, -6.4_671], [-6.3_967, -13.9_114, -1.1_212], [-7.7_812, -13.9_516, -3.7_406]]] )
									self.assertTrue(torch.allclose(output[:, :3, :3] ,   lowerCAmelCase__ ,   atol=1E-4 ) )






							@slow
							def        __a  (  self )      ->					str:
								with torch.no_grad():
									a :						Optional[int]      =							EsmModel.from_pretrained("facebook/esm2_t6_8M_UR50D" )
									model.eval()

									a :						str      =							torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]] )
									a :						Tuple      =							model(lowerCAmelCase__ )[0]
									# compare the actual values for a slice.
									a :						Union[str, Any]      =							torch.tensor(
									    [[[0.1_444, 0.5_413, 0.3_248], [0.3_034, 0.0_053, 0.3_108], [0.3_228, -0.2_499, 0.3_415]]] )
									self.assertTrue(torch.allclose(output[:, :3, :3] ,   lowerCAmelCase__ ,   atol=1E-4 ) )


