

'''simple docstring'''



import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
      MODEL_FOR_QUESTION_ANSWERING_MAPPING,
      MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
      MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
      LayoutLMvaConfig,
      LayoutLMvaForQuestionAnswering,
      LayoutLMvaForSequenceClassification,
      LayoutLMvaForTokenClassification,
      LayoutLMvaModel,
  )
  from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
  from PIL import Image

  from transformers import LayoutLMvaImageProcessor







class 						lowerCAmelCase__   :



 """simple docstring"""



 def __init__(		self					:       List[str]					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       Dict=2					,							__SCREAMING_SNAKE_CASE					:       int=3					,							__SCREAMING_SNAKE_CASE					:       Any=4					,							__SCREAMING_SNAKE_CASE					:       List[Any]=2					,							__SCREAMING_SNAKE_CASE					:       Dict=7					,							__SCREAMING_SNAKE_CASE					:       str=True					,							__SCREAMING_SNAKE_CASE					:       Optional[int]=True					,							__SCREAMING_SNAKE_CASE					:       List[str]=True					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=True					,							__SCREAMING_SNAKE_CASE					:       int=99					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]=36					,							__SCREAMING_SNAKE_CASE					:       Optional[int]=3					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=4					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]=37					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="gelu"					,							__SCREAMING_SNAKE_CASE					:       Tuple=0.1					,							__SCREAMING_SNAKE_CASE					:       Dict=0.1					,							__SCREAMING_SNAKE_CASE					:       Tuple=512					,							__SCREAMING_SNAKE_CASE					:       Tuple=16					,							__SCREAMING_SNAKE_CASE					:       Dict=2					,							__SCREAMING_SNAKE_CASE					:       int=0.02					,							__SCREAMING_SNAKE_CASE					:       Tuple=6					,							__SCREAMING_SNAKE_CASE					:       int=6					,							__SCREAMING_SNAKE_CASE					:       str=3					,							__SCREAMING_SNAKE_CASE					:       List[Any]=4					,							__SCREAMING_SNAKE_CASE					:       List[Any]=None					,							__SCREAMING_SNAKE_CASE					:       Tuple=1_000					,							)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      parent
      __SCREAMING_SNAKE_CASE     =      batch_size
      __SCREAMING_SNAKE_CASE     =      num_channels
      __SCREAMING_SNAKE_CASE     =      image_size
      __SCREAMING_SNAKE_CASE     =      patch_size
      __SCREAMING_SNAKE_CASE     =      text_seq_length
      __SCREAMING_SNAKE_CASE     =      is_training
      __SCREAMING_SNAKE_CASE     =      use_input_mask
      __SCREAMING_SNAKE_CASE     =      use_token_type_ids
      __SCREAMING_SNAKE_CASE     =      use_labels
      __SCREAMING_SNAKE_CASE     =      vocab_size
      __SCREAMING_SNAKE_CASE     =      hidden_size
      __SCREAMING_SNAKE_CASE     =      num_hidden_layers
      __SCREAMING_SNAKE_CASE     =      num_attention_heads
      __SCREAMING_SNAKE_CASE     =      intermediate_size
      __SCREAMING_SNAKE_CASE     =      hidden_act
      __SCREAMING_SNAKE_CASE     =      hidden_dropout_prob
      __SCREAMING_SNAKE_CASE     =      attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE     =      max_position_embeddings
      __SCREAMING_SNAKE_CASE     =      type_vocab_size
      __SCREAMING_SNAKE_CASE     =      type_sequence_label_size
      __SCREAMING_SNAKE_CASE     =      initializer_range
      __SCREAMING_SNAKE_CASE     =      coordinate_size
      __SCREAMING_SNAKE_CASE     =      shape_size
      __SCREAMING_SNAKE_CASE     =      num_labels
      __SCREAMING_SNAKE_CASE     =      num_choices
      __SCREAMING_SNAKE_CASE     =      scope
      __SCREAMING_SNAKE_CASE     =      range_bbox

      # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
      __SCREAMING_SNAKE_CASE     =      text_seq_length
      __SCREAMING_SNAKE_CASE     =      (image_size // patch_size) ** 2 + 1
      __SCREAMING_SNAKE_CASE     =      self.text_seq_length + self.image_seq_length
 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      ids_tensor([self.batch_size, self.text_seq_length]					,							self.vocab_size			)

      __SCREAMING_SNAKE_CASE     =      ids_tensor([self.batch_size, self.text_seq_length, 4]					,							self.range_bbox			)
      # Ensure that bbox is legal
      for i in range(bbox.shape[0]			):
           for j in range(bbox.shape[1]			):
                if bbox[i, j, 3] < bbox[i, j, 1]:
                     __SCREAMING_SNAKE_CASE     =      bbox[i, j, 3]
                     __SCREAMING_SNAKE_CASE     =      bbox[i, j, 1]
                     __SCREAMING_SNAKE_CASE     =      t
                if bbox[i, j, 2] < bbox[i, j, 0]:
                     __SCREAMING_SNAKE_CASE     =      bbox[i, j, 2]
                     __SCREAMING_SNAKE_CASE     =      bbox[i, j, 0]
                     __SCREAMING_SNAKE_CASE     =      t

      __SCREAMING_SNAKE_CASE     =      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

      __SCREAMING_SNAKE_CASE     =      None
      if self.use_input_mask:
           __SCREAMING_SNAKE_CASE     =      random_attention_mask([self.batch_size, self.text_seq_length]			)

      __SCREAMING_SNAKE_CASE     =      None
      if self.use_token_type_ids:
           __SCREAMING_SNAKE_CASE     =      ids_tensor([self.batch_size, self.text_seq_length]					,							self.type_vocab_size			)

      __SCREAMING_SNAKE_CASE     =      None
      __SCREAMING_SNAKE_CASE     =      None
      if self.use_labels:
           __SCREAMING_SNAKE_CASE     =      ids_tensor([self.batch_size]					,							self.type_sequence_label_size			)
           __SCREAMING_SNAKE_CASE     =      ids_tensor([self.batch_size, self.text_seq_length]					,							self.num_labels			)

      __SCREAMING_SNAKE_CASE     =      LayoutLMvaConfig(
          vocab_size=self.vocab_size					,							hidden_size=self.hidden_size					,							num_hidden_layers=self.num_hidden_layers					,							num_attention_heads=self.num_attention_heads					,							intermediate_size=self.intermediate_size					,							hidden_act=self.hidden_act					,							hidden_dropout_prob=self.hidden_dropout_prob					,							attention_probs_dropout_prob=self.attention_probs_dropout_prob					,							max_position_embeddings=self.max_position_embeddings					,							type_vocab_size=self.type_vocab_size					,							initializer_range=self.initializer_range					,							coordinate_size=self.coordinate_size					,							shape_size=self.shape_size					,							input_size=self.image_size					,							patch_size=self.patch_size					,							)

      return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels
 def       UpperCAmelCase__		(		self					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Dict					,							__SCREAMING_SNAKE_CASE					:       List[str]					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      LayoutLMvaModel(config=__SCREAMING_SNAKE_CASE			)
      model.to(__SCREAMING_SNAKE_CASE			)
      model.eval()

      # text + image
      __SCREAMING_SNAKE_CASE     =      model(__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      model(
          __SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE					,							attention_mask=__SCREAMING_SNAKE_CASE					,							token_type_ids=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      model(__SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE					,							token_type_ids=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      model(__SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE			)

      self.parent.assertEqual(result.last_hidden_state.shape					,							(self.batch_size, self.seq_length, self.hidden_size)			)

      # text only
      __SCREAMING_SNAKE_CASE     =      model(__SCREAMING_SNAKE_CASE			)
      self.parent.assertEqual(
          result.last_hidden_state.shape					,							(self.batch_size, self.text_seq_length, self.hidden_size)			)

      # image only
      __SCREAMING_SNAKE_CASE     =      model(pixel_values=__SCREAMING_SNAKE_CASE			)
      self.parent.assertEqual(
          result.last_hidden_state.shape					,							(self.batch_size, self.image_seq_length, self.hidden_size)			)
 def       UpperCAmelCase__		(		self					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Any					,							__SCREAMING_SNAKE_CASE					:       Dict					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       List[Any]			)  -> Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.num_labels
      __SCREAMING_SNAKE_CASE     =      LayoutLMvaForSequenceClassification(__SCREAMING_SNAKE_CASE			)
      model.to(__SCREAMING_SNAKE_CASE			)
      model.eval()
      __SCREAMING_SNAKE_CASE     =      model(
          __SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE					,							attention_mask=__SCREAMING_SNAKE_CASE					,							token_type_ids=__SCREAMING_SNAKE_CASE					,							labels=__SCREAMING_SNAKE_CASE					,							)
      self.parent.assertEqual(result.logits.shape					,							(self.batch_size, self.num_labels)			)
 def       UpperCAmelCase__		(		self					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       Dict					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       Any					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       Any					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       str			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.num_labels
      __SCREAMING_SNAKE_CASE     =      LayoutLMvaForTokenClassification(config=__SCREAMING_SNAKE_CASE			)
      model.to(__SCREAMING_SNAKE_CASE			)
      model.eval()
      __SCREAMING_SNAKE_CASE     =      model(
          __SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE					,							attention_mask=__SCREAMING_SNAKE_CASE					,							token_type_ids=__SCREAMING_SNAKE_CASE					,							labels=__SCREAMING_SNAKE_CASE					,							)
      self.parent.assertEqual(result.logits.shape					,							(self.batch_size, self.text_seq_length, self.num_labels)			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       List[Any]					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      LayoutLMvaForQuestionAnswering(config=__SCREAMING_SNAKE_CASE			)
      model.to(__SCREAMING_SNAKE_CASE			)
      model.eval()
      __SCREAMING_SNAKE_CASE     =      model(
          __SCREAMING_SNAKE_CASE					,							bbox=__SCREAMING_SNAKE_CASE					,							pixel_values=__SCREAMING_SNAKE_CASE					,							attention_mask=__SCREAMING_SNAKE_CASE					,							token_type_ids=__SCREAMING_SNAKE_CASE					,							start_positions=__SCREAMING_SNAKE_CASE					,							end_positions=__SCREAMING_SNAKE_CASE					,							)
      self.parent.assertEqual(result.start_logits.shape					,							(self.batch_size, self.seq_length)			)
      self.parent.assertEqual(result.end_logits.shape					,							(self.batch_size, self.seq_length)			)


 def       UpperCAmelCase__		(		self					:       Tuple			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.prepare_config_and_inputs()
      (
          (
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			(
          __SCREAMING_SNAKE_CASE
      )   ,			
      )     =      config_and_inputs
      __SCREAMING_SNAKE_CASE     =      {
          """input_ids""": input_ids,
          """bbox""": bbox,
          """pixel_values""": pixel_values,
          """token_type_ids""": token_type_ids,
          """attention_mask""": input_mask,
      }
      return config, inputs_dict







@require_torch
class 						lowerCAmelCase__   (      a ,					a ,					unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      False
 lowerCAmelCase__  					=      False
 lowerCAmelCase__  					=      False

 lowerCAmelCase__  					=      (
     (
         LayoutLMvaModel,
         LayoutLMvaForSequenceClassification,
         LayoutLMvaForTokenClassification,
         LayoutLMvaForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 lowerCAmelCase__  					=      (
     {"document-question-answering": LayoutLMvaForQuestionAnswering, "feature-extraction": LayoutLMvaModel}
     if is_torch_available()
     else {}
 )
 def       UpperCAmelCase__		(		self					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       List[Any]					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       int					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]			)  -> Tuple:


      """simple docstring"""




      return True
 def       UpperCAmelCase__		(		self					:       Any			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      LayoutLMvaModelTester(self			)
      __SCREAMING_SNAKE_CASE     =      ConfigTester(self					,							config_class=__SCREAMING_SNAKE_CASE					,							hidden_size=37			)
 def       UpperCAmelCase__		(		self					:       Dict					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       List[Any]					,							__SCREAMING_SNAKE_CASE					:       str=False			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      copy.deepcopy(__SCREAMING_SNAKE_CASE			)
      if model_class in get_values(__SCREAMING_SNAKE_CASE			):
           __SCREAMING_SNAKE_CASE     =      {
               k: v.unsqueeze(1			).expand(-1					,							self.model_tester.num_choices					,							-1			).contiguous()
               if isinstance(__SCREAMING_SNAKE_CASE					,							torch.Tensor			) and v.ndim > 1
               else v
               for k, v in inputs_dict.items()
           }
      if return_labels:
           if model_class in get_values(__SCREAMING_SNAKE_CASE			):
                __SCREAMING_SNAKE_CASE     =      torch.ones(self.model_tester.batch_size					,							dtype=torch.long					,							device=__SCREAMING_SNAKE_CASE			)
           elif model_class in get_values(__SCREAMING_SNAKE_CASE			):
                __SCREAMING_SNAKE_CASE     =      torch.zeros(
                    self.model_tester.batch_size					,							dtype=torch.long					,							device=__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      torch.zeros(
                    self.model_tester.batch_size					,							dtype=torch.long					,							device=__SCREAMING_SNAKE_CASE			)
           elif model_class in [
               *get_values(__SCREAMING_SNAKE_CASE			),
           ]:
                __SCREAMING_SNAKE_CASE     =      torch.zeros(
                    self.model_tester.batch_size					,							dtype=torch.long					,							device=__SCREAMING_SNAKE_CASE			)
           elif model_class in [
               *get_values(__SCREAMING_SNAKE_CASE			),
           ]:
                __SCREAMING_SNAKE_CASE     =      torch.zeros(
                    (self.model_tester.batch_size, self.model_tester.text_seq_length)					,							dtype=torch.long					,							device=__SCREAMING_SNAKE_CASE					,							)

      return inputs_dict
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Union[str, Any]:


      """simple docstring"""




      self.config_tester.run_common_tests()
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
           __SCREAMING_SNAKE_CASE     =      type
           self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       int			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       int			)  -> Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE			)


 @slow
 def       UpperCAmelCase__		(		self					:       str			)  -> Tuple:


      """simple docstring"""




      for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           __SCREAMING_SNAKE_CASE     =      LayoutLMvaModel.from_pretrained(__SCREAMING_SNAKE_CASE			)
           self.assertIsNotNone(__SCREAMING_SNAKE_CASE			)







def    a__       (  ):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						)
     return image







@require_torch
class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 @cached_property
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Optional[Any]:


      """simple docstring"""




      return LayoutLMvaImageProcessor(apply_ocr=__SCREAMING_SNAKE_CASE			) if is_vision_available() else None


 @slow
 def       UpperCAmelCase__		(		self					:       int			)  -> List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      LayoutLMvaModel.from_pretrained("""microsoft/layoutlmv3-base"""			).to(__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      self.default_image_processor
      __SCREAMING_SNAKE_CASE     =      prepare_img()
      __SCREAMING_SNAKE_CASE     =      image_processor(images=__SCREAMING_SNAKE_CASE					,							return_tensors="""pt"""			).pixel_values.to(__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      torch.tensor([[1, 2]]			)
      __SCREAMING_SNAKE_CASE     =      torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]			).unsqueeze(0			)

      # forward pass
      __SCREAMING_SNAKE_CASE     =      model(
          input_ids=input_ids.to(__SCREAMING_SNAKE_CASE			)					,							bbox=bbox.to(__SCREAMING_SNAKE_CASE			)					,							pixel_values=pixel_values.to(__SCREAMING_SNAKE_CASE			)					,							)

      # verify the logits
      __SCREAMING_SNAKE_CASE     =      torch.Size((1, 199, 768)			)
      self.assertEqual(outputs.last_hidden_state.shape					,							__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      torch.tensor(
          [[-0.0529, 0.3618, 0.1632], [-0.1587, -0.1667, -0.0400], [-0.1557, -0.1671, -0.0505]]			).to(__SCREAMING_SNAKE_CASE			)

      self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]					,							__SCREAMING_SNAKE_CASE					,							atol=1E-4			)			)









'''simple docstring'''



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase :       Optional[int]			   =     get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase :       Optional[int]			   =     2_5_6_0_4_7
UpperCAmelCase :       Union[str, Any]			   =     2_5_6_1_4_5







@require_sentencepiece
@require_tokenizers
class 						lowerCAmelCase__   (      a ,					unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      NllbTokenizer
 lowerCAmelCase__  					=      NllbTokenizerFast
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      {}
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> int:


      """simple docstring"""




      super().setUp()

      # We have a SentencePiece fixture for testing
      __SCREAMING_SNAKE_CASE     =      NllbTokenizer(__SCREAMING_SNAKE_CASE					,							keep_accents=__SCREAMING_SNAKE_CASE			)
      tokenizer.save_pretrained(self.tmpdirname			)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      NllbTokenizer(__SCREAMING_SNAKE_CASE					,							keep_accents=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize("""This is a test"""			)
      self.assertListEqual(__SCREAMING_SNAKE_CASE					,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE			)					,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,							)

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """9""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """é""",
              """.""",
          ]					,							)
      __SCREAMING_SNAKE_CASE     =      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]					,							)

      __SCREAMING_SNAKE_CASE     =      tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """<unk>""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """<unk>""",
              """.""",
          ]					,							)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)

                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)
                __SCREAMING_SNAKE_CASE     =      tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f			)
                self.assertSequenceEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)

                # Save tokenizer rust, legacy_format=True
                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE					,							legacy_format=__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it save with the same files
                self.assertSequenceEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)

                # Save tokenizer rust, legacy_format=False
                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE					,							legacy_format=__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it saved the tokenizer.json file
                self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Any:


      """simple docstring"""




      if not self.test_seqaseq:
           return

      __SCREAMING_SNAKE_CASE     =      self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'			):
                # Longer text that will definitely require truncation.
                __SCREAMING_SNAKE_CASE     =      [
                    """ UN Chief Says There Is No Military Solution in Syria""",
                    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                    """ will only worsen the violence and misery for millions of people.""",
                ]
                __SCREAMING_SNAKE_CASE     =      [
                    """Şeful ONU declară că nu există o soluţie militară în Siria""",
                    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                ]
                try:
                     __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                         src_texts=__SCREAMING_SNAKE_CASE					,							tgt_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							max_target_length=10					,							return_tensors="""pt"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""ron_Latn"""					,							)
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]					,							3			)
                self.assertEqual(batch.labels.shape[1]					,							10			)
                # max_target_length will default to max_length if not specified
                __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                    __SCREAMING_SNAKE_CASE					,							tgt_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							return_tensors="""pt"""			)
                self.assertEqual(batch.input_ids.shape[1]					,							3			)
                self.assertEqual(batch.labels.shape[1]					,							3			)

                __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                    src_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							max_target_length=10					,							return_tensors="""pt"""			)
                self.assertEqual(batch_encoder_only.input_ids.shape[1]					,							3			)
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]					,							3			)
                self.assertNotIn("""decoder_input_ids"""					,							__SCREAMING_SNAKE_CASE			)
 @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[int]:


      """simple docstring"""




      pass


 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Dict:


      """simple docstring"""




      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      [AddedToken("""<special>"""					,							lstrip=__SCREAMING_SNAKE_CASE			)]

                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(
                    __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode("""Hey this is a <special> token"""			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode("""<special>"""					,							add_special_tokens=__SCREAMING_SNAKE_CASE			)[0]

                self.assertTrue(special_token_id in r_output			)

                if self.test_slow_tokenizer:
                     __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(
                         __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE					,							)
                     __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(
                         __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)

                     __SCREAMING_SNAKE_CASE     =      tokenizer_p.encode("""Hey this is a <special> token"""			)

                     __SCREAMING_SNAKE_CASE     =      tokenizer_cr.encode("""Hey this is a <special> token"""			)

                     self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
                     self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
                     self.assertTrue(special_token_id in p_output			)
                     self.assertTrue(special_token_id in cr_output			)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      "facebook/nllb-200-distilled-600M"
 lowerCAmelCase__  					=      [
     " UN Chief Says There Is No Military Solution in Syria",
     " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
 ]
 lowerCAmelCase__  					=      [
     "Şeful ONU declară că nu există o soluţie militară în Siria",
     "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
     " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
     " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
 ]
 lowerCAmelCase__  					=      [
     256047,
     16297,
     134408,
     8165,
     248066,
     14734,
     950,
     1135,
     105721,
     3573,
     83,
     27352,
     108,
     49486,
     2,
 ]
 @classmethod
 def       UpperCAmelCase__		(		cls					:       List[Any]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      NllbTokenizer.from_pretrained(
          cls.checkpoint_name					,							src_lang="""eng_Latn"""					,							tgt_lang="""ron_Latn"""			)
      __SCREAMING_SNAKE_CASE     =      1
      return cls
 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> List[str]:


      """simple docstring"""




      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]					,							256_001			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]					,							256_002			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]					,							256_057			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
      self.assertListEqual(self.expected_src_tokens					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Any			)  -> int:


      """simple docstring"""




      self.assertIn(__SCREAMING_SNAKE_CASE					,							self.tokenizer.all_special_ids			)
      # fmt: off
      __SCREAMING_SNAKE_CASE     =      [RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
      # fmt: on

      __SCREAMING_SNAKE_CASE     =      self.tokenizer.decode(__SCREAMING_SNAKE_CASE					,							skip_special_tokens=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      self.tokenizer.decode(generated_ids[1:]					,							skip_special_tokens=__SCREAMING_SNAKE_CASE			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
      self.assertNotIn(self.tokenizer.eos_token					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      ["""this is gunna be a long sentence """ * 20]
      assert isinstance(src_text[0]					,							__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      10
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(__SCREAMING_SNAKE_CASE					,							max_length=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE			).input_ids[0]
      self.assertEqual(ids[-1]					,							2			)
      self.assertEqual(ids[0]					,							__SCREAMING_SNAKE_CASE			)
      self.assertEqual(len(__SCREAMING_SNAKE_CASE			)					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       int			)  -> List[Any]:


      """simple docstring"""




      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]			)					,							[256_203, 3]			)
 def       UpperCAmelCase__		(		self					:       str			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()
      __SCREAMING_SNAKE_CASE     =      self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      NllbTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,							__SCREAMING_SNAKE_CASE			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       str			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          self.src_text					,							text_target=self.tgt_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=len(self.expected_src_tokens			)					,							return_tensors="""pt"""					,							)
      __SCREAMING_SNAKE_CASE     =      shift_tokens_right(
          batch["""labels"""]					,							self.tokenizer.pad_token_id					,							self.tokenizer.lang_code_to_id["""ron_Latn"""]			)

      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      self.assertEqual((2, 15)					,							batch.input_ids.shape			)
      self.assertEqual((2, 15)					,							batch.attention_mask.shape			)
      __SCREAMING_SNAKE_CASE     =      batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens					,							__SCREAMING_SNAKE_CASE			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							batch.decoder_input_ids[0, 0]			)  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens					,							[EN_CODE]			)
      self.assertEqual(self.tokenizer.suffix_tokens					,							[self.tokenizer.eos_token_id]			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer(self.src_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=3					,							return_tensors="""pt"""			)
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          text_target=self.tgt_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=10					,							return_tensors="""pt"""			)
      __SCREAMING_SNAKE_CASE     =      targets["""input_ids"""]
      __SCREAMING_SNAKE_CASE     =      shift_tokens_right(
          __SCREAMING_SNAKE_CASE					,							self.tokenizer.pad_token_id					,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					,							)

      self.assertEqual(batch.input_ids.shape[1]					,							3			)
      self.assertEqual(batch.decoder_input_ids.shape[1]					,							10			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       int			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer._build_translation_inputs(
          """A test"""					,							return_tensors="""pt"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)

      self.assertEqual(
          nested_simplify(__SCREAMING_SNAKE_CASE			)					,							{
              # A, test, EOS, en_XX
              """input_ids""": [[256_047, 70, 7_356, 2]],
              """attention_mask""": [[1, 1, 1, 1]],
              # ar_AR
              """forced_bos_token_id""": 256_057,
          }					,							)


 @require_torch
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      True
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          """UN Chief says there is no military solution in Syria"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)
      self.assertEqual(
          inputs.input_ids					,							[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]			)

      __SCREAMING_SNAKE_CASE     =      False
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          """UN Chief says there is no military solution in Syria"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)
      self.assertEqual(
          inputs.input_ids					,							[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]			)






