import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import (
     FocalNetBackbone,
     FocalNetForImageClassification,
     FocalNetForMaskedImageModeling,
     FocalNetModel,
 )
 from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor

class 					_lowercase    :






    '''simple docstring'''




    def __init__(    self       ,						lowerCamelCase__       ,						lowerCamelCase__=13       ,						lowerCamelCase__=32       ,						lowerCamelCase__=2       ,						lowerCamelCase__=3       ,						lowerCamelCase__=16       ,						lowerCamelCase__=[32, 64, 128]       ,						lowerCamelCase__=[1, 2, 1]       ,						lowerCamelCase__=[2, 2, 4]       ,						lowerCamelCase__=2       ,						lowerCamelCase__=2.0       ,						lowerCamelCase__=True       ,						lowerCamelCase__=0.0       ,						lowerCamelCase__=0.0       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__="gelu"       ,						lowerCamelCase__=False       ,						lowerCamelCase__=True       ,						lowerCamelCase__=0.0_2       ,						lowerCamelCase__=1E-5       ,						lowerCamelCase__=True       ,						lowerCamelCase__=None       ,						lowerCamelCase__=True       ,						lowerCamelCase__=10       ,						lowerCamelCase__=8       ,						lowerCamelCase__=["stage1", "stage2"]       ,						lowerCamelCase__=[1, 2]       ,						):
       lowerCAmelCase_:							Dict       =	parent
       lowerCAmelCase_:							str       =	batch_size
       lowerCAmelCase_:							Tuple       =	image_size
       lowerCAmelCase_:							List[Any]       =	patch_size
       lowerCAmelCase_:							Any       =	num_channels
       lowerCAmelCase_:							List[Any]       =	embed_dim
       lowerCAmelCase_:							Optional[int]       =	hidden_sizes
       lowerCAmelCase_:							Union[str, Any]       =	depths
       lowerCAmelCase_:							Tuple       =	num_heads
       lowerCAmelCase_:							Union[str, Any]       =	window_size
       lowerCAmelCase_:							str       =	mlp_ratio
       lowerCAmelCase_:							Any       =	qkv_bias
       lowerCAmelCase_:							Tuple       =	hidden_dropout_prob
       lowerCAmelCase_:							Union[str, Any]       =	attention_probs_dropout_prob
       lowerCAmelCase_:							int       =	drop_path_rate
       lowerCAmelCase_:							Any       =	hidden_act
       lowerCAmelCase_:							str       =	use_absolute_embeddings
       lowerCAmelCase_:							Tuple       =	patch_norm
       lowerCAmelCase_:							Tuple       =	layer_norm_eps
       lowerCAmelCase_:							List[Any]       =	initializer_range
       lowerCAmelCase_:							List[str]       =	is_training
       lowerCAmelCase_:							Tuple       =	scope
       lowerCAmelCase_:							str       =	use_labels
       lowerCAmelCase_:							Optional[int]       =	type_sequence_label_size
       lowerCAmelCase_:							Optional[int]       =	encoder_stride
       lowerCAmelCase_:							Tuple       =	out_features
       lowerCAmelCase_:							Any       =	out_indices




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

       lowerCAmelCase_:							int       =	None
       if self.use_labels:
          lowerCAmelCase_:							Dict       =	ids_tensor([self.batch_size]       ,						self.type_sequence_label_size )

       lowerCAmelCase_:							Tuple       =	self.get_config()

       return config, pixel_values, labels




    def      _a    (    self ):
       return FocalNetConfig(
           image_size=self.image_size       ,						patch_size=self.patch_size       ,						num_channels=self.num_channels       ,						embed_dim=self.embed_dim       ,						hidden_sizes=self.hidden_sizes       ,						depths=self.depths       ,						num_heads=self.num_heads       ,						window_size=self.window_size       ,						mlp_ratio=self.mlp_ratio       ,						qkv_bias=self.qkv_bias       ,						hidden_dropout_prob=self.hidden_dropout_prob       ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,						drop_path_rate=self.drop_path_rate       ,						hidden_act=self.hidden_act       ,						use_absolute_embeddings=self.use_absolute_embeddings       ,						path_norm=self.patch_norm       ,						layer_norm_eps=self.layer_norm_eps       ,						initializer_range=self.initializer_range       ,						encoder_stride=self.encoder_stride       ,						out_features=self.out_features       ,						out_indices=self.out_indices       ,						)




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	FocalNetModel(config=lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()
       lowerCAmelCase_:							int       =	model(lowerCamelCase__ )

       lowerCAmelCase_:							Dict       =	((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths ) - 1))
       lowerCAmelCase_:							List[str]       =	int(config.embed_dim * 2 ** (len(config.depths ) - 1) )

       self.parent.assertEqual(result.last_hidden_state.shape       ,						(self.batch_size, expected_seq_len, expected_dim) )




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Any       =	FocalNetBackbone(config=lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()
       lowerCAmelCase_:							Optional[int]       =	model(lowerCamelCase__ )

       # verify feature maps
       self.parent.assertEqual(len(result.feature_maps )       ,						len(config.out_features ) )
       self.parent.assertListEqual(list(result.feature_maps[0].shape )       ,						[self.batch_size, self.image_size, 8, 8] )

       # verify channels
       self.parent.assertEqual(len(model.channels )       ,						len(config.out_features ) )
       self.parent.assertListEqual(model.channels       ,						config.hidden_sizes[:-1] )

       # verify backbone works with out_features=None
       lowerCAmelCase_:							str       =	None
       lowerCAmelCase_:							str       =	FocalNetBackbone(config=lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()
       lowerCAmelCase_:							str       =	model(lowerCamelCase__ )

       # verify feature maps
       self.parent.assertEqual(len(result.feature_maps )       ,						1 )
       self.parent.assertListEqual(list(result.feature_maps[0].shape )       ,						[self.batch_size, self.image_size * 2, 4, 4] )

       # verify channels
       self.parent.assertEqual(len(model.channels )       ,						1 )
       self.parent.assertListEqual(model.channels       ,						[config.hidden_sizes[-1]] )




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Dict       =	FocalNetForMaskedImageModeling(config=lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()
       lowerCAmelCase_:							List[str]       =	model(lowerCamelCase__ )
       self.parent.assertEqual(
           result.reconstruction.shape       ,						(self.batch_size, self.num_channels, self.image_size, self.image_size) )

       # test greyscale images
       lowerCAmelCase_:							Dict       =	1
       lowerCAmelCase_:							Optional[Any]       =	FocalNetForMaskedImageModeling(lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()

       lowerCAmelCase_:							Optional[int]       =	floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
       lowerCAmelCase_:							int       =	model(lowerCamelCase__ )
       self.parent.assertEqual(result.reconstruction.shape       ,						(self.batch_size, 1, self.image_size, self.image_size) )




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							int       =	self.type_sequence_label_size
       lowerCAmelCase_:							Dict       =	FocalNetForImageClassification(lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()
       lowerCAmelCase_:							Optional[int]       =	model(lowerCamelCase__       ,						labels=lowerCamelCase__ )
       self.parent.assertEqual(result.logits.shape       ,						(self.batch_size, self.type_sequence_label_size) )

       # test greyscale images
       lowerCAmelCase_:							List[str]       =	1
       lowerCAmelCase_:							Union[str, Any]       =	FocalNetForImageClassification(lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()

       lowerCAmelCase_:							List[Any]       =	floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
       lowerCAmelCase_:							str       =	model(lowerCamelCase__ )
       self.parent.assertEqual(result.logits.shape       ,						(self.batch_size, self.type_sequence_label_size) )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	self.prepare_config_and_inputs()

       lowerCAmelCase_  ,	lowerCAmelCase_  ,	lowerCAmelCase_:							Tuple       =	config_and_inputs
       lowerCAmelCase_:							int       =	{"pixel_values": pixel_values}
       return config, inputs_dict





@require_torch
class 					_lowercase    (     UpperCAmelCase__ ,  UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				List[Any]          =    (
        (
            FocalNetModel,
            FocalNetForImageClassification,
            FocalNetForMaskedImageModeling,
            FocalNetBackbone,
        )
        if is_torch_available()
        else ()
    )
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    (
        {'feature-extraction': FocalNetModel, 'image-classification': FocalNetForImageClassification}
        if is_torch_available()
        else {}
    )
    SCREAMING_SNAKE_CASE:				int          =    False

    SCREAMING_SNAKE_CASE:				Optional[int]          =    False
    SCREAMING_SNAKE_CASE:				Any          =    False
    SCREAMING_SNAKE_CASE:				List[Any]          =    False
    SCREAMING_SNAKE_CASE:				Dict          =    False




    def      _a    (    self ):
       lowerCAmelCase_:							int       =	FocalNetModelTester(self )
       lowerCAmelCase_:							str       =	ConfigTester(self       ,						config_class=lowerCamelCase__       ,						embed_dim=37       ,						has_text_modality=lowerCamelCase__ )




    def      _a    (    self ):
       self.create_and_test_config_common_properties()
       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()




    def      _a    (    self ):
       return




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_backbone(*lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_image_modeling(*lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*lowerCamelCase__ )




    @unittest.skip(reason="FocalNet does not use inputs_embeds" )
    def      _a    (    self ):
       pass




    @unittest.skip(reason="FocalNet does not use feedforward chunking" )
    def      _a    (    self ):
       pass




    def      _a    (    self ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							str       =	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes[:-1]:
          lowerCAmelCase_:							int       =	model_class(lowerCamelCase__ )
          self.assertIsInstance(model.get_input_embeddings()       ,						(nn.Module) )
          lowerCAmelCase_:							List[Any]       =	model.get_output_embeddings()
          self.assertTrue(x is None or isinstance(lowerCamelCase__       ,						nn.Linear ) )




    def      _a    (    self ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							Optional[Any]       =	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes[:-1]:
          lowerCAmelCase_:							Union[str, Any]       =	model_class(lowerCamelCase__ )
          lowerCAmelCase_:							Union[str, Any]       =	inspect.signature(model.forward )
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          lowerCAmelCase_:							List[str]       =	[*signature.parameters.keys()]

          lowerCAmelCase_:							Optional[Any]       =	["pixel_values"]
          self.assertListEqual(arg_names[:1]       ,						lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	model_class(lowerCamelCase__ )
       model.to(lowerCamelCase__ )
       model.eval()

       with torch.no_grad():
          lowerCAmelCase_:							List[str]       =	model(**self._prepare_for_class(lowerCamelCase__       ,						lowerCamelCase__ ) )

       lowerCAmelCase_:							Tuple       =	outputs.hidden_states

       lowerCAmelCase_:							Optional[int]       =	getattr(
           self.model_tester       ,						"expected_num_hidden_layers"       ,						len(self.model_tester.depths ) + 1 )
       self.assertEqual(len(lowerCamelCase__ )       ,						lowerCamelCase__ )

       # FocalNet has a different seq_length
       lowerCAmelCase_:							List[Any]       =	(
           config.patch_size
           if isinstance(config.patch_size       ,						collections.abc.Iterable )
           else (config.patch_size, config.patch_size)
       )

       lowerCAmelCase_:							int       =	(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

       self.assertListEqual(
           list(hidden_states[0].shape[-2:] )       ,						[num_patches, self.model_tester.embed_dim]       ,						)

       lowerCAmelCase_:							Optional[int]       =	outputs.reshaped_hidden_states
       self.assertEqual(len(lowerCamelCase__ )       ,						lowerCamelCase__ )

       lowerCAmelCase_  ,	lowerCAmelCase_  ,	lowerCAmelCase_  ,	lowerCAmelCase_:							List[str]       =	reshaped_hidden_states[0].shape
       lowerCAmelCase_:							Optional[Any]       =	(
           reshaped_hidden_states[0].view(lowerCamelCase__       ,						lowerCamelCase__       ,						height * width ).permute(0       ,						2       ,						1 )
       )
       self.assertListEqual(
           list(reshaped_hidden_states.shape[-2:] )       ,						[num_patches, self.model_tester.embed_dim]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							Any       =	self.model_tester.prepare_config_and_inputs_for_common()

       lowerCAmelCase_:							Dict       =	(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size       ,						collections.abc.Iterable )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )

       for model_class in self.all_model_classes[:-1]:
          lowerCAmelCase_:							str       =	True
          self.check_hidden_states_output(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          lowerCAmelCase_:							Union[str, Any]       =	True

          self.check_hidden_states_output(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							Tuple       =	self.model_tester.prepare_config_and_inputs_for_common()
       lowerCAmelCase_:							Any       =	3

       lowerCAmelCase_:							int       =	(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size       ,						collections.abc.Iterable )
           else (self.model_tester.image_size, self.model_tester.image_size)
       )
       lowerCAmelCase_:							Any       =	(
           config.patch_size
           if isinstance(config.patch_size       ,						collections.abc.Iterable )
           else (config.patch_size, config.patch_size)
       )

       lowerCAmelCase_:							List[str]       =	image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
       lowerCAmelCase_:							int       =	image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

       for model_class in self.all_model_classes[:-1]:
          lowerCAmelCase_:							Dict       =	True
          self.check_hidden_states_output(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						(padded_height, padded_width) )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          lowerCAmelCase_:							int       =	True
          self.check_hidden_states_output(lowerCamelCase__       ,						lowerCamelCase__       ,						lowerCamelCase__       ,						(padded_height, padded_width) )




    @slow
    def      _a    (    self ):
       for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase_:							List[Any]       =	FocalNetModel.from_pretrained(lowerCamelCase__ )
          self.assertIsNotNone(lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							List[Any]       =	self.model_tester.prepare_config_and_inputs_for_common()

       lowerCAmelCase_:							Union[str, Any]       =	_config_zero_init(lowerCamelCase__ )
       for model_class in self.all_model_classes:
          lowerCAmelCase_:							List[str]       =	model_class(config=lowerCamelCase__ )
          for name, param in model.named_parameters():
             if "embeddings" not in name and param.requires_grad:
                self.assertIn(
                    ((param.data.mean() * 1E9).round() / 1E9).item()       ,						[0.0, 1.0]       ,						msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''       ,						)





@require_vision
@require_torch
class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    @cached_property
    def      _a    (    self ):
       # TODO update organization
       return AutoImageProcessor.from_pretrained("microsoft/focalnet-tiny" ) if is_vision_available() else None




    @slow
    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	FocalNetForImageClassification.from_pretrained("microsoft/focalnet-tiny" ).to(lowerCamelCase__ )
       lowerCAmelCase_:							Union[str, Any]       =	self.default_image_processor

       lowerCAmelCase_:							Any       =	Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png" )
       lowerCAmelCase_:							int       =	image_processor(images=lowerCamelCase__       ,						return_tensors="pt" ).to(lowerCamelCase__ )

       # forward pass
       with torch.no_grad():
          lowerCAmelCase_:							Optional[Any]       =	model(**lowerCamelCase__ )

       # verify the logits
       lowerCAmelCase_:							int       =	torch.Size((1, 1_000) )
       self.assertEqual(outputs.logits.shape       ,						lowerCamelCase__ )
       lowerCAmelCase_:							Union[str, Any]       =	torch.tensor([0.2_1_6_6, -0.4_3_6_8, 0.2_1_9_1] ).to(lowerCamelCase__ )
       self.assertTrue(torch.allclose(outputs.logits[0, :3]       ,						lowerCamelCase__       ,						atol=1E-4 ) )
       self.assertTrue(outputs.logits.argmax(dim=-1 ).item()       ,						281 )





@require_torch
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Any          =    (FocalNetBackbone,) if is_torch_available() else ()
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    FocalNetConfig

    SCREAMING_SNAKE_CASE:				Any          =    False




    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	FocalNetModelTester(self )
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
 from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


a  :      List[Any]            =    get_tests_dir("""fixtures/test_sentencepiece.model""")

a  :      Union[str, Any]            =    {"""target_lang""": """fi""", """source_lang""": """en"""}
a  :      Optional[int]            =    """>>zh<<"""
a  :      Optional[int]            =    """Helsinki-NLP/"""

if is_torch_available():
 a  :      str            =    """pt"""
elif is_tf_available():
 a  :      List[Any]            =    """tf"""
else:
 a  :      str            =    """jax"""

@require_sentencepiece
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				str          =    MarianTokenizer
    SCREAMING_SNAKE_CASE:				Tuple          =    False
    SCREAMING_SNAKE_CASE:				Optional[int]          =    True




    def      _a    (    self ):
       super().setUp()
       lowerCAmelCase_:							Optional[Any]       =	["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
       lowerCAmelCase_:							Optional[Any]       =	dict(zip(lowerCamelCase__       ,						range(len(lowerCamelCase__ ) ) ) )
       lowerCAmelCase_:							Any       =	Path(self.tmpdirname )
       save_json(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["vocab"] )
       save_json(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["tokenizer_config_file"] )
       if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
          copyfile(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["source_spm"] )
          copyfile(lowerCamelCase__       ,						save_dir / VOCAB_FILES_NAMES["target_spm"] )

       lowerCAmelCase_:							Any       =	MarianTokenizer.from_pretrained(self.tmpdirname )
       tokenizer.save_pretrained(self.tmpdirname )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return MarianTokenizer.from_pretrained(self.tmpdirname       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       return (
           "This is a test",
           "This is a test",
       )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	"</s>"
       lowerCAmelCase_:							Tuple       =	0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__ )       ,						lowerCamelCase__ )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__ )       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	list(self.get_tokenizer().get_vocab().keys() )

       self.assertEqual(vocab_keys[0]       ,						"</s>" )
       self.assertEqual(vocab_keys[1]       ,						"<unk>" )
       self.assertEqual(vocab_keys[-1]       ,						"<pad>" )
       self.assertEqual(len(lowerCamelCase__ )       ,						9 )




    def      _a    (    self ):
       self.assertEqual(self.get_tokenizer().vocab_size       ,						9 )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	MarianTokenizer.from_pretrained(F'''{ORG_NAME}opus-mt-en-de''' )
       lowerCAmelCase_:							List[str]       =	en_de_tokenizer(["I am a small frog"]       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	[38, 121, 14, 697, 38_848, 0]
       self.assertListEqual(lowerCamelCase__       ,						batch.input_ids[0] )

       lowerCAmelCase_:							int       =	tempfile.mkdtemp()
       en_de_tokenizer.save_pretrained(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	[x.name for x in Path(lowerCamelCase__ ).glob("*" )]
       self.assertIn("source.spm"       ,						lowerCamelCase__ )
       MarianTokenizer.from_pretrained(lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.get_tokenizer()

       lowerCAmelCase_:							Tuple       =	tok(
           ["I am a small frog" * 1_000, "I am a small frog"]       ,						padding=lowerCamelCase__       ,						truncation=lowerCamelCase__       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       self.assertEqual(batch.input_ids.shape       ,						(2, 512) )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.get_tokenizer()
       lowerCAmelCase_:							List[Any]       =	tok(["I am a tiny frog", "I am a small frog"]       ,						padding=lowerCamelCase__       ,						return_tensors=lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )
       self.assertEqual(batch_smaller.input_ids.shape       ,						(2, 10) )




    @slow
    def      _a    (    self ):
       # fmt: off
       lowerCAmelCase_:							int       =	{"input_ids": [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=lowerCamelCase__       ,						model_name="Helsinki-NLP/opus-mt-en-de"       ,						revision="1a8c2263da11e68e50938f97e10cd57820bd504c"       ,						decode_kwargs={"use_source_tokenizer": True}       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	MarianTokenizer.from_pretrained("hf-internal-testing/test-marian-two-vocabs" )

       lowerCAmelCase_:							Any       =	"Tämä on testi"
       lowerCAmelCase_:							Optional[int]       =	"This is a test"

       lowerCAmelCase_:							List[str]       =	[76, 7, 2_047, 2]
       lowerCAmelCase_:							str       =	[69, 12, 11, 940, 2]

       lowerCAmelCase_:							Dict       =	tokenizer(lowerCamelCase__ ).input_ids
       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer(text_target=lowerCamelCase__ ).input_ids
       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	tokenizer.decode(lowerCamelCase__       ,						skip_special_tokens=lowerCamelCase__ )
       self.assertEqual(lowerCamelCase__       ,						lowerCamelCase__ )