


'''simple docstring'''







import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
   from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


A__:    Union[str, Any]					    =       get_tests_dir('''fixtures/test_sentencepiece.model''')

A__:    int					    =       {'''target_lang''': '''fi''', '''source_lang''': '''en'''}
A__:    Dict					    =       '''>>zh<<'''
A__:    Optional[Any]					    =       '''Helsinki-NLP/'''

if is_torch_available():
   A__:    Optional[int]					    =       '''pt'''
elif is_tf_available():
   A__:    int					    =       '''tf'''
else:
   A__:    Optional[Any]					    =       '''jax'''







@require_sentencepiece
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):
      __UpperCamelCase       :  int	       =							MarianTokenizer
      __UpperCamelCase       :  str	       =							False
      __UpperCamelCase       :  int	       =							True






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Any:




             '''simple docstring'''
             super().setUp()
             _a    : Tuple     					=["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
             _a    : str     					=dict(zip(SCREAMING_SNAKE_CASE     ,						range(len(SCREAMING_SNAKE_CASE	)	)	)	)
             _a    : Optional[Any]     					=Path(self.tmpdirname	)
             save_json(SCREAMING_SNAKE_CASE     ,						save_dir / VOCAB_FILES_NAMES["""vocab"""]	)
             save_json(SCREAMING_SNAKE_CASE     ,						save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""]	)
             if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
                    copyfile(SCREAMING_SNAKE_CASE     ,						save_dir / VOCAB_FILES_NAMES["""source_spm"""]	)
                    copyfile(SCREAMING_SNAKE_CASE     ,						save_dir / VOCAB_FILES_NAMES["""target_spm"""]	)

             _a    : Union[str, Any]     					=MarianTokenizer.from_pretrained(self.tmpdirname	)
             tokenizer.save_pretrained(self.tmpdirname	)






      def __UpperCAmelCase       ( self			:Tuple     ,						**SCREAMING_SNAKE_CASE			:List[Any]	)       ->					MarianTokenizer:




             '''simple docstring'''
             return MarianTokenizer.from_pretrained(self.tmpdirname     ,						**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Tuple	)       ->					Any:




             '''simple docstring'''
             return (
                 "This is a test",
                 "This is a test",
             )






      def __UpperCAmelCase       ( self			:Any	)       ->					Dict:




             '''simple docstring'''
             _a    : Tuple     					="""</s>"""
             _a    : Tuple     					=0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Dict     					=list(self.get_tokenizer().get_vocab().keys()	)

             self.assertEqual(vocab_keys[0]     ,						"""</s>"""	)
             self.assertEqual(vocab_keys[1]     ,						"""<unk>"""	)
             self.assertEqual(vocab_keys[-1]     ,						"""<pad>"""	)
             self.assertEqual(len(SCREAMING_SNAKE_CASE	)     ,						9	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Dict:




             '''simple docstring'''
             self.assertEqual(self.get_tokenizer().vocab_size     ,						9	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Union[str, Any]     					=MarianTokenizer.from_pretrained(f"{ORG_NAME}opus-mt-en-de"	)
             _a    : List[str]     					=en_de_tokenizer(["""I am a small frog"""]     ,						return_tensors=SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : Union[str, Any]     					=[3_8, 1_2_1, 1_4, 6_9_7, 3_8_8_4_8, 0]
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						batch.input_ids[0]	)

             _a    : int     					=tempfile.mkdtemp()
             en_de_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
             _a    : List[Any]     					=[x.name for x in Path(SCREAMING_SNAKE_CASE	).glob("""*"""	)]
             self.assertIn("""source.spm"""     ,						SCREAMING_SNAKE_CASE	)
             MarianTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Any	)       ->					str:




             '''simple docstring'''
             _a    : Union[str, Any]     					=self.get_tokenizer()

             _a    : List[str]     					=tok(
                 ["""I am a small frog""" * 1_0_0_0, """I am a small frog"""]     ,						padding=SCREAMING_SNAKE_CASE     ,						truncation=SCREAMING_SNAKE_CASE     ,						return_tensors=SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             self.assertEqual(batch.input_ids.shape     ,						(2, 5_1_2)	)






      def __UpperCAmelCase       ( self			:str	)       ->					int:




             '''simple docstring'''
             _a    : Dict     					=self.get_tokenizer()
             _a    : Optional[Any]     					=tok(["""I am a tiny frog""", """I am a small frog"""]     ,						padding=SCREAMING_SNAKE_CASE     ,						return_tensors=SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             self.assertEqual(batch_smaller.input_ids.shape     ,						(2, 1_0)	)






      @slow
      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Optional[Any]:




             '''simple docstring'''
             # fmt: off
             _a    : List[Any]     					={"""input_ids""": [[4_3_4_9_5, 4_6_2, 2_0, 4_2_1_6_4, 1_3_6_9, 5_2, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 7_4_9_1, 3_8_9_9_9, 6, 8, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 4_6_6_9, 3_7_8_6_7, 1_3, 7_5_2_5, 2_7, 1_5_9_3, 9_8_8, 1_3, 3_3_9_7_2, 7_0_2_9, 6, 2_0, 8_2_5_1, 3_8_3, 2, 2_7_0, 5_8_6_6, 3_7_8_8, 2, 2_3_5_3, 8_2_5_1, 1_2_3_3_8, 2, 1_3_9_5_8, 3_8_7, 2, 3_6_2_9, 6_9_5_3, 1_8_8, 2_9_0_0, 2, 1_3_9_5_8, 8_0_1_1, 1_1_5_0_1, 2_3, 8_4_6_0, 4_0_7_3, 3_4_0_0_9, 2_0, 4_3_5, 1_1_4_3_9, 2_7, 8, 8_4_6_0, 4_0_7_3, 6_0_0_4, 2_0, 9_9_8_8, 3_7_5, 2_7, 3_3, 2_6_6, 1_9_4_5, 1_0_7_6, 1_3_5_0, 3_7_8_6_7, 3_2_8_8, 5, 5_7_7, 1_0_7_6, 4_3_7_4, 8, 5_0_8_2, 5, 2_6_4_5_3, 2_5_7, 5_5_6, 4_0_3, 2, 2_4_2, 1_3_2, 3_8_3, 3_1_6, 4_9_2, 8, 1_0_7_6_7, 6, 3_1_6, 3_0_4, 4_2_3_9, 3, 0], [1_4_8, 1_5_7_2_2, 1_9, 1_8_3_9, 1_2, 1_3_5_0, 1_3, 2_2_3_2_7, 5_0_8_2, 5_4_1_8, 4_7_5_6_7, 3_5_9_3_8, 5_9, 3_1_8, 1_9_5_5_2, 1_0_8, 2_1_8_3, 5_4, 1_4_9_7_6, 4_8_3_5, 3_2, 5_4_7, 1_1_1_4, 8, 3_1_5, 2_4_1_7, 5, 9_2, 1_9_0_8_8, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0], [3_6, 6_3_9_5, 1_2_5_7_0, 3_9_1_4_7, 1_1_5_9_7, 6, 2_6_6, 4, 4_5_4_0_5, 7_2_9_6, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=SCREAMING_SNAKE_CASE     ,						model_name="""Helsinki-NLP/opus-mt-en-de"""     ,						revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""     ,						decode_kwargs={"""use_source_tokenizer""": True}     ,						)






      def __UpperCAmelCase       ( self			:Tuple	)       ->					str:




             '''simple docstring'''
             _a    : Union[str, Any]     					=MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs"""	)

             _a    : Dict     					="""Tämä on testi"""
             _a    : Tuple     					="""This is a test"""

             _a    : Any     					=[7_6, 7, 2_0_4_7, 2]
             _a    : int     					=[6_9, 1_2, 1_1, 9_4_0, 2]

             _a    : List[Any]     					=tokenizer(SCREAMING_SNAKE_CASE	).input_ids
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             _a    : str     					=tokenizer(text_target=SCREAMING_SNAKE_CASE	).input_ids
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             _a    : Union[str, Any]     					=tokenizer.decode(SCREAMING_SNAKE_CASE     ,						skip_special_tokens=SCREAMING_SNAKE_CASE	)
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






'''simple docstring'''







# NOTE: This file is deprecated and will be removed in a future version.
# It only exists so that temporarely `from diffusers.pipelines import DiffusionPipeline` works
from ...utils import deprecate
from ..controlnet.multicontrolnet import MultiControlNetModel  # noqa: F401
from ..controlnet.pipeline_controlnet import StableDiffusionControlNetPipeline  # noqa: F401


deprecate(
    '''stable diffusion controlnet''',
    '''0.22.0''',
    '''Importing `StableDiffusionControlNetPipeline` or `MultiControlNetModel` from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet is deprecated. Please import `from diffusers import StableDiffusionControlNetPipeline` instead.''',
    standard_warn=False,
    stacklevel=3,
)


