

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A      :   int          =	logging.get_logger(__name__)

__A      :   Optional[Any]          =	{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__A      :   List[Any]          =	[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

__A      :   Optional[int]          =	{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
__A      :   Optional[int]          =	{F'funnel-transformer/{name}': 512 for name in _model_names}
__A      :   Optional[int]          =	{F'funnel-transformer/{name}': {"""do_lower_case""": True} for name in _model_names}






class       __A (							UpperCamelCase__       ):
    lowerCAmelCase_							:						str						 =							VOCAB_FILES_NAMES
    lowerCAmelCase_							:						List[Any]						 =							PRETRAINED_VOCAB_FILES_MAP
    lowerCAmelCase_							:						Dict						 =							PRETRAINED_INIT_CONFIGURATION
    lowerCAmelCase_							:						Union[str, Any]						 =							FunnelTokenizer
    lowerCAmelCase_							:						Any						 =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCAmelCase_							:						int						 =							2






    def __init__(		self	:			str			, UpperCAmelCase_	:			Union[str, Any]=None			, UpperCAmelCase_	:			Any=None			, UpperCAmelCase_	:			int=True			, UpperCAmelCase_	:			List[Any]="<unk>"			, UpperCAmelCase_	:			Union[str, Any]="<sep>"			, UpperCAmelCase_	:			Tuple="<pad>"			, UpperCAmelCase_	:			str="<cls>"			, UpperCAmelCase_	:			Optional[Any]="<mask>"			, UpperCAmelCase_	:			Optional[Any]="<s>"			, UpperCAmelCase_	:			Optional[int]="</s>"			, UpperCAmelCase_	:			Any=True			, UpperCAmelCase_	:			Any=True			, UpperCAmelCase_	:			str=None			, UpperCAmelCase_	:			Optional[Any]="##"			, **UpperCAmelCase_	:			List[str]			, ):
        super().__init__(
            lowerCAmelCase__			, tokenizer_file=lowerCAmelCase__			, do_lower_case=lowerCAmelCase__			, unk_token=lowerCAmelCase__			, sep_token=lowerCAmelCase__			, pad_token=lowerCAmelCase__			, cls_token=lowerCAmelCase__			, mask_token=lowerCAmelCase__			, bos_token=lowerCAmelCase__			, eos_token=lowerCAmelCase__			, clean_text=lowerCAmelCase__			, tokenize_chinese_chars=lowerCAmelCase__			, strip_accents=lowerCAmelCase__			, wordpieces_prefix=lowerCAmelCase__			, **lowerCAmelCase__			, )

        lowerCAmelCase :	Optional[Any]				=					json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
        if (
            normalizer_state.get('lowercase'			, lowerCAmelCase__			) != do_lower_case
            or normalizer_state.get('strip_accents'			, lowerCAmelCase__			) != strip_accents
            or normalizer_state.get('handle_chinese_chars'			, lowerCAmelCase__			) != tokenize_chinese_chars
        ):
            lowerCAmelCase :	List[str]				=					getattr(lowerCAmelCase__			, normalizer_state.pop('type'			)			)
            lowerCAmelCase :	Union[str, Any]				=					do_lower_case
            lowerCAmelCase :	Any				=					strip_accents
            lowerCAmelCase :	Optional[Any]				=					tokenize_chinese_chars
            lowerCAmelCase :	Dict				=					normalizer_class(**lowerCAmelCase__			)

        lowerCAmelCase :	Any				=					do_lower_case






    def 		lowercase__       (		self	:			List[str]			, UpperCAmelCase_	:			int			, UpperCAmelCase_	:			int=None			):
        lowerCAmelCase :	Dict				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output






    def 		lowercase__       (		self	:			int			, UpperCAmelCase_	:			str			, UpperCAmelCase_	:			str = None			):
        lowerCAmelCase :	Optional[int]				=					[self.sep_token_id]
        lowerCAmelCase :	Union[str, Any]				=					[self.cls_token_id]
        if token_ids_a is None:
            return len(cls			) * [self.cls_token_type_id] + len(token_ids_a + sep			) * [0]
        return len(cls			) * [self.cls_token_type_id] + len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]






    def 		lowercase__       (		self	:			Tuple			, UpperCAmelCase_	:			List[Any]			, UpperCAmelCase_	:			int = None			):
        lowerCAmelCase :	Tuple				=					self._tokenizer.model.save(lowerCAmelCase__			, name=lowerCAmelCase__			)
        return tuple(lowerCAmelCase__			)












import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


UpperCAmelCase	:	int		     =       False







class 				__lowerCAmelCase       (    unittest.TestCase):
      pass







@nightly
@require_torch_gpu
class 				__lowerCAmelCase       (    unittest.TestCase):







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					str										=VersatileDiffusionPipeline.from_pretrained("shi-labs/versatile-diffusion"		,      torch_dtype=torch.floataa							)
        pipe.to(lowerCAmelCase__							)
        pipe.set_progress_bar_config(disable=lowerCAmelCase__							)

        a__	:					int										=load_image(
            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg"							)

        a__	:					Optional[Any]										=torch.manual_seed(0							)
        a__	:					Optional[Any]										=pipe.dual_guided(
            prompt="first prompt"		,      image=lowerCAmelCase__		,      text_to_image_strength=0.75		,      generator=lowerCAmelCase__		,      guidance_scale=7.5		,      num_inference_steps=2		,      output_type="numpy"		,      ).images

        with tempfile.TemporaryDirectory() as tmpdirname:
          pipe.save_pretrained(lowerCAmelCase__							)
          a__	:					str										=VersatileDiffusionPipeline.from_pretrained(lowerCAmelCase__		,      torch_dtype=torch.floataa							)
        pipe.to(lowerCAmelCase__							)
        pipe.set_progress_bar_config(disable=lowerCAmelCase__							)

        a__	:					Optional[Any]										=generator.manual_seed(0							)
        a__	:					Tuple										=pipe.dual_guided(
            prompt="first prompt"		,      image=lowerCAmelCase__		,      text_to_image_strength=0.75		,      generator=lowerCAmelCase__		,      guidance_scale=7.5		,      num_inference_steps=2		,      output_type="numpy"		,      ).images

        assert np.abs(image - new_image							).sum() < 1E-5, "Models don't have the same forward pass"







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					str										=VersatileDiffusionPipeline.from_pretrained("shi-labs/versatile-diffusion"		,      torch_dtype=torch.floataa							)
        pipe.to(lowerCAmelCase__							)
        pipe.set_progress_bar_config(disable=lowerCAmelCase__							)

        a__	:					Optional[Any]										="cyberpunk 2077"
        a__	:					int										=load_image(
            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg"							)
        a__	:					Union[str, Any]										=torch.manual_seed(0							)
        a__	:					Tuple										=pipe.dual_guided(
            prompt=lowerCAmelCase__		,      image=lowerCAmelCase__		,      text_to_image_strength=0.75		,      generator=lowerCAmelCase__		,      guidance_scale=7.5		,      num_inference_steps=5_0		,      output_type="numpy"		,      ).images

        a__	:					int										=image[0, 2_5_3:2_5_6, 2_5_3:2_5_6, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        a__	:					Any										=np.array([0.14_48, 0.16_19, 0.17_41, 0.10_86, 0.11_47, 0.11_28, 0.11_99, 0.11_65, 0.10_01]							)

        assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1

        a__	:					str										="A painting of a squirrel eating a burger "
        a__	:					Optional[int]										=torch.manual_seed(0							)
        a__	:					str										=pipe.text_to_image(
            prompt=lowerCAmelCase__		,      generator=lowerCAmelCase__		,      guidance_scale=7.5		,      num_inference_steps=5_0		,      output_type="numpy"							).images

        a__	:					Any										=image[0, 2_5_3:2_5_6, 2_5_3:2_5_6, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        a__	:					Optional[int]										=np.array([0.33_67, 0.31_69, 0.26_56, 0.38_70, 0.47_90, 0.37_96, 0.40_09, 0.48_78, 0.47_78]							)

        assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1

        a__	:					Optional[Any]										=pipe.image_variation(lowerCAmelCase__		,      generator=lowerCAmelCase__		,      output_type="numpy"							).images

        a__	:					Union[str, Any]										=image[0, 2_5_3:2_5_6, 2_5_3:2_5_6, -1]

        assert image.shape == (1, 5_1_2, 5_1_2, 3)
        a__	:					Any										=np.array([0.30_76, 0.31_23, 0.32_84, 0.37_82, 0.37_70, 0.38_94, 0.42_97, 0.43_31, 0.44_56]							)

        assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1






