"""simple docstring"""






import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin
class 				snake_case       ( __UpperCAmelCase      ,    unittest.TestCase				):




					"""simple docstring"""

					snake_case__				    = BlenderbotSmallTokenizer
					snake_case__				    = False







					def 		__lowerCAmelCase						(				self   :	List[str]			):
									super().setUp()

									UpperCAmelCase__             =							['__start__', 'adapt', 'act', 'ap@@', 'te', '__end__', '__unk__']
									UpperCAmelCase__             =							dict(zip(lowerCamelCase__    ,range(len(lowerCamelCase__			)			)			)			)

									UpperCAmelCase__             =							['#version: 0.2', 'a p', 't e</w>', 'ap t</w>', 'a d', 'ad apt</w>', 'a c', 'ac t</w>', '']
									UpperCAmelCase__             =							{'unk_token': '__unk__', 'bos_token': '__start__', 'eos_token': '__end__'}

									UpperCAmelCase__             =							os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['vocab_file']			)
									UpperCAmelCase__             =							os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['merges_file']			)
									with open(self.vocab_file    ,'w'    ,encoding='utf-8'			) as fp:
													fp.write(json.dumps(lowerCamelCase__			) + '\n'			)
									with open(self.merges_file    ,'w'    ,encoding='utf-8'			) as fp:
													fp.write('\n'.join(lowerCamelCase__			)			)







					def 		__lowerCAmelCase						(				self   :	Optional[int]    ,**lowerCamelCase__   :	Any			):
									kwargs.update(self.special_tokens_map			)
									return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname    ,**lowerCamelCase__			)







					def 		__lowerCAmelCase						(				self   :	Any    ,lowerCamelCase__   :	Optional[int]			):
									UpperCAmelCase__             =							'adapt act apte'
									UpperCAmelCase__             =							'adapt act apte'
									return input_text, output_text







					def 		__lowerCAmelCase						(				self   :	List[str]			):
									UpperCAmelCase__             =							BlenderbotSmallTokenizer(self.vocab_file    ,self.merges_file    ,**self.special_tokens_map			)
									UpperCAmelCase__             =							'adapt act apte'
									UpperCAmelCase__             =							['adapt', 'act', 'ap@@', 'te']
									UpperCAmelCase__             =							tokenizer.tokenize(lowerCamelCase__			)
									self.assertListEqual(lowerCamelCase__    ,lowerCamelCase__			)

									UpperCAmelCase__             =							[tokenizer.bos_token] + tokens + [tokenizer.eos_token]

									UpperCAmelCase__             =							[0, 1, 2, 3, 4, 5]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__			)    ,lowerCamelCase__			)







					def 		__lowerCAmelCase						(				self   :	Union[str, Any]			):
									UpperCAmelCase__             =							BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'			)
									assert tok('sam'			).input_ids == [1_384]
									UpperCAmelCase__             =							'I am a small frog.'
									UpperCAmelCase__             =							tok([src_text]    ,padding=lowerCamelCase__    ,truncation=lowerCamelCase__			)['input_ids']
									UpperCAmelCase__             =							tok.batch_decode(lowerCamelCase__    ,skip_special_tokens=lowerCamelCase__    ,clean_up_tokenization_spaces=lowerCamelCase__			)[0]
									assert src_text != decoded  # I wish it did!
									assert decoded == "i am a small frog ."







					def 		__lowerCAmelCase						(				self   :	int			):
									UpperCAmelCase__             =							BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'			)
									UpperCAmelCase__             =							'I am a small frog .'
									UpperCAmelCase__             =							'.'
									UpperCAmelCase__             =							tok(lowerCamelCase__			)['input_ids']
									UpperCAmelCase__             =							tok(lowerCamelCase__			)['input_ids']

									assert encoded[-1] == encoded_dot[0]








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_:Dict             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:int             =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_:Optional[Any]             =						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

SCREAMING_SNAKE_CASE_:int             =						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
SCREAMING_SNAKE_CASE_:int             =						{F"""funnel-transformer/{name}""": 512 for name in _model_names}
SCREAMING_SNAKE_CASE_:Dict             =						{F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      VOCAB_FILES_NAMES
    __lowerCamelCase			: Union[str, Any]					=      PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase			: str					=      PRETRAINED_INIT_CONFIGURATION
    __lowerCamelCase			: Tuple					=      FunnelTokenizer
    __lowerCamelCase			: str					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase			: int					=      2






    def __init__(       self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__="<unk>",		lowerCamelCase__="<sep>",		lowerCamelCase__="<pad>",		lowerCamelCase__="<cls>",		lowerCamelCase__="<mask>",		lowerCamelCase__="<s>",		lowerCamelCase__="</s>",		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__="##",		**lowerCamelCase__,		):
        super().__init__(
            lowerCamelCase__,		tokenizer_file=lowerCamelCase__,		do_lower_case=lowerCamelCase__,		unk_token=lowerCamelCase__,		sep_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		cls_token=lowerCamelCase__,		mask_token=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		clean_text=lowerCamelCase__,		tokenize_chinese_chars=lowerCamelCase__,		strip_accents=lowerCamelCase__,		wordpieces_prefix=lowerCamelCase__,		**lowerCamelCase__,		)

        A :       List[str]          =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
        if (
            normalizer_state.get("""lowercase""",		lowerCamelCase__		) != do_lower_case
            or normalizer_state.get("""strip_accents""",		lowerCamelCase__		) != strip_accents
            or normalizer_state.get("""handle_chinese_chars""",		lowerCamelCase__		) != tokenize_chinese_chars
        ):
            A :       Optional[Any]          =   getattr(lowerCamelCase__,		normalizer_state.pop("""type"""		)		)
            A :       Union[str, Any]          =   do_lower_case
            A :       List[Any]          =   strip_accents
            A :       Optional[int]          =   tokenize_chinese_chars
            A :       Dict          =   normalizer_class(**lowerCamelCase__		)

        A :       List[Any]          =   do_lower_case






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__=None		):
        A :       Union[str, Any]          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       Any          =   [self.sep_token_id]
        A :       Any          =   [self.cls_token_id]
        if token_ids_a is None:
            return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
        return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       int          =   self._tokenizer.model.save(lowerCamelCase__,		name=lowerCamelCase__		)
        return tuple(lowerCamelCase__		)


