import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class        UpperCamelCase						(				__a	,	unittest.TestCase				):
						a__		:List[str]      =			None
						a__		:int      =			BloomTokenizerFast
						a__		:Any      =			BloomTokenizerFast
						a__		:List[str]      =			True
						a__		:str      =			False
						a__		:Dict      =			'''tokenizer_file'''
						a__		:Optional[int]      =			{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}





						def        A_							(self						)				->				Any:
												super().setUp()
												UpperCamelCase_		:							List[Any]  =		BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""						)
												tokenizer.save_pretrained(self.tmpdirname						)





						def        A_							(self			,     **__UpperCamelCase						)				->				Optional[Any]:
												kwargs.update(self.special_tokens_map						)
												return BloomTokenizerFast.from_pretrained(self.tmpdirname			,     **__UpperCamelCase						)





						def        A_							(self						)				->				List[Any]:
												UpperCamelCase_		:							List[str]  =		self.get_rust_tokenizer()

												UpperCamelCase_		:							int  =		["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
												UpperCamelCase_		:							Dict  =		[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

												UpperCamelCase_		:							Optional[Any]  =		tokenizer.batch_encode_plus(__UpperCamelCase						)["""input_ids"""]
												self.assertListEqual(__UpperCamelCase			,     __UpperCamelCase						)

												UpperCamelCase_		:							Any  =		tokenizer.batch_decode(__UpperCamelCase						)
												self.assertListEqual(__UpperCamelCase			,     __UpperCamelCase						)





						def        A_							(self			,     __UpperCamelCase=6						)				->				str:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																								UpperCamelCase_		:							int  =		self.rust_tokenizer_class.from_pretrained(__UpperCamelCase			,     **__UpperCamelCase						)
																								# tokenizer_r.pad_token = None # Hotfixing padding = None
																								# Simple input
																								UpperCamelCase_		:							str  =		"""This is a simple input"""
																								UpperCamelCase_		:							Tuple  =		["""This is a simple input 1""", """This is a simple input 2"""]
																								UpperCamelCase_		:							List[Any]  =		("""This is a simple input""", """This is a pair""")
																								UpperCamelCase_		:							List[str]  =		[
																								    ("""This is a simple input 1""", """This is a simple input 2"""),
																								    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																								]

																								# Simple input tests
																								try:
																														tokenizer_r.encode(__UpperCamelCase			,     max_length=__UpperCamelCase						)
																														tokenizer_r.encode_plus(__UpperCamelCase			,     max_length=__UpperCamelCase						)

																														tokenizer_r.batch_encode_plus(__UpperCamelCase			,     max_length=__UpperCamelCase						)
																														tokenizer_r.encode(__UpperCamelCase			,     max_length=__UpperCamelCase						)
																														tokenizer_r.batch_encode_plus(__UpperCamelCase			,     max_length=__UpperCamelCase						)
																								except ValueError:
																														self.fail("""Bloom Tokenizer should be able to deal with padding"""						)

																								UpperCamelCase_		:							Dict  =		None  # Hotfixing padding = None
																								self.assertRaises(__UpperCamelCase			,     tokenizer_r.encode			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""						)

																								# Simple input
																								self.assertRaises(__UpperCamelCase			,     tokenizer_r.encode_plus			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""						)

																								# Simple input
																								self.assertRaises(
																								    __UpperCamelCase			,     tokenizer_r.batch_encode_plus			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""			,     )

																								# Pair input
																								self.assertRaises(__UpperCamelCase			,     tokenizer_r.encode			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""						)

																								# Pair input
																								self.assertRaises(__UpperCamelCase			,     tokenizer_r.encode_plus			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""						)

																								# Pair input
																								self.assertRaises(
																								    __UpperCamelCase			,     tokenizer_r.batch_encode_plus			,     __UpperCamelCase			,     max_length=__UpperCamelCase			,     padding="""max_length"""			,     )





						def        A_							(self						)				->				Tuple:
												UpperCamelCase_		:							Optional[Any]  =		self.get_rust_tokenizer()
												UpperCamelCase_		:							List[str]  =		load_dataset("""xnli"""			,     """all_languages"""			,     split="""test"""			,     streaming=__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		next(iter(__UpperCamelCase						)						)["""premise"""]  # pick up one data
												UpperCamelCase_		:							List[str]  =		list(sample_data.values()						)

												UpperCamelCase_		:							int  =		list(map(tokenizer.encode			,     __UpperCamelCase						)						)
												UpperCamelCase_		:							Tuple  =		[tokenizer.decode(__UpperCamelCase			,     clean_up_tokenization_spaces=__UpperCamelCase						) for x in output_tokens]
												self.assertListEqual(__UpperCamelCase			,     __UpperCamelCase						)





						def        A_							(self						)				->				Tuple:
												# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
												# any sequence length constraints. This test of the parent class will fail since it relies on the
												# maximum sequence length of the positoonal embeddings.
												self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map						)			,     1						)
												self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()						)[0]						)			,     1						)







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE		:     Dict         =	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

SCREAMING_SNAKE_CASE		:     int         =	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
SCREAMING_SNAKE_CASE		:     Any         =	{F'''funnel-transformer/{name}''': 512 for name in _model_names}
SCREAMING_SNAKE_CASE		:     Tuple         =	{F'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class        UpperCamelCase						(				__a				):
						a__		:Dict      =			VOCAB_FILES_NAMES
						a__		:Dict      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Dict      =			PRETRAINED_INIT_CONFIGURATION
						a__		:str      =			FunnelTokenizer
						a__		:List[str]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:int      =			2





						def __init__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=True			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase="<sep>"			,     __UpperCamelCase="<pad>"			,     __UpperCamelCase="<cls>"			,     __UpperCamelCase="<mask>"			,     __UpperCamelCase="<s>"			,     __UpperCamelCase="</s>"			,     __UpperCamelCase=True			,     __UpperCamelCase=True			,     __UpperCamelCase=None			,     __UpperCamelCase="##"			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(
												    __UpperCamelCase			,     tokenizer_file=__UpperCamelCase			,     do_lower_case=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     sep_token=__UpperCamelCase			,     pad_token=__UpperCamelCase			,     cls_token=__UpperCamelCase			,     mask_token=__UpperCamelCase			,     bos_token=__UpperCamelCase			,     eos_token=__UpperCamelCase			,     clean_text=__UpperCamelCase			,     tokenize_chinese_chars=__UpperCamelCase			,     strip_accents=__UpperCamelCase			,     wordpieces_prefix=__UpperCamelCase			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
												if (
												    normalizer_state.get("""lowercase"""			,     __UpperCamelCase						) != do_lower_case
												    or normalizer_state.get("""strip_accents"""			,     __UpperCamelCase						) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""			,     __UpperCamelCase						) != tokenize_chinese_chars
												):
																		UpperCamelCase_		:							List[Any]  =		getattr(__UpperCamelCase			,     normalizer_state.pop("""type"""						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		do_lower_case
																		UpperCamelCase_		:							Tuple  =		strip_accents
																		UpperCamelCase_		:							Dict  =		tokenize_chinese_chars
																		UpperCamelCase_		:							Union[str, Any]  =		normalizer_class(**__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		do_lower_case





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None						)				->				Dict:
												UpperCamelCase_		:							Any  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				List[int]:
												UpperCamelCase_		:							Optional[Any]  =		[self.sep_token_id]
												UpperCamelCase_		:							Any  =		[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
												return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												UpperCamelCase_		:							int  =		self._tokenizer.model.save(__UpperCamelCase			,     name=__UpperCamelCase						)
												return tuple(__UpperCamelCase						)






