






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     _SCREAMING_SNAKE_CASE       (			SCREAMING_SNAKE_CASE__		,   unittest.TestCase      ):
	snake_case__							:     Optional[Any]     	=  MobileBertTokenizer
	snake_case__							:     Tuple     	=  MobileBertTokenizerFast
	snake_case__							:     Optional[int]     	=  True
	snake_case__							:     str     	=  True
	snake_case__							:     Dict     	=  filter_non_english
	snake_case__							:     Union[str, Any]     	=  '''google/mobilebert-uncased'''





	def 						_A (		self				:					Optional[int]	):
		super().setUp()

		UpperCamelCase     :Union[str, Any]          =	[
		    """[UNK]""",
		    """[CLS]""",
		    """[SEP]""",
		    """[PAD]""",
		    """[MASK]""",
		    """want""",
		    """##want""",
		    """##ed""",
		    """wa""",
		    """un""",
		    """runn""",
		    """##ing""",
		    """,""",
		    """low""",
		    """lowest""",
		]
		UpperCamelCase     :Tuple          =	os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES["""vocab_file"""]	)
		with open(self.vocab_file ,  """w""" ,  encoding="""utf-8"""	) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)

		UpperCamelCase     :Optional[Any]          =	[
		    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
		    for tokenizer_def in self.tokenizers_list
		]





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					Dict	):
		UpperCamelCase     :List[str]          =	"""UNwant\u00E9d,running"""
		UpperCamelCase     :Tuple          =	"""unwanted, running"""
		return input_text, output_text





	def 						_A (		self				:					Any	):
		UpperCamelCase     :Any          =	self.tokenizer_class(self.vocab_file	)

		UpperCamelCase     :Tuple          =	tokenizer.tokenize("""UNwant\u00E9d,running"""	)
		self.assertListEqual(A__ ,  ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(A__	) ,  [9, 6, 7, 12, 10, 11]	)





	def 						_A (		self				:					Dict	):
		if not self.test_rust_tokenizer:
			return

		UpperCamelCase     :Any          =	self.get_tokenizer()
		UpperCamelCase     :List[Any]          =	self.get_rust_tokenizer()

		UpperCamelCase     :List[Any]          =	"""UNwant\u00E9d,running"""

		UpperCamelCase     :List[str]          =	tokenizer.tokenize(A__	)
		UpperCamelCase     :Optional[Any]          =	rust_tokenizer.tokenize(A__	)
		self.assertListEqual(A__ ,  A__	)

		UpperCamelCase     :Union[str, Any]          =	tokenizer.encode(A__ ,  add_special_tokens=A__	)
		UpperCamelCase     :Union[str, Any]          =	rust_tokenizer.encode(A__ ,  add_special_tokens=A__	)
		self.assertListEqual(A__ ,  A__	)

		UpperCamelCase     :Optional[int]          =	self.get_rust_tokenizer()
		UpperCamelCase     :Tuple          =	tokenizer.encode(A__	)
		UpperCamelCase     :Optional[Any]          =	rust_tokenizer.encode(A__	)
		self.assertListEqual(A__ ,  A__	)

		# With lower casing
		UpperCamelCase     :int          =	self.get_tokenizer(do_lower_case=A__	)
		UpperCamelCase     :Tuple          =	self.get_rust_tokenizer(do_lower_case=A__	)

		UpperCamelCase     :Tuple          =	"""UNwant\u00E9d,running"""

		UpperCamelCase     :Tuple          =	tokenizer.tokenize(A__	)
		UpperCamelCase     :Tuple          =	rust_tokenizer.tokenize(A__	)
		self.assertListEqual(A__ ,  A__	)

		UpperCamelCase     :Tuple          =	tokenizer.encode(A__ ,  add_special_tokens=A__	)
		UpperCamelCase     :Dict          =	rust_tokenizer.encode(A__ ,  add_special_tokens=A__	)
		self.assertListEqual(A__ ,  A__	)

		UpperCamelCase     :str          =	self.get_rust_tokenizer()
		UpperCamelCase     :Tuple          =	tokenizer.encode(A__	)
		UpperCamelCase     :List[Any]          =	rust_tokenizer.encode(A__	)
		self.assertListEqual(A__ ,  A__	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Optional[Any]          =	BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	) ,  ["""ah""", """\u535A""", """\u63A8""", """zz"""]	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :List[str]          =	BasicTokenizer(do_lower_case=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	) ,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,  ["""hello"""]	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Union[str, Any]          =	BasicTokenizer(do_lower_case=A__ ,  strip_accents=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,  ["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,  ["""h\u00E9llo"""]	)





	def 						_A (		self				:					Any	):
		UpperCamelCase     :int          =	BasicTokenizer(do_lower_case=A__ ,  strip_accents=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,  ["""hello"""]	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Dict          =	BasicTokenizer(do_lower_case=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	) ,  ["""hello"""]	)





	def 						_A (		self				:					Dict	):
		UpperCamelCase     :Dict          =	BasicTokenizer(do_lower_case=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	) ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :Optional[Any]          =	BasicTokenizer(do_lower_case=A__ ,  strip_accents=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,  ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)





	def 						_A (		self				:					str	):
		UpperCamelCase     :str          =	BasicTokenizer(do_lower_case=A__ ,  strip_accents=A__	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	) ,  ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Tuple          =	BasicTokenizer(do_lower_case=A__ ,  never_split=["""[UNK]"""]	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	) ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :str          =	["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

		UpperCamelCase     :Union[str, Any]          =	{}
		for i, token in enumerate(A__	):
			UpperCamelCase     :Optional[int]          =	i
		UpperCamelCase     :Dict          =	WordpieceTokenizer(vocab=A__ ,  unk_token="""[UNK]"""	)

		self.assertListEqual(tokenizer.tokenize(""""""	) ,  []	)

		self.assertListEqual(tokenizer.tokenize("""unwanted running"""	) ,  ["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

		self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	) ,  ["""[UNK]""", """runn""", """##ing"""]	)





	def 						_A (		self				:					int	):
		self.assertTrue(_is_whitespace(""" """	)	)
		self.assertTrue(_is_whitespace("""\t"""	)	)
		self.assertTrue(_is_whitespace("""\r"""	)	)
		self.assertTrue(_is_whitespace("""\n"""	)	)
		self.assertTrue(_is_whitespace("""\u00A0"""	)	)

		self.assertFalse(_is_whitespace("""A"""	)	)
		self.assertFalse(_is_whitespace("""-"""	)	)





	def 						_A (		self				:					Union[str, Any]	):
		self.assertTrue(_is_control("""\u0005"""	)	)

		self.assertFalse(_is_control("""A"""	)	)
		self.assertFalse(_is_control(""" """	)	)
		self.assertFalse(_is_control("""\t"""	)	)
		self.assertFalse(_is_control("""\r"""	)	)





	def 						_A (		self				:					List[str]	):
		self.assertTrue(_is_punctuation("""-"""	)	)
		self.assertTrue(_is_punctuation("""$"""	)	)
		self.assertTrue(_is_punctuation("""`"""	)	)
		self.assertTrue(_is_punctuation("""."""	)	)

		self.assertFalse(_is_punctuation("""A"""	)	)
		self.assertFalse(_is_punctuation(""" """	)	)





	def 						_A (		self				:					Optional[int]	):
		UpperCamelCase     :Optional[int]          =	self.get_tokenizer()
		UpperCamelCase     :List[Any]          =	self.get_rust_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(A__	) for t in ["""Test""", """\xad""", """test"""]] ,  [["""[UNK]"""], [], ["""[UNK]"""]]	)

		self.assertListEqual(
		    [rust_tokenizer.tokenize(A__	) for t in ["""Test""", """\xad""", """test"""]] ,  [["""[UNK]"""], [], ["""[UNK]"""]]	)





	@slow
	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :str          =	self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""	)

		UpperCamelCase     :Optional[int]          =	tokenizer.encode("""sequence builders""" ,  add_special_tokens=A__	)
		UpperCamelCase     :Optional[int]          =	tokenizer.encode("""multi-sequence build""" ,  add_special_tokens=A__	)

		UpperCamelCase     :Optional[int]          =	tokenizer.build_inputs_with_special_tokens(A__	)
		UpperCamelCase     :Union[str, Any]          =	tokenizer.build_inputs_with_special_tokens(A__ ,  A__	)

		assert encoded_sentence == [101] + text + [102]
		assert encoded_pair == [101] + text + [102] + text_a + [102]





	def 						_A (		self				:					str	):
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
				UpperCamelCase     :Any          =	self.rust_tokenizer_class.from_pretrained(A__ ,  **A__	)

				UpperCamelCase     :Dict          =	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
				UpperCamelCase     :Optional[int]          =	tokenizer_r.encode_plus(
				    A__ ,  return_attention_mask=A__ ,  return_token_type_ids=A__ ,  return_offsets_mapping=A__ ,  add_special_tokens=A__ ,  )

				UpperCamelCase     :Optional[Any]          =	tokenizer_r.do_lower_case if hasattr(A__ ,  """do_lower_case"""	) else False
				UpperCamelCase     :Tuple          =	(
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), """A"""),
				        ((1, 2), ""","""),
				        ((3, 5), """na"""),
				        ((5, 6), """##ï"""),
				        ((6, 8), """##ve"""),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), """Allen"""),
				        ((21, 23), """##NL"""),
				        ((23, 24), """##P"""),
				        ((25, 33), """sentence"""),
				        ((33, 34), """."""),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), """a"""),
				        ((1, 2), ""","""),
				        ((3, 8), """naive"""),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), """allen"""),
				        ((21, 23), """##nl"""),
				        ((23, 24), """##p"""),
				        ((25, 33), """sentence"""),
				        ((33, 34), """."""),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results] ,  tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]	)	)
				self.assertEqual([e[0] for e in expected_results] ,  tokens["""offset_mapping"""]	)





	def 						_A (		self				:					int	):
		UpperCamelCase     :Optional[int]          =	["""的""", """人""", """有"""]
		UpperCamelCase     :Union[str, Any]          =	"""""".join(A__	)
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
				UpperCamelCase     :str          =	True
				UpperCamelCase     :Tuple          =	self.tokenizer_class.from_pretrained(A__ ,  **A__	)
				UpperCamelCase     :Union[str, Any]          =	self.rust_tokenizer_class.from_pretrained(A__ ,  **A__	)

				UpperCamelCase     :Optional[Any]          =	tokenizer_p.encode(A__ ,  add_special_tokens=A__	)
				UpperCamelCase     :Optional[int]          =	tokenizer_r.encode(A__ ,  add_special_tokens=A__	)

				UpperCamelCase     :Any          =	tokenizer_r.convert_ids_to_tokens(A__	)
				UpperCamelCase     :List[str]          =	tokenizer_p.convert_ids_to_tokens(A__	)

				# it is expected that each Chinese character is not preceded by "##"
				self.assertListEqual(A__ ,  A__	)
				self.assertListEqual(A__ ,  A__	)

				UpperCamelCase     :Tuple          =	False
				UpperCamelCase     :str          =	self.rust_tokenizer_class.from_pretrained(A__ ,  **A__	)
				UpperCamelCase     :List[str]          =	self.tokenizer_class.from_pretrained(A__ ,  **A__	)

				UpperCamelCase     :Optional[Any]          =	tokenizer_r.encode(A__ ,  add_special_tokens=A__	)
				UpperCamelCase     :int          =	tokenizer_p.encode(A__ ,  add_special_tokens=A__	)

				UpperCamelCase     :Optional[int]          =	tokenizer_r.convert_ids_to_tokens(A__	)
				UpperCamelCase     :Union[str, Any]          =	tokenizer_p.convert_ids_to_tokens(A__	)

				# it is expected that only the first Chinese character is not preceded by "##".
				UpperCamelCase     :Union[str, Any]          =	[
				    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__	)
				]
				self.assertListEqual(A__ ,  A__	)
				self.assertListEqual(A__ ,  A__	)


from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowercase__		:int   =    logging.get_logger(__name__)

lowercase__		:Any   =    {
    "sail/poolformer_s12": "https://huggingface.co/sail/poolformer_s12/resolve/main/config.json",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       Optional[Any]       					='''poolformer'''




							def __init__(				self   ,A__=3   ,A__=1_6   ,A__=1_6   ,A__=3   ,A__=4.0   ,A__=[2, 2, 6, 2]   ,A__=[6_4, 1_2_8, 3_2_0, 5_1_2]   ,A__=[7, 3, 3, 3]   ,A__=[4, 2, 2, 2]   ,A__=[2, 1, 1, 1]   ,A__=4   ,A__=0.0   ,A__="gelu"   ,A__=True   ,A__=1E-5   ,A__=0.02   ,**A__   ,):
												lowercase							=						num_channels
												lowercase							=						patch_size
												lowercase							=						stride
												lowercase							=						padding
												lowercase							=						pool_size
												lowercase							=						hidden_sizes
												lowercase							=						mlp_ratio
												lowercase							=						depths
												lowercase							=						patch_sizes
												lowercase							=						strides
												lowercase							=						num_encoder_blocks
												lowercase							=						drop_path_rate
												lowercase							=						hidden_act
												lowercase							=						use_layer_scale
												lowercase							=						layer_scale_init_value
												lowercase							=						initializer_range
												super().__init__(**A__)







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       List[Any]       					=version.parse('''1.11'''		)




							@property
							def  A__      (				self):
												return OrderedDict(
												    [
												        ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
												    ])





							@property
							def  A__      (				self):
												return 2E-3




