



'''simple docstring'''






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				SCREAMING_SNAKE_CASE(							A__							,		unittest.TestCase			):







	"""simple docstring"""






	lowerCamelCase__			 =       MobileBertTokenizer
	lowerCamelCase__			 =       MobileBertTokenizerFast
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       True
	lowerCamelCase__			 =       filter_non_english
	lowerCamelCase__			 =       """google/mobilebert-uncased"""







	def      A	(						self			:     Any		)			->		int:
								super().setUp()

								UpperCAmelCase :       str 							=		[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								UpperCAmelCase :       Dict 							=		os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file''']		)
								with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8'''		) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)

								UpperCAmelCase :       List[Any] 							=		[
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Optional[Any] 							=		'''UNwant\u00E9d,running'''
								UpperCAmelCase :       int 							=		'''unwanted, running'''
								return input_text, output_text







	def      A	(						self			:     Optional[Any]		)			->		Dict:
								UpperCAmelCase :       List[str] 							=		self.tokenizer_class(self.vocab_file		)

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize('''UNwant\u00E9d,running'''		)
								self.assertListEqual(__snake_case		,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case		)		,			[9, 6, 7, 12, 10, 11]		)







	def      A	(						self			:     Any		)			->		int:
								if not self.test_rust_tokenizer:
															return

								UpperCAmelCase :       Optional[int] 							=		self.get_tokenizer()
								UpperCAmelCase :       Any 							=		self.get_rust_tokenizer()

								UpperCAmelCase :       int 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       Dict 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       Optional[int] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[int] 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       int 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       str 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								# With lower casing
								UpperCAmelCase :       Dict 							=		self.get_tokenizer(do_lower_case=__snake_case		)
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer(do_lower_case=__snake_case		)

								UpperCAmelCase :       Dict 							=		'''UNwant\u00E9d,running'''

								UpperCAmelCase :       int 							=		tokenizer.tokenize(__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.tokenize(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       int 							=		tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[Any] 							=		rust_tokenizer.encode(__snake_case		,			add_special_tokens=__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)

								UpperCAmelCase :       Optional[Any] 							=		self.get_rust_tokenizer()
								UpperCAmelCase :       Union[str, Any] 							=		tokenizer.encode(__snake_case		)
								UpperCAmelCase :       Union[str, Any] 							=		rust_tokenizer.encode(__snake_case		)
								self.assertListEqual(__snake_case		,			__snake_case		)







	def      A	(						self			:     List[str]		)			->		List[str]:
								UpperCAmelCase :       str 							=		BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''		)		,			['''ah''', '''\u535A''', '''\u63A8''', '''zz''']		)







	def      A	(						self			:     Optional[int]		)			->		Any:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Optional[int]		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''h\u00E9llo''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[Any]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     str		)			->		Optional[int]:
								UpperCAmelCase :       Any 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']		)
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)		,			['''hello''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Tuple		)			->		Any:
								UpperCAmelCase :       Dict 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[int]		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		BasicTokenizer(do_lower_case=__snake_case		,			strip_accents=__snake_case		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''		)		,			['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']		)







	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       Tuple 							=		BasicTokenizer(do_lower_case=__snake_case		,			never_split=['''[UNK]''']		)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''		)		,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']		)







	def      A	(						self			:     List[Any]		)			->		Dict:
								UpperCAmelCase :       List[Any] 							=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								UpperCAmelCase :       Tuple 							=		{}
								for i, token in enumerate(__snake_case		):
															UpperCAmelCase :       List[str] 							=		i
								UpperCAmelCase :       str 							=		WordpieceTokenizer(vocab=__snake_case		,			unk_token='''[UNK]'''		)

								self.assertListEqual(tokenizer.tokenize(''''''		)		,			[]		)

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''		)		,			['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']		)

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''		)		,			['''[UNK]''', '''runn''', '''##ing''']		)







	def      A	(						self			:     Union[str, Any]		)			->		Tuple:
								self.assertTrue(_is_whitespace(''' '''		)		)
								self.assertTrue(_is_whitespace('''\t'''		)		)
								self.assertTrue(_is_whitespace('''\r'''		)		)
								self.assertTrue(_is_whitespace('''\n'''		)		)
								self.assertTrue(_is_whitespace('''\u00A0'''		)		)

								self.assertFalse(_is_whitespace('''A'''		)		)
								self.assertFalse(_is_whitespace('''-'''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		Any:
								self.assertTrue(_is_control('''\u0005'''		)		)

								self.assertFalse(_is_control('''A'''		)		)
								self.assertFalse(_is_control(''' '''		)		)
								self.assertFalse(_is_control('''\t'''		)		)
								self.assertFalse(_is_control('''\r'''		)		)







	def      A	(						self			:     Optional[int]		)			->		Tuple:
								self.assertTrue(_is_punctuation('''-'''		)		)
								self.assertTrue(_is_punctuation('''$'''		)		)
								self.assertTrue(_is_punctuation('''`'''		)		)
								self.assertTrue(_is_punctuation('''.'''		)		)

								self.assertFalse(_is_punctuation('''A'''		)		)
								self.assertFalse(_is_punctuation(''' '''		)		)







	def      A	(						self			:     Union[str, Any]		)			->		List[str]:
								UpperCAmelCase :       Dict 							=		self.get_tokenizer()
								UpperCAmelCase :       int 							=		self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(__snake_case		) for t in ['''Test''', '''\xad''', '''test''']]		,			[['''[UNK]'''], [], ['''[UNK]''']]		)







	@slow
	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								UpperCAmelCase :       Optional[Any] 							=		self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''		)

								UpperCAmelCase :       Dict 							=		tokenizer.encode('''sequence builders'''		,			add_special_tokens=__snake_case		)
								UpperCAmelCase :       List[str] 							=		tokenizer.encode('''multi-sequence build'''		,			add_special_tokens=__snake_case		)

								UpperCAmelCase :       Any 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		)
								UpperCAmelCase :       Tuple 							=		tokenizer.build_inputs_with_special_tokens(__snake_case		,			__snake_case		)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]







	def      A	(						self			:     Optional[Any]		)			->		Any:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Dict 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       str 							=		F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_r.encode_plus(
																						    __snake_case		,			return_attention_mask=__snake_case		,			return_token_type_ids=__snake_case		,			return_offsets_mapping=__snake_case		,			add_special_tokens=__snake_case		,			)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.do_lower_case if hasattr(__snake_case		,			'''do_lower_case'''		) else False
																						UpperCAmelCase :       str 							=		(
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''Allen'''),
																						        ((21, 23), '''##NL'''),
																						        ((23, 24), '''##P'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''allen'''),
																						        ((21, 23), '''##nl'''),
																						        ((23, 24), '''##p'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]		,			tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']		)		)
																						self.assertEqual([e[0] for e in expected_results]		,			tokens['''offset_mapping''']		)







	def      A	(						self			:     Optional[int]		)			->		str:
								UpperCAmelCase :       str 							=		['''的''', '''人''', '''有''']
								UpperCAmelCase :       List[Any] 							=		''''''.join(__snake_case		)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																						UpperCAmelCase :       Union[str, Any] 							=		True
																						UpperCAmelCase :       List[str] 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       Any 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)

																						UpperCAmelCase :       Union[str, Any] 							=		False
																						UpperCAmelCase :       int 							=		self.rust_tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)
																						UpperCAmelCase :       int 							=		self.tokenizer_class.from_pretrained(__snake_case		,			**__snake_case		)

																						UpperCAmelCase :       List[Any] 							=		tokenizer_r.encode(__snake_case		,			add_special_tokens=__snake_case		)
																						UpperCAmelCase :       Tuple 							=		tokenizer_p.encode(__snake_case		,			add_special_tokens=__snake_case		)

																						UpperCAmelCase :       List[str] 							=		tokenizer_r.convert_ids_to_tokens(__snake_case		)
																						UpperCAmelCase :       Optional[int] 							=		tokenizer_p.convert_ids_to_tokens(__snake_case		)

																						# it is expected that only the first Chinese character is not preceded by "##".
																						UpperCAmelCase :       Optional[Any] 							=		[
																						    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__snake_case		)
																						]
																						self.assertListEqual(__snake_case		,			__snake_case		)
																						self.assertListEqual(__snake_case		,			__snake_case		)












# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_snake_case  :    Dict						    =  {'configuration_mra': ['MRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MraConfig']}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _snake_case  :    Optional[int]						    =  [
         'MRA_PRETRAINED_MODEL_ARCHIVE_LIST',
         'MraForMaskedLM',
         'MraForMultipleChoice',
         'MraForQuestionAnswering',
         'MraForSequenceClassification',
         'MraForTokenClassification',
         'MraLayer',
         'MraModel',
         'MraPreTrainedModel',
     ]


if TYPE_CHECKING:
     from .configuration_mra import MRA_PRETRAINED_CONFIG_ARCHIVE_MAP, MraConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mra import (
              MRA_PRETRAINED_MODEL_ARCHIVE_LIST,
              MraForMaskedLM,
              MraForMultipleChoice,
              MraForQuestionAnswering,
              MraForSequenceClassification,
              MraForTokenClassification,
              MraLayer,
              MraModel,
              MraPreTrainedModel,
          )
else:
     import sys

     _snake_case  :    Tuple						    =  _LazyModule(__name__, globals()['__file__'], _import_structure)

