
"""simple docstring"""



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class       lowerCAmelCase__				(						A_ ,	unittest.TestCase	):
     __a						=					MobileBertTokenizer
     __a						=					MobileBertTokenizerFast
     __a						=					True
     __a						=					True
     __a						=					filter_non_english
     __a						=					"""google/mobilebert-uncased"""







     def      lowercase	(							self			:			Optional[int]			):
       super().setUp()

       _snake_case					     =       [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       _snake_case					     =       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''vocab_file''']			)
       with open(self.vocab_file    ,      '''w'''    ,      encoding='''utf-8'''			) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)

       _snake_case					     =       [
           (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
           for tokenizer_def in self.tokenizers_list
       ]







     def      lowercase	(							self			:			Any    ,      _lowerCamelCase			:			List[str]			):
       _snake_case					     =       '''UNwant\u00E9d,running'''
       _snake_case					     =       '''unwanted, running'''
       return input_text, output_text







     def      lowercase	(							self			:			Any			):
       _snake_case					     =       self.tokenizer_class(self.vocab_file			)

       _snake_case					     =       tokenizer.tokenize('''UNwant\u00E9d,running'''			)
       self.assertListEqual(_lowerCamelCase    ,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase			)    ,      [9, 6, 7, 12, 10, 11]			)







     def      lowercase	(							self			:			Optional[Any]			):
       if not self.test_rust_tokenizer:
         return

       _snake_case					     =       self.get_tokenizer()
       _snake_case					     =       self.get_rust_tokenizer()

       _snake_case					     =       '''UNwant\u00E9d,running'''

       _snake_case					     =       tokenizer.tokenize(_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.tokenize(_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

       _snake_case					     =       tokenizer.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

       _snake_case					     =       self.get_rust_tokenizer()
       _snake_case					     =       tokenizer.encode(_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.encode(_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

       # With lower casing
       _snake_case					     =       self.get_tokenizer(do_lower_case=_lowerCamelCase			)
       _snake_case					     =       self.get_rust_tokenizer(do_lower_case=_lowerCamelCase			)

       _snake_case					     =       '''UNwant\u00E9d,running'''

       _snake_case					     =       tokenizer.tokenize(_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.tokenize(_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

       _snake_case					     =       tokenizer.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

       _snake_case					     =       self.get_rust_tokenizer()
       _snake_case					     =       tokenizer.encode(_lowerCamelCase			)
       _snake_case					     =       rust_tokenizer.encode(_lowerCamelCase			)
       self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)







     def      lowercase	(							self			:			Any			):
       _snake_case					     =       BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)    ,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)







     def      lowercase	(							self			:			int			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)    ,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,      ['''hello''']			)







     def      lowercase	(							self			:			Tuple			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase    ,      strip_accents=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,      ['''h\u00E9llo''']			)







     def      lowercase	(							self			:			Optional[int]			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase    ,      strip_accents=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,      ['''hello''']			)







     def      lowercase	(							self			:			Tuple			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)    ,      ['''hello''']			)







     def      lowercase	(							self			:			str			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)







     def      lowercase	(							self			:			Optional[Any]			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase    ,      strip_accents=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)







     def      lowercase	(							self			:			str			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase    ,      strip_accents=_lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)    ,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)







     def      lowercase	(							self			:			Dict			):
       _snake_case					     =       BasicTokenizer(do_lower_case=_lowerCamelCase    ,      never_split=['''[UNK]''']			)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)







     def      lowercase	(							self			:			Dict			):
       _snake_case					     =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       _snake_case					     =       {}
       for i, token in enumerate(_lowerCamelCase			):
         _snake_case					     =       i
       _snake_case					     =       WordpieceTokenizer(vocab=_lowerCamelCase    ,      unk_token='''[UNK]'''			)

       self.assertListEqual(tokenizer.tokenize(''''''			)    ,      []			)

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)    ,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)    ,      ['''[UNK]''', '''runn''', '''##ing''']			)







     def      lowercase	(							self			:			str			):
       self.assertTrue(_is_whitespace(''' '''			)			)
       self.assertTrue(_is_whitespace('''\t'''			)			)
       self.assertTrue(_is_whitespace('''\r'''			)			)
       self.assertTrue(_is_whitespace('''\n'''			)			)
       self.assertTrue(_is_whitespace('''\u00A0'''			)			)

       self.assertFalse(_is_whitespace('''A'''			)			)
       self.assertFalse(_is_whitespace('''-'''			)			)







     def      lowercase	(							self			:			Any			):
       self.assertTrue(_is_control('''\u0005'''			)			)

       self.assertFalse(_is_control('''A'''			)			)
       self.assertFalse(_is_control(''' '''			)			)
       self.assertFalse(_is_control('''\t'''			)			)
       self.assertFalse(_is_control('''\r'''			)			)







     def      lowercase	(							self			:			int			):
       self.assertTrue(_is_punctuation('''-'''			)			)
       self.assertTrue(_is_punctuation('''$'''			)			)
       self.assertTrue(_is_punctuation('''`'''			)			)
       self.assertTrue(_is_punctuation('''.'''			)			)

       self.assertFalse(_is_punctuation('''A'''			)			)
       self.assertFalse(_is_punctuation(''' '''			)			)







     def      lowercase	(							self			:			Dict			):
       _snake_case					     =       self.get_tokenizer()
       _snake_case					     =       self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_lowerCamelCase			) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]			)

       self.assertListEqual(
           [rust_tokenizer.tokenize(_lowerCamelCase			) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]			)







     @slow
     def      lowercase	(							self			:			Optional[Any]			):
       _snake_case					     =       self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''			)

       _snake_case					     =       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=_lowerCamelCase			)
       _snake_case					     =       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=_lowerCamelCase			)

       _snake_case					     =       tokenizer.build_inputs_with_special_tokens(_lowerCamelCase			)
       _snake_case					     =       tokenizer.build_inputs_with_special_tokens(_lowerCamelCase    ,      _lowerCamelCase			)

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]







     def      lowercase	(							self			:			List[str]			):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
           _snake_case					     =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,      **_lowerCamelCase			)

           _snake_case					     =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
           _snake_case					     =       tokenizer_r.encode_plus(
               _lowerCamelCase    ,      return_attention_mask=_lowerCamelCase    ,      return_token_type_ids=_lowerCamelCase    ,      return_offsets_mapping=_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase    ,      )

           _snake_case					     =       tokenizer_r.do_lower_case if hasattr(_lowerCamelCase    ,      '''do_lower_case'''			) else False
           _snake_case					     =       (
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''Allen'''),
                   ((21, 23), '''##NL'''),
                   ((23, 24), '''##P'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''allen'''),
                   ((21, 23), '''##nl'''),
                   ((23, 24), '''##p'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]    ,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
           self.assertEqual([e[0] for e in expected_results]    ,      tokens['''offset_mapping''']			)







     def      lowercase	(							self			:			List[str]			):
       _snake_case					     =       ['''的''', '''人''', '''有''']
       _snake_case					     =       ''''''.join(_lowerCamelCase			)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
           _snake_case					     =       True
           _snake_case					     =       self.tokenizer_class.from_pretrained(_lowerCamelCase    ,      **_lowerCamelCase			)
           _snake_case					     =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,      **_lowerCamelCase			)

           _snake_case					     =       tokenizer_p.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
           _snake_case					     =       tokenizer_r.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)

           _snake_case					     =       tokenizer_r.convert_ids_to_tokens(_lowerCamelCase			)
           _snake_case					     =       tokenizer_p.convert_ids_to_tokens(_lowerCamelCase			)

           # it is expected that each Chinese character is not preceded by "##"
           self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)
           self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)

           _snake_case					     =       False
           _snake_case					     =       self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,      **_lowerCamelCase			)
           _snake_case					     =       self.tokenizer_class.from_pretrained(_lowerCamelCase    ,      **_lowerCamelCase			)

           _snake_case					     =       tokenizer_r.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)
           _snake_case					     =       tokenizer_p.encode(_lowerCamelCase    ,      add_special_tokens=_lowerCamelCase			)

           _snake_case					     =       tokenizer_r.convert_ids_to_tokens(_lowerCamelCase			)
           _snake_case					     =       tokenizer_p.convert_ids_to_tokens(_lowerCamelCase			)

           # it is expected that only the first Chinese character is not preceded by "##".
           _snake_case					     =       [
               f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase			)
           ]
           self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)
           self.assertListEqual(_lowerCamelCase    ,      _lowerCamelCase			)





"""simple docstring"""



def 					_UpperCAmelCase      ( __lowerCamelCase   :       list ,					__lowerCamelCase   :       int = 0		)     ->					list:
  _snake_case					     =       length or len(__lowerCamelCase		)
  _snake_case					     =       False
  for i in range(length - 1		):
    if list_data[i] > list_data[i + 1]:
      _snake_case     ,					_snake_case					     =       list_data[i + 1], list_data[i]
      _snake_case					     =       True

  return list_data if not swapped else bubble_sort(__lowerCamelCase ,					length - 1		)


if __name__ == "__main__":
 import doctest

 doctest.testmod()



