


'''simple docstring'''







import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 						UpperCamelCase__       (			_a     ,       unittest.TestCase						):



     """simple docstring"""






     __magic_name__           =     RoCBertTokenizer
     __magic_name__           =     None
     __magic_name__           =     False
     __magic_name__           =     True
     __magic_name__           =     filter_non_english







     def a						(			self ):

            '''simple docstring'''

            super().setUp()

            _lowerCAmelCase     :				str       =							['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
            _lowerCAmelCase     :				Optional[Any]       =							{}
            _lowerCAmelCase     :				str       =							{}
            for i, value in enumerate(__lowerCamelCase ):
                   _lowerCAmelCase     :				Tuple       =							i
                   _lowerCAmelCase     :				Optional[Any]       =							i
            _lowerCAmelCase     :				int       =							os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['vocab_file'] )
            _lowerCAmelCase     :				List[Any]       =							os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['word_shape_file'] )
            _lowerCAmelCase     :				int       =							os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['word_pronunciation_file'] )
            with open(self.vocab_file   ,						'w'   ,						encoding='utf-8' ) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )
            with open(self.word_shape_file   ,						'w'   ,						encoding='utf-8' ) as word_shape_writer:
                   json.dump(__lowerCamelCase   ,						__lowerCamelCase   ,						ensure_ascii=__lowerCamelCase )
            with open(self.word_pronunciation_file   ,						'w'   ,						encoding='utf-8' ) as word_pronunciation_writer:
                   json.dump(__lowerCamelCase   ,						__lowerCamelCase   ,						ensure_ascii=__lowerCamelCase )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Dict       =							self.tokenizer_class(self.vocab_file   ,						self.word_shape_file   ,						self.word_pronunciation_file )

            _lowerCAmelCase     :				Tuple       =							tokenizer.tokenize('你好[SEP]你是谁' )
            self.assertListEqual(__lowerCamelCase   ,						['你', '好', '[SEP]', '你', '是', '谁'] )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase )   ,						[5, 6, 2, 5, 7, 8] )
            self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__lowerCamelCase )   ,						[5, 6, 2, 5, 7, 8] )
            self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__lowerCamelCase )   ,						[5, 6, 2, 5, 7, 8] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							RoCBertBasicTokenizer()

            self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )   ,						['ah', '\u535A', '\u63A8', 'zz'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[Any]       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )   ,						['hello', '!', 'how', 'are', 'you', '?'] )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hällo', '!', 'how', 'are', 'you', '?'] )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['h\u00E9llo'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Dict       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hallo', '!', 'how', 'are', 'you', '?'] )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hallo', '!', 'how', 'are', 'you', '?'] )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )   ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[Any]       =							RoCBertBasicTokenizer(do_lower_case=__lowerCamelCase   ,						never_split=['[UNK]'] )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )   ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[Any]       =							['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

            _lowerCAmelCase     :				List[Any]       =							{}
            for i, token in enumerate(__lowerCamelCase ):
                   _lowerCAmelCase     :				Tuple       =							i
            _lowerCAmelCase     :				List[Any]       =							RoCBertWordpieceTokenizer(vocab=__lowerCamelCase   ,						unk_token='[UNK]' )

            self.assertListEqual(tokenizer.tokenize('' )   ,						[] )

            self.assertListEqual(tokenizer.tokenize('unwanted running' )   ,						['un', '##want', '##ed', 'runn', '##ing'] )

            self.assertListEqual(tokenizer.tokenize('unwantedX running' )   ,						['[UNK]', 'runn', '##ing'] )







     def a						(			self ):

            '''simple docstring'''

            self.assertTrue(_is_whitespace(' ' ) )
            self.assertTrue(_is_whitespace('\t' ) )
            self.assertTrue(_is_whitespace('\r' ) )
            self.assertTrue(_is_whitespace('\n' ) )
            self.assertTrue(_is_whitespace('\u00A0' ) )

            self.assertFalse(_is_whitespace('A' ) )
            self.assertFalse(_is_whitespace('-' ) )







     def a						(			self ):

            '''simple docstring'''

            self.assertTrue(_is_control('\u0005' ) )

            self.assertFalse(_is_control('A' ) )
            self.assertFalse(_is_control(' ' ) )
            self.assertFalse(_is_control('\t' ) )
            self.assertFalse(_is_control('\r' ) )







     def a						(			self ):

            '''simple docstring'''

            self.assertTrue(_is_punctuation('-' ) )
            self.assertTrue(_is_punctuation('$' ) )
            self.assertTrue(_is_punctuation('`' ) )
            self.assertTrue(_is_punctuation('.' ) )

            self.assertFalse(_is_punctuation('A' ) )
            self.assertFalse(_is_punctuation(' ' ) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							self.get_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(__lowerCamelCase ) for t in ['Test', '\xad', 'test']]   ,						[['[UNK]'], [], ['[UNK]']] )

            if self.test_rust_tokenizer:
                   _lowerCAmelCase     :				str       =							self.get_rust_tokenizer()
                   self.assertListEqual(
                       [rust_tokenizer.tokenize(__lowerCamelCase ) for t in ['Test', '\xad', 'test']]   ,						[['[UNK]'], [], ['[UNK]']] )







     def a						(			self ):

            '''simple docstring'''

            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
                          _lowerCAmelCase     :				str       =							self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase )

                          _lowerCAmelCase     :				Optional[int]       =							F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                          _lowerCAmelCase     :				List[Any]       =							tokenizer_r.encode_plus(
                              __lowerCamelCase   ,						return_attention_mask=__lowerCamelCase   ,						return_token_type_ids=__lowerCamelCase   ,						return_offsets_mapping=__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase   ,						)

                          _lowerCAmelCase     :				List[str]       =							tokenizer_r.do_lower_case if hasattr(__lowerCamelCase   ,						'do_lower_case' ) else False
                          _lowerCAmelCase     :				Optional[Any]       =							(
                              [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), 'A'),
                                  ((1, 2), ','),
                                  ((3, 5), 'na'),
                                  ((5, 6), '##ï'),
                                  ((6, 8), '##ve'),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), 'Allen'),
                                  ((21, 23), '##NL'),
                                  ((23, 24), '##P'),
                                  ((25, 33), 'sentence'),
                                  ((33, 34), '.'),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                              if not do_lower_case
                              else [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), 'a'),
                                  ((1, 2), ','),
                                  ((3, 8), 'naive'),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), 'allen'),
                                  ((21, 23), '##nl'),
                                  ((23, 24), '##p'),
                                  ((25, 33), 'sentence'),
                                  ((33, 34), '.'),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                          )

                          self.assertEqual(
                              [e[1] for e in expected_results]   ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
                          self.assertEqual([e[0] for e in expected_results]   ,						tokens['offset_mapping'] )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							['的', '人', '有']
            _lowerCAmelCase     :				Union[str, Any]       =							''.join(__lowerCamelCase )
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
                          _lowerCAmelCase     :				Optional[Any]       =							True
                          _lowerCAmelCase     :				Dict       =							self.tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase )
                          _lowerCAmelCase     :				Dict       =							self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase )

                          _lowerCAmelCase     :				Union[str, Any]       =							tokenizer_p.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )
                          _lowerCAmelCase     :				Tuple       =							tokenizer_r.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )

                          _lowerCAmelCase     :				Dict       =							tokenizer_r.convert_ids_to_tokens(__lowerCamelCase )
                          _lowerCAmelCase     :				List[Any]       =							tokenizer_p.convert_ids_to_tokens(__lowerCamelCase )

                          # it is expected that each Chinese character is not preceded by "##"
                          self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase )
                          self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase )

                          _lowerCAmelCase     :				Union[str, Any]       =							False
                          _lowerCAmelCase     :				int       =							self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase )
                          _lowerCAmelCase     :				Dict       =							self.tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase )

                          _lowerCAmelCase     :				str       =							tokenizer_r.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )
                          _lowerCAmelCase     :				List[Any]       =							tokenizer_p.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )

                          _lowerCAmelCase     :				Dict       =							tokenizer_r.convert_ids_to_tokens(__lowerCamelCase )
                          _lowerCAmelCase     :				int       =							tokenizer_p.convert_ids_to_tokens(__lowerCamelCase )

                          # it is expected that only the first Chinese character is not preceded by "##".
                          _lowerCAmelCase     :				List[str]       =							[
                              F'##{token}' if idx != 0 else token for idx, token in enumerate(__lowerCamelCase )
                          ]
                          self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase )
                          self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase )







     @slow
     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							self.tokenizer_class(self.vocab_file   ,						self.word_shape_file   ,						self.word_pronunciation_file )

            _lowerCAmelCase     :				List[Any]       =							tokenizer.encode('你好'   ,						add_special_tokens=__lowerCamelCase )
            _lowerCAmelCase     :				List[str]       =							tokenizer.encode('你是谁'   ,						add_special_tokens=__lowerCamelCase )

            _lowerCAmelCase     :				List[str]       =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase )
            _lowerCAmelCase     :				Optional[int]       =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   ,						__lowerCamelCase )

            assert encoded_sentence == [1] + text + [2]
            assert encoded_pair == [1] + text + [2] + text_a + [2]







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							self.get_tokenizers(do_lower_case=__lowerCamelCase )
            for tokenizer in tokenizers:
                   with self.subTest(F'{tokenizer.__class__.__name__}' ):
                          _lowerCAmelCase     :				Union[str, Any]       =							'你好，你是谁'
                          _lowerCAmelCase     :				List[str]       =							tokenizer.tokenize(__lowerCamelCase )
                          _lowerCAmelCase     :				Optional[Any]       =							tokenizer.convert_tokens_to_ids(__lowerCamelCase )
                          _lowerCAmelCase     :				Optional[int]       =							tokenizer.convert_tokens_to_shape_ids(__lowerCamelCase )
                          _lowerCAmelCase     :				str       =							tokenizer.convert_tokens_to_pronunciation_ids(__lowerCamelCase )
                          _lowerCAmelCase     :				Tuple       =							tokenizer.prepare_for_model(
                              __lowerCamelCase   ,						__lowerCamelCase   ,						__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )

                          _lowerCAmelCase     :				Optional[int]       =							tokenizer.encode_plus(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase )

                          self.assertEqual(__lowerCamelCase   ,						__lowerCamelCase )







import webbrowser
from sys import argv
from urllib.parse import parse_qs, quote

import requests
from bsa import BeautifulSoup
from fake_useragent import UserAgent

if __name__ == "__main__":
      __magic_name__  :  List[Any]					=       """%20""".join(argv[1:]) if len(argv) > 1 else quote(str(input("""Search: """)))

      print("""Googling.....""")

      __magic_name__  :  List[Any]					=       F"https://www.google.com/search?q={query}&num=100"

      __magic_name__  :  Union[str, Any]					=       requests.get(
          url,
          headers={"""User-Agent""": str(UserAgent().random)},
      )

      try:
            __magic_name__  :  Optional[Any]					=       (
                BeautifulSoup(res.text, """html.parser""")
                .find("""div""", attrs={"""class""": """yuRUbf"""})
                .find("""a""")
                .get("""href""")
            )

      except AttributeError:
            __magic_name__  :  List[Any]					=       parse_qs(
                BeautifulSoup(res.text, """html.parser""")
                .find("""div""", attrs={"""class""": """kCrYT"""})
                .find("""a""")
                .get("""href""")
            )["""url"""][0]

      webbrowser.open(link)

