


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class        _a							(			UpperCamelCase__     ,	unittest.TestCase			):
							_lowercase	:			Union[str, Any]        = RoCBertTokenizer
							_lowercase	:			str        = None
							_lowercase	:			Any        = False
							_lowercase	:			str        = True
							_lowercase	:			List[Any]        = filter_non_english

							def       lowerCamelCase_ (							self:	Any							)			->   Dict:





									"""simple docstring"""







									super().setUp()

									lowercase__            =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
									lowercase__            =      {}
									lowercase__            =      {}
									for i, value in enumerate(UpperCamelCase_							):
											lowercase__            =      i
											lowercase__            =      i
									lowercase__            =      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['''vocab_file''']							)
									lowercase__            =      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['''word_shape_file''']							)
									lowercase__            =      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['''word_pronunciation_file''']							)
									with open(self.vocab_file    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
									with open(self.word_shape_file    ,						'''w'''    ,						encoding='''utf-8'''							) as word_shape_writer:
											json.dump(UpperCamelCase_    ,						UpperCamelCase_    ,						ensure_ascii=UpperCamelCase_							)
									with open(self.word_pronunciation_file    ,						'''w'''    ,						encoding='''utf-8'''							) as word_pronunciation_writer:
											json.dump(UpperCamelCase_    ,						UpperCamelCase_    ,						ensure_ascii=UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Optional[int]							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      self.tokenizer_class(self.vocab_file    ,						self.word_shape_file    ,						self.word_pronunciation_file							)

									lowercase__            =      tokenizer.tokenize('''你好[SEP]你是谁'''							)
									self.assertListEqual(UpperCamelCase_    ,						['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']							)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)    ,						[5, 6, 2, 5, 7, 8]							)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)    ,						[5, 6, 2, 5, 7, 8]							)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)    ,						[5, 6, 2, 5, 7, 8]							)

							def       lowerCamelCase_ (							self:	Dict							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)    ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)

							def       lowerCamelCase_ (							self:	Any							)			->   Any:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)    ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)    ,						['''hello''']							)

							def       lowerCamelCase_ (							self:	Dict							)			->   Tuple:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_    ,						strip_accents=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)    ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)    ,						['''h\u00E9llo''']							)

							def       lowerCamelCase_ (							self:	List[Any]							)			->   Tuple:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_    ,						strip_accents=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)    ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)    ,						['''hello''']							)

							def       lowerCamelCase_ (							self:	Any							)			->   Dict:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)    ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)    ,						['''hello''']							)

							def       lowerCamelCase_ (							self:	int							)			->   int:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)    ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def       lowerCamelCase_ (							self:	Tuple							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_    ,						strip_accents=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)    ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def       lowerCamelCase_ (							self:	List[str]							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_    ,						strip_accents=UpperCamelCase_							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)    ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)

							def       lowerCamelCase_ (							self:	Tuple							)			->   str:





									"""simple docstring"""







									lowercase__            =      RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_    ,						never_split=['''[UNK]''']							)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)    ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)

							def       lowerCamelCase_ (							self:	Tuple							)			->   Dict:





									"""simple docstring"""







									lowercase__            =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowercase__            =      {}
									for i, token in enumerate(UpperCamelCase_							):
											lowercase__            =      i
									lowercase__            =      RoCBertWordpieceTokenizer(vocab=UpperCamelCase_    ,						unk_token='''[UNK]'''							)

									self.assertListEqual(tokenizer.tokenize(''''''							)    ,						[]							)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)    ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)    ,						['''[UNK]''', '''runn''', '''##ing''']							)

							def       lowerCamelCase_ (							self:	Any							)			->   int:





									"""simple docstring"""







									self.assertTrue(_is_whitespace(''' '''							)							)
									self.assertTrue(_is_whitespace('''\t'''							)							)
									self.assertTrue(_is_whitespace('''\r'''							)							)
									self.assertTrue(_is_whitespace('''\n'''							)							)
									self.assertTrue(_is_whitespace('''\u00A0'''							)							)

									self.assertFalse(_is_whitespace('''A'''							)							)
									self.assertFalse(_is_whitespace('''-'''							)							)

							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   int:





									"""simple docstring"""







									self.assertTrue(_is_control('''\u0005'''							)							)

									self.assertFalse(_is_control('''A'''							)							)
									self.assertFalse(_is_control(''' '''							)							)
									self.assertFalse(_is_control('''\t'''							)							)
									self.assertFalse(_is_control('''\r'''							)							)

							def       lowerCamelCase_ (							self:	Dict							)			->   Dict:





									"""simple docstring"""







									self.assertTrue(_is_punctuation('''-'''							)							)
									self.assertTrue(_is_punctuation('''$'''							)							)
									self.assertTrue(_is_punctuation('''`'''							)							)
									self.assertTrue(_is_punctuation('''.'''							)							)

									self.assertFalse(_is_punctuation('''A'''							)							)
									self.assertFalse(_is_punctuation(''' '''							)							)

							def       lowerCamelCase_ (							self:	Optional[int]							)			->   int:





									"""simple docstring"""







									lowercase__            =      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]    ,						[['''[UNK]'''], [], ['''[UNK]''']]							)

									if self.test_rust_tokenizer:
											lowercase__            =      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]    ,						[['''[UNK]'''], [], ['''[UNK]''']]							)

							def       lowerCamelCase_ (							self:	List[str]							)			->   List[Any]:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__            =      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_    ,						**UpperCamelCase_							)

													lowercase__            =      f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													lowercase__            =      tokenizer_r.encode_plus(
													    UpperCamelCase_    ,						return_attention_mask=UpperCamelCase_    ,						return_token_type_ids=UpperCamelCase_    ,						return_offsets_mapping=UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_    ,						)

													lowercase__            =      tokenizer_r.do_lower_case if hasattr(UpperCamelCase_    ,						'''do_lower_case'''							) else False
													lowercase__            =      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]    ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
													self.assertEqual([e[0] for e in expected_results]    ,						tokens['''offset_mapping''']							)

							def       lowerCamelCase_ (							self:	List[str]							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      ['''的''', '''人''', '''有''']
									lowercase__            =      ''''''.join(UpperCamelCase_							)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'							):
													lowercase__            =      True
													lowercase__            =      self.tokenizer_class.from_pretrained(UpperCamelCase_    ,						**UpperCamelCase_							)
													lowercase__            =      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_    ,						**UpperCamelCase_							)

													lowercase__            =      tokenizer_p.encode(UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)
													lowercase__            =      tokenizer_r.encode(UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)

													lowercase__            =      tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
													lowercase__            =      tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCamelCase_    ,						UpperCamelCase_							)
													self.assertListEqual(UpperCamelCase_    ,						UpperCamelCase_							)

													lowercase__            =      False
													lowercase__            =      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_    ,						**UpperCamelCase_							)
													lowercase__            =      self.tokenizer_class.from_pretrained(UpperCamelCase_    ,						**UpperCamelCase_							)

													lowercase__            =      tokenizer_r.encode(UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)
													lowercase__            =      tokenizer_p.encode(UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)

													lowercase__            =      tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
													lowercase__            =      tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

													# it is expected that only the first Chinese character is not preceded by "##".
													lowercase__            =      [
													    f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_							)
													]
													self.assertListEqual(UpperCamelCase_    ,						UpperCamelCase_							)
													self.assertListEqual(UpperCamelCase_    ,						UpperCamelCase_							)

							@slow
							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   Any:





									"""simple docstring"""







									lowercase__            =      self.tokenizer_class(self.vocab_file    ,						self.word_shape_file    ,						self.word_pronunciation_file							)

									lowercase__            =      tokenizer.encode('''你好'''    ,						add_special_tokens=UpperCamelCase_							)
									lowercase__            =      tokenizer.encode('''你是谁'''    ,						add_special_tokens=UpperCamelCase_							)

									lowercase__            =      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
									lowercase__            =      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_    ,						UpperCamelCase_							)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def       lowerCamelCase_ (							self:	Tuple							)			->   List[Any]:





									"""simple docstring"""







									lowercase__            =      self.get_tokenizers(do_lower_case=UpperCamelCase_							)
									for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'							):
													lowercase__            =      '''你好，你是谁'''
													lowercase__            =      tokenizer.tokenize(UpperCamelCase_							)
													lowercase__            =      tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
													lowercase__            =      tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)
													lowercase__            =      tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)
													lowercase__            =      tokenizer.prepare_for_model(
													    UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)

													lowercase__            =      tokenizer.encode_plus(UpperCamelCase_    ,						add_special_tokens=UpperCamelCase_							)

													self.assertEqual(UpperCamelCase_    ,						UpperCamelCase_							)






from ..utils import DummyObject, requires_backends


class        _a							(			metaclass=UpperCamelCase__			):
							_lowercase	:			Any        = ['''torch''', '''scipy''']

							def __init__(							self:	int    ,						*UpperCamelCase_:	Any    ,						**UpperCamelCase_:	Optional[Any]							)			->   List[str]:





									"""simple docstring"""







									requires_backends(self    ,						['''torch''', '''scipy''']							)

							@classmethod
							def       lowerCamelCase_ (							cls:	Optional[int]    ,						*UpperCamelCase_:	Any    ,						**UpperCamelCase_:	List[Any]							)			->   int:





									"""simple docstring"""







									requires_backends(cls    ,						['''torch''', '''scipy''']							)


							@classmethod
							def       lowerCamelCase_ (							cls:	Any    ,						*UpperCamelCase_:	Any    ,						**UpperCamelCase_:	Dict							)			->   Optional[int]:





									"""simple docstring"""







									requires_backends(cls    ,						['''torch''', '''scipy''']							)


