





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class   A			(SCREAMING_SNAKE_CASE   ,     unittest.TestCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       Tuple							=    RoCBertTokenizer
			__lowerCamelCase	:       str							=    None
			__lowerCamelCase	:       Any							=    False
			__lowerCamelCase	:       Tuple							=    True
			__lowerCamelCase	:       List[str]							=    filter_non_english






			def    a_       (			self   :    List[Any]   )	->	Optional[Any]:







					"""simple docstring"""






					super().setUp()

					A__        =   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
					A__        =   {}
					A__        =   {}
					for i, value in enumerate(__lowerCAmelCase   ):
							A__        =   i
							A__        =   i
					A__        =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["""vocab_file"""]   )
					A__        =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["""word_shape_file"""]   )
					A__        =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["""word_pronunciation_file"""]   )
					with open(self.vocab_file							,	"""w"""							,	encoding="""utf-8"""   ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )
					with open(self.word_shape_file							,	"""w"""							,	encoding="""utf-8"""   ) as word_shape_writer:
							json.dump(__lowerCAmelCase							,	__lowerCAmelCase							,	ensure_ascii=__lowerCAmelCase   )
					with open(self.word_pronunciation_file							,	"""w"""							,	encoding="""utf-8"""   ) as word_pronunciation_writer:
							json.dump(__lowerCAmelCase							,	__lowerCAmelCase							,	ensure_ascii=__lowerCAmelCase   )






			def    a_       (			self   :    List[str]   )	->	Tuple:







					"""simple docstring"""






					A__        =   self.tokenizer_class(self.vocab_file							,	self.word_shape_file							,	self.word_pronunciation_file   )

					A__        =   tokenizer.tokenize("""你好[SEP]你是谁"""   )
					self.assertListEqual(__lowerCAmelCase							,	["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]   )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase   )							,	[5, 6, 2, 5, 7, 8]   )
					self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__lowerCAmelCase   )							,	[5, 6, 2, 5, 7, 8]   )
					self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__lowerCAmelCase   )							,	[5, 6, 2, 5, 7, 8]   )






			def    a_       (			self   :    Union[str, Any]   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""   )							,	["""ah""", """\u535A""", """\u63A8""", """zz"""]   )






			def    a_       (			self   :    Any   )	->	Any:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """   )							,	["""hello""", """!""", """how""", """are""", """you""", """?"""]   )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )							,	["""hello"""]   )






			def    a_       (			self   :    Tuple   )	->	int:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase							,	strip_accents=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )							,	["""hällo""", """!""", """how""", """are""", """you""", """?"""]   )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )							,	["""h\u00E9llo"""]   )






			def    a_       (			self   :    Optional[Any]   )	->	str:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase							,	strip_accents=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )							,	["""hallo""", """!""", """how""", """are""", """you""", """?"""]   )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )							,	["""hello"""]   )






			def    a_       (			self   :    List[str]   )	->	List[str]:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )							,	["""hallo""", """!""", """how""", """are""", """you""", """?"""]   )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )							,	["""hello"""]   )






			def    a_       (			self   :    Dict   )	->	Dict:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """   )							,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )






			def    a_       (			self   :    Union[str, Any]   )	->	List[str]:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase							,	strip_accents=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )							,	["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )






			def    a_       (			self   :    int   )	->	List[str]:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase							,	strip_accents=__lowerCAmelCase   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )							,	["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )






			def    a_       (			self   :    List[str]   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   RoCBertBasicTokenizer(do_lower_case=__lowerCAmelCase							,	never_split=["""[UNK]"""]   )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""   )							,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]   )






			def    a_       (			self   :    int   )	->	str:







					"""simple docstring"""






					A__        =   ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

					A__        =   {}
					for i, token in enumerate(__lowerCAmelCase   ):
							A__        =   i
					A__        =   RoCBertWordpieceTokenizer(vocab=__lowerCAmelCase							,	unk_token="""[UNK]"""   )

					self.assertListEqual(tokenizer.tokenize(""""""   )							,	[]   )

					self.assertListEqual(tokenizer.tokenize("""unwanted running"""   )							,	["""un""", """##want""", """##ed""", """runn""", """##ing"""]   )

					self.assertListEqual(tokenizer.tokenize("""unwantedX running"""   )							,	["""[UNK]""", """runn""", """##ing"""]   )






			def    a_       (			self   :    List[Any]   )	->	Union[str, Any]:







					"""simple docstring"""






					self.assertTrue(_is_whitespace(""" """   )   )
					self.assertTrue(_is_whitespace("""\t"""   )   )
					self.assertTrue(_is_whitespace("""\r"""   )   )
					self.assertTrue(_is_whitespace("""\n"""   )   )
					self.assertTrue(_is_whitespace("""\u00A0"""   )   )

					self.assertFalse(_is_whitespace("""A"""   )   )
					self.assertFalse(_is_whitespace("""-"""   )   )






			def    a_       (			self   :    int   )	->	Union[str, Any]:







					"""simple docstring"""






					self.assertTrue(_is_control("""\u0005"""   )   )

					self.assertFalse(_is_control("""A"""   )   )
					self.assertFalse(_is_control(""" """   )   )
					self.assertFalse(_is_control("""\t"""   )   )
					self.assertFalse(_is_control("""\r"""   )   )






			def    a_       (			self   :    str   )	->	List[Any]:







					"""simple docstring"""






					self.assertTrue(_is_punctuation("""-"""   )   )
					self.assertTrue(_is_punctuation("""$"""   )   )
					self.assertTrue(_is_punctuation("""`"""   )   )
					self.assertTrue(_is_punctuation("""."""   )   )

					self.assertFalse(_is_punctuation("""A"""   )   )
					self.assertFalse(_is_punctuation(""" """   )   )






			def    a_       (			self   :    List[str]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   self.get_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(__lowerCAmelCase   ) for t in ["""Test""", """\xad""", """test"""]]							,	[["""[UNK]"""], [], ["""[UNK]"""]]   )

					if self.test_rust_tokenizer:
							A__        =   self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(__lowerCAmelCase   ) for t in ["""Test""", """\xad""", """test"""]]							,	[["""[UNK]"""], [], ["""[UNK]"""]]   )






			def    a_       (			self   :    Any   )	->	Dict:







					"""simple docstring"""






					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
									A__        =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,	**__lowerCAmelCase   )

									A__        =   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
									A__        =   tokenizer_r.encode_plus(
									    __lowerCAmelCase							,	return_attention_mask=__lowerCAmelCase							,	return_token_type_ids=__lowerCAmelCase							,	return_offsets_mapping=__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase							,	)

									A__        =   tokenizer_r.do_lower_case if hasattr(__lowerCAmelCase							,	"""do_lower_case"""   ) else False
									A__        =   (
									    [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), """A"""),
									        ((1, 2), ""","""),
									        ((3, 5), """na"""),
									        ((5, 6), """##ï"""),
									        ((6, 8), """##ve"""),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), """Allen"""),
									        ((21, 23), """##NL"""),
									        ((23, 24), """##P"""),
									        ((25, 33), """sentence"""),
									        ((33, 34), """."""),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									    if not do_lower_case
									    else [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), """a"""),
									        ((1, 2), ""","""),
									        ((3, 8), """naive"""),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), """allen"""),
									        ((21, 23), """##nl"""),
									        ((23, 24), """##p"""),
									        ((25, 33), """sentence"""),
									        ((33, 34), """."""),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									)

									self.assertEqual(
									    [e[1] for e in expected_results]							,	tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]   )   )
									self.assertEqual([e[0] for e in expected_results]							,	tokens["""offset_mapping"""]   )






			def    a_       (			self   :    Dict   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   ["""的""", """人""", """有"""]
					A__        =   """""".join(__lowerCAmelCase   )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
									A__        =   True
									A__        =   self.tokenizer_class.from_pretrained(__lowerCAmelCase							,	**__lowerCAmelCase   )
									A__        =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,	**__lowerCAmelCase   )

									A__        =   tokenizer_p.encode(__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )
									A__        =   tokenizer_r.encode(__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )

									A__        =   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase   )
									A__        =   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase   )

									# it is expected that each Chinese character is not preceded by "##"
									self.assertListEqual(__lowerCAmelCase							,	__lowerCAmelCase   )
									self.assertListEqual(__lowerCAmelCase							,	__lowerCAmelCase   )

									A__        =   False
									A__        =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,	**__lowerCAmelCase   )
									A__        =   self.tokenizer_class.from_pretrained(__lowerCAmelCase							,	**__lowerCAmelCase   )

									A__        =   tokenizer_r.encode(__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )
									A__        =   tokenizer_p.encode(__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )

									A__        =   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase   )
									A__        =   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase   )

									# it is expected that only the first Chinese character is not preceded by "##".
									A__        =   [
									    f'##{token}' if idx != 0 else token for idx, token in enumerate(__lowerCAmelCase   )
									]
									self.assertListEqual(__lowerCAmelCase							,	__lowerCAmelCase   )
									self.assertListEqual(__lowerCAmelCase							,	__lowerCAmelCase   )






			@slow
			def    a_       (			self   :    Union[str, Any]   )	->	str:







					"""simple docstring"""






					A__        =   self.tokenizer_class(self.vocab_file							,	self.word_shape_file							,	self.word_pronunciation_file   )

					A__        =   tokenizer.encode("""你好"""							,	add_special_tokens=__lowerCAmelCase   )
					A__        =   tokenizer.encode("""你是谁"""							,	add_special_tokens=__lowerCAmelCase   )

					A__        =   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase   )
					A__        =   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase							,	__lowerCAmelCase   )

					assert encoded_sentence == [1] + text + [2]
					assert encoded_pair == [1] + text + [2] + text_a + [2]







			def    a_       (			self   :    Dict   )	->	int:







					"""simple docstring"""






					A__        =   self.get_tokenizers(do_lower_case=__lowerCAmelCase   )
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'   ):
									A__        =   """你好，你是谁"""
									A__        =   tokenizer.tokenize(__lowerCAmelCase   )
									A__        =   tokenizer.convert_tokens_to_ids(__lowerCAmelCase   )
									A__        =   tokenizer.convert_tokens_to_shape_ids(__lowerCAmelCase   )
									A__        =   tokenizer.convert_tokens_to_pronunciation_ids(__lowerCAmelCase   )
									A__        =   tokenizer.prepare_for_model(
									    __lowerCAmelCase							,	__lowerCAmelCase							,	__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )

									A__        =   tokenizer.encode_plus(__lowerCAmelCase							,	add_special_tokens=__lowerCAmelCase   )

									self.assertEqual(__lowerCAmelCase							,	__lowerCAmelCase   )










import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
					import jax.numpy as jnp

					from transformers.models.albert.modeling_flax_albert import (
					    FlaxAlbertForMaskedLM,
					    FlaxAlbertForMultipleChoice,
					    FlaxAlbertForPreTraining,
					    FlaxAlbertForQuestionAnswering,
					    FlaxAlbertForSequenceClassification,
					    FlaxAlbertForTokenClassification,
					    FlaxAlbertModel,
					)


class   A			(unittest.TestCase    ):
			'''simple docstring'''




			def __init__(			self   :    List[str]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    List[str]=13							,	__lowerCAmelCase   :    int=7							,	__lowerCAmelCase   :    Tuple=True							,	__lowerCAmelCase   :    int=True							,	__lowerCAmelCase   :    Dict=True							,	__lowerCAmelCase   :    Union[str, Any]=True							,	__lowerCAmelCase   :    List[Any]=99							,	__lowerCAmelCase   :    Optional[Any]=32							,	__lowerCAmelCase   :    Optional[Any]=5							,	__lowerCAmelCase   :    Tuple=4							,	__lowerCAmelCase   :    Any=37							,	__lowerCAmelCase   :    str="gelu"							,	__lowerCAmelCase   :    Optional[Any]=0.1							,	__lowerCAmelCase   :    Union[str, Any]=0.1							,	__lowerCAmelCase   :    List[str]=5_12							,	__lowerCAmelCase   :    int=16							,	__lowerCAmelCase   :    Optional[int]=2							,	__lowerCAmelCase   :    List[Any]=0.0_2							,	__lowerCAmelCase   :    Tuple=4							,	)	->	Dict:







					"""simple docstring"""






					A__        =   parent
					A__        =   batch_size
					A__        =   seq_length
					A__        =   is_training
					A__        =   use_attention_mask
					A__        =   use_token_type_ids
					A__        =   use_labels
					A__        =   vocab_size
					A__        =   hidden_size
					A__        =   num_hidden_layers
					A__        =   num_attention_heads
					A__        =   intermediate_size
					A__        =   hidden_act
					A__        =   hidden_dropout_prob
					A__        =   attention_probs_dropout_prob
					A__        =   max_position_embeddings
					A__        =   type_vocab_size
					A__        =   type_sequence_label_size
					A__        =   initializer_range
					A__        =   num_choices






			def    a_       (			self   :    Any   )	->	str:







					"""simple docstring"""






					A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.vocab_size   )

					A__        =   None
					if self.use_attention_mask:
							A__        =   random_attention_mask([self.batch_size, self.seq_length]   )

					A__        =   None
					if self.use_token_type_ids:
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.type_vocab_size   )

					A__        =   AlbertConfig(
					    vocab_size=self.vocab_size							,	hidden_size=self.hidden_size							,	num_hidden_layers=self.num_hidden_layers							,	num_attention_heads=self.num_attention_heads							,	intermediate_size=self.intermediate_size							,	hidden_act=self.hidden_act							,	hidden_dropout_prob=self.hidden_dropout_prob							,	attention_probs_dropout_prob=self.attention_probs_dropout_prob							,	max_position_embeddings=self.max_position_embeddings							,	type_vocab_size=self.type_vocab_size							,	is_decoder=__lowerCAmelCase							,	initializer_range=self.initializer_range							,	)

					return config, input_ids, token_type_ids, attention_mask







			def    a_       (			self   :    Optional[int]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   self.prepare_config_and_inputs()
					A__      ,			A__      ,			A__      ,			A__        =   config_and_inputs
					A__        =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
					return config, inputs_dict


@require_flax
class   A			(SCREAMING_SNAKE_CASE   ,     unittest.TestCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       str							=    (
			    (
			        FlaxAlbertModel,
			        FlaxAlbertForPreTraining,
			        FlaxAlbertForMaskedLM,
			        FlaxAlbertForMultipleChoice,
			        FlaxAlbertForQuestionAnswering,
			        FlaxAlbertForSequenceClassification,
			        FlaxAlbertForTokenClassification,
			        FlaxAlbertForQuestionAnswering,
			    )
			    if is_flax_available()
			    else ()
			)






			def    a_       (			self   :    str   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   FlaxAlbertModelTester(self   )







			@slow
			def    a_       (			self   :    int   )	->	Tuple:







					"""simple docstring"""






					for model_class_name in self.all_model_classes:
							A__        =   model_class_name.from_pretrained("""albert-base-v2"""   )
							A__        =   model(np.ones((1, 1)   )   )
							self.assertIsNotNone(__lowerCAmelCase   )


@require_flax
class   A			(unittest.TestCase    ):
			'''simple docstring'''




			@slow
			def    a_       (			self   :    Dict   )	->	List[Any]:







					"""simple docstring"""






					A__        =   FlaxAlbertModel.from_pretrained("""albert-base-v2"""   )
					A__        =   np.array([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]   )
					A__        =   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase   )[0]
					A__        =   (1, 11, 7_68)
					self.assertEqual(output.shape							,	__lowerCAmelCase   )
					A__        =   np.array(
					    [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]   )

					self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]							,	__lowerCAmelCase							,	atol=1e-4   )   )



