





"""simple docstring"""






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin



class 					__UpperCAmelCase							(						_UpperCamelCase						,  unittest.TestCase  ):
				__lowerCamelCase				:	Optional[int]				     =				ProphetNetTokenizer
				__lowerCamelCase				:	List[Any]				     =				False
				def       UpperCAmelCase      (		self    :			Dict    )	->	Tuple:



							'''simple docstring'''



							super().setUp()

							a__  :	List[Any]		 =			[
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							a__  :	List[Any]		 =			os.path.join(self.tmpdirname	, VOCAB_FILES_NAMES["vocab_file"]    )
							with open(self.vocab_file	, "w"	, encoding="utf-8"    ) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )
				def       UpperCAmelCase      (		self    :			Optional[Any]	, a_    :			Optional[Any]    )	->	Dict:



							'''simple docstring'''



							a__  :	Dict		 =			"UNwant\u00E9d,running"
							a__  :	Dict		 =			"unwanted, running"
							return input_text, output_text
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Dict:



							'''simple docstring'''



							a__  :	str		 =			self.tokenizer_class(self.vocab_file    )

							a__  :	Optional[int]		 =			tokenizer.tokenize("UNwant\u00E9d,running"    )
							self.assertListEqual(a_	, ["un", "##want", "##ed", ",", "runn", "##ing"]    )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(a_    )	, [9, 6, 7, 12, 10, 11]    )
				def       UpperCAmelCase      (		self    :			Any    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	Tuple		 =			BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"    )	, ["ah", "\u535A", "\u63A8", "zz"]    )
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Dict		 =			BasicTokenizer(do_lower_case=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )	, ["hello", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )	, ["hello"]    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	str:



							'''simple docstring'''



							a__  :	Any		 =			BasicTokenizer(do_lower_case=a_	, strip_accents=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )	, ["hällo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )	, ["h\u00E9llo"]    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Union[str, Any]		 =			BasicTokenizer(do_lower_case=a_	, strip_accents=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )	, ["hallo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )	, ["hello"]    )
				def       UpperCAmelCase      (		self    :			str    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			BasicTokenizer(do_lower_case=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )	, ["hallo", "!", "how", "are", "you", "?"]    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )	, ["hello"]    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Any:



							'''simple docstring'''



							a__  :	Any		 =			BasicTokenizer(do_lower_case=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "    )	, ["HeLLo", "!", "how", "Are", "yoU", "?"]    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Optional[int]		 =			BasicTokenizer(do_lower_case=a_	, strip_accents=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )	, ["HäLLo", "!", "how", "Are", "yoU", "?"]    )
				def       UpperCAmelCase      (		self    :			int    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			BasicTokenizer(do_lower_case=a_	, strip_accents=a_    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "    )	, ["HaLLo", "!", "how", "Are", "yoU", "?"]    )
				def       UpperCAmelCase      (		self    :			Any    )	->	List[str]:



							'''simple docstring'''



							a__  :	List[Any]		 =			BasicTokenizer(do_lower_case=a_	, never_split=["[UNK]"]    )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"    )	, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]    )
				def       UpperCAmelCase      (		self    :			Any    )	->	Tuple:



							'''simple docstring'''



							a__  :	Union[str, Any]		 =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							a__  :	int		 =			{}
							for i, token in enumerate(a_    ):
										a__  :	Dict		 =			i
							a__  :	List[Any]		 =			WordpieceTokenizer(vocab=a_	, unk_token="[UNK]"    )

							self.assertListEqual(tokenizer.tokenize(""    )	, []    )

							self.assertListEqual(tokenizer.tokenize("unwanted running"    )	, ["un", "##want", "##ed", "runn", "##ing"]    )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"    )	, ["[UNK]", "runn", "##ing"]    )
				@require_torch
				def       UpperCAmelCase      (		self    :			Tuple    )	->	Any:



							'''simple docstring'''



							a__  :	Any		 =			self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"    )

							a__  :	str		 =			["A long paragraph for summarization.", "Another paragraph for summarization."]
							a__  :	int		 =			[10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
							a__  :	Union[str, Any]		 =			tokenizer(a_	, padding=a_	, return_tensors="pt"    )
							self.assertIsInstance(a_	, a_    )
							a__  :	Any		 =			list(batch.input_ids.numpy()[0]    )
							self.assertListEqual(a_	, a_    )

							self.assertEqual((2, 9)	, batch.input_ids.shape    )
							self.assertEqual((2, 9)	, batch.attention_mask.shape    )
				def       UpperCAmelCase      (		self    :			str    )	->	int:



							'''simple docstring'''



							self.assertTrue(_is_whitespace(" "    )    )
							self.assertTrue(_is_whitespace("\t"    )    )
							self.assertTrue(_is_whitespace("\r"    )    )
							self.assertTrue(_is_whitespace("\n"    )    )
							self.assertTrue(_is_whitespace("\u00A0"    )    )

							self.assertFalse(_is_whitespace("A"    )    )
							self.assertFalse(_is_whitespace("-"    )    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	int:



							'''simple docstring'''



							self.assertTrue(_is_control("\u0005"    )    )

							self.assertFalse(_is_control("A"    )    )
							self.assertFalse(_is_control(" "    )    )
							self.assertFalse(_is_control("\t"    )    )
							self.assertFalse(_is_control("\r"    )    )
				def       UpperCAmelCase      (		self    :			int    )	->	Optional[int]:



							'''simple docstring'''



							self.assertTrue(_is_punctuation("-"    )    )
							self.assertTrue(_is_punctuation("$"    )    )
							self.assertTrue(_is_punctuation("`"    )    )
							self.assertTrue(_is_punctuation("."    )    )

							self.assertFalse(_is_punctuation("A"    )    )
							self.assertFalse(_is_punctuation(" "    )    )






				@slow
				def       UpperCAmelCase      (		self    :			List[str]    )	->	Tuple:



							'''simple docstring'''



							a__  :	List[Any]		 =			self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"    )

							a__  :	Dict		 =			tokenizer.encode("sequence builders"	, add_special_tokens=a_    )
							a__  :	List[str]		 =			tokenizer.encode("multi-sequence build"	, add_special_tokens=a_    )

							a__  :	Optional[Any]		 =			tokenizer.build_inputs_with_special_tokens(a_    )
							a__  :	Optional[Any]		 =			tokenizer.build_inputs_with_special_tokens(a_	, a_    )

							assert encoded_sentence == text + [1_02]
							assert encoded_pair == text + [1_02] + text_a + [1_02]






"""simple docstring"""






import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin



class 					__UpperCAmelCase							(						unittest.TestCase						,  _UpperCamelCase  ):
				def       UpperCAmelCase      (		self    :			Dict    )	->	List[Any]:



							'''simple docstring'''



							a__  :	Any		 =			load_tool("text-classification"    )
							self.tool.setup()
							a__  :	List[Any]		 =			load_tool("text-classification"	, remote=a_    )
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	str		 =			self.tool("That's quite cool"	, ["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	List[str]		 =			self.remote_tool("That's quite cool"	, ["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	Any:



							'''simple docstring'''



							a__  :	List[str]		 =			self.tool(text="That's quite cool"	, labels=["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )






				def       UpperCAmelCase      (		self    :			List[Any]    )	->	List[Any]:



							'''simple docstring'''



							a__  :	Tuple		 =			self.remote_tool(text="That's quite cool"	, labels=["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )