






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class    lowercase__   (	snake_case_,    unittest.TestCase		):
	a_   			=BertTokenizer
	a_   			=BertTokenizerFast
	a_   			=True
	a_   			=True
	a_   			=filter_non_english

	def 						UpperCAmelCase     (							self				)->    List[Any]:







							'''simple docstring'''





							super().setUp()

							lowerCAmelCase__       					=  [
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """[PAD]""",
							    """[MASK]""",
							    """want""",
							    """##want""",
							    """##ed""",
							    """wa""",
							    """un""",
							    """runn""",
							    """##ing""",
							    """,""",
							    """low""",
							    """lowest""",
							]
							lowerCAmelCase__       					=  os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES["vocab_file"]				)
							with open(self.vocab_file						,			"w"						,			encoding="utf-8"				) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)

	def 						UpperCAmelCase     (							self						,			__UpperCAmelCase				)->    Any:







							'''simple docstring'''





							lowerCAmelCase__       					=  """UNwant\u00E9d,running"""
							lowerCAmelCase__       					=  """unwanted, running"""
							return input_text, output_text

	def 						UpperCAmelCase     (							self				)->    Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__       					=  self.tokenizer_class(self.vocab_file				)

							lowerCAmelCase__       					=  tokenizer.tokenize("UNwant\u00E9d,running"				)
							self.assertListEqual(UpperCAmelCase__						,			["un", "##want", "##ed", ",", "runn", "##ing"]				)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__				)						,			[9, 6, 7, 12, 10, 11]				)

	def 						UpperCAmelCase     (							self				)->    List[Any]:







							'''simple docstring'''





							if not self.test_rust_tokenizer:
													return

							lowerCAmelCase__       					=  self.get_tokenizer()
							lowerCAmelCase__       					=  self.get_rust_tokenizer()

							lowerCAmelCase__       					=  """UNwant\u00E9d,running"""

							lowerCAmelCase__       					=  tokenizer.tokenize(UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.tokenize(UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

							lowerCAmelCase__       					=  tokenizer.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

							lowerCAmelCase__       					=  self.get_rust_tokenizer()
							lowerCAmelCase__       					=  tokenizer.encode(UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.encode(UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

							# With lower casing
							lowerCAmelCase__       					=  self.get_tokenizer(do_lower_case=UpperCAmelCase__				)
							lowerCAmelCase__       					=  self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__				)

							lowerCAmelCase__       					=  """UNwant\u00E9d,running"""

							lowerCAmelCase__       					=  tokenizer.tokenize(UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.tokenize(UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

							lowerCAmelCase__       					=  tokenizer.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

							lowerCAmelCase__       					=  self.get_rust_tokenizer()
							lowerCAmelCase__       					=  tokenizer.encode(UpperCAmelCase__				)
							lowerCAmelCase__       					=  rust_tokenizer.encode(UpperCAmelCase__				)
							self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

	def 						UpperCAmelCase     (							self				)->    List[str]:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)						,			["ah", "\u535A", "\u63A8", "zz"]				)

	def 						UpperCAmelCase     (							self				)->    Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)						,			["hello", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)						,			["hello"]				)

	def 						UpperCAmelCase     (							self				)->    Tuple:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__						,			strip_accents=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)						,			["hällo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)						,			["h\u00E9llo"]				)

	def 						UpperCAmelCase     (							self				)->    Dict:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__						,			strip_accents=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)						,			["hallo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)						,			["hello"]				)

	def 						UpperCAmelCase     (							self				)->    Dict:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)						,			["hallo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)						,			["hello"]				)

	def 						UpperCAmelCase     (							self				)->    List[str]:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)						,			["HeLLo", "!", "how", "Are", "yoU", "?"]				)

	def 						UpperCAmelCase     (							self				)->    int:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__						,			strip_accents=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)						,			["HäLLo", "!", "how", "Are", "yoU", "?"]				)

	def 						UpperCAmelCase     (							self				)->    Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__						,			strip_accents=UpperCAmelCase__				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)						,			["HaLLo", "!", "how", "Are", "yoU", "?"]				)

	def 						UpperCAmelCase     (							self				)->    Tuple:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer(do_lower_case=UpperCAmelCase__						,			never_split=["[UNK]"]				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)						,			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)

	def 						UpperCAmelCase     (							self				)->    int:







							'''simple docstring'''





							lowerCAmelCase__       					=  BasicTokenizer()
							lowerCAmelCase__       					=  """a\n'll !!to?'d of, can't."""
							lowerCAmelCase__       					=  ["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
							self.assertListEqual(tokenizer.tokenize(UpperCAmelCase__				)						,			UpperCAmelCase__				)

	def 						UpperCAmelCase     (							self				)->    Dict:







							'''simple docstring'''





							lowerCAmelCase__       					=  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

							lowerCAmelCase__       					=  {}
							for i, token in enumerate(UpperCAmelCase__				):
													lowerCAmelCase__       					=  i
							lowerCAmelCase__       					=  WordpieceTokenizer(vocab=UpperCAmelCase__						,			unk_token="[UNK]"				)

							self.assertListEqual(tokenizer.tokenize(""				)						,			[]				)

							self.assertListEqual(tokenizer.tokenize("unwanted running"				)						,			["un", "##want", "##ed", "runn", "##ing"]				)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"				)						,			["[UNK]", "runn", "##ing"]				)

	def 						UpperCAmelCase     (							self				)->    List[str]:







							'''simple docstring'''





							self.assertTrue(_is_whitespace(" "				)				)
							self.assertTrue(_is_whitespace("\t"				)				)
							self.assertTrue(_is_whitespace("\r"				)				)
							self.assertTrue(_is_whitespace("\n"				)				)
							self.assertTrue(_is_whitespace("\u00A0"				)				)

							self.assertFalse(_is_whitespace("A"				)				)
							self.assertFalse(_is_whitespace("-"				)				)

	def 						UpperCAmelCase     (							self				)->    str:







							'''simple docstring'''





							self.assertTrue(_is_control("\u0005"				)				)

							self.assertFalse(_is_control("A"				)				)
							self.assertFalse(_is_control(" "				)				)
							self.assertFalse(_is_control("\t"				)				)
							self.assertFalse(_is_control("\r"				)				)

	def 						UpperCAmelCase     (							self				)->    str:







							'''simple docstring'''





							self.assertTrue(_is_punctuation("-"				)				)
							self.assertTrue(_is_punctuation("$"				)				)
							self.assertTrue(_is_punctuation("`"				)				)
							self.assertTrue(_is_punctuation("."				)				)

							self.assertFalse(_is_punctuation("A"				)				)
							self.assertFalse(_is_punctuation(" "				)				)

	def 						UpperCAmelCase     (							self				)->    Tuple:







							'''simple docstring'''





							lowerCAmelCase__       					=  self.get_tokenizer()
							lowerCAmelCase__       					=  self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__				) for t in ["Test", "\xad", "test"]]						,			[["[UNK]"], [], ["[UNK]"]]				)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(UpperCAmelCase__				) for t in ["Test", "\xad", "test"]]						,			[["[UNK]"], [], ["[UNK]"]]				)

	@slow
	def 						UpperCAmelCase     (							self				)->    Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       					=  self.tokenizer_class.from_pretrained("bert-base-uncased"				)

							lowerCAmelCase__       					=  tokenizer.encode("sequence builders"						,			add_special_tokens=UpperCAmelCase__				)
							lowerCAmelCase__       					=  tokenizer.encode("multi-sequence build"						,			add_special_tokens=UpperCAmelCase__				)

							lowerCAmelCase__       					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__				)
							lowerCAmelCase__       					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__						,			UpperCAmelCase__				)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]

	def 						UpperCAmelCase     (							self				)->    int:







							'''simple docstring'''





							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
																			lowerCAmelCase__       					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__						,			**UpperCAmelCase__				)

																			lowerCAmelCase__       					=  F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																			lowerCAmelCase__       					=  tokenizer_r.encode_plus(
																			    UpperCAmelCase__						,			return_attention_mask=UpperCAmelCase__						,			return_token_type_ids=UpperCAmelCase__						,			return_offsets_mapping=UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__						,			)

																			lowerCAmelCase__       					=  tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__						,			"do_lower_case"				) else False
																			lowerCAmelCase__       					=  (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), """A"""),
																			        ((1, 2), ""","""),
																			        ((3, 5), """na"""),
																			        ((5, 6), """##ï"""),
																			        ((6, 8), """##ve"""),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), """Allen"""),
																			        ((21, 23), """##NL"""),
																			        ((23, 24), """##P"""),
																			        ((25, 33), """sentence"""),
																			        ((33, 34), """."""),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), """a"""),
																			        ((1, 2), ""","""),
																			        ((3, 8), """naive"""),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), """allen"""),
																			        ((21, 23), """##nl"""),
																			        ((23, 24), """##p"""),
																			        ((25, 33), """sentence"""),
																			        ((33, 34), """."""),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]						,			tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
																			self.assertEqual([e[0] for e in expected_results]						,			tokens["offset_mapping"]				)


	def 						UpperCAmelCase     (							self				)->    Any:







							'''simple docstring'''





							lowerCAmelCase__       					=  ["""的""", """人""", """有"""]
							lowerCAmelCase__       					=  """""".join(UpperCAmelCase__				)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
																			lowerCAmelCase__       					=  True
																			lowerCAmelCase__       					=  self.tokenizer_class.from_pretrained(UpperCAmelCase__						,			**UpperCAmelCase__				)
																			lowerCAmelCase__       					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__						,			**UpperCAmelCase__				)

																			lowerCAmelCase__       					=  tokenizer_p.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
																			lowerCAmelCase__       					=  tokenizer_r.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)

																			lowerCAmelCase__       					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__				)
																			lowerCAmelCase__       					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__				)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)
																			self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)

																			lowerCAmelCase__       					=  False
																			lowerCAmelCase__       					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__						,			**UpperCAmelCase__				)
																			lowerCAmelCase__       					=  self.tokenizer_class.from_pretrained(UpperCAmelCase__						,			**UpperCAmelCase__				)

																			lowerCAmelCase__       					=  tokenizer_r.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)
																			lowerCAmelCase__       					=  tokenizer_p.encode(UpperCAmelCase__						,			add_special_tokens=UpperCAmelCase__				)

																			lowerCAmelCase__       					=  tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__				)
																			lowerCAmelCase__       					=  tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__				)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			lowerCAmelCase__       					=  [
																			    F"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__				)
																			]
																			self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)
																			self.assertListEqual(UpperCAmelCase__						,			UpperCAmelCase__				)














"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_snake_case          =   {
    'configuration_perceiver': ['PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'PerceiverConfig', 'PerceiverOnnxConfig'],
    'tokenization_perceiver': ['PerceiverTokenizer'],
}

try:
 if not is_vision_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _snake_case          =   ['PerceiverFeatureExtractor']
 _snake_case          =   ['PerceiverImageProcessor']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _snake_case          =   [
     'PERCEIVER_PRETRAINED_MODEL_ARCHIVE_LIST',
     'PerceiverForImageClassificationConvProcessing',
     'PerceiverForImageClassificationFourier',
     'PerceiverForImageClassificationLearned',
     'PerceiverForMaskedLM',
     'PerceiverForMultimodalAutoencoding',
     'PerceiverForOpticalFlow',
     'PerceiverForSequenceClassification',
     'PerceiverLayer',
     'PerceiverModel',
     'PerceiverPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_perceiver import PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP, PerceiverConfig, PerceiverOnnxConfig
 from .tokenization_perceiver import PerceiverTokenizer

 try:
  if not is_vision_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .feature_extraction_perceiver import PerceiverFeatureExtractor
  from .image_processing_perceiver import PerceiverImageProcessor

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_perceiver import (
      PERCEIVER_PRETRAINED_MODEL_ARCHIVE_LIST,
      PerceiverForImageClassificationConvProcessing,
      PerceiverForImageClassificationFourier,
      PerceiverForImageClassificationLearned,
      PerceiverForMaskedLM,
      PerceiverForMultimodalAutoencoding,
      PerceiverForOpticalFlow,
      PerceiverForSequenceClassification,
      PerceiverLayer,
      PerceiverModel,
      PerceiverPreTrainedModel,
  )

else:
 import sys

 _snake_case          =   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

