






'''simple docstring'''
import warnings

from ...utils import is_sklearn_available, requires_backends


if is_sklearn_available():
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import fa_score, matthews_corrcoef


UpperCamelCase__			=      (
    'This metric will be removed from the library soon, metrics should be handled with the 🤗 Evaluate '
    'library. You can have a look at this example script for pointers: '
    'https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py'
)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   warnings.warn(_UpperCamelCase    ,		_UpperCamelCase				)
   requires_backends(_UpperCamelCase    ,		"sklearn"				)
   return (preds == labels).mean()




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   warnings.warn(_UpperCamelCase    ,		_UpperCamelCase				)
   requires_backends(_UpperCamelCase    ,		"sklearn"				)
   lowercase_						:     Optional[int]							     =  simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)
   lowercase_						:     Dict							     =  fa_score(y_true=_UpperCamelCase    ,		y_pred=_UpperCamelCase				)
   return {
       "acc": acc,
       "f1": fa,
       "acc_and_f1": (acc + fa) / 2,
   }




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   warnings.warn(_UpperCamelCase    ,		_UpperCamelCase				)
   requires_backends(_UpperCamelCase    ,		"sklearn"				)
   lowercase_						:     Union[str, Any]							     =  pearsonr(_UpperCamelCase    ,		_UpperCamelCase				)[0]
   lowercase_						:     int							     =  spearmanr(_UpperCamelCase    ,		_UpperCamelCase				)[0]
   return {
       "pearson": pearson_corr,
       "spearmanr": spearman_corr,
       "corr": (pearson_corr + spearman_corr) / 2,
   }




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   warnings.warn(_UpperCamelCase    ,		_UpperCamelCase				)
   requires_backends(_UpperCamelCase    ,		"sklearn"				)
   assert len(_UpperCamelCase				) == len(_UpperCamelCase				), F"""Predictions and labels have mismatched lengths {len(_UpperCamelCase				)} and {len(_UpperCamelCase				)}"""
   if task_name == "cola":
      return {"mcc": matthews_corrcoef(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "sst-2":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "mrpc":
      return acc_and_fa(_UpperCamelCase    ,		_UpperCamelCase				)
   elif task_name == "sts-b":
      return pearson_and_spearman(_UpperCamelCase    ,		_UpperCamelCase				)
   elif task_name == "qqp":
      return acc_and_fa(_UpperCamelCase    ,		_UpperCamelCase				)
   elif task_name == "mnli":
      return {"mnli/acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "mnli-mm":
      return {"mnli-mm/acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "qnli":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "rte":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "wnli":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   elif task_name == "hans":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   else:
      raise KeyError(_UpperCamelCase				)





def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   warnings.warn(_UpperCamelCase    ,		_UpperCamelCase				)
   requires_backends(_UpperCamelCase    ,		"sklearn"				)
   if len(_UpperCamelCase				) != len(_UpperCamelCase				):
      raise ValueError(F"""Predictions and labels have mismatched lengths {len(_UpperCamelCase				)} and {len(_UpperCamelCase				)}"""				)
   if task_name == "xnli":
      return {"acc": simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)}
   else:
      raise KeyError(_UpperCamelCase				)












'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							_UpperCAmelCase    (		snake_case					,       unittest.TestCase      ):
    __lowerCamelCase: Optional[int]									=     BertTokenizer
    __lowerCamelCase: Optional[int]									=     BertTokenizerFast
    __lowerCamelCase: Union[str, Any]									=     True
    __lowerCamelCase: int									=     True
    __lowerCamelCase: Optional[Any]									=     filter_non_english




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       super().setUp()

       lowercase_						:     Optional[Any]							     =  [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase_						:     Optional[Any]							     =  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  "UNwant\u00E9d,running"
       lowercase_						:     List[str]							     =  "unwanted, running"
       return input_text, output_text




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  self.tokenizer_class(self.vocab_file      )

       lowercase_						:     Union[str, Any]							     =  tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(a	,  ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a      )	,  [9, 6, 7, 1_2, 1_0, 1_1]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       if not self.test_rust_tokenizer:
          return

       lowercase_						:     Optional[Any]							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       lowercase_						:     int							     =  "UNwant\u00E9d,running"

       lowercase_						:     Dict							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     List[Any]							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     List[Any]							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Optional[int]							     =  self.get_rust_tokenizer()
       lowercase_						:     Optional[int]							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )

       # With lower casing
       lowercase_						:     Tuple							     =  self.get_tokenizer(do_lower_case=a      )
       lowercase_						:     Dict							     =  self.get_rust_tokenizer(do_lower_case=a      )

       lowercase_						:     List[Any]							     =  "UNwant\u00E9d,running"

       lowercase_						:     str							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Any							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     str							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Union[str, Any]							     =  self.get_rust_tokenizer()
       lowercase_						:     Tuple							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )	,  ["ah", "\u535A", "\u63A8", "zz"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["h\u00E9llo"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     str							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     int							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HäLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HaLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer(do_lower_case=a	,  never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer()
       lowercase_						:     Optional[int]							     =  "a\n'll !!to?'d of, can't."
       lowercase_						:     List[Any]							     =  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(a      )	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase_						:     int							     =  {}
       for i, token in enumerate(a      ):
          lowercase_						:     Any							     =  i
       lowercase_						:     Union[str, Any]							     =  WordpieceTokenizer(vocab=a	,  unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )	,  []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      )	,  ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      )	,  ["[UNK]", "runn", "##ing"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )

       self.assertListEqual(
           [rust_tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )




    @slow
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     str							     =  self.tokenizer_class.from_pretrained("bert-base-uncased"      )

       lowercase_						:     Tuple							     =  tokenizer.encode("sequence builders"	,  add_special_tokens=a      )
       lowercase_						:     Union[str, Any]							     =  tokenizer.encode("multi-sequence build"	,  add_special_tokens=a      )

       lowercase_						:     Dict							     =  tokenizer.build_inputs_with_special_tokens(a      )
       lowercase_						:     Optional[Any]							     =  tokenizer.build_inputs_with_special_tokens(a	,  a      )

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Dict							     =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             lowercase_						:     int							     =  tokenizer_r.encode_plus(
                 a	,  return_attention_mask=a	,  return_token_type_ids=a	,  return_offsets_mapping=a	,  add_special_tokens=a	,  )

             lowercase_						:     Tuple							     =  tokenizer_r.do_lower_case if hasattr(a	,  "do_lower_case"      ) else False
             lowercase_						:     List[Any]							     =  (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]	,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
             self.assertEqual([e[0] for e in expected_results]	,  tokens["offset_mapping"]      )





    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  ["的", "人", "有"]
       lowercase_						:     int							     =  "".join(a      )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  True
             lowercase_						:     Dict							     =  self.tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )

             lowercase_						:     str							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )

             lowercase_						:     Tuple							     =  False
             lowercase_						:     Union[str, Any]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )

             lowercase_						:     int							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     List[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase_						:     int							     =  [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a      )
             ]
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )




