






"""simple docstring"""



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase		:  Dict						=		get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class        a     (   lowerCAmelCase__,				unittest.TestCase	):
    UpperCAmelCase_ :     str	     =XLMRobertaTokenizer
    UpperCAmelCase_ :     int	     =XLMRobertaTokenizerFast
    UpperCAmelCase_ :     str	     =True
    UpperCAmelCase_ :     Dict	     =True







    def 			UpperCamelCase_      (			self       ):
      super().setUp()

      # We have a SentencePiece fixture for testing
      lowercase							    =       XLMRobertaTokenizer(lowercase_							,		keep_accents=lowercase_       )
      tokenizer.save_pretrained(self.tmpdirname       )







    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       '''<pad>'''
      lowercase							    =       1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_       )							,		lowercase_       )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_       )							,		lowercase_       )







    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       list(self.get_tokenizer().get_vocab().keys()       )

      self.assertEqual(vocab_keys[0]							,		'<s>'       )
      self.assertEqual(vocab_keys[1]							,		'<pad>'       )
      self.assertEqual(vocab_keys[-1]							,		'<mask>'       )
      self.assertEqual(len(lowercase_       )							,		1_0_0_2       )







    def 			UpperCamelCase_      (			self       ):
      self.assertEqual(self.get_tokenizer().vocab_size							,		1_0_0_2       )







    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       XLMRobertaTokenizer(lowercase_							,		keep_accents=lowercase_       )

      lowercase							    =       tokenizer.tokenize('This is a test'       )
      self.assertListEqual(lowercase_							,		['▁This', '▁is', '▁a', '▁t', 'est']       )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowercase_       )							,		[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,		)

      lowercase							    =       tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
      self.assertListEqual(
          lowercase_							,		[
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '9',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              'é',
              '.',
          ]							,		)
      lowercase							    =       tokenizer.convert_tokens_to_ids(lowercase_       )
      self.assertListEqual(
          lowercase_							,		[
              value + tokenizer.fairseq_offset
              for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
              #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
          ]							,		)

      lowercase							    =       tokenizer.convert_ids_to_tokens(lowercase_       )
      self.assertListEqual(
          lowercase_							,		[
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '<unk>',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              '<unk>',
              '.',
          ]							,		)







    def 			UpperCamelCase_      (			self       ):
      if not self.test_slow_tokenizer:
        # as we don't have a slow version, we can't compare the outputs between slow and fast versions
        return

      lowercase							    =       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-xlm-roberta''', {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
          lowercase							    =       self.rust_tokenizer_class.from_pretrained(lowercase_							,		**lowercase_       )
          lowercase							    =       self.tokenizer_class.from_pretrained(lowercase_							,		**lowercase_       )

          lowercase							    =       tempfile.mkdtemp()

          lowercase							    =       tokenizer_r.save_pretrained(lowercase_       )
          lowercase							    =       tokenizer_p.save_pretrained(lowercase_       )

          # Checks it save with the same files + the tokenizer.json file for the fast one
          self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )
          lowercase							    =       tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f       )
          self.assertSequenceEqual(lowercase_							,		lowercase_       )

          # Checks everything loads correctly in the same way
          lowercase							    =       tokenizer_r.from_pretrained(lowercase_       )
          lowercase							    =       tokenizer_p.from_pretrained(lowercase_       )

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
            self.assertTrue(hasattr(lowercase_							,		lowercase_       )       )
            # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
            # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

          shutil.rmtree(lowercase_       )

          # Save tokenizer rust, legacy_format=True
          lowercase							    =       tempfile.mkdtemp()

          lowercase							    =       tokenizer_r.save_pretrained(lowercase_							,		legacy_format=lowercase_       )
          lowercase							    =       tokenizer_p.save_pretrained(lowercase_       )

          # Checks it save with the same files
          self.assertSequenceEqual(lowercase_							,		lowercase_       )

          # Checks everything loads correctly in the same way
          lowercase							    =       tokenizer_r.from_pretrained(lowercase_       )
          lowercase							    =       tokenizer_p.from_pretrained(lowercase_       )

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
            self.assertTrue(hasattr(lowercase_							,		lowercase_       )       )

          shutil.rmtree(lowercase_       )

          # Save tokenizer rust, legacy_format=False
          lowercase							    =       tempfile.mkdtemp()

          lowercase							    =       tokenizer_r.save_pretrained(lowercase_							,		legacy_format=lowercase_       )
          lowercase							    =       tokenizer_p.save_pretrained(lowercase_       )

          # Checks it saved the tokenizer.json file
          self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )

          # Checks everything loads correctly in the same way
          lowercase							    =       tokenizer_r.from_pretrained(lowercase_       )
          lowercase							    =       tokenizer_p.from_pretrained(lowercase_       )

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
            self.assertTrue(hasattr(lowercase_							,		lowercase_       )       )

          shutil.rmtree(lowercase_       )







    @cached_property
    def 			UpperCamelCase_      (			self       ):
      return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'       )







    def 			UpperCamelCase_      (			self       ):
      with tempfile.NamedTemporaryFile() as f:
        shutil.copyfile(lowercase_							,		f.name       )
        lowercase							    =       XLMRobertaTokenizer(f.name							,		keep_accents=lowercase_       )
        lowercase							    =       pickle.dumps(lowercase_       )
      pickle.loads(lowercase_       )







    def 			UpperCamelCase_      (			self       ):
      if not self.test_rust_tokenizer:
        return

      lowercase							    =       self.get_tokenizer()
      lowercase							    =       self.get_rust_tokenizer()

      lowercase							    =       '''I was born in 92000, and this is falsé.'''

      lowercase							    =       tokenizer.tokenize(lowercase_       )
      lowercase							    =       rust_tokenizer.tokenize(lowercase_       )
      self.assertListEqual(lowercase_							,		lowercase_       )

      lowercase							    =       tokenizer.encode(lowercase_							,		add_special_tokens=lowercase_       )
      lowercase							    =       rust_tokenizer.encode(lowercase_							,		add_special_tokens=lowercase_       )
      self.assertListEqual(lowercase_							,		lowercase_       )

      lowercase							    =       self.get_rust_tokenizer()
      lowercase							    =       tokenizer.encode(lowercase_       )
      lowercase							    =       rust_tokenizer.encode(lowercase_       )
      self.assertListEqual(lowercase_							,		lowercase_       )







    @slow
    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       '''Hello World!'''
      lowercase							    =       [0, 3_5_3_7_8, 6_6_6_1, 3_8, 2]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(lowercase_							,		self.big_tokenizer.encode(lowercase_       )       )







    @slow
    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       (
          '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
          ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
      )
      lowercase							    =       [
          0,
          3_2_9_3,
          8_3,
          1_0,
          4_5_5_2,
          4_9_8_9,
          7_9_8_6,
          6_7_8,
          1_0,
          5_9_1_5,
          1_1_1,
          1_7_9_4_5_9,
          1_2_4_8_5_0,
          4,
          6_0_4_4,
          2_3_7,
          1_2,
          6,
          5,
          6,
          4,
          6_7_8_0,
          7_0_5,
          1_5,
          1_3_8_8,
          4_4,
          3_7_8,
          1_0_1_1_4,
          7_1_1,
          1_5_2,
          2_0,
          6,
          5,
          2_2_3_7_6,
          6_4_2,
          1_2_2_1,
          1_5_1_9_0,
          3_4_1_5_3,
          4_5_0,
          5_6_0_8,
          9_5_9,
          1_1_1_9,
          5_7_7_0_2,
          1_3_6,
          1_8_6,
          4_7,
          1_0_9_8,
          2_9_3_6_7,
          4_7,
          # 4426, # What fairseq tokenizes from "<unk>": "_<"
          # 3678, # What fairseq tokenizes from "<unk>": "unk"
          # 2740, # What fairseq tokenizes from "<unk>": ">"
          3,  # What we tokenize from "<unk>": "<unk>"
          6,  # Residue from the tokenization: an extra sentencepiece underline
          4,
          6_0_4_4,
          2_3_7,
          6_2_8_4,
          5_0_9_0_1,
          5_2_8,
          3_1,
          9_0,
          3_4,
          9_2_7,
          2,
      ]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(lowercase_							,		self.big_tokenizer.encode(lowercase_       )       )







    @slow
    def 			UpperCamelCase_      (			self       ):
      # fmt: off
      lowercase							    =       {'''input_ids''': [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=lowercase_							,		model_name='xlm-roberta-base'							,		revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'							,		)







"""simple docstring"""

from functools import lru_cache




@lru_cache
def __lowercase			(							_a    ):
    if num < 0:
        raise ValueError('''Number should not be negative.'''    )

    return 1 if num in (0, 1) else num * factorial(num - 1    )


if __name__ == "__main__":
     import doctest

     doctest.testmod()





