


'''simple docstring'''


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__								=   get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase		(				__SCREAMING_SNAKE_CASE	,			unittest.TestCase     ):


  '''simple docstring'''







  a_	       =				ReformerTokenizer
  a_	       =				ReformerTokenizerFast
  a_	       =				True
  a_	       =				False
  a_	       =				True






  def 							_a				(				self			:							Optional[Any]			):

        '''simple docstring'''




        super().setUp()

        A_    :	int							 =					ReformerTokenizer(UpperCamelCase__						,keep_accents=UpperCamelCase__			)
        tokenizer.save_pretrained(self.tmpdirname			)






  def 							_a				(				self			:							Optional[Any]			):

        '''simple docstring'''




        A_    :	int							 =					'''<s>'''
        A_    :	str							 =					1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__			)						,UpperCamelCase__			)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__			)						,UpperCamelCase__			)






  def 							_a				(				self			:							Tuple			):

        '''simple docstring'''




        A_    :	Tuple							 =					list(self.get_tokenizer().get_vocab().keys()			)

        self.assertEqual(vocab_keys[0]						,"""<unk>"""			)
        self.assertEqual(vocab_keys[1]						,"""<s>"""			)
        self.assertEqual(vocab_keys[-1]						,"""j"""			)
        self.assertEqual(len(UpperCamelCase__			)						,1000			)






  def 							_a				(				self			:							Tuple			):

        '''simple docstring'''




        self.assertEqual(self.get_tokenizer().vocab_size						,1000			)






  def 							_a				(				self			:							Any			):

        '''simple docstring'''




        if not self.test_rust_tokenizer:
              return

        A_    :	Tuple							 =					self.get_tokenizer()
        A_    :	int							 =					self.get_rust_tokenizer()

        A_    :	str							 =					'''I was born in 92000, and this is falsé.'''

        A_    :	int							 =					tokenizer.tokenize(UpperCamelCase__			)
        A_    :	str							 =					rust_tokenizer.tokenize(UpperCamelCase__			)
        self.assertListEqual(UpperCamelCase__						,UpperCamelCase__			)

        A_    :	Optional[Any]							 =					tokenizer.encode(UpperCamelCase__						,add_special_tokens=UpperCamelCase__			)
        A_    :	str							 =					rust_tokenizer.encode(UpperCamelCase__						,add_special_tokens=UpperCamelCase__			)
        self.assertListEqual(UpperCamelCase__						,UpperCamelCase__			)

        A_    :	str							 =					self.get_rust_tokenizer()
        A_    :	int							 =					tokenizer.encode(UpperCamelCase__			)
        A_    :	Any							 =					rust_tokenizer.encode(UpperCamelCase__			)
        self.assertListEqual(UpperCamelCase__						,UpperCamelCase__			)






  def 							_a				(				self			:							List[str]						,_a			:							List[str]=15			):

        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                    A_    :	Optional[Any]							 =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase__						,**UpperCamelCase__			)

                    # Simple input
                    A_    :	Union[str, Any]							 =					'''This is a simple input'''
                    A_    :	str							 =					['''This is a simple input 1''', '''This is a simple input 2''']
                    A_    :	List[Any]							 =					('''This is a simple input''', '''This is a pair''')
                    A_    :	List[str]							 =					[
                        ('''This is a simple input 1''', '''This is a simple input 2'''),
                        ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                    ]

                    # Simple input tests
                    self.assertRaises(UpperCamelCase__						,tokenizer_r.encode						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""			)

                    # Simple input
                    self.assertRaises(UpperCamelCase__						,tokenizer_r.encode_plus						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""			)

                    # Simple input
                    self.assertRaises(
                        UpperCamelCase__						,tokenizer_r.batch_encode_plus						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""						,)

                    # Pair input
                    self.assertRaises(UpperCamelCase__						,tokenizer_r.encode						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""			)

                    # Pair input
                    self.assertRaises(UpperCamelCase__						,tokenizer_r.encode_plus						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""			)

                    # Pair input
                    self.assertRaises(
                        UpperCamelCase__						,tokenizer_r.batch_encode_plus						,UpperCamelCase__						,max_length=UpperCamelCase__						,padding="""max_length"""						,)






  def 							_a				(				self			:							List[Any]			):

        '''simple docstring'''




        pass






  def 							_a				(				self			:							str			):

        '''simple docstring'''




        A_    :	List[Any]							 =					ReformerTokenizer(UpperCamelCase__						,keep_accents=UpperCamelCase__			)

        A_    :	Tuple							 =					tokenizer.tokenize("""This is a test"""			)
        self.assertListEqual(UpperCamelCase__						,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(UpperCamelCase__			)						,[285, 46, 10, 170, 382]						,)

        A_    :	List[Any]							 =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
        self.assertListEqual(
            UpperCamelCase__						,[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]						,)
        A_    :	Tuple							 =					tokenizer.convert_tokens_to_ids(UpperCamelCase__			)
        self.assertListEqual(
            UpperCamelCase__						,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						,)

        A_    :	Union[str, Any]							 =					tokenizer.convert_ids_to_tokens(UpperCamelCase__			)
        self.assertListEqual(
            UpperCamelCase__						,[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]						,)






  @cached_property
  def 							_a				(				self			:							List[Any]			):

        '''simple docstring'''




        return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""			)






  @slow
  def 							_a				(				self			:							Any			):

        '''simple docstring'''




        A_    :	Tuple							 =					'''Hello World!'''
        A_    :	List[str]							 =					[126, 32, 262, 152, 38, 72, 287]

        self.assertListEqual(UpperCamelCase__						,self.big_tokenizer.encode(UpperCamelCase__			)			)






  @slow
  def 							_a				(				self			:							Tuple			):

        '''simple docstring'''




        A_    :	Any							 =					(
            '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
            ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
        )
        A_    :	List[Any]							 =					[
            108,
            265,
            24,
            111,
            4,
            258,
            156,
            35,
            28,
            275,
            3,
            259,
            297,
            260,
            84,
            4,
            35,
            110,
            44,
            8,
            259,
            91,
            268,
            21,
            11,
            209,
            274,
            109,
            266,
            277,
            117,
            86,
            93,
            315,
            258,
            278,
            258,
            277,
            258,
            0,
            258,
            288,
            258,
            319,
            258,
            0,
            258,
            0,
            258,
            0,
            258,
            0,
            258,
            287,
            258,
            315,
            258,
            289,
            258,
            278,
            99,
            269,
            266,
            262,
            8,
            259,
            241,
            4,
            217,
            230,
            268,
            266,
            55,
            168,
            106,
            75,
            193,
            266,
            223,
            27,
            49,
            26,
            282,
            25,
            264,
            299,
            19,
            26,
            0,
            258,
            277,
            117,
            86,
            93,
            176,
            183,
            270,
            11,
            262,
            42,
            61,
            265,
        ]

        self.assertListEqual(UpperCamelCase__						,self.big_tokenizer.encode(UpperCamelCase__			)			)






  @require_torch
  @slow
  def 							_a				(				self			:							Optional[Any]			):

        '''simple docstring'''




        import torch

        from transformers import ReformerConfig, ReformerModel

        # Build sequence
        A_    :	Tuple							 =					list(self.big_tokenizer.get_vocab().keys()			)[:10]
        A_    :	Optional[Any]							 =					''' '''.join(UpperCamelCase__			)
        A_    :	Dict							 =					self.big_tokenizer.encode_plus(UpperCamelCase__						,return_tensors="""pt"""			)
        A_    :	Union[str, Any]							 =					self.big_tokenizer.batch_encode_plus([sequence, sequence]						,return_tensors="""pt"""			)

        A_    :	Union[str, Any]							 =					ReformerConfig()
        # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
        A_    :	Optional[Any]							 =					encoded_sequence['''input_ids'''].shape
        A_    :	Any							 =					ReformerModel(UpperCamelCase__			)

        # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
              model(**UpperCamelCase__			)
              model(**UpperCamelCase__			)






  @slow
  def 							_a				(				self			:							Dict			):

        '''simple docstring'''




        A_    :	Any							 =					{'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        # This tokenizer does not know some characters like ")".
        # That is the reason why we use very simple texts here.
        # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
        A_    :	Tuple							 =					[
            '''This is a very simple sentence.''',
            '''The quick brown fox jumps over the lazy dog.''',
        ]

        self.tokenizer_integration_test_util(
            expected_encoding=UpperCamelCase__						,model_name="""google/reformer-crime-and-punishment"""						,revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""						,padding=UpperCamelCase__						,sequences=UpperCamelCase__						,)


'''simple docstring'''
from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)


