





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			tempfile.mkdtemp()
        SCREAMING_SNAKE_CASE__											=			5

        # Realm tok
        SCREAMING_SNAKE_CASE__											=			[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "[PAD]",
            "[MASK]",
            "test",
            "question",
            "this",
            "is",
            "the",
            "first",
            "second",
            "third",
            "fourth",
            "fifth",
            "record",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				'''realm_tokenizer'''     )
        os.makedirs(A_   ,				exist_ok=A_     )
        SCREAMING_SNAKE_CASE__											=			os.path.join(A_   ,				VOCAB_FILES_NAMES['''vocab_file''']     )
        with open(self.vocab_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				'''realm_block_records'''     )
        os.makedirs(A_   ,				exist_ok=A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,				'''realm_tokenizer'''     )     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        shutil.rmtree(self.tmpdirname     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			RealmConfig(num_block_records=self.num_block_records     )
        return config



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			Dataset.from_dict(
            {
                '''id''': ['''0''', '''1'''],
                '''question''': ['''foo''', '''bar'''],
                '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
            }     )
        return dataset



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			np.array(
            [
                b'''This is the first record''',
                b'''This is the second record''',
                b'''This is the third record''',
                b'''This is the fourth record''',
                b'''This is the fifth record''',
                b'''This is a longer longer longer record''',
            ]   ,				dtype=A_   ,				)
        return block_records



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			RealmRetriever(
            block_records=self.get_dummy_block_records()   ,				tokenizer=self.get_tokenizer()   ,				)
        return retriever



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.get_config()
        SCREAMING_SNAKE_CASE__											=			self.get_dummy_retriever()
        SCREAMING_SNAKE_CASE__											=			retriever.tokenizer

        SCREAMING_SNAKE_CASE__											=			np.array([0, 3]   ,				dtype='''long'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(['''Test question''']     ).input_ids
        SCREAMING_SNAKE_CASE__											=			tokenizer(
            ['''the fourth''']   ,				add_special_tokens=A_   ,				return_token_type_ids=A_   ,				return_attention_mask=A_   ,				).input_ids
        SCREAMING_SNAKE_CASE__											=			config.reader_seq_len

        SCREAMING_SNAKE_CASE__											=			retriever(
            A_   ,				A_   ,				answer_ids=A_   ,				max_length=A_   ,				return_tensors='''np'''     )

        self.assertEqual(len(A_     )   ,				2     )
        self.assertEqual(len(A_     )   ,				2     )
        self.assertEqual(len(A_     )   ,				2     )
        self.assertEqual(concat_inputs.input_ids.shape   ,				(2, 10)     )
        self.assertEqual(concat_inputs.attention_mask.shape   ,				(2, 10)     )
        self.assertEqual(concat_inputs.token_type_ids.shape   ,				(2, 10)     )
        self.assertEqual(concat_inputs.special_tokens_mask.shape   ,				(2, 10)     )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]     )   ,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']   ,				)
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]     )   ,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']   ,				)



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.get_config()
        SCREAMING_SNAKE_CASE__											=			self.get_dummy_retriever()
        SCREAMING_SNAKE_CASE__											=			retriever.tokenizer

        SCREAMING_SNAKE_CASE__											=			np.array([0, 3, 5]   ,				dtype='''long'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(['''Test question''']     ).input_ids
        SCREAMING_SNAKE_CASE__											=			tokenizer(
            ['''the fourth''', '''longer longer''']   ,				add_special_tokens=A_   ,				return_token_type_ids=A_   ,				return_attention_mask=A_   ,				).input_ids
        SCREAMING_SNAKE_CASE__											=			config.reader_seq_len

        SCREAMING_SNAKE_CASE__											=			retriever(
            A_   ,				A_   ,				answer_ids=A_   ,				max_length=A_   ,				return_tensors='''np'''     )

        self.assertEqual([False, True, True]   ,				A_     )
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]   ,				A_     )
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]   ,				A_     )




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname   ,				'''realm_block_records'''     )     )

        # Test local path
        SCREAMING_SNAKE_CASE__											=			retriever.from_pretrained(os.path.join(self.tmpdirname   ,				'''realm_block_records'''     )     )
        self.assertEqual(retriever.block_records[0]   ,				b'''This is the first record'''     )

        # Test mocked remote path
        with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''     ) as mock_hf_hub_download:
            SCREAMING_SNAKE_CASE__											=			os.path.join(
                os.path.join(self.tmpdirname   ,				'''realm_block_records'''     )   ,				_REALM_BLOCK_RECORDS_FILENAME     )
            SCREAMING_SNAKE_CASE__											=			RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''     )

        self.assertEqual(retriever.block_records[0]   ,				b'''This is the first record'''     )








'''simple docstring'''
def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase = 100				):


   """simple docstring"""




   lowercase_						:     Dict							     =  n * (n + 1) * (2 * n + 1) / 6
   lowercase_						:     int							     =  (n * (n + 1) / 2) ** 2
   return int(square_of_sum - sum_of_squares				)


if __name__ == "__main__":
 print(f"""{solution() = }""")




