





"""simple docstring"""




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Any										=  tempfile.mkdtemp()
        A_  :       str										=  5

        # Realm tok
        A_  :       Union[str, Any]										=  [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'test',
            'question',
            'this',
            'is',
            'the',
            'first',
            'second',
            'third',
            'fourth',
            'fifth',
            'record',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        A_  :       Union[str, Any]										=  os.path.join(self.tmpdirname					,      'realm_tokenizer'  )
        os.makedirs(snake_case_					,      exist_ok=snake_case_  )
        A_  :       int										=  os.path.join(snake_case_					,      VOCAB_FILES_NAMES['vocab_file']  )
        with open(self.vocab_file					,      'w'					,      encoding='utf-8'  ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )

        A_  :       List[str]										=  os.path.join(self.tmpdirname					,      'realm_block_records'  )
        os.makedirs(snake_case_					,      exist_ok=snake_case_  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,      'realm_tokenizer'  )  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        shutil.rmtree(self.tmpdirname  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Optional[int]										=  RealmConfig(num_block_records=self.num_block_records  )
        return config

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Any										=  Dataset.from_dict(
            {
                'id': ['0', '1'],
                'question': ['foo', 'bar'],
                'answers': [['Foo', 'Bar'], ['Bar']],
            }  )
        return dataset

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       str										=  np.array(
            [
                B'This is the first record',
                B'This is the second record',
                B'This is the third record',
                B'This is the fourth record',
                B'This is the fifth record',
                B'This is a longer longer longer record',
            ]					,      dtype=snake_case_					,      )
        return block_records

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Optional[Any]										=  RealmRetriever(
            block_records=self.get_dummy_block_records()					,      tokenizer=self.get_tokenizer()					,      )
        return retriever

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Dict										=  self.get_config()
        A_  :       Tuple										=  self.get_dummy_retriever()
        A_  :       Optional[Any]										=  retriever.tokenizer

        A_  :       Dict										=  np.array([0, 3]					,      dtype='long'  )
        A_  :       int										=  tokenizer(['Test question']  ).input_ids
        A_  :       Dict										=  tokenizer(
            ['the fourth']					,      add_special_tokens=snake_case_					,      return_token_type_ids=snake_case_					,      return_attention_mask=snake_case_					,      ).input_ids
        A_  :       Tuple										=  config.reader_seq_len

        A_						,       A_						,       A_						,       A_  :       int										=  retriever(
            snake_case_					,      snake_case_					,      answer_ids=snake_case_					,      max_length=snake_case_					,      return_tensors='np'  )

        self.assertEqual(len(snake_case_  )					,      2  )
        self.assertEqual(len(snake_case_  )					,      2  )
        self.assertEqual(len(snake_case_  )					,      2  )
        self.assertEqual(concat_inputs.input_ids.shape					,      (2, 1_0)  )
        self.assertEqual(concat_inputs.attention_mask.shape					,      (2, 1_0)  )
        self.assertEqual(concat_inputs.token_type_ids.shape					,      (2, 1_0)  )
        self.assertEqual(concat_inputs.special_tokens_mask.shape					,      (2, 1_0)  )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]  )					,      ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']					,      )
        self.assertEqual(
            tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]  )					,      ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']					,      )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       str										=  self.get_config()
        A_  :       int										=  self.get_dummy_retriever()
        A_  :       Dict										=  retriever.tokenizer

        A_  :       int										=  np.array([0, 3, 5]					,      dtype='long'  )
        A_  :       Union[str, Any]										=  tokenizer(['Test question']  ).input_ids
        A_  :       str										=  tokenizer(
            ['the fourth', 'longer longer']					,      add_special_tokens=snake_case_					,      return_token_type_ids=snake_case_					,      return_attention_mask=snake_case_					,      ).input_ids
        A_  :       Any										=  config.reader_seq_len

        A_						,       A_						,       A_						,       A_  :       str										=  retriever(
            snake_case_					,      snake_case_					,      answer_ids=snake_case_					,      max_length=snake_case_					,      return_tensors='np'  )

        self.assertEqual([False, True, True]					,      snake_case_  )
        self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,      snake_case_  )
        self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,      snake_case_  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       int										=  self.get_dummy_retriever()
        retriever.save_pretrained(os.path.join(self.tmpdirname					,      'realm_block_records'  )  )

        # Test local path
        A_  :       str										=  retriever.from_pretrained(os.path.join(self.tmpdirname					,      'realm_block_records'  )  )
        self.assertEqual(retriever.block_records[0]					,      B'This is the first record'  )

        # Test mocked remote path
        with patch('transformers.models.realm.retrieval_realm.hf_hub_download'  ) as mock_hf_hub_download:
          A_  :       Any										=  os.path.join(
              os.path.join(self.tmpdirname					,      'realm_block_records'  )					,      _REALM_BLOCK_RECORDS_FILENAME  )
          A_  :       Union[str, Any]										=  RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'  )

        self.assertEqual(retriever.block_records[0]					,      B'This is the first record'  )






"""simple docstring"""




def 						UpperCAmelCase__		(     _UpperCAmelCase					):






  """simple docstring"""





  A_  :       List[str]										=  hex_num.strip()
  if not hex_num:
    raise ValueError('No value was passed to the function'					)

  A_  :       Any										=  hex_num[0] == '-'
  if is_negative:
    A_  :       Optional[Any]										=  hex_num[1:]

  try:
    A_  :       Tuple										=  int(_UpperCAmelCase ,					16					)
  except ValueError:
    raise ValueError('Invalid value was passed to the function'					)

  A_  :       Union[str, Any]										=  ''
  while int_num > 0:
    A_  :       Optional[Any]										=  str(int_num % 2					) + bin_str
    int_num >>= 1

  return int(('-' + bin_str) if is_negative else bin_str					)


if __name__ == "__main__":
   import doctest

   doctest.testmod()