


import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer


class        _a							(			UpperCamelCase__			):

							def       lowerCamelCase_ (							self:	List[Any]							)			->   Optional[int]:





									"""simple docstring"""







									lowercase__            =      tempfile.mkdtemp()
									lowercase__            =      5

									# Realm tok
									lowercase__            =      [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''test''',
									    '''question''',
									    '''this''',
									    '''is''',
									    '''the''',
									    '''first''',
									    '''second''',
									    '''third''',
									    '''fourth''',
									    '''fifth''',
									    '''record''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowercase__            =      os.path.join(self.tmpdirname    ,						'''realm_tokenizer'''							)
									os.makedirs(UpperCamelCase_    ,						exist_ok=UpperCamelCase_							)
									lowercase__            =      os.path.join(UpperCamelCase_    ,						VOCAB_FILES_NAMES['''vocab_file''']							)
									with open(self.vocab_file    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)

									lowercase__            =      os.path.join(self.tmpdirname    ,						'''realm_block_records'''							)
									os.makedirs(UpperCamelCase_    ,						exist_ok=UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Any							)			->   RealmTokenizer:





									"""simple docstring"""







									return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,						'''realm_tokenizer'''							)							)

							def       lowerCamelCase_ (							self:	Tuple							)			->   Optional[int]:





									"""simple docstring"""







									shutil.rmtree(self.tmpdirname							)

							def       lowerCamelCase_ (							self:	List[str]							)			->   str:





									"""simple docstring"""







									lowercase__            =      RealmConfig(num_block_records=self.num_block_records							)
									return config

							def       lowerCamelCase_ (							self:	List[Any]							)			->   Tuple:





									"""simple docstring"""







									lowercase__            =      Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''question''': ['''foo''', '''bar'''],
									        '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
									    }							)
									return dataset

							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Optional[int]:





									"""simple docstring"""







									lowercase__            =      np.array(
									    [
									        b'''This is the first record''',
									        b'''This is the second record''',
									        b'''This is the third record''',
									        b'''This is the fourth record''',
									        b'''This is the fifth record''',
									        b'''This is a longer longer longer record''',
									    ]    ,						dtype=UpperCamelCase_    ,						)
									return block_records

							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      RealmRetriever(
									    block_records=self.get_dummy_block_records()    ,						tokenizer=self.get_tokenizer()    ,						)
									return retriever

							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      self.get_config()
									lowercase__            =      self.get_dummy_retriever()
									lowercase__            =      retriever.tokenizer

									lowercase__            =      np.array([0, 3]    ,						dtype='''long'''							)
									lowercase__            =      tokenizer(['''Test question''']							).input_ids
									lowercase__            =      tokenizer(
									    ['''the fourth''']    ,						add_special_tokens=UpperCamelCase_    ,						return_token_type_ids=UpperCamelCase_    ,						return_attention_mask=UpperCamelCase_    ,						).input_ids
									lowercase__            =      config.reader_seq_len

									lowercase__     ,		lowercase__     ,		lowercase__     ,		lowercase__            =      retriever(
									    UpperCamelCase_    ,						UpperCamelCase_    ,						answer_ids=UpperCamelCase_    ,						max_length=UpperCamelCase_    ,						return_tensors='''np'''							)

									self.assertEqual(len(UpperCamelCase_							)    ,						2							)
									self.assertEqual(len(UpperCamelCase_							)    ,						2							)
									self.assertEqual(len(UpperCamelCase_							)    ,						2							)
									self.assertEqual(concat_inputs.input_ids.shape    ,						(2, 10)							)
									self.assertEqual(concat_inputs.attention_mask.shape    ,						(2, 10)							)
									self.assertEqual(concat_inputs.token_type_ids.shape    ,						(2, 10)							)
									self.assertEqual(concat_inputs.special_tokens_mask.shape    ,						(2, 10)							)
									self.assertEqual(
									    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]							)    ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']    ,						)
									self.assertEqual(
									    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]							)    ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']    ,						)

							def       lowerCamelCase_ (							self:	List[str]							)			->   List[str]:





									"""simple docstring"""







									lowercase__            =      self.get_config()
									lowercase__            =      self.get_dummy_retriever()
									lowercase__            =      retriever.tokenizer

									lowercase__            =      np.array([0, 3, 5]    ,						dtype='''long'''							)
									lowercase__            =      tokenizer(['''Test question''']							).input_ids
									lowercase__            =      tokenizer(
									    ['''the fourth''', '''longer longer''']    ,						add_special_tokens=UpperCamelCase_    ,						return_token_type_ids=UpperCamelCase_    ,						return_attention_mask=UpperCamelCase_    ,						).input_ids
									lowercase__            =      config.reader_seq_len

									lowercase__     ,		lowercase__     ,		lowercase__     ,		lowercase__            =      retriever(
									    UpperCamelCase_    ,						UpperCamelCase_    ,						answer_ids=UpperCamelCase_    ,						max_length=UpperCamelCase_    ,						return_tensors='''np'''							)

									self.assertEqual([False, True, True]    ,						UpperCamelCase_							)
									self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]    ,						UpperCamelCase_							)
									self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]    ,						UpperCamelCase_							)


							def       lowerCamelCase_ (							self:	Optional[int]							)			->   str:





									"""simple docstring"""







									lowercase__            =      self.get_dummy_retriever()
									retriever.save_pretrained(os.path.join(self.tmpdirname    ,						'''realm_block_records'''							)							)

									# Test local path
									lowercase__            =      retriever.from_pretrained(os.path.join(self.tmpdirname    ,						'''realm_block_records'''							)							)
									self.assertEqual(retriever.block_records[0]    ,						b'''This is the first record'''							)

									# Test mocked remote path
									with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''							) as mock_hf_hub_download:
											lowercase__            =      os.path.join(
											    os.path.join(self.tmpdirname    ,						'''realm_block_records'''							)    ,						_REALM_BLOCK_RECORDS_FILENAME							)
											lowercase__            =      RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''							)

									self.assertEqual(retriever.block_records[0]    ,						b'''This is the first record'''							)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCAmelCase       =							{
    'configuration_tapas': ['TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TapasConfig'],
    'tokenization_tapas': ['TapasTokenizer'],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase       =							[
			    'TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'TapasForMaskedLM',
			    'TapasForQuestionAnswering',
			    'TapasForSequenceClassification',
			    'TapasModel',
			    'TapasPreTrainedModel',
			    'load_tf_weights_in_tapas',
			]
try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase       =							[
			    'TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'TFTapasForMaskedLM',
			    'TFTapasForQuestionAnswering',
			    'TFTapasForSequenceClassification',
			    'TFTapasModel',
			    'TFTapasPreTrainedModel',
			]


if TYPE_CHECKING:
			from .configuration_tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig
			from .tokenization_tapas import TapasTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tapas import (
						    TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TapasForMaskedLM,
						    TapasForQuestionAnswering,
						    TapasForSequenceClassification,
						    TapasModel,
						    TapasPreTrainedModel,
						    load_tf_weights_in_tapas,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_tapas import (
						    TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFTapasForMaskedLM,
						    TFTapasForQuestionAnswering,
						    TFTapasForSequenceClassification,
						    TFTapasModel,
						    TFTapasPreTrainedModel,
						)


else:
			import sys

			lowerCAmelCase       =							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


