



"""simple docstring"""

import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer




class        _UpperCAmelCase       (			lowerCAmelCase__):

 def 						_snake_case					(     self		:       Optional[int]				):
     snake_case_ :     str				=	tempfile.mkdtemp()
     snake_case_ :     List[Any]				=	5

     # Realm tok
     snake_case_ :     List[str]				=	[
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''[PAD]''',
         '''[MASK]''',
         '''test''',
         '''question''',
         '''this''',
         '''is''',
         '''the''',
         '''first''',
         '''second''',
         '''third''',
         '''fourth''',
         '''fifth''',
         '''record''',
         '''want''',
         '''##want''',
         '''##ed''',
         '''wa''',
         '''un''',
         '''runn''',
         '''##ing''',
         ''',''',
         '''low''',
         '''lowest''',
     ]
     snake_case_ :     Union[str, Any]				=	os.path.join(self.tmpdirname  ,     '''realm_tokenizer'''				)
     os.makedirs(lowercase_  ,     exist_ok=lowercase_				)
     snake_case_ :     Tuple				=	os.path.join(lowercase_  ,     VOCAB_FILES_NAMES['''vocab_file''']				)
     with open(self.vocab_file  ,     '''w'''  ,     encoding='''utf-8'''				) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)

     snake_case_ :     Tuple				=	os.path.join(self.tmpdirname  ,     '''realm_block_records'''				)
     os.makedirs(lowercase_  ,     exist_ok=lowercase_				)

 def 						_snake_case					(     self		:       Union[str, Any]				):
     return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,     '''realm_tokenizer'''				)				)

 def 						_snake_case					(     self		:       Optional[Any]				):
     shutil.rmtree(self.tmpdirname				)

 def 						_snake_case					(     self		:       Tuple				):
     snake_case_ :     str				=	RealmConfig(num_block_records=self.num_block_records				)
     return config

 def 						_snake_case					(     self		:       Optional[Any]				):
     snake_case_ :     Any				=	Dataset.from_dict(
         {
             '''id''': ['''0''', '''1'''],
             '''question''': ['''foo''', '''bar'''],
             '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
         }				)
     return dataset

 def 						_snake_case					(     self		:       Dict				):
     snake_case_ :     Union[str, Any]				=	np.array(
         [
             B'''This is the first record''',
             B'''This is the second record''',
             B'''This is the third record''',
             B'''This is the fourth record''',
             B'''This is the fifth record''',
             B'''This is a longer longer longer record''',
         ]  ,     dtype=lowercase_  ,     )
     return block_records

 def 						_snake_case					(     self		:       Optional[int]				):
     snake_case_ :     List[Any]				=	RealmRetriever(
         block_records=self.get_dummy_block_records()  ,     tokenizer=self.get_tokenizer()  ,     )
     return retriever

 def 						_snake_case					(     self		:       Optional[Any]				):
     snake_case_ :     Dict				=	self.get_config()
     snake_case_ :     int				=	self.get_dummy_retriever()
     snake_case_ :     Dict				=	retriever.tokenizer

     snake_case_ :     List[Any]				=	np.array([0, 3]  ,     dtype='''long'''				)
     snake_case_ :     Any				=	tokenizer(['''Test question''']				).input_ids
     snake_case_ :     Any				=	tokenizer(
         ['''the fourth''']  ,     add_special_tokens=lowercase_  ,     return_token_type_ids=lowercase_  ,     return_attention_mask=lowercase_  ,     ).input_ids
     snake_case_ :     List[Any]				=	config.reader_seq_len

     snake_case_,	snake_case_,	snake_case_,	snake_case_ :     Optional[Any]				=	retriever(
         lowercase_  ,     lowercase_  ,     answer_ids=lowercase_  ,     max_length=lowercase_  ,     return_tensors='''np'''				)

     self.assertEqual(len(lowercase_				)  ,     2				)
     self.assertEqual(len(lowercase_				)  ,     2				)
     self.assertEqual(len(lowercase_				)  ,     2				)
     self.assertEqual(concat_inputs.input_ids.shape  ,     (2, 10)				)
     self.assertEqual(concat_inputs.attention_mask.shape  ,     (2, 10)				)
     self.assertEqual(concat_inputs.token_type_ids.shape  ,     (2, 10)				)
     self.assertEqual(concat_inputs.special_tokens_mask.shape  ,     (2, 10)				)
     self.assertEqual(
         tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]				)  ,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']  ,     )
     self.assertEqual(
         tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]				)  ,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']  ,     )

 def 						_snake_case					(     self		:       Optional[Any]				):
     snake_case_ :     Any				=	self.get_config()
     snake_case_ :     Tuple				=	self.get_dummy_retriever()
     snake_case_ :     int				=	retriever.tokenizer

     snake_case_ :     Any				=	np.array([0, 3, 5]  ,     dtype='''long'''				)
     snake_case_ :     Tuple				=	tokenizer(['''Test question''']				).input_ids
     snake_case_ :     List[Any]				=	tokenizer(
         ['''the fourth''', '''longer longer''']  ,     add_special_tokens=lowercase_  ,     return_token_type_ids=lowercase_  ,     return_attention_mask=lowercase_  ,     ).input_ids
     snake_case_ :     Dict				=	config.reader_seq_len

     snake_case_,	snake_case_,	snake_case_,	snake_case_ :     List[str]				=	retriever(
         lowercase_  ,     lowercase_  ,     answer_ids=lowercase_  ,     max_length=lowercase_  ,     return_tensors='''np'''				)

     self.assertEqual([False, True, True]  ,     lowercase_				)
     self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]  ,     lowercase_				)
     self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]  ,     lowercase_				)






 def 						_snake_case					(     self		:       str				):
     snake_case_ :     int				=	self.get_dummy_retriever()
     retriever.save_pretrained(os.path.join(self.tmpdirname  ,     '''realm_block_records'''				)				)

     # Test local path
     snake_case_ :     Optional[int]				=	retriever.from_pretrained(os.path.join(self.tmpdirname  ,     '''realm_block_records'''				)				)
     self.assertEqual(retriever.block_records[0]  ,     B'''This is the first record'''				)

     # Test mocked remote path
     with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''				) as mock_hf_hub_download:
         snake_case_ :     Optional[Any]				=	os.path.join(
             os.path.join(self.tmpdirname  ,     '''realm_block_records'''				)  ,     _REALM_BLOCK_RECORDS_FILENAME				)
         snake_case_ :     List[Any]				=	RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''				)

     self.assertEqual(retriever.block_records[0]  ,     B'''This is the first record'''				)










"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__    :  List[Any]							    =	{
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Any							    =	['''DistilBertTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  int							    =	[
         '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''DistilBertForMaskedLM''',
         '''DistilBertForMultipleChoice''',
         '''DistilBertForQuestionAnswering''',
         '''DistilBertForSequenceClassification''',
         '''DistilBertForTokenClassification''',
         '''DistilBertModel''',
         '''DistilBertPreTrainedModel''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Dict							    =	[
         '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''TFDistilBertForMaskedLM''',
         '''TFDistilBertForMultipleChoice''',
         '''TFDistilBertForQuestionAnswering''',
         '''TFDistilBertForSequenceClassification''',
         '''TFDistilBertForTokenClassification''',
         '''TFDistilBertMainLayer''',
         '''TFDistilBertModel''',
         '''TFDistilBertPreTrainedModel''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Tuple							    =	[
         '''FlaxDistilBertForMaskedLM''',
         '''FlaxDistilBertForMultipleChoice''',
         '''FlaxDistilBertForQuestionAnswering''',
         '''FlaxDistilBertForSequenceClassification''',
         '''FlaxDistilBertForTokenClassification''',
         '''FlaxDistilBertModel''',
         '''FlaxDistilBertPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_distilbert import (
         DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DistilBertConfig,
         DistilBertOnnxConfig,
     )
     from .tokenization_distilbert import DistilBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_distilbert_fast import DistilBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_distilbert import (
              DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
              DistilBertModel,
              DistilBertPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_distilbert import (
              TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFDistilBertForMaskedLM,
              TFDistilBertForMultipleChoice,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertMainLayer,
              TFDistilBertModel,
              TFDistilBertPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_distilbert import (
              FlaxDistilBertForMaskedLM,
              FlaxDistilBertForMultipleChoice,
              FlaxDistilBertForQuestionAnswering,
              FlaxDistilBertForSequenceClassification,
              FlaxDistilBertForTokenClassification,
              FlaxDistilBertModel,
              FlaxDistilBertPreTrainedModel,
          )

else:
     import sys

     lowercase__    :  Union[str, Any]							    =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





