




import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
      import faiss





@require_faiss
class     SCREAMING_SNAKE_CASE__		(_a			):
     def        A__      (						self	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      tempfile.mkdtemp()
       lowerCAmelCase__										=      8

       # DPR tok
       lowerCAmelCase__										=      [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''dpr_tokenizer'''						)
       os.makedirs(__lowerCamelCase       ,						exist_ok=__lowerCamelCase						)
       lowerCAmelCase__										=      os.path.join(__lowerCamelCase       ,						DPR_VOCAB_FILES_NAMES['''vocab_file''']						)
       with open(self.vocab_file       ,						'''w'''       ,						encoding='''utf-8'''						) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)

       # BART tok
       lowerCAmelCase__										=      [
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''<unk>''',
       ]
       lowerCAmelCase__										=      dict(zip(__lowerCamelCase       ,						range(len(__lowerCamelCase						)						)						)						)
       lowerCAmelCase__										=      ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       lowerCAmelCase__										=      {'''unk_token''': '''<unk>'''}

       lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''bart_tokenizer'''						)
       os.makedirs(__lowerCamelCase       ,						exist_ok=__lowerCamelCase						)
       lowerCAmelCase__										=      os.path.join(__lowerCamelCase       ,						BART_VOCAB_FILES_NAMES['''vocab_file''']						)
       lowerCAmelCase__										=      os.path.join(__lowerCamelCase       ,						BART_VOCAB_FILES_NAMES['''merges_file''']						)
       with open(self.vocab_file       ,						'''w'''       ,						encoding='''utf-8'''						) as fp:
         fp.write(json.dumps(__lowerCamelCase						) + '''\n'''						)
       with open(self.merges_file       ,						'''w'''       ,						encoding='''utf-8'''						) as fp:
         fp.write('''\n'''.join(__lowerCamelCase						)						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,						'''dpr_tokenizer'''						)						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,						'''dpr_tokenizer'''						)						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,						'''bart_tokenizer'''						)						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       shutil.rmtree(self.tmpdirname						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      Dataset.from_dict(
           {
               '''id''': ['''0''', '''1'''],
               '''text''': ['''foo''', '''bar'''],
               '''title''': ['''Foo''', '''Bar'''],
               '''embeddings''': [np.ones(self.retrieval_vector_size						), 2 * np.ones(self.retrieval_vector_size						)],
           }						)
       dataset.add_faiss_index('''embeddings'''       ,						string_factory='''Flat'''       ,						metric_type=faiss.METRIC_INNER_PRODUCT						)
       return dataset
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_dataset()
       lowerCAmelCase__										=      RagConfig(
           retrieval_vector_size=self.retrieval_vector_size       ,						question_encoder=DPRConfig().to_dict()       ,						generator=BartConfig().to_dict()       ,						)
       with patch('''transformers.models.rag.retrieval_rag.load_dataset'''						) as mock_load_dataset:
         lowerCAmelCase__										=      dataset
         lowerCAmelCase__										=      RagRetriever(
             __lowerCamelCase       ,						question_encoder_tokenizer=self.get_dpr_tokenizer()       ,						generator_tokenizer=self.get_bart_tokenizer()       ,						)
       return retriever
     def        A__      (						self	:		Optional[int]       ,						__lowerCamelCase	:		bool						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_dataset()
       lowerCAmelCase__										=      RagConfig(
           retrieval_vector_size=self.retrieval_vector_size       ,						question_encoder=DPRConfig().to_dict()       ,						generator=BartConfig().to_dict()       ,						index_name='''custom'''       ,						)
       if from_disk:
         lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''dataset'''						)
         lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''index.faiss'''						)
         dataset.get_index('''embeddings'''						).save(os.path.join(self.tmpdirname       ,						'''index.faiss'''						)						)
         dataset.drop_index('''embeddings'''						)
         dataset.save_to_disk(os.path.join(self.tmpdirname       ,						'''dataset'''						)						)
         del dataset
         lowerCAmelCase__										=      RagRetriever(
             __lowerCamelCase       ,						question_encoder_tokenizer=self.get_dpr_tokenizer()       ,						generator_tokenizer=self.get_bart_tokenizer()       ,						)
       else:
         lowerCAmelCase__										=      RagRetriever(
             __lowerCamelCase       ,						question_encoder_tokenizer=self.get_dpr_tokenizer()       ,						generator_tokenizer=self.get_bart_tokenizer()       ,						index=CustomHFIndex(config.retrieval_vector_size       ,						__lowerCamelCase						)       ,						)
       return retriever
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      Dataset.from_dict(
           {
               '''id''': ['''0''', '''1'''],
               '''text''': ['''foo''', '''bar'''],
               '''title''': ['''Foo''', '''Bar'''],
               '''embeddings''': [np.ones(self.retrieval_vector_size + 1						), 2 * np.ones(self.retrieval_vector_size + 1						)],
           }						)
       dataset.add_faiss_index('''embeddings'''       ,						string_factory='''Flat'''       ,						metric_type=faiss.METRIC_INNER_PRODUCT						)

       lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''						)
       dataset.save_faiss_index('''embeddings'''       ,						index_file_name + '''.index.dpr'''						)
       pickle.dump(dataset['''id''']       ,						open(index_file_name + '''.index_meta.dpr'''       ,						'''wb'''						)						)

       lowerCAmelCase__										=      os.path.join(self.tmpdirname       ,						'''psgs_w100.tsv.pkl'''						)
       lowerCAmelCase__										=      {sample['''id''']: [sample['''text'''], sample['''title''']] for sample in dataset}
       pickle.dump(__lowerCamelCase       ,						open(__lowerCamelCase       ,						'''wb'''						)						)

       lowerCAmelCase__										=      RagConfig(
           retrieval_vector_size=self.retrieval_vector_size       ,						question_encoder=DPRConfig().to_dict()       ,						generator=BartConfig().to_dict()       ,						index_name='''legacy'''       ,						index_path=self.tmpdirname       ,						)
       lowerCAmelCase__										=      RagRetriever(
           __lowerCamelCase       ,						question_encoder_tokenizer=self.get_dpr_tokenizer()       ,						generator_tokenizer=self.get_bart_tokenizer()						)
       return retriever
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_canonical_hf_index_retriever()
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=__lowerCamelCase						)
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertEqual(len(__lowerCamelCase						)       ,						2						)
       self.assertEqual(sorted(doc_dicts[0]						)       ,						['''embeddings''', '''id''', '''text''', '''title''']						)
       self.assertEqual(len(doc_dicts[0]['''id''']						)       ,						__lowerCamelCase						)
       self.assertEqual(doc_dicts[0]['''id'''][0]       ,						'''1'''						)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['''id'''][0]       ,						'''0'''						)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()       ,						[[1], [0]]						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_canonical_hf_index_retriever()
       with tempfile.TemporaryDirectory() as tmp_dirname:
         with patch('''transformers.models.rag.retrieval_rag.load_dataset'''						) as mock_load_dataset:
           lowerCAmelCase__										=      self.get_dummy_dataset()
           retriever.save_pretrained(__lowerCamelCase						)
           lowerCAmelCase__										=      RagRetriever.from_pretrained(__lowerCamelCase						)
           self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
           lowerCAmelCase__										=      np.array(
               [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
           lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=1						)
           self.assertTrue(out is not None						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase						)
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=__lowerCamelCase						)
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertEqual(len(__lowerCamelCase						)       ,						2						)
       self.assertEqual(sorted(doc_dicts[0]						)       ,						['''embeddings''', '''id''', '''text''', '''title''']						)
       self.assertEqual(len(doc_dicts[0]['''id''']						)       ,						__lowerCamelCase						)
       self.assertEqual(doc_dicts[0]['''id'''][0]       ,						'''1'''						)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['''id'''][0]       ,						'''0'''						)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()       ,						[[1], [0]]						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase						)
       with tempfile.TemporaryDirectory() as tmp_dirname:
         retriever.save_pretrained(__lowerCamelCase						)
         lowerCAmelCase__										=      RagRetriever.from_pretrained(__lowerCamelCase						)
         self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
         lowerCAmelCase__										=      np.array(
             [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
         lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=1						)
         self.assertTrue(out is not None						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase						)
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=__lowerCamelCase						)
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertEqual(len(__lowerCamelCase						)       ,						2						)
       self.assertEqual(sorted(doc_dicts[0]						)       ,						['''embeddings''', '''id''', '''text''', '''title''']						)
       self.assertEqual(len(doc_dicts[0]['''id''']						)       ,						__lowerCamelCase						)
       self.assertEqual(doc_dicts[0]['''id'''][0]       ,						'''1'''						)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['''id'''][0]       ,						'''0'''						)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()       ,						[[1], [0]]						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase						)
       with tempfile.TemporaryDirectory() as tmp_dirname:
         retriever.save_pretrained(__lowerCamelCase						)
         lowerCAmelCase__										=      RagRetriever.from_pretrained(__lowerCamelCase						)
         self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
         lowerCAmelCase__										=      np.array(
             [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
         lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=1						)
         self.assertTrue(out is not None						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_legacy_index_retriever()
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=__lowerCamelCase						)
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertEqual(len(__lowerCamelCase						)       ,						2						)
       self.assertEqual(sorted(doc_dicts[0]						)       ,						['''text''', '''title''']						)
       self.assertEqual(len(doc_dicts[0]['''text''']						)       ,						__lowerCamelCase						)
       self.assertEqual(doc_dicts[0]['''text'''][0]       ,						'''bar'''						)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['''text'''][0]       ,						'''foo'''						)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()       ,						[[1], [0]]						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dummy_legacy_index_retriever()
       with tempfile.TemporaryDirectory() as tmp_dirname:
         retriever.save_pretrained(__lowerCamelCase						)
         lowerCAmelCase__										=      RagRetriever.from_pretrained(__lowerCamelCase						)
         self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
         lowerCAmelCase__										=      np.array(
             [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
         lowerCAmelCase__										=      retriever.retrieve(__lowerCamelCase       ,						n_docs=1						)
         self.assertTrue(out is not None						)
     @require_torch
     @require_tokenizers
     @require_sentencepiece
     def        A__      (						self	:		int						):



       """simple docstring"""
       import torch

       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_canonical_hf_index_retriever()
       lowerCAmelCase__										=      [[5, 7], [10, 11]]
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__										=      retriever(__lowerCamelCase       ,						__lowerCamelCase       ,						prefix=retriever.config.generator.prefix       ,						n_docs=__lowerCamelCase						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      (
           out['''context_input_ids'''],
           out['''context_attention_mask'''],
           out['''retrieved_doc_embeds'''],
       )
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
       self.assertIsInstance(__lowerCamelCase       ,						__lowerCamelCase						)
       self.assertIsInstance(__lowerCamelCase       ,						np.ndarray						)

       lowerCAmelCase__										=      retriever(
           __lowerCamelCase       ,						__lowerCamelCase       ,						prefix=retriever.config.generator.prefix       ,						n_docs=__lowerCamelCase       ,						return_tensors='''pt'''       ,						)
       lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      (  # noqa: F841
           out['''context_input_ids'''],
           out['''context_attention_mask'''],
           out['''retrieved_doc_embeds'''],
           out['''doc_ids'''],
       )
       self.assertEqual(retrieved_doc_embeds.shape       ,						(2, n_docs, self.retrieval_vector_size)						)
       self.assertIsInstance(__lowerCamelCase       ,						torch.Tensor						)
       self.assertIsInstance(__lowerCamelCase       ,						torch.Tensor						)
       self.assertIsInstance(__lowerCamelCase       ,						torch.Tensor						)



     @require_torch
     @require_tokenizers
     @require_sentencepiece
     def        A__      (						self	:		List[str]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_dpr_ctx_encoder_tokenizer()
       lowerCAmelCase__										=      1
       lowerCAmelCase__										=      self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase						)
       retriever.set_ctx_encoder_tokenizer(__lowerCamelCase						)

       lowerCAmelCase__										=      [[5, 7], [10, 11]]
       lowerCAmelCase__										=      np.array(
           [np.ones(self.retrieval_vector_size						), -np.ones(self.retrieval_vector_size						)]       ,						dtype=np.floataa						)
       lowerCAmelCase__										=      retriever(__lowerCamelCase       ,						__lowerCamelCase       ,						prefix=retriever.config.generator.prefix       ,						n_docs=__lowerCamelCase						)

       self.assertEqual(
           len(__lowerCamelCase						)       ,						6						)  # check whether the retriever output consist of 6 attributes including tokenized docs
       self.assertEqual(
           all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')						)       ,						__lowerCamelCase						)  # check for doc token related keys in dictionary.







import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__  :  Optional[int]					=       get_tests_dir("""fixtures/spiece.model""")





@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(_a    ,					unittest.TestCase			):
     lowercase_ :      Optional[Any]				= DebertaVaTokenizer
     lowercase_ :      Any				= DebertaVaTokenizerFast
     lowercase_ :      List[str]				= True
     lowercase_ :      str				= True
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						unk_token='''<unk>'''						)
       tokenizer.save_pretrained(self.tmpdirname						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      '''this is a test'''
       lowerCAmelCase__										=      '''this is a test'''
       return input_text, output_text
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      '''<pad>'''
       lowerCAmelCase__										=      0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase						)       ,						__lowerCamelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase						)       ,						__lowerCamelCase						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      list(self.get_tokenizer().get_vocab().keys()						)
       self.assertEqual(vocab_keys[0]       ,						'''<pad>'''						)
       self.assertEqual(vocab_keys[1]       ,						'''<unk>'''						)
       self.assertEqual(vocab_keys[-1]       ,						'''[PAD]'''						)
       self.assertEqual(len(__lowerCamelCase						)       ,						3_00_01						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       self.assertEqual(self.get_tokenizer().vocab_size       ,						3_00_00						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		List[str]						):



       """simple docstring"""
       pass
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       pass
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_tokenizer()
       lowerCAmelCase__										=      self.get_rust_tokenizer()

       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''

       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      self.get_rust_tokenizer()
       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      '''This is a test'''
       lowerCAmelCase__										=      [13, 1, 43_98, 25, 21, 12_89]
       lowerCAmelCase__										=      ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)
       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      [13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
       lowerCAmelCase__										=      ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode('''sequence builders'''						)
       lowerCAmelCase__										=      tokenizer.encode('''multi-sequence build'''						)

       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,						__lowerCamelCase						)

       self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]       ,						__lowerCamelCase						)
       self.assertEqual(
           [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]       ,						__lowerCamelCase       ,						)



     @slow
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      {'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__lowerCamelCase       ,						model_name='''microsoft/deberta-v2-xlarge'''       ,						revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''       ,						)

