





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					A_					(   UpperCAmelCase       ,						unittest.TestCase						):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_   :			Any  			=	CLIPTokenizer
       SCREAMING_SNAKE_CASE_   :			Optional[int]  			=	CLIPTokenizerFast
       SCREAMING_SNAKE_CASE_   :			Dict  			=	True
       SCREAMING_SNAKE_CASE_   :			Tuple  			=	{}
       SCREAMING_SNAKE_CASE_   :			str  			=	False







       def  __UpperCAmelCase			(     self						:						Optional[int]	)    ->       int:
         super().setUp()

         # fmt: off
         _lowercase						    =			['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
         # fmt: on
         _lowercase						    =			dict(zip(__A    ,range(len(__A	)	)	)	)
         _lowercase						    =			['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
         _lowercase						    =			{'unk_token': '<unk>'}

         _lowercase						    =			os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['vocab_file']	)
         _lowercase						    =			os.path.join(self.tmpdirname    ,VOCAB_FILES_NAMES['merges_file']	)
         with open(self.vocab_file    ,'w'    ,encoding='utf-8'	) as fp:
           fp.write(json.dumps(__A	) + '\n'	)
         with open(self.merges_file    ,'w'    ,encoding='utf-8'	) as fp:
           fp.write('\n'.join(__A	)	)







       def  __UpperCAmelCase			(     self						:						Tuple    ,**__A						:						Tuple	)    ->       str:
         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizer.from_pretrained(self.tmpdirname    ,**__A	)







       def  __UpperCAmelCase			(     self						:						Union[str, Any]    ,**__A						:						List[str]	)    ->       Dict:
         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,**__A	)







       def  __UpperCAmelCase			(     self						:						List[str]    ,__A						:						Any	)    ->       Optional[Any]:
         _lowercase						    =			'lower newer'
         _lowercase						    =			'lower newer'
         return input_text, output_text







       def  __UpperCAmelCase			(     self						:						Dict	)    ->       List[str]:
         _lowercase						    =			CLIPTokenizer(self.vocab_file    ,self.merges_file    ,**self.special_tokens_map	)
         _lowercase						    =			'lower newer'
         _lowercase						    =			['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
         _lowercase						    =			tokenizer.tokenize(__A	)
         self.assertListEqual(__A    ,__A	)

         _lowercase						    =			tokens + [tokenizer.unk_token]
         _lowercase						    =			[10, 2, 16, 9, 3, 2, 16, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)    ,__A	)







       @require_ftfy
       def  __UpperCAmelCase			(     self						:						int	)    ->       List[Any]:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
             _lowercase						    =			self.tokenizer_class.from_pretrained(__A    ,**__A	)
             _lowercase						    =			self.rust_tokenizer_class.from_pretrained(__A    ,**__A	)

             _lowercase						    =			'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
             _lowercase						    =			tokenizer_s.tokenize(__A	)
             _lowercase						    =			tokenizer_r.tokenize(__A	)

             self.assertListEqual(__A    ,__A	)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             _lowercase						    =			'xa\u0303y' + ' ' + 'x\xe3y'
             _lowercase						    =			tokenizer_s.tokenize(__A	)
             _lowercase						    =			tokenizer_r.tokenize(__A	)

             self.assertListEqual(__A    ,__A	)

             # Test that the tokenization is identical on unicode of space type
             _lowercase						    =			[
                 '\u0009',  # (horizontal tab, '\t')
                 '\u000B',  # (vertical tab)
                 '\u000C',  # (form feed)
                 '\u0020',  # (space, ' ')
                 '\u200E',  # (left-to-right mark):w
                 '\u200F',  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
               _lowercase						    =			tokenizer_s.tokenize(__A	)
               _lowercase						    =			tokenizer_r.tokenize(__A	)

               self.assertListEqual(__A    ,__A	)

             # Test that the tokenization is identical on unicode of line break type
             _lowercase						    =			[
                 '\u000A',  # (line feed, '\n')
                 '\r\n',  # (carriage return and line feed, '\r\n')
                 '\u000D',  # (carriage return, '\r')
                 '\r',  # (carriage return, '\r')
                 '\u000D',  # (carriage return, '\r')
                 '\u2028',  # (line separator)
                 '\u2029',  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
               _lowercase						    =			tokenizer_s.tokenize(__A	)
               _lowercase						    =			tokenizer_r.tokenize(__A	)

               self.assertListEqual(__A    ,__A	)







       def  __UpperCAmelCase			(     self						:						Dict	)    ->       Union[str, Any]:
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
             _lowercase						    =			'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             _lowercase						    =			F"""{text_of_1_token} {text_of_1_token}"""

             _lowercase						    =			self.rust_tokenizer_class.from_pretrained(
                 __A    ,use_fast=__A    ,)
             _lowercase						    =			tokenizer_r(__A    ,return_offsets_mapping=__A    ,add_special_tokens=__A	)
             self.assertEqual(encoding.offset_mapping[0]    ,(0, len(__A	))	)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,(len(__A	) + 1, len(__A	) + 1 + len(__A	))    ,)

             _lowercase						    =			F""" {text}"""

             _lowercase						    =			self.rust_tokenizer_class.from_pretrained(
                 __A    ,use_fast=__A    ,)
             _lowercase						    =			tokenizer_r(__A    ,return_offsets_mapping=__A    ,add_special_tokens=__A	)
             self.assertEqual(encoding.offset_mapping[0]    ,(1, 1 + len(__A	))	)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,(1 + len(__A	) + 1, 1 + len(__A	) + 1 + len(__A	))    ,)







       def  __UpperCAmelCase			(     self						:						int	)    ->       Any:
         # Test related to the breaking change introduced in transformers v4.17.0
         # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
         with self.assertRaises(__A	) as context:
           self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'	)

         self.assertTrue(
             context.exception.args[0].startswith(
                 'The `backend_tokenizer` provided does not match the expected format.'	)	)







       @require_ftfy
       def  __UpperCAmelCase			(     self						:						List[Any]	)    ->       str:
         super().test_tokenization_python_rust_equals()







       def  __UpperCAmelCase			(     self						:						int	)    ->       Union[str, Any]:
         # CLIP always lower cases letters
         pass






import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
					from transformers.models.rag.configuration_rag import RagConfig
					from transformers.models.rag.tokenization_rag import RagTokenizer

@require_faiss
@require_torch
class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      tempfile.mkdtemp()
														UpperCamelCase__					  =      8

														# DPR tok
														UpperCamelCase__					  =      [
														    """[UNK]""",
														    """[CLS]""",
														    """[SEP]""",
														    """[PAD]""",
														    """[MASK]""",
														    """want""",
														    """##want""",
														    """##ed""",
														    """wa""",
														    """un""",
														    """runn""",
														    """##ing""",
														    """,""",
														    """low""",
														    """lowest""",
														]
														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			"""dpr_tokenizer"""	)
														os.makedirs(__lowerCAmelCase	,			exist_ok=__lowerCAmelCase	)
														UpperCamelCase__					  =      os.path.join(__lowerCAmelCase	,			DPR_VOCAB_FILES_NAMES["""vocab_file"""]	)
														with open(self.vocab_file	,			"""w"""	,			encoding="""utf-8"""	) as vocab_writer:
																					vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)

														# BART tok
														UpperCamelCase__					  =      [
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														]
														UpperCamelCase__					  =      dict(zip(__lowerCAmelCase	,			range(len(__lowerCAmelCase	)	)	)	)
														UpperCamelCase__					  =      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														UpperCamelCase__					  =      {"""unk_token""": """<unk>"""}

														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			"""bart_tokenizer"""	)
														os.makedirs(__lowerCAmelCase	,			exist_ok=__lowerCAmelCase	)
														UpperCamelCase__					  =      os.path.join(__lowerCAmelCase	,			BART_VOCAB_FILES_NAMES["""vocab_file"""]	)
														UpperCamelCase__					  =      os.path.join(__lowerCAmelCase	,			BART_VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write(json.dumps(__lowerCAmelCase	) + """\n"""	)
														with open(self.merges_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write("""\n""".join(__lowerCAmelCase	)	)






							def 					_lowerCamelCase		(    self	):
														return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname	,			"""dpr_tokenizer"""	)	)






							def 					_lowerCamelCase		(    self	):
														return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname	,			"""bart_tokenizer"""	)	)






							def 					_lowerCamelCase		(    self	):
														shutil.rmtree(self.tmpdirname	)






							@require_tokenizers
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			"""rag_tokenizer"""	)
														UpperCamelCase__					  =      RagConfig(question_encoder=DPRConfig().to_dict()	,			generator=BartConfig().to_dict()	)
														UpperCamelCase__					  =      RagTokenizer(question_encoder=self.get_dpr_tokenizer()	,			generator=self.get_bart_tokenizer()	)
														rag_config.save_pretrained(__lowerCAmelCase	)
														rag_tokenizer.save_pretrained(__lowerCAmelCase	)
														UpperCamelCase__					  =      RagTokenizer.from_pretrained(__lowerCAmelCase	,			config=__lowerCAmelCase	)
														self.assertIsInstance(new_rag_tokenizer.question_encoder	,			__lowerCAmelCase	)
														self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()	,			rag_tokenizer.question_encoder.get_vocab()	)
														self.assertIsInstance(new_rag_tokenizer.generator	,			__lowerCAmelCase	)
														self.assertEqual(new_rag_tokenizer.generator.get_vocab()	,			rag_tokenizer.generator.get_vocab()	)






							@slow
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      RagTokenizer.from_pretrained("""facebook/rag-token-nq"""	)
														UpperCamelCase__					  =      [
														    """who got the first nobel prize in physics""",
														    """when is the next deadpool movie being released""",
														    """which mode is used for short wave broadcast service""",
														    """who is the owner of reading football club""",
														    """when is the next scandal episode coming out""",
														    """when is the last time the philadelphia won the superbowl""",
														    """what is the most current adobe flash player version""",
														    """how many episodes are there in dragon ball z""",
														    """what is the first step in the evolution of the eye""",
														    """where is gall bladder situated in human body""",
														    """what is the main mineral in lithium batteries""",
														    """who is the president of usa right now""",
														    """where do the greasers live in the outsiders""",
														    """panda is a national animal of which country""",
														    """what is the name of manchester united stadium""",
														]
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)
														self.assertIsNotNone(__lowerCAmelCase	)







							@slow
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      RagTokenizer.from_pretrained("""facebook/rag-sequence-nq"""	)
														UpperCamelCase__					  =      [
														    """who got the first nobel prize in physics""",
														    """when is the next deadpool movie being released""",
														    """which mode is used for short wave broadcast service""",
														    """who is the owner of reading football club""",
														    """when is the next scandal episode coming out""",
														    """when is the last time the philadelphia won the superbowl""",
														    """what is the most current adobe flash player version""",
														    """how many episodes are there in dragon ball z""",
														    """what is the first step in the evolution of the eye""",
														    """where is gall bladder situated in human body""",
														    """what is the main mineral in lithium batteries""",
														    """who is the president of usa right now""",
														    """where do the greasers live in the outsiders""",
														    """panda is a national animal of which country""",
														    """what is the name of manchester united stadium""",
														]
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)
														self.assertIsNotNone(__lowerCAmelCase	)




