


"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast
class 					lowerCAmelCase	(					unittest.TestCase    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							List[str]:
								# A mock response for an HTTP head request to emulate server down
								SCREAMING_SNAKE_CASE													=					mock.Mock()
								SCREAMING_SNAKE_CASE													=					500
								SCREAMING_SNAKE_CASE													=					{}
								SCREAMING_SNAKE_CASE													=					HTTPError
								SCREAMING_SNAKE_CASE													=					{}

								# Download this model to make sure it's in the cache.
								SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'						,       return_value=lowerCAmelCase__				) as mock_head:
														SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)
														# This check we did call the fake head request
														mock_head.assert_called()







		@require_tokenizers
		def 							__A						(	self				)     ->							List[str]:
								# A mock response for an HTTP head request to emulate server down
								SCREAMING_SNAKE_CASE													=					mock.Mock()
								SCREAMING_SNAKE_CASE													=					500
								SCREAMING_SNAKE_CASE													=					{}
								SCREAMING_SNAKE_CASE													=					HTTPError
								SCREAMING_SNAKE_CASE													=					{}

								# Download this model to make sure it's in the cache.
								SCREAMING_SNAKE_CASE													=					GPTaTokenizerFast.from_pretrained('gpt2'				)

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('requests.Session.request'						,       return_value=lowerCAmelCase__				) as mock_head:
														SCREAMING_SNAKE_CASE													=					GPTaTokenizerFast.from_pretrained('gpt2'				)
														# This check we did call the fake head request
														mock_head.assert_called()







		def 							__A						(	self				)     ->							Optional[int]:
								# This test is for deprecated behavior and can be removed in v5
								try:
														SCREAMING_SNAKE_CASE													=					tempfile.mktemp()
														with open(lowerCAmelCase__						,       'wb'				) as f:
																				http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						,       lowerCAmelCase__				)

														SCREAMING_SNAKE_CASE													=					AlbertTokenizer.from_pretrained(lowerCAmelCase__				)
								finally:
														os.remove(lowerCAmelCase__				)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('tokenizer.json'				):
														# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
														return
								try:
														with open('tokenizer.json'						,       'wb'				) as f:
																				http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'						,       lowerCAmelCase__				)
														SCREAMING_SNAKE_CASE													=					AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'				)
														# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
														self.assertEqual(tokenizer.vocab_size						,       1_000				)
														# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
														os.remove('tokenizer.json'				)







		def 							__A						(	self				)     ->							Dict:
								# This test is for deprecated behavior and can be removed in v5
								SCREAMING_SNAKE_CASE													=					AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				)





@is_staging_test
class 					lowerCAmelCase	(					unittest.TestCase    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Tuple								=     ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]







		@classmethod
		def 							__A						(	cls				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					TOKEN
								HfFolder.save_token(lowerCAmelCase__				)







		@classmethod
		def 							__A						(	cls				)     ->							List[Any]:
								try:
														delete_repo(token=cls._token						,       repo_id='test-tokenizer'				)
								except HTTPError:
														pass

								try:
														delete_repo(token=cls._token						,       repo_id='valid_org/test-tokenizer-org'				)
								except HTTPError:
														pass

								try:
														delete_repo(token=cls._token						,       repo_id='test-dynamic-tokenizer'				)
								except HTTPError:
														pass







		def 							__A						(	self				)     ->							List[str]:
								with tempfile.TemporaryDirectory() as tmp_dir:
														SCREAMING_SNAKE_CASE													=					os.path.join(lowerCAmelCase__						,       'vocab.txt'				)
														with open(lowerCAmelCase__						,       'w'						,       encoding='utf-8'				) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
														SCREAMING_SNAKE_CASE													=					BertTokenizer(lowerCAmelCase__				)

								tokenizer.push_to_hub('test-tokenizer'						,       use_auth_token=self._token				)
								SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'				)
								self.assertDictEqual(new_tokenizer.vocab						,       tokenizer.vocab				)

								# Reset repo
								delete_repo(token=self._token						,       repo_id='test-tokenizer'				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(lowerCAmelCase__						,       repo_id='test-tokenizer'						,       push_to_hub=lowerCAmelCase__						,       use_auth_token=self._token				)

								SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'				)
								self.assertDictEqual(new_tokenizer.vocab						,       tokenizer.vocab				)







		def 							__A						(	self				)     ->							Tuple:
								with tempfile.TemporaryDirectory() as tmp_dir:
														SCREAMING_SNAKE_CASE													=					os.path.join(lowerCAmelCase__						,       'vocab.txt'				)
														with open(lowerCAmelCase__						,       'w'						,       encoding='utf-8'				) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
														SCREAMING_SNAKE_CASE													=					BertTokenizer(lowerCAmelCase__				)

								tokenizer.push_to_hub('valid_org/test-tokenizer-org'						,       use_auth_token=self._token				)
								SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
								self.assertDictEqual(new_tokenizer.vocab						,       tokenizer.vocab				)

								# Reset repo
								delete_repo(token=self._token						,       repo_id='valid_org/test-tokenizer-org'				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(
														    lowerCAmelCase__						,       repo_id='valid_org/test-tokenizer-org'						,       push_to_hub=lowerCAmelCase__						,       use_auth_token=self._token				)

								SCREAMING_SNAKE_CASE													=					BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
								self.assertDictEqual(new_tokenizer.vocab						,       tokenizer.vocab				)







		@require_tokenizers
		def 							__A						(	self				)     ->							List[Any]:
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
														SCREAMING_SNAKE_CASE													=					os.path.join(lowerCAmelCase__						,       'vocab.txt'				)
														with open(lowerCAmelCase__						,       'w'						,       encoding='utf-8'				) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
														SCREAMING_SNAKE_CASE													=					CustomTokenizer(lowerCAmelCase__				)

								# No fast custom tokenizer
								tokenizer.push_to_hub('test-dynamic-tokenizer'						,       use_auth_token=self._token				)

								SCREAMING_SNAKE_CASE													=					AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,       trust_remote_code=lowerCAmelCase__				)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__						,       'CustomTokenizer'				)

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
														SCREAMING_SNAKE_CASE													=					os.path.join(lowerCAmelCase__						,       'vocab.txt'				)
														with open(lowerCAmelCase__						,       'w'						,       encoding='utf-8'				) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)

														SCREAMING_SNAKE_CASE													=					BertTokenizerFast.from_pretrained(lowerCAmelCase__				)
														bert_tokenizer.save_pretrained(lowerCAmelCase__				)
														SCREAMING_SNAKE_CASE													=					CustomTokenizerFast.from_pretrained(lowerCAmelCase__				)

								tokenizer.push_to_hub('test-dynamic-tokenizer'						,       use_auth_token=self._token				)

								SCREAMING_SNAKE_CASE													=					AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,       trust_remote_code=lowerCAmelCase__				)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__						,       'CustomTokenizerFast'				)
								SCREAMING_SNAKE_CASE													=					AutoTokenizer.from_pretrained(
								    F'{USER}/test-dynamic-tokenizer'						,       use_fast=lowerCAmelCase__						,       trust_remote_code=lowerCAmelCase__				)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__						,       'CustomTokenizer'				)





class 					lowerCAmelCase	(					unittest.TestCase    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('Hello 友達'				)
								self.assertEqual(trie.data						,       {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}				)
								trie.add('Hello'				)
								trie.data
								self.assertEqual(trie.data						,       {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}				)







		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					Trie()
								self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)						,       ['[CLS] This is a extra_id_100']				)
								trie.add('[CLS]'				)
								trie.add('extra_id_1'				)
								trie.add('extra_id_100'				)
								self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)						,       ['[CLS]', ' This is a ', 'extra_id_100']				)







		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('A'				)
								self.assertEqual(trie.split('ABC'				)						,       ['A', 'BC']				)
								self.assertEqual(trie.split('BCA'				)						,       ['BC', 'A']				)







		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('TOKEN]'				)
								trie.add('[SPECIAL_TOKEN]'				)
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)						,       ['This is something ', '[SPECIAL_TOKEN]']				)







		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('A'				)
								trie.add('P'				)
								trie.add('[SPECIAL_TOKEN]'				)
								self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)						,       ['This is something ', '[SPECIAL_TOKEN]']				)







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('AB'				)
								trie.add('B'				)
								trie.add('C'				)
								self.assertEqual(trie.split('ABC'				)						,       ['AB', 'C']				)







		def 							__A						(	self				)     ->							Tuple:
								SCREAMING_SNAKE_CASE													=					Trie()
								trie.add('ABC'				)
								trie.add('B'				)
								trie.add('CD'				)
								self.assertEqual(trie.split('ABCD'				)						,       ['ABC', 'D']				)







		def 							__A						(	self				)     ->							int:
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								SCREAMING_SNAKE_CASE													=					Trie()
								SCREAMING_SNAKE_CASE													=					trie.cut_text('ABC'						,       [0, 0, 2, 1, 2, 3]				)
								self.assertEqual(lowerCAmelCase__						,       ['AB', 'C']				)






"""simple docstring"""




__UpperCamelCase						=      frozenset(
    [
        '''prompt''',
        '''height''',
        '''width''',
        '''guidance_scale''',
        '''negative_prompt''',
        '''prompt_embeds''',
        '''negative_prompt_embeds''',
        '''cross_attention_kwargs''',
    ]
)

__UpperCamelCase						=      frozenset(['''prompt''', '''negative_prompt'''])

__UpperCamelCase						=      frozenset([])

__UpperCamelCase						=      frozenset(['''image'''])

__UpperCamelCase						=      frozenset(
    [
        '''image''',
        '''height''',
        '''width''',
        '''guidance_scale''',
    ]
)

__UpperCamelCase						=      frozenset(['''image'''])

__UpperCamelCase						=      frozenset(
    [
        '''prompt''',
        '''image''',
        '''height''',
        '''width''',
        '''guidance_scale''',
        '''negative_prompt''',
        '''prompt_embeds''',
        '''negative_prompt_embeds''',
    ]
)

__UpperCamelCase						=      frozenset(['''prompt''', '''image''', '''negative_prompt'''])

__UpperCamelCase						=      frozenset(
    [
        # Text guided image variation with an image mask
        '''prompt''',
        '''image''',
        '''mask_image''',
        '''height''',
        '''width''',
        '''guidance_scale''',
        '''negative_prompt''',
        '''prompt_embeds''',
        '''negative_prompt_embeds''',
    ]
)

__UpperCamelCase						=      frozenset(['''prompt''', '''image''', '''mask_image''', '''negative_prompt'''])

__UpperCamelCase						=      frozenset(
    [
        # image variation with an image mask
        '''image''',
        '''mask_image''',
        '''height''',
        '''width''',
        '''guidance_scale''',
    ]
)

__UpperCamelCase						=      frozenset(['''image''', '''mask_image'''])

__UpperCamelCase						=      frozenset(
    [
        '''example_image''',
        '''image''',
        '''mask_image''',
        '''height''',
        '''width''',
        '''guidance_scale''',
    ]
)

__UpperCamelCase						=      frozenset(['''example_image''', '''image''', '''mask_image'''])

__UpperCamelCase						=      frozenset(['''class_labels'''])

__UpperCamelCase						=      frozenset(['''class_labels'''])

__UpperCamelCase						=      frozenset(['''batch_size'''])

__UpperCamelCase						=      frozenset([])

__UpperCamelCase						=      frozenset(['''batch_size'''])

__UpperCamelCase						=      frozenset([])

__UpperCamelCase						=      frozenset(
    [
        '''prompt''',
        '''audio_length_in_s''',
        '''guidance_scale''',
        '''negative_prompt''',
        '''prompt_embeds''',
        '''negative_prompt_embeds''',
        '''cross_attention_kwargs''',
    ]
)

__UpperCamelCase						=      frozenset(['''prompt''', '''negative_prompt'''])
__UpperCamelCase						=      frozenset(['''input_tokens'''])

__UpperCamelCase						=      frozenset(['''input_tokens'''])


