



"""simple docstring"""







import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class       UpperCamelCase__			(   _lowerCAmelCase       ,		unittest.TestCase      ):

			"""simple docstring"""


			A__	:					str							  =     GPTaTokenizer
			A__	:					int							  =     GPTaTokenizerFast
			A__	:					Optional[int]							  =     True
			A__	:					List[str]							  =     {"add_prefix_space": True}
			A__	:					List[str]							  =     False


			def   snake_case__	(      self		)      ->	Tuple:
				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				A__											=					[
				    "l",
				    "o",
				    "w",
				    "e",
				    "r",
				    "s",
				    "t",
				    "i",
				    "d",
				    "n",
				    "\u0120",
				    "\u0120l",
				    "\u0120n",
				    "\u0120lo",
				    "\u0120low",
				    "er",
				    "\u0120lowest",
				    "\u0120newer",
				    "\u0120wider",
				    "<unk>",
				    "<|endoftext|>",
				]
				A__											=					dict(zip(SCREAMING_SNAKE_CASE__						,	range(len(SCREAMING_SNAKE_CASE__		)		)		)		)
				A__											=					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
				A__											=					{"unk_token": "<unk>"}

				A__											=					os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]		)
				A__											=					os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]		)
				with open(self.vocab_file						,	"w"						,	encoding="utf-8"		) as fp:
					fp.write(json.dumps(SCREAMING_SNAKE_CASE__		) + "\n"		)
				with open(self.merges_file						,	"w"						,	encoding="utf-8"		) as fp:
					fp.write("\n".join(SCREAMING_SNAKE_CASE__		)		)


			def   snake_case__	(      self						,	**SCREAMING_SNAKE_CASE__		)      ->	Union[str, Any]:
				kwargs.update(self.special_tokens_map		)
				return GPTaTokenizer.from_pretrained(self.tmpdirname						,	**SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	**SCREAMING_SNAKE_CASE__		)      ->	Dict:
				kwargs.update(self.special_tokens_map		)
				return GPTaTokenizerFast.from_pretrained(self.tmpdirname						,	**SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	Optional[int]:
				A__											=					"lower newer"
				A__											=					"lower newer"
				return input_text, output_text


			def   snake_case__	(      self		)      ->	Any:
				A__											=					GPTaTokenizer(self.vocab_file						,	self.merges_file						,	**self.special_tokens_map		)
				A__											=					"lower newer"
				A__											=					["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
				A__											=					tokenizer.tokenize(SCREAMING_SNAKE_CASE__						,	add_prefix_space=SCREAMING_SNAKE_CASE__		)
				self.assertListEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)

				A__											=					tokens + [tokenizer.unk_token]
				A__											=					[14, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__		)						,	SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self		)      ->	Union[str, Any]:
				if not self.test_rust_tokenizer:
					return

				A__											=					self.get_tokenizer()
				A__											=					self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__		)

				A__											=					"lower newer"

				# Testing tokenization
				A__											=					tokenizer.tokenize(SCREAMING_SNAKE_CASE__						,	add_prefix_space=SCREAMING_SNAKE_CASE__		)
				A__											=					rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__		)
				self.assertListEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)

				# Testing conversion to ids without special tokens
				A__											=					tokenizer.encode(SCREAMING_SNAKE_CASE__						,	add_special_tokens=SCREAMING_SNAKE_CASE__						,	add_prefix_space=SCREAMING_SNAKE_CASE__		)
				A__											=					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__						,	add_special_tokens=SCREAMING_SNAKE_CASE__		)
				self.assertListEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)

				# Testing conversion to ids with special tokens
				A__											=					self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__		)
				A__											=					tokenizer.encode(SCREAMING_SNAKE_CASE__						,	add_prefix_space=SCREAMING_SNAKE_CASE__		)
				A__											=					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__		)
				self.assertListEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)

				# Testing the unknown token
				A__											=					tokens + [rust_tokenizer.unk_token]
				A__											=					[14, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__		)						,	SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	*SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__		)      ->	Optional[int]:
				# It's very difficult to mix/test pretokenization with byte-level
				# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
				pass


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__=15		)      ->	Optional[int]:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
						A__											=					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__		)

						# Simple input
						A__											=					"This is a simple input"
						A__											=					["This is a simple input 1", "This is a simple input 2"]
						A__											=					("This is a simple input", "This is a pair")
						A__											=					[
						    ("This is a simple input 1", "This is a simple input 2"),
						    ("This is a simple pair 1", "This is a simple pair 2"),
						]

						# Simple input tests
						self.assertRaises(SCREAMING_SNAKE_CASE__						,	tokenizer_r.encode						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"		)

						# Simple input
						self.assertRaises(SCREAMING_SNAKE_CASE__						,	tokenizer_r.encode_plus						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"		)

						# Simple input
						self.assertRaises(
						    SCREAMING_SNAKE_CASE__						,	tokenizer_r.batch_encode_plus						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"						,	)

						# Pair input
						self.assertRaises(SCREAMING_SNAKE_CASE__						,	tokenizer_r.encode						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"		)

						# Pair input
						self.assertRaises(SCREAMING_SNAKE_CASE__						,	tokenizer_r.encode_plus						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"		)

						# Pair input
						self.assertRaises(
						    SCREAMING_SNAKE_CASE__						,	tokenizer_r.batch_encode_plus						,	SCREAMING_SNAKE_CASE__						,	max_length=SCREAMING_SNAKE_CASE__						,	padding="max_length"						,	)


			def   snake_case__	(      self		)      ->	Optional[int]:
				A__											=					GPTaTokenizer.from_pretrained(self.tmpdirname						,	pad_token="<pad>"		)

				# Simple input
				A__											=					"This is a simple input"
				A__											=					["This is a simple input looooooooong", "This is a simple input"]
				A__											=					("This is a simple input", "This is a pair")
				A__											=					[
				    ("This is a simple input loooooong", "This is a simple input"),
				    ("This is a simple pair loooooong", "This is a simple pair"),
				]

				A__											=					tokenizer.pad_token_id

				A__											=					tokenizer(SCREAMING_SNAKE_CASE__						,	padding="max_length"						,	max_length=30						,	return_tensors="np"		)
				A__											=					tokenizer(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	truncate=SCREAMING_SNAKE_CASE__						,	return_tensors="np"		)
				A__											=					tokenizer(*SCREAMING_SNAKE_CASE__						,	padding="max_length"						,	max_length=60						,	return_tensors="np"		)
				A__											=					tokenizer(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	truncate=SCREAMING_SNAKE_CASE__						,	return_tensors="np"		)

				# s
				# test single string max_length padding
				self.assertEqual(out_s["input_ids"].shape[-1]						,	30		)
				self.assertTrue(pad_token_id in out_s["input_ids"]		)
				self.assertTrue(0 in out_s["attention_mask"]		)

				# s2
				# test automatic padding
				self.assertEqual(out_sa["input_ids"].shape[-1]						,	33		)
				# long slice doesn't have padding
				self.assertFalse(pad_token_id in out_sa["input_ids"][0]		)
				self.assertFalse(0 in out_sa["attention_mask"][0]		)
				# short slice does have padding
				self.assertTrue(pad_token_id in out_sa["input_ids"][1]		)
				self.assertTrue(0 in out_sa["attention_mask"][1]		)

				# p
				# test single pair max_length padding
				self.assertEqual(out_p["input_ids"].shape[-1]						,	60		)
				self.assertTrue(pad_token_id in out_p["input_ids"]		)
				self.assertTrue(0 in out_p["attention_mask"]		)

				# p2
				# test automatic padding pair
				self.assertEqual(out_pa["input_ids"].shape[-1]						,	52		)
				# long slice pair doesn't have padding
				self.assertFalse(pad_token_id in out_pa["input_ids"][0]		)
				self.assertFalse(0 in out_pa["attention_mask"][0]		)
				# short slice pair does have padding
				self.assertTrue(pad_token_id in out_pa["input_ids"][1]		)
				self.assertTrue(0 in out_pa["attention_mask"][1]		)


			def   snake_case__	(      self		)      ->	Optional[Any]:
				A__											=					"$$$"
				A__											=					GPTaTokenizer.from_pretrained(self.tmpdirname						,	bos_token=SCREAMING_SNAKE_CASE__						,	add_bos_token=SCREAMING_SNAKE_CASE__		)

				A__											=					"This is a simple input"
				A__											=					["This is a simple input 1", "This is a simple input 2"]

				A__											=					tokenizer.bos_token_id

				A__											=					tokenizer(SCREAMING_SNAKE_CASE__		)
				A__											=					tokenizer(SCREAMING_SNAKE_CASE__		)

				self.assertEqual(out_s.input_ids[0]						,	SCREAMING_SNAKE_CASE__		)
				self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

				A__											=					tokenizer.decode(out_s.input_ids		)
				A__											=					tokenizer.batch_decode(out_sa.input_ids		)

				self.assertEqual(decode_s.split()[0]						,	SCREAMING_SNAKE_CASE__		)
				self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


			def   snake_case__	(      self		)      ->	Any:
				pass







			def   snake_case__	(      self		)      ->	Optional[int]:
				# TODO: change to self.get_tokenizers() when the fast version is implemented
				A__											=					[self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE__						,	add_bos_token=SCREAMING_SNAKE_CASE__		)]
				for tokenizer in tokenizers:
					with self.subTest(f"""{tokenizer.__class__.__name__}"""		):
						A__											=					"Encode this."
						A__											=					"This one too please."
						A__											=					tokenizer.encode(SCREAMING_SNAKE_CASE__						,	add_special_tokens=SCREAMING_SNAKE_CASE__		)
						encoded_sequence += tokenizer.encode(SCREAMING_SNAKE_CASE__						,	add_special_tokens=SCREAMING_SNAKE_CASE__		)
						A__											=					tokenizer.encode_plus(
						    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	add_special_tokens=SCREAMING_SNAKE_CASE__						,	return_special_tokens_mask=SCREAMING_SNAKE_CASE__						,	)
						A__											=					encoded_sequence_dict["input_ids"]
						A__											=					encoded_sequence_dict["special_tokens_mask"]
						self.assertEqual(len(SCREAMING_SNAKE_CASE__		)						,	len(SCREAMING_SNAKE_CASE__		)		)

						A__											=					[
						    (x if not special_tokens_mask[i] else None) for i, x in enumerate(SCREAMING_SNAKE_CASE__		)
						]
						A__											=					[x for x in filtered_sequence if x is not None]
						self.assertEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)






@require_tokenizers
class       UpperCamelCase__			(   unittest.TestCase      ):

			"""simple docstring"""


			def   snake_case__	(      self		)      ->	List[str]:
				# More context:
				# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
				# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
				# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

				A__											=					AutoTokenizer.from_pretrained("facebook/opt-350m"						,	from_slow=SCREAMING_SNAKE_CASE__		)
				A__											=					"A photo of a cat"

				A__											=					tokenizer.encode(
				    SCREAMING_SNAKE_CASE__						,	)
				self.assertEqual(SCREAMING_SNAKE_CASE__						,	[2, 250, 1345, 9, 10, 4758]		)
				tokenizer.save_pretrained("test_opt"		)

				A__											=					AutoTokenizer.from_pretrained("./test_opt"		)
				A__											=					tokenizer.encode(
				    SCREAMING_SNAKE_CASE__						,	)
				self.assertEqual(SCREAMING_SNAKE_CASE__						,	[2, 250, 1345, 9, 10, 4758]		)


			def   snake_case__	(      self		)      ->	List[str]:
				A__											=					AutoTokenizer.from_pretrained("facebook/opt-350m"						,	use_slow=SCREAMING_SNAKE_CASE__		)
				A__											=					"A photo of a cat"

				A__											=					tokenizer.encode(
				    SCREAMING_SNAKE_CASE__						,	)
				# Same as above
				self.assertEqual(SCREAMING_SNAKE_CASE__						,	[2, 250, 1345, 9, 10, 4758]		)







			@unittest.skip("This test is failing because of a bug in the fast tokenizer"		)
			def   snake_case__	(      self		)      ->	List[Any]:
				A__											=					AutoTokenizer.from_pretrained("facebook/opt-350m"						,	from_slow=SCREAMING_SNAKE_CASE__		)

				A__											=					"bos"
				A__											=					tokenizer.get_vocab()["bos"]

				A__											=					"A photo of a cat"
				A__											=					tokenizer.encode(
				    SCREAMING_SNAKE_CASE__						,	)
				# We changed the bos token
				self.assertEqual(SCREAMING_SNAKE_CASE__						,	[31957, 250, 1345, 9, 10, 4758]		)
				tokenizer.save_pretrained("./tok"		)
				A__											=					AutoTokenizer.from_pretrained("./tok"		)
				self.assertTrue(tokenizer.is_fast		)
				A__											=					tokenizer.encode(
				    SCREAMING_SNAKE_CASE__						,	)
				self.assertEqual(SCREAMING_SNAKE_CASE__						,	[31957, 250, 1345, 9, 10, 4758]		)








import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


lowerCAmelCase__    :Optional[Any]       =				get_tests_dir('''fixtures''')

class 						__a (							unittest.TestCase     ):
		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     mock.Mock()
									_UpperCAmelCase											=     500
									_UpperCAmelCase											=     {}
									_UpperCAmelCase											=     HTTPError
									_UpperCAmelCase											=     {}

									# Download this model to make sure it's in the cache.
									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('hf-internal-testing/tiny-random-wav2vec2'		)
									# Under the mock environment we get a 500 error when trying to reach the model.
									with mock.patch('requests.Session.request'     ,       return_value=_SCREAMING_SNAKE_CASE		) as mock_head:
																_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('hf-internal-testing/tiny-random-wav2vec2'		)
																# This check we did call the fake head request
																mock_head.assert_called()






		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(
									    'https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json'		)



@is_staging_test
class 						__a (							unittest.TestCase     ):
		@classmethod
		def       UpperCAmelCase__							(     cls		)					->			Tuple:




									"""simple docstring"""





									_UpperCAmelCase											=     TOKEN
									HfFolder.save_token(_SCREAMING_SNAKE_CASE		)
		@classmethod
		def       UpperCAmelCase__							(     cls		)					->			Tuple:




									"""simple docstring"""





									try:
																delete_repo(token=cls._token     ,       repo_id='test-feature-extractor'		)
									except HTTPError:
																pass

									try:
																delete_repo(token=cls._token     ,       repo_id='valid_org/test-feature-extractor-org'		)
									except HTTPError:
																pass

									try:
																delete_repo(token=cls._token     ,       repo_id='test-dynamic-feature-extractor'		)
									except HTTPError:
																pass
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)
									feature_extractor.push_to_hub('test-feature-extractor'     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(f'''{USER}/test-feature-extractor'''		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)

									# Reset repo
									delete_repo(token=self._token     ,       repo_id='test-feature-extractor'		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
																feature_extractor.save_pretrained(
																    _SCREAMING_SNAKE_CASE     ,       repo_id='test-feature-extractor'     ,       push_to_hub=_SCREAMING_SNAKE_CASE     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(f'''{USER}/test-feature-extractor'''		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)
		def       UpperCAmelCase__							(     self		)					->			Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)
									feature_extractor.push_to_hub('valid_org/test-feature-extractor'     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('valid_org/test-feature-extractor'		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)

									# Reset repo
									delete_repo(token=self._token     ,       repo_id='valid_org/test-feature-extractor'		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
																feature_extractor.save_pretrained(
																    _SCREAMING_SNAKE_CASE     ,       repo_id='valid_org/test-feature-extractor-org'     ,       push_to_hub=_SCREAMING_SNAKE_CASE     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('valid_org/test-feature-extractor-org'		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)






		def       UpperCAmelCase__							(     self		)					->			Tuple:




									"""simple docstring"""





									CustomFeatureExtractor.register_for_auto_class()
									_UpperCAmelCase											=     CustomFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)

									feature_extractor.push_to_hub('test-dynamic-feature-extractor'     ,       use_auth_token=self._token		)

									# This has added the proper auto_map field to the config
									self.assertDictEqual(
									    feature_extractor.auto_map     ,       {'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor'}     ,       )

									_UpperCAmelCase											=     AutoFeatureExtractor.from_pretrained(
									    f'''{USER}/test-dynamic-feature-extractor'''     ,       trust_remote_code=_SCREAMING_SNAKE_CASE		)
									# Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
									self.assertEqual(new_feature_extractor.__class__.__name__     ,       'CustomFeatureExtractor'		)
