




import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Optional[Any]					=    GPTaTokenizer
						_a							:    Dict					=    GPTaTokenizerFast
						_a							:    Tuple					=    True
						_a							:    List[Any]					=    {"""add_prefix_space""": True}
						_a							:    Optional[int]					=    False






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											__lowerCAmelCase				   =       [
											    "l",
											    "o",
											    "w",
											    "e",
											    "r",
											    "s",
											    "t",
											    "i",
											    "d",
											    "n",
											    "\u0120",
											    "\u0120l",
											    "\u0120n",
											    "\u0120lo",
											    "\u0120low",
											    "er",
											    "\u0120lowest",
											    "\u0120newer",
											    "\u0120wider",
											    "<unk>",
											    "<|endoftext|>",
											]
											__lowerCAmelCase				   =       dict(zip(_A		,			range(len(_A			)			)			)			)
											__lowerCAmelCase				   =       ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
											__lowerCAmelCase				   =       {"unk_token": "<unk>"}

											__lowerCAmelCase				   =       os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES["vocab_file"]			)
											__lowerCAmelCase				   =       os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES["merges_file"]			)
											with open(self.vocab_file		,			"w"		,			encoding="utf-8"			) as fp:
																fp.write(json.dumps(_A			) + "\n"			)
											with open(self.merges_file		,			"w"		,			encoding="utf-8"			) as fp:
																fp.write("\n".join(_A			)			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			**_A			):




											"""simple docstring"""



											kwargs.update(self.special_tokens_map			)
											return GPTaTokenizer.from_pretrained(self.tmpdirname		,			**_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			**_A			):




											"""simple docstring"""



											kwargs.update(self.special_tokens_map			)
											return GPTaTokenizerFast.from_pretrained(self.tmpdirname		,			**_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "lower newer"
											__lowerCAmelCase				   =       "lower newer"
											return input_text, output_text






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       GPTaTokenizer(self.vocab_file		,			self.merges_file		,			**self.special_tokens_map			)
											__lowerCAmelCase				   =       "lower newer"
											__lowerCAmelCase				   =       ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
											__lowerCAmelCase				   =       tokenizer.tokenize(_A		,			add_prefix_space=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokens + [tokenizer.unk_token]
											__lowerCAmelCase				   =       [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											if not self.test_rust_tokenizer:
																return

											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer(add_prefix_space=_A			)

											__lowerCAmelCase				   =       "lower newer"

											# Testing tokenization
											__lowerCAmelCase				   =       tokenizer.tokenize(_A		,			add_prefix_space=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)

											# Testing conversion to ids without special tokens
											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A		,			add_prefix_space=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											# Testing conversion to ids with special tokens
											__lowerCAmelCase				   =       self.get_rust_tokenizer(add_prefix_space=_A			)
											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_prefix_space=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)

											# Testing the unknown token
											__lowerCAmelCase				   =       tokens + [rust_tokenizer.unk_token]
											__lowerCAmelCase				   =       [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
											self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			*_A		,			**_A			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A=1_5			):




											"""simple docstring"""



											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																					__lowerCAmelCase				   =       self.rust_tokenizer_class.from_pretrained(_A		,			**_A			)

																					# Simple input
																					__lowerCAmelCase				   =       "This is a simple input"
																					__lowerCAmelCase				   =       ["This is a simple input 1", "This is a simple input 2"]
																					__lowerCAmelCase				   =       ("This is a simple input", "This is a pair")
																					__lowerCAmelCase				   =       [
																					    ("This is a simple input 1", "This is a simple input 2"),
																					    ("This is a simple pair 1", "This is a simple pair 2"),
																					]

																					# Simple input tests
																					self.assertRaises(_A		,			tokenizer_r.encode		,			_A		,			max_length=_A		,			padding="max_length"			)

																					# Simple input
																					self.assertRaises(_A		,			tokenizer_r.encode_plus		,			_A		,			max_length=_A		,			padding="max_length"			)

																					# Simple input
																					self.assertRaises(
																					    _A		,			tokenizer_r.batch_encode_plus		,			_A		,			max_length=_A		,			padding="max_length"		,			)

																					# Pair input
																					self.assertRaises(_A		,			tokenizer_r.encode		,			_A		,			max_length=_A		,			padding="max_length"			)

																					# Pair input
																					self.assertRaises(_A		,			tokenizer_r.encode_plus		,			_A		,			max_length=_A		,			padding="max_length"			)

																					# Pair input
																					self.assertRaises(
																					    _A		,			tokenizer_r.batch_encode_plus		,			_A		,			max_length=_A		,			padding="max_length"		,			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       GPTaTokenizer.from_pretrained(self.tmpdirname		,			pad_token="<pad>"			)

											# Simple input
											__lowerCAmelCase				   =       "This is a simple input"
											__lowerCAmelCase				   =       ["This is a simple input looooooooong", "This is a simple input"]
											__lowerCAmelCase				   =       ("This is a simple input", "This is a pair")
											__lowerCAmelCase				   =       [
											    ("This is a simple input loooooong", "This is a simple input"),
											    ("This is a simple pair loooooong", "This is a simple pair"),
											]

											__lowerCAmelCase				   =       tokenizer.pad_token_id

											__lowerCAmelCase				   =       tokenizer(_A		,			padding="max_length"		,			max_length=3_0		,			return_tensors="np"			)
											__lowerCAmelCase				   =       tokenizer(_A		,			padding=_A		,			truncate=_A		,			return_tensors="np"			)
											__lowerCAmelCase				   =       tokenizer(*_A		,			padding="max_length"		,			max_length=6_0		,			return_tensors="np"			)
											__lowerCAmelCase				   =       tokenizer(_A		,			padding=_A		,			truncate=_A		,			return_tensors="np"			)

											# s
											# test single string max_length padding
											self.assertEqual(out_s["input_ids"].shape[-1]		,			3_0			)
											self.assertTrue(pad_token_id in out_s["input_ids"]			)
											self.assertTrue(0 in out_s["attention_mask"]			)

											# s2
											# test automatic padding
											self.assertEqual(out_sa["input_ids"].shape[-1]		,			3_3			)
											# long slice doesn't have padding
											self.assertFalse(pad_token_id in out_sa["input_ids"][0]			)
											self.assertFalse(0 in out_sa["attention_mask"][0]			)
											# short slice does have padding
											self.assertTrue(pad_token_id in out_sa["input_ids"][1]			)
											self.assertTrue(0 in out_sa["attention_mask"][1]			)

											# p
											# test single pair max_length padding
											self.assertEqual(out_p["input_ids"].shape[-1]		,			6_0			)
											self.assertTrue(pad_token_id in out_p["input_ids"]			)
											self.assertTrue(0 in out_p["attention_mask"]			)

											# p2
											# test automatic padding pair
											self.assertEqual(out_pa["input_ids"].shape[-1]		,			5_2			)
											# long slice pair doesn't have padding
											self.assertFalse(pad_token_id in out_pa["input_ids"][0]			)
											self.assertFalse(0 in out_pa["attention_mask"][0]			)
											# short slice pair does have padding
											self.assertTrue(pad_token_id in out_pa["input_ids"][1]			)
											self.assertTrue(0 in out_pa["attention_mask"][1]			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "$$$"
											__lowerCAmelCase				   =       GPTaTokenizer.from_pretrained(self.tmpdirname		,			bos_token=_A		,			add_bos_token=_A			)

											__lowerCAmelCase				   =       "This is a simple input"
											__lowerCAmelCase				   =       ["This is a simple input 1", "This is a simple input 2"]

											__lowerCAmelCase				   =       tokenizer.bos_token_id

											__lowerCAmelCase				   =       tokenizer(_A			)
											__lowerCAmelCase				   =       tokenizer(_A			)

											self.assertEqual(out_s.input_ids[0]		,			_A			)
											self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

											__lowerCAmelCase				   =       tokenizer.decode(out_s.input_ids			)
											__lowerCAmelCase				   =       tokenizer.batch_decode(out_sa.input_ids			)

											self.assertEqual(decode_s.split()[0]		,			_A			)
											self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       [self.get_tokenizer(do_lower_case=_A		,			add_bos_token=_A			)]
											for tokenizer in tokenizers:
																with self.subTest(f"""{tokenizer.__class__.__name__}"""			):
																					__lowerCAmelCase				   =       "Encode this."
																					__lowerCAmelCase				   =       "This one too please."
																					__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
																					encoded_sequence += tokenizer.encode(_A		,			add_special_tokens=_A			)
																					__lowerCAmelCase				   =       tokenizer.encode_plus(
																					    _A		,			_A		,			add_special_tokens=_A		,			return_special_tokens_mask=_A		,			)
																					__lowerCAmelCase				   =       encoded_sequence_dict["input_ids"]
																					__lowerCAmelCase				   =       encoded_sequence_dict["special_tokens_mask"]
																					self.assertEqual(len(_A			)		,			len(_A			)			)

																					__lowerCAmelCase				   =       [
																					    (x if not special_tokens_mask[i] else None) for i, x in enumerate(_A			)
																					]
																					__lowerCAmelCase				   =       [x for x in filtered_sequence if x is not None]
																					self.assertEqual(_A		,			_A			)






@require_tokenizers
class 			a__    (					unittest.TestCase      ):






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       AutoTokenizer.from_pretrained("facebook/opt-350m"		,			from_slow=_A			)
											__lowerCAmelCase				   =       "A photo of a cat"

											__lowerCAmelCase				   =       tokenizer.encode(
											    _A		,			)
											self.assertEqual(_A		,			[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)
											tokenizer.save_pretrained("test_opt"			)

											__lowerCAmelCase				   =       AutoTokenizer.from_pretrained("./test_opt"			)
											__lowerCAmelCase				   =       tokenizer.encode(
											    _A		,			)
											self.assertEqual(_A		,			[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       AutoTokenizer.from_pretrained("facebook/opt-350m"		,			use_slow=_A			)
											__lowerCAmelCase				   =       "A photo of a cat"

											__lowerCAmelCase				   =       tokenizer.encode(
											    _A		,			)
											# Same as above
											self.assertEqual(_A		,			[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)






						@unittest.skip("This test is failing because of a bug in the fast tokenizer"			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       AutoTokenizer.from_pretrained("facebook/opt-350m"		,			from_slow=_A			)

											__lowerCAmelCase				   =       "bos"
											__lowerCAmelCase				   =       tokenizer.get_vocab()["bos"]

											__lowerCAmelCase				   =       "A photo of a cat"
											__lowerCAmelCase				   =       tokenizer.encode(
											    _A		,			)
											# We changed the bos token
											self.assertEqual(_A		,			[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)
											tokenizer.save_pretrained("./tok"			)
											__lowerCAmelCase				   =       AutoTokenizer.from_pretrained("./tok"			)
											self.assertTrue(tokenizer.is_fast			)
											__lowerCAmelCase				   =       tokenizer.encode(
											    _A		,			)
											self.assertEqual(_A		,			[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]			)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


UpperCamelCase__    			=			{
    """configuration_audio_spectrogram_transformer""": [
        """AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """ASTConfig""",
    ]
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__    			=			[
					    """AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """ASTForAudioClassification""",
					    """ASTModel""",
					    """ASTPreTrainedModel""",
					]

try:
					if not is_speech_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__    			=			["""ASTFeatureExtractor"""]

if TYPE_CHECKING:
					from .configuration_audio_spectrogram_transformer import (
					    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    ASTConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_audio_spectrogram_transformer import (
										    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    ASTForAudioClassification,
										    ASTModel,
										    ASTPreTrainedModel,
										)

					try:
										if not is_speech_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
					import sys

					UpperCamelCase__    			=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




