






"""simple docstring"""
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class _lowerCAmelCase						(    __UpperCAmelCase   ,  unittest.TestCase       ):
						SCREAMING_SNAKE_CASE_:					Optional[Any] 					=				GPTaTokenizer
						SCREAMING_SNAKE_CASE_:					List[str] 					=				GPTaTokenizerFast
						SCREAMING_SNAKE_CASE_:					Optional[Any] 					=				True
						SCREAMING_SNAKE_CASE_:					str 					=				{'add_prefix_space': True}
						SCREAMING_SNAKE_CASE_:					List[Any] 					=				False







						def 			A      (    self							)		->							Optional[Any]:
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    [
										    'l',
										    'o',
										    'w',
										    'e',
										    'r',
										    's',
										    't',
										    'i',
										    'd',
										    'n',
										    '\u0120',
										    '\u0120l',
										    '\u0120n',
										    '\u0120lo',
										    '\u0120low',
										    'er',
										    '\u0120lowest',
										    '\u0120newer',
										    '\u0120wider',
										    '<unk>',
										    '<|endoftext|>',
										]
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    dict(zip(lowerCAmelCase_   ,				range(len(lowerCAmelCase_							)							)							)							)
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    {'unk_token': '<unk>'}

										_SCREAMING_SNAKE_CASE   :		Any       		=    os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['vocab_file']							)
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['merges_file']							)
										with open(self.vocab_file   ,				'w'   ,				encoding='utf-8'							) as fp:
														fp.write(json.dumps(lowerCAmelCase_							) + '\n'							)
										with open(self.merges_file   ,				'w'   ,				encoding='utf-8'							) as fp:
														fp.write('\n'.join(lowerCAmelCase_							)							)







						def 			A      (    self   ,				**lowerCAmelCase_							)		->							Optional[int]:
										kwargs.update(self.special_tokens_map							)
										return GPTaTokenizer.from_pretrained(self.tmpdirname   ,				**lowerCAmelCase_							)







						def 			A      (    self   ,				**lowerCAmelCase_							)		->							str:
										kwargs.update(self.special_tokens_map							)
										return GPTaTokenizerFast.from_pretrained(self.tmpdirname   ,				**lowerCAmelCase_							)







						def 			A      (    self   ,				lowerCAmelCase_							)		->							List[Any]:
										_SCREAMING_SNAKE_CASE   :		int       		=    'lower newer'
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    'lower newer'
										return input_text, output_text







						def 			A      (    self							)		->							List[Any]:
										_SCREAMING_SNAKE_CASE   :		Any       		=    GPTaTokenizer(self.vocab_file   ,				self.merges_file   ,				**self.special_tokens_map							)
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    'lower newer'
										_SCREAMING_SNAKE_CASE   :		str       		=    ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    tokenizer.tokenize(lowerCAmelCase_   ,				add_prefix_space=lowerCAmelCase_							)
										self.assertListEqual(lowerCAmelCase_   ,				lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		int       		=    tokens + [tokenizer.unk_token]
										_SCREAMING_SNAKE_CASE   :		int       		=    [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_							)   ,				lowerCAmelCase_							)







						def 			A      (    self							)		->							Tuple:
										if not self.test_rust_tokenizer:
														return

										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    self.get_tokenizer()
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    'lower newer'

										# Testing tokenization
										_SCREAMING_SNAKE_CASE   :		int       		=    tokenizer.tokenize(lowerCAmelCase_   ,				add_prefix_space=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    rust_tokenizer.tokenize(lowerCAmelCase_							)
										self.assertListEqual(lowerCAmelCase_   ,				lowerCAmelCase_							)

										# Testing conversion to ids without special tokens
										_SCREAMING_SNAKE_CASE   :		Dict       		=    tokenizer.encode(lowerCAmelCase_   ,				add_special_tokens=lowerCAmelCase_   ,				add_prefix_space=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		str       		=    rust_tokenizer.encode(lowerCAmelCase_   ,				add_special_tokens=lowerCAmelCase_							)
										self.assertListEqual(lowerCAmelCase_   ,				lowerCAmelCase_							)

										# Testing conversion to ids with special tokens
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Any       		=    tokenizer.encode(lowerCAmelCase_   ,				add_prefix_space=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    rust_tokenizer.encode(lowerCAmelCase_							)
										self.assertListEqual(lowerCAmelCase_   ,				lowerCAmelCase_							)

										# Testing the unknown token
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    tokens + [rust_tokenizer.unk_token]
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
										self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_							)   ,				lowerCAmelCase_							)







						def 			A      (    self   ,				*lowerCAmelCase_   ,				**lowerCAmelCase_							)		->							Any:
										# It's very difficult to mix/test pretokenization with byte-level
										# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
										pass







						def 			A      (    self   ,				lowerCAmelCase_=1_5							)		->							int:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		_SCREAMING_SNAKE_CASE   :		Tuple       		=    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_   ,				**lowerCAmelCase_							)

																		# Simple input
																		_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    'This is a simple input'
																		_SCREAMING_SNAKE_CASE   :		Tuple       		=    ['This is a simple input 1', 'This is a simple input 2']
																		_SCREAMING_SNAKE_CASE   :		str       		=    ('This is a simple input', 'This is a pair')
																		_SCREAMING_SNAKE_CASE   :		Tuple       		=    [
																		    ('This is a simple input 1', 'This is a simple input 2'),
																		    ('This is a simple pair 1', 'This is a simple pair 2'),
																		]

																		# Simple input tests
																		self.assertRaises(lowerCAmelCase_   ,				tokenizer_r.encode   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'							)

																		# Simple input
																		self.assertRaises(lowerCAmelCase_   ,				tokenizer_r.encode_plus   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'							)

																		# Simple input
																		self.assertRaises(
																		    lowerCAmelCase_   ,				tokenizer_r.batch_encode_plus   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'   ,				)

																		# Pair input
																		self.assertRaises(lowerCAmelCase_   ,				tokenizer_r.encode   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'							)

																		# Pair input
																		self.assertRaises(lowerCAmelCase_   ,				tokenizer_r.encode_plus   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'							)

																		# Pair input
																		self.assertRaises(
																		    lowerCAmelCase_   ,				tokenizer_r.batch_encode_plus   ,				lowerCAmelCase_   ,				max_length=lowerCAmelCase_   ,				padding='max_length'   ,				)







						def 			A      (    self							)		->							Dict:
										_SCREAMING_SNAKE_CASE   :		Dict       		=    GPTaTokenizer.from_pretrained(self.tmpdirname   ,				pad_token='<pad>'							)

										# Simple input
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    'This is a simple input'
										_SCREAMING_SNAKE_CASE   :		Dict       		=    ['This is a simple input looooooooong', 'This is a simple input']
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    ('This is a simple input', 'This is a pair')
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    [
										    ('This is a simple input loooooong', 'This is a simple input'),
										    ('This is a simple pair loooooong', 'This is a simple pair'),
										]

										_SCREAMING_SNAKE_CASE   :		str       		=    tokenizer.pad_token_id

										_SCREAMING_SNAKE_CASE   :		List[str]       		=    tokenizer(lowerCAmelCase_   ,				padding='max_length'   ,				max_length=3_0   ,				return_tensors='np'							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    tokenizer(lowerCAmelCase_   ,				padding=lowerCAmelCase_   ,				truncate=lowerCAmelCase_   ,				return_tensors='np'							)
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    tokenizer(*lowerCAmelCase_   ,				padding='max_length'   ,				max_length=6_0   ,				return_tensors='np'							)
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    tokenizer(lowerCAmelCase_   ,				padding=lowerCAmelCase_   ,				truncate=lowerCAmelCase_   ,				return_tensors='np'							)

										# s
										# test single string max_length padding
										self.assertEqual(out_s['input_ids'].shape[-1]   ,				3_0							)
										self.assertTrue(pad_token_id in out_s['input_ids']							)
										self.assertTrue(0 in out_s['attention_mask']							)

										# s2
										# test automatic padding
										self.assertEqual(out_sa['input_ids'].shape[-1]   ,				3_3							)
										# long slice doesn't have padding
										self.assertFalse(pad_token_id in out_sa['input_ids'][0]							)
										self.assertFalse(0 in out_sa['attention_mask'][0]							)
										# short slice does have padding
										self.assertTrue(pad_token_id in out_sa['input_ids'][1]							)
										self.assertTrue(0 in out_sa['attention_mask'][1]							)

										# p
										# test single pair max_length padding
										self.assertEqual(out_p['input_ids'].shape[-1]   ,				6_0							)
										self.assertTrue(pad_token_id in out_p['input_ids']							)
										self.assertTrue(0 in out_p['attention_mask']							)

										# p2
										# test automatic padding pair
										self.assertEqual(out_pa['input_ids'].shape[-1]   ,				5_2							)
										# long slice pair doesn't have padding
										self.assertFalse(pad_token_id in out_pa['input_ids'][0]							)
										self.assertFalse(0 in out_pa['attention_mask'][0]							)
										# short slice pair does have padding
										self.assertTrue(pad_token_id in out_pa['input_ids'][1]							)
										self.assertTrue(0 in out_pa['attention_mask'][1]							)







						def 			A      (    self							)		->							int:
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    '$$$'
										_SCREAMING_SNAKE_CASE   :		Dict       		=    GPTaTokenizer.from_pretrained(self.tmpdirname   ,				bos_token=lowerCAmelCase_   ,				add_bos_token=lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    'This is a simple input'
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    ['This is a simple input 1', 'This is a simple input 2']

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    tokenizer.bos_token_id

										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    tokenizer(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		int       		=    tokenizer(lowerCAmelCase_							)

										self.assertEqual(out_s.input_ids[0]   ,				lowerCAmelCase_							)
										self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

										_SCREAMING_SNAKE_CASE   :		Any       		=    tokenizer.decode(out_s.input_ids							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    tokenizer.batch_decode(out_sa.input_ids							)

										self.assertEqual(decode_s.split()[0]   ,				lowerCAmelCase_							)
										self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)







						def 			A      (    self							)		->							List[str]:
										pass







						def 			A      (    self							)		->							Optional[Any]:
										# TODO: change to self.get_tokenizers() when the fast version is implemented
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    [self.get_tokenizer(do_lower_case=lowerCAmelCase_   ,				add_bos_token=lowerCAmelCase_							)]
										for tokenizer in tokenizers:
														with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
																		_SCREAMING_SNAKE_CASE   :		List[Any]       		=    'Encode this.'
																		_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    'This one too please.'
																		_SCREAMING_SNAKE_CASE   :		int       		=    tokenizer.encode(lowerCAmelCase_   ,				add_special_tokens=lowerCAmelCase_							)
																		encoded_sequence += tokenizer.encode(lowerCAmelCase_   ,				add_special_tokens=lowerCAmelCase_							)
																		_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    tokenizer.encode_plus(
																		    lowerCAmelCase_   ,				lowerCAmelCase_   ,				add_special_tokens=lowerCAmelCase_   ,				return_special_tokens_mask=lowerCAmelCase_   ,				)
																		_SCREAMING_SNAKE_CASE   :		str       		=    encoded_sequence_dict['input_ids']
																		_SCREAMING_SNAKE_CASE   :		str       		=    encoded_sequence_dict['special_tokens_mask']
																		self.assertEqual(len(lowerCAmelCase_							)   ,				len(lowerCAmelCase_							)							)

																		_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    [
																		    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCAmelCase_							)
																		]
																		_SCREAMING_SNAKE_CASE   :		Any       		=    [x for x in filtered_sequence if x is not None]
																		self.assertEqual(lowerCAmelCase_   ,				lowerCAmelCase_							)







@require_tokenizers
class _lowerCAmelCase						(    unittest.TestCase       ):







						def 			A      (    self							)		->							Any:
										# More context:
										# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
										# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
										# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

										_SCREAMING_SNAKE_CASE   :		Any       		=    AutoTokenizer.from_pretrained('facebook/opt-350m'   ,				from_slow=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    'A photo of a cat'

										_SCREAMING_SNAKE_CASE   :		str       		=    tokenizer.encode(
										    lowerCAmelCase_   ,				)
										self.assertEqual(lowerCAmelCase_   ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]							)
										tokenizer.save_pretrained('test_opt'							)

										_SCREAMING_SNAKE_CASE   :		str       		=    AutoTokenizer.from_pretrained('./test_opt'							)
										_SCREAMING_SNAKE_CASE   :		Any       		=    tokenizer.encode(
										    lowerCAmelCase_   ,				)
										self.assertEqual(lowerCAmelCase_   ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]							)







						def 			A      (    self							)		->							Dict:
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    AutoTokenizer.from_pretrained('facebook/opt-350m'   ,				use_slow=lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    'A photo of a cat'

										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    tokenizer.encode(
										    lowerCAmelCase_   ,				)
										# Same as above
										self.assertEqual(lowerCAmelCase_   ,				[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]							)







						@unittest.skip('This test is failing because of a bug in the fast tokenizer'							)
						def 			A      (    self							)		->							Optional[Any]:
										_SCREAMING_SNAKE_CASE   :		str       		=    AutoTokenizer.from_pretrained('facebook/opt-350m'   ,				from_slow=lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    'bos'
										_SCREAMING_SNAKE_CASE   :		Dict       		=    tokenizer.get_vocab()['bos']

										_SCREAMING_SNAKE_CASE   :		Dict       		=    'A photo of a cat'
										_SCREAMING_SNAKE_CASE   :		int       		=    tokenizer.encode(
										    lowerCAmelCase_   ,				)
										# We changed the bos token
										self.assertEqual(lowerCAmelCase_   ,				[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]							)
										tokenizer.save_pretrained('./tok'							)
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    AutoTokenizer.from_pretrained('./tok'							)
										self.assertTrue(tokenizer.is_fast							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    tokenizer.encode(
										    lowerCAmelCase_   ,				)
										self.assertEqual(lowerCAmelCase_   ,				[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]							)









"""simple docstring"""
lowerCAmelCase__    						=			'\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

lowerCAmelCase__    						=			[{'type': 'code', 'content': INSTALL_CONTENT}]
lowerCAmelCase__    						=			{
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}

