



"""simple docstring"""
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							_UpperCAmelCase							( a		,unittest.TestCase      ):





     '''simple docstring'''



     a__									=GPTaTokenizer
     a__									=GPTaTokenizerFast
     a__									=True
     a__									={'''add_prefix_space''': True}
     a__									=False





     def  __lowerCAmelCase	(					self		)							->					Optional[Any]:
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _UpperCAmelCase							:  Union[str, Any]											=    [
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''\u0120''',
             '''\u0120l''',
             '''\u0120n''',
             '''\u0120lo''',
             '''\u0120low''',
             '''er''',
             '''\u0120lowest''',
             '''\u0120newer''',
             '''\u0120wider''',
             '''<unk>''',
             '''<|endoftext|>''',
         ]
         _UpperCAmelCase							:  List[str]											=    dict(zip(A       ,	range(len(A		)		)		)		)
         _UpperCAmelCase							:  int											=    ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
         _UpperCAmelCase							:  Dict											=    {'''unk_token''': '''<unk>'''}

         _UpperCAmelCase							:  Tuple											=    os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['''vocab_file''']		)
         _UpperCAmelCase							:  Dict											=    os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['''merges_file''']		)
         with open(self.vocab_file       ,	'''w'''       ,	encoding='''utf-8'''		) as fp:
             fp.write(json.dumps(A		) + '''\n'''		)
         with open(self.merges_file       ,	'''w'''       ,	encoding='''utf-8'''		) as fp:
             fp.write('''\n'''.join(A		)		)





     def  __lowerCAmelCase	(					self       ,	**A		)							->					Dict:
         kwargs.update(self.special_tokens_map		)
         return GPTaTokenizer.from_pretrained(self.tmpdirname       ,	**A		)





     def  __lowerCAmelCase	(					self       ,	**A		)							->					List[str]:
         kwargs.update(self.special_tokens_map		)
         return GPTaTokenizerFast.from_pretrained(self.tmpdirname       ,	**A		)





     def  __lowerCAmelCase	(					self       ,	A		)							->					List[Any]:
         _UpperCAmelCase							:  List[Any]											=    '''lower newer'''
         _UpperCAmelCase							:  List[str]											=    '''lower newer'''
         return input_text, output_text





     def  __lowerCAmelCase	(					self		)							->					str:
         _UpperCAmelCase							:  Union[str, Any]											=    GPTaTokenizer(self.vocab_file       ,	self.merges_file       ,	**self.special_tokens_map		)
         _UpperCAmelCase							:  Optional[Any]											=    '''lower newer'''
         _UpperCAmelCase							:  str											=    ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
         _UpperCAmelCase							:  List[str]											=    tokenizer.tokenize(A       ,	add_prefix_space=A		)
         self.assertListEqual(A       ,	A		)

         _UpperCAmelCase							:  Tuple											=    tokens + [tokenizer.unk_token]
         _UpperCAmelCase							:  Optional[int]											=    [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A		)       ,	A		)





     def  __lowerCAmelCase	(					self		)							->					str:
         if not self.test_rust_tokenizer:
             return

         _UpperCAmelCase							:  Union[str, Any]											=    self.get_tokenizer()
         _UpperCAmelCase							:  Union[str, Any]											=    self.get_rust_tokenizer(add_prefix_space=A		)

         _UpperCAmelCase							:  Any											=    '''lower newer'''

         # Testing tokenization
         _UpperCAmelCase							:  int											=    tokenizer.tokenize(A       ,	add_prefix_space=A		)
         _UpperCAmelCase							:  List[str]											=    rust_tokenizer.tokenize(A		)
         self.assertListEqual(A       ,	A		)

         # Testing conversion to ids without special tokens
         _UpperCAmelCase							:  Optional[int]											=    tokenizer.encode(A       ,	add_special_tokens=A       ,	add_prefix_space=A		)
         _UpperCAmelCase							:  List[str]											=    rust_tokenizer.encode(A       ,	add_special_tokens=A		)
         self.assertListEqual(A       ,	A		)

         # Testing conversion to ids with special tokens
         _UpperCAmelCase							:  Optional[Any]											=    self.get_rust_tokenizer(add_prefix_space=A		)
         _UpperCAmelCase							:  List[str]											=    tokenizer.encode(A       ,	add_prefix_space=A		)
         _UpperCAmelCase							:  Optional[int]											=    rust_tokenizer.encode(A		)
         self.assertListEqual(A       ,	A		)

         # Testing the unknown token
         _UpperCAmelCase							:  List[str]											=    tokens + [rust_tokenizer.unk_token]
         _UpperCAmelCase							:  List[str]											=    [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A		)       ,	A		)





     def  __lowerCAmelCase	(					self       ,	*A       ,	**A		)							->					Tuple:
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass





     def  __lowerCAmelCase	(					self       ,	A=1_5		)							->					List[Any]:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                 _UpperCAmelCase							:  Tuple											=    self.rust_tokenizer_class.from_pretrained(A       ,	**A		)

                 # Simple input
                 _UpperCAmelCase							:  Optional[int]											=    '''This is a simple input'''
                 _UpperCAmelCase							:  Optional[Any]											=    ['''This is a simple input 1''', '''This is a simple input 2''']
                 _UpperCAmelCase							:  Dict											=    ('''This is a simple input''', '''This is a pair''')
                 _UpperCAmelCase							:  Optional[int]											=    [
                     ('''This is a simple input 1''', '''This is a simple input 2'''),
                     ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                 ]

                 # Simple input tests
                 self.assertRaises(A       ,	tokenizer_r.encode       ,	A       ,	max_length=A       ,	padding='''max_length'''		)

                 # Simple input
                 self.assertRaises(A       ,	tokenizer_r.encode_plus       ,	A       ,	max_length=A       ,	padding='''max_length'''		)

                 # Simple input
                 self.assertRaises(
                     A       ,	tokenizer_r.batch_encode_plus       ,	A       ,	max_length=A       ,	padding='''max_length'''       ,	)

                 # Pair input
                 self.assertRaises(A       ,	tokenizer_r.encode       ,	A       ,	max_length=A       ,	padding='''max_length'''		)

                 # Pair input
                 self.assertRaises(A       ,	tokenizer_r.encode_plus       ,	A       ,	max_length=A       ,	padding='''max_length'''		)

                 # Pair input
                 self.assertRaises(
                     A       ,	tokenizer_r.batch_encode_plus       ,	A       ,	max_length=A       ,	padding='''max_length'''       ,	)





     def  __lowerCAmelCase	(					self		)							->					Optional[Any]:
         _UpperCAmelCase							:  str											=    GPTaTokenizer.from_pretrained(self.tmpdirname       ,	pad_token='''<pad>'''		)

         # Simple input
         _UpperCAmelCase							:  Optional[int]											=    '''This is a simple input'''
         _UpperCAmelCase							:  List[str]											=    ['''This is a simple input looooooooong''', '''This is a simple input''']
         _UpperCAmelCase							:  List[Any]											=    ('''This is a simple input''', '''This is a pair''')
         _UpperCAmelCase							:  List[str]											=    [
             ('''This is a simple input loooooong''', '''This is a simple input'''),
             ('''This is a simple pair loooooong''', '''This is a simple pair'''),
         ]

         _UpperCAmelCase							:  int											=    tokenizer.pad_token_id

         _UpperCAmelCase							:  str											=    tokenizer(A       ,	padding='''max_length'''       ,	max_length=3_0       ,	return_tensors='''np'''		)
         _UpperCAmelCase							:  List[str]											=    tokenizer(A       ,	padding=A       ,	truncate=A       ,	return_tensors='''np'''		)
         _UpperCAmelCase							:  List[str]											=    tokenizer(*A       ,	padding='''max_length'''       ,	max_length=6_0       ,	return_tensors='''np'''		)
         _UpperCAmelCase							:  Any											=    tokenizer(A       ,	padding=A       ,	truncate=A       ,	return_tensors='''np'''		)

         # s
         # test single string max_length padding
         self.assertEqual(out_s['''input_ids'''].shape[-1]       ,	3_0		)
         self.assertTrue(pad_token_id in out_s['''input_ids''']		)
         self.assertTrue(0 in out_s['''attention_mask''']		)

         # s2
         # test automatic padding
         self.assertEqual(out_sa['''input_ids'''].shape[-1]       ,	3_3		)
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]		)
         self.assertFalse(0 in out_sa['''attention_mask'''][0]		)
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]		)
         self.assertTrue(0 in out_sa['''attention_mask'''][1]		)

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['''input_ids'''].shape[-1]       ,	6_0		)
         self.assertTrue(pad_token_id in out_p['''input_ids''']		)
         self.assertTrue(0 in out_p['''attention_mask''']		)

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['''input_ids'''].shape[-1]       ,	5_2		)
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]		)
         self.assertFalse(0 in out_pa['''attention_mask'''][0]		)
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]		)
         self.assertTrue(0 in out_pa['''attention_mask'''][1]		)





     def  __lowerCAmelCase	(					self		)							->					Any:
         _UpperCAmelCase							:  Tuple											=    '''$$$'''
         _UpperCAmelCase							:  Tuple											=    GPTaTokenizer.from_pretrained(self.tmpdirname       ,	bos_token=A       ,	add_bos_token=A		)

         _UpperCAmelCase							:  Optional[Any]											=    '''This is a simple input'''
         _UpperCAmelCase							:  Tuple											=    ['''This is a simple input 1''', '''This is a simple input 2''']

         _UpperCAmelCase							:  str											=    tokenizer.bos_token_id

         _UpperCAmelCase							:  List[Any]											=    tokenizer(A		)
         _UpperCAmelCase							:  Union[str, Any]											=    tokenizer(A		)

         self.assertEqual(out_s.input_ids[0]       ,	A		)
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

         _UpperCAmelCase							:  Dict											=    tokenizer.decode(out_s.input_ids		)
         _UpperCAmelCase							:  Tuple											=    tokenizer.batch_decode(out_sa.input_ids		)

         self.assertEqual(decode_s.split()[0]       ,	A		)
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)





     def  __lowerCAmelCase	(					self		)							->					Tuple:
         pass





     def  __lowerCAmelCase	(					self		)							->					List[Any]:
         # TODO: change to self.get_tokenizers() when the fast version is implemented
         _UpperCAmelCase							:  Any											=    [self.get_tokenizer(do_lower_case=A       ,	add_bos_token=A		)]
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'		):
                 _UpperCAmelCase							:  List[str]											=    '''Encode this.'''
                 _UpperCAmelCase							:  List[Any]											=    '''This one too please.'''
                 _UpperCAmelCase							:  str											=    tokenizer.encode(A       ,	add_special_tokens=A		)
                 encoded_sequence += tokenizer.encode(A       ,	add_special_tokens=A		)
                 _UpperCAmelCase							:  List[Any]											=    tokenizer.encode_plus(
                     A       ,	A       ,	add_special_tokens=A       ,	return_special_tokens_mask=A       ,	)
                 _UpperCAmelCase							:  Optional[int]											=    encoded_sequence_dict['''input_ids''']
                 _UpperCAmelCase							:  str											=    encoded_sequence_dict['''special_tokens_mask''']
                 self.assertEqual(len(A		)       ,	len(A		)		)

                 _UpperCAmelCase							:  Union[str, Any]											=    [
                     (x if not special_tokens_mask[i] else None) for i, x in enumerate(A		)
                 ]
                 _UpperCAmelCase							:  Optional[int]											=    [x for x in filtered_sequence if x is not None]
                 self.assertEqual(A       ,	A		)



@require_tokenizers
class 							_UpperCAmelCase							( unittest.TestCase      ):





     '''simple docstring'''



     def  __lowerCAmelCase	(					self		)							->					int:
         # More context:
         # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
         # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
         # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

         _UpperCAmelCase							:  Dict											=    AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,	from_slow=A		)
         _UpperCAmelCase							:  Tuple											=    '''A photo of a cat'''

         _UpperCAmelCase							:  str											=    tokenizer.encode(
             A       ,	)
         self.assertEqual(A       ,	[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]		)
         tokenizer.save_pretrained('''test_opt'''		)

         _UpperCAmelCase							:  Union[str, Any]											=    AutoTokenizer.from_pretrained('''./test_opt'''		)
         _UpperCAmelCase							:  Dict											=    tokenizer.encode(
             A       ,	)
         self.assertEqual(A       ,	[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]		)





     def  __lowerCAmelCase	(					self		)							->					Optional[Any]:
         _UpperCAmelCase							:  Dict											=    AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,	use_slow=A		)
         _UpperCAmelCase							:  Optional[Any]											=    '''A photo of a cat'''

         _UpperCAmelCase							:  Union[str, Any]											=    tokenizer.encode(
             A       ,	)
         # Same as above
         self.assertEqual(A       ,	[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]		)





     @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''		)
     def  __lowerCAmelCase	(					self		)							->					Optional[Any]:
         _UpperCAmelCase							:  int											=    AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,	from_slow=A		)

         _UpperCAmelCase							:  Optional[Any]											=    '''bos'''
         _UpperCAmelCase							:  Optional[int]											=    tokenizer.get_vocab()['''bos''']

         _UpperCAmelCase							:  Tuple											=    '''A photo of a cat'''
         _UpperCAmelCase							:  Optional[Any]											=    tokenizer.encode(
             A       ,	)
         # We changed the bos token
         self.assertEqual(A       ,	[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]		)
         tokenizer.save_pretrained('''./tok'''		)
         _UpperCAmelCase							:  List[str]											=    AutoTokenizer.from_pretrained('''./tok'''		)
         self.assertTrue(tokenizer.is_fast		)
         _UpperCAmelCase							:  List[str]											=    tokenizer.encode(
             A       ,	)
         self.assertEqual(A       ,	[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8]		)







"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow
@is_pipeline_test
@require_torch
class 							_UpperCAmelCase							( unittest.TestCase      ):





     '''simple docstring'''



     @require_torch
     def  __lowerCAmelCase	(					self		)							->					Any:
         _UpperCAmelCase							:  str											=    pipeline(
             task='''zero-shot-audio-classification'''       ,	model='''hf-internal-testing/tiny-clap-htsat-unfused'''		)
         _UpperCAmelCase							:  List[Any]											=    load_dataset('''ashraq/esc50'''		)
         _UpperCAmelCase							:  Optional[int]											=    dataset['''train''']['''audio'''][-1]['''array''']
         _UpperCAmelCase							:  str											=    audio_classifier(A       ,	candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner''']		)
         self.assertEqual(
             nested_simplify(A		)       ,	[{'''score''': 0.501, '''label''': '''Sound of a dog'''}, {'''score''': 0.499, '''label''': '''Sound of vaccum cleaner'''}]       ,	)





     @unittest.skip('''No models are available in TF'''		)
     def  __lowerCAmelCase	(					self		)							->					Union[str, Any]:
         pass





     @slow
     @require_torch
     def  __lowerCAmelCase	(					self		)							->					str:
         _UpperCAmelCase							:  Union[str, Any]											=    pipeline(
             task='''zero-shot-audio-classification'''       ,	model='''laion/clap-htsat-unfused'''       ,	)
         # This is an audio of a dog
         _UpperCAmelCase							:  List[Any]											=    load_dataset('''ashraq/esc50'''		)
         _UpperCAmelCase							:  Optional[int]											=    dataset['''train''']['''audio'''][-1]['''array''']
         _UpperCAmelCase							:  Any											=    audio_classifier(A       ,	candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner''']		)

         self.assertEqual(
             nested_simplify(A		)       ,	[
                 {'''score''': 0.999, '''label''': '''Sound of a dog'''},
                 {'''score''': 0.001, '''label''': '''Sound of vaccum cleaner'''},
             ]       ,	)

         _UpperCAmelCase							:  List[Any]											=    audio_classifier([audio] * 5       ,	candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner''']		)
         self.assertEqual(
             nested_simplify(A		)       ,	[
                 [
                     {'''score''': 0.999, '''label''': '''Sound of a dog'''},
                     {'''score''': 0.001, '''label''': '''Sound of vaccum cleaner'''},
                 ],
             ]
             * 5       ,	)
         _UpperCAmelCase							:  Tuple											=    audio_classifier(
             [audio] * 5       ,	candidate_labels=['''Sound of a dog''', '''Sound of vaccum cleaner''']       ,	batch_size=5		)
         self.assertEqual(
             nested_simplify(A		)       ,	[
                 [
                     {'''score''': 0.999, '''label''': '''Sound of a dog'''},
                     {'''score''': 0.001, '''label''': '''Sound of vaccum cleaner'''},
                 ],
             ]
             * 5       ,	)





     @unittest.skip('''No models are available in TF'''		)
     def  __lowerCAmelCase	(					self		)							->					int:
         pass


