

'''simple docstring'''

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class        A_ (				__lowerCAmelCase       ,   unittest.TestCase							):






     '''simple docstring'''



     UpperCAmelCase_						:	str  		=  CodeGenTokenizer
     UpperCAmelCase_						:	List[Any]  		=  CodeGenTokenizerFast
     UpperCAmelCase_						:	Dict  		=  True
     UpperCAmelCase_						:	Dict  		=  {"""add_prefix_space""": True}
     UpperCAmelCase_						:	Optional[Any]  		=  False


     def    UpperCAmelCase_       (		self :     Union[str, Any]       )						->	Any:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      UpperCAmelCase       :	int											= [
          'l',
          'o',
          'w',
          'e',
          'r',
          's',
          't',
          'i',
          'd',
          'n',
          '\u0120',
          '\u0120l',
          '\u0120n',
          '\u0120lo',
          '\u0120low',
          'er',
          '\u0120lowest',
          '\u0120newer',
          '\u0120wider',
          '<unk>',
          '<|endoftext|>',
      ]
      UpperCAmelCase       :	List[str]											= dict(zip(lowercase_     ,			range(len(lowercase_       )       )       )       )
      UpperCAmelCase       :	Optional[int]											= ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
      UpperCAmelCase       :	Tuple											= {'unk_token': '<unk>'}

      UpperCAmelCase       :	Optional[Any]											= os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES['vocab_file']       )
      UpperCAmelCase       :	Dict											= os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES['merges_file']       )
      with open(self.vocab_file     ,			'w'     ,			encoding='utf-8'       ) as fp:
       fp.write(json.dumps(lowercase_       ) + '\n'       )
      with open(self.merges_file     ,			'w'     ,			encoding='utf-8'       ) as fp:
       fp.write('\n'.join(lowercase_       )       )


     def    UpperCAmelCase_       (		self :     str     ,			**lowercase_ :     Optional[int]       )						->	Optional[int]:
      kwargs.update(self.special_tokens_map       )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,			**lowercase_       )


     def    UpperCAmelCase_       (		self :     Union[str, Any]     ,			**lowercase_ :     Dict       )						->	Tuple:
      kwargs.update(self.special_tokens_map       )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,			**lowercase_       )


     def    UpperCAmelCase_       (		self :     int     ,			lowercase_ :     Optional[int]       )						->	Optional[int]:
      UpperCAmelCase       :	List[str]											= 'lower newer'
      UpperCAmelCase       :	str											= 'lower newer'
      return input_text, output_text


     def    UpperCAmelCase_       (		self :     Optional[Any]       )						->	Union[str, Any]:
      UpperCAmelCase       :	List[Any]											= CodeGenTokenizer(self.vocab_file     ,			self.merges_file     ,			**self.special_tokens_map       )
      UpperCAmelCase       :	Union[str, Any]											= 'lower newer'
      UpperCAmelCase       :	Any											= ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
      UpperCAmelCase       :	Tuple											= tokenizer.tokenize(lowercase_     ,			add_prefix_space=lowercase_       )
      self.assertListEqual(lowercase_     ,			lowercase_       )

      UpperCAmelCase       :	int											= tokens + [tokenizer.unk_token]
      UpperCAmelCase       :	Any											= [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_       )     ,			lowercase_       )


     def    UpperCAmelCase_       (		self :     Any       )						->	Tuple:





      if not self.test_rust_tokenizer:
       return

      UpperCAmelCase       :	Union[str, Any]											= self.get_tokenizer()
      UpperCAmelCase       :	int											= self.get_rust_tokenizer(add_prefix_space=lowercase_       )

      UpperCAmelCase       :	Any											= 'lower newer'

      # Testing tokenization
      UpperCAmelCase       :	Any											= tokenizer.tokenize(lowercase_     ,			add_prefix_space=lowercase_       )
      UpperCAmelCase       :	Tuple											= rust_tokenizer.tokenize(lowercase_       )
      self.assertListEqual(lowercase_     ,			lowercase_       )

      # Testing conversion to ids without special tokens
      UpperCAmelCase       :	Optional[Any]											= tokenizer.encode(lowercase_     ,			add_special_tokens=lowercase_     ,			add_prefix_space=lowercase_       )
      UpperCAmelCase       :	Dict											= rust_tokenizer.encode(lowercase_     ,			add_special_tokens=lowercase_       )
      self.assertListEqual(lowercase_     ,			lowercase_       )

      # Testing conversion to ids with special tokens
      UpperCAmelCase       :	Dict											= self.get_rust_tokenizer(add_prefix_space=lowercase_       )
      UpperCAmelCase       :	Tuple											= tokenizer.encode(lowercase_     ,			add_prefix_space=lowercase_       )
      UpperCAmelCase       :	Tuple											= rust_tokenizer.encode(lowercase_       )
      self.assertListEqual(lowercase_     ,			lowercase_       )

      # Testing the unknown token
      UpperCAmelCase       :	Any											= tokens + [rust_tokenizer.unk_token]
      UpperCAmelCase       :	Any											= [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_       )     ,			lowercase_       )


     def    UpperCAmelCase_       (		self :     int     ,			*lowercase_ :     List[Any]     ,			**lowercase_ :     Union[str, Any]       )						->	List[Any]:
      pass


     def    UpperCAmelCase_       (		self :     Optional[Any]     ,			lowercase_ :     List[Any]=15       )						->	int:





      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
        UpperCAmelCase       :	Optional[Any]											= self.rust_tokenizer_class.from_pretrained(lowercase_     ,			**lowercase_       )

        # Simple input
        UpperCAmelCase       :	Any											= 'This is a simple input'
        UpperCAmelCase       :	str											= ['This is a simple input 1', 'This is a simple input 2']
        UpperCAmelCase       :	Any											= ('This is a simple input', 'This is a pair')
        UpperCAmelCase       :	int											= [
            ('This is a simple input 1', 'This is a simple input 2'),
            ('This is a simple pair 1', 'This is a simple pair 2'),
        ]

        # Simple input tests
        self.assertRaises(lowercase_     ,			tokenizer_r.encode     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'       )

        # Simple input
        self.assertRaises(lowercase_     ,			tokenizer_r.encode_plus     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'       )

        # Simple input
        self.assertRaises(
            lowercase_     ,			tokenizer_r.batch_encode_plus     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'     ,			)

        # Pair input
        self.assertRaises(lowercase_     ,			tokenizer_r.encode     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'       )

        # Pair input
        self.assertRaises(lowercase_     ,			tokenizer_r.encode_plus     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'       )

        # Pair input
        self.assertRaises(
            lowercase_     ,			tokenizer_r.batch_encode_plus     ,			lowercase_     ,			max_length=lowercase_     ,			padding='max_length'     ,			)


     def    UpperCAmelCase_       (		self :     Any       )						->	str:
      UpperCAmelCase       :	str											= CodeGenTokenizer.from_pretrained(self.tmpdirname     ,			pad_token='<pad>'       )

      # Simple input
      UpperCAmelCase       :	str											= 'This is a simple input'
      UpperCAmelCase       :	List[Any]											= ['This is a simple input looooooooong', 'This is a simple input']
      UpperCAmelCase       :	int											= ('This is a simple input', 'This is a pair')
      UpperCAmelCase       :	str											= [
          ('This is a simple input loooooong', 'This is a simple input'),
          ('This is a simple pair loooooong', 'This is a simple pair'),
      ]

      UpperCAmelCase       :	List[Any]											= tokenizer.pad_token_id

      UpperCAmelCase       :	Optional[Any]											= tokenizer(lowercase_     ,			padding='max_length'     ,			max_length=30     ,			return_tensors='np'       )
      UpperCAmelCase       :	Union[str, Any]											= tokenizer(lowercase_     ,			padding=lowercase_     ,			truncate=lowercase_     ,			return_tensors='np'       )
      UpperCAmelCase       :	Union[str, Any]											= tokenizer(*lowercase_     ,			padding='max_length'     ,			max_length=60     ,			return_tensors='np'       )
      UpperCAmelCase       :	Optional[Any]											= tokenizer(lowercase_     ,			padding=lowercase_     ,			truncate=lowercase_     ,			return_tensors='np'       )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['input_ids'].shape[-1]     ,			30       )
      self.assertTrue(pad_token_id in out_s['input_ids']       )
      self.assertTrue(0 in out_s['attention_mask']       )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['input_ids'].shape[-1]     ,			33       )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['input_ids'][0]       )
      self.assertFalse(0 in out_sa['attention_mask'][0]       )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['input_ids'][1]       )
      self.assertTrue(0 in out_sa['attention_mask'][1]       )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['input_ids'].shape[-1]     ,			60       )
      self.assertTrue(pad_token_id in out_p['input_ids']       )
      self.assertTrue(0 in out_p['attention_mask']       )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['input_ids'].shape[-1]     ,			52       )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['input_ids'][0]       )
      self.assertFalse(0 in out_pa['attention_mask'][0]       )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['input_ids'][1]       )
      self.assertTrue(0 in out_pa['attention_mask'][1]       )


     def    UpperCAmelCase_       (		self :     List[str]       )						->	Tuple:
      UpperCAmelCase       :	Any											= '$$$'
      UpperCAmelCase       :	Union[str, Any]											= CodeGenTokenizer.from_pretrained(self.tmpdirname     ,			bos_token=lowercase_     ,			add_bos_token=lowercase_       )

      UpperCAmelCase       :	List[str]											= 'This is a simple input'
      UpperCAmelCase       :	List[Any]											= ['This is a simple input 1', 'This is a simple input 2']

      UpperCAmelCase       :	Tuple											= tokenizer.bos_token_id

      UpperCAmelCase       :	Optional[Any]											= tokenizer(lowercase_       )
      UpperCAmelCase       :	Dict											= tokenizer(lowercase_       )

      self.assertEqual(out_s.input_ids[0]     ,			lowercase_       )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

      UpperCAmelCase       :	str											= tokenizer.decode(out_s.input_ids       )
      UpperCAmelCase       :	List[str]											= tokenizer.batch_decode(out_sa.input_ids       )

      self.assertEqual(decode_s.split()[0]     ,			lowercase_       )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )


     @slow
     def    UpperCAmelCase_       (		self :     Union[str, Any]       )						->	Union[str, Any]:
      UpperCAmelCase       :	Optional[int]											= CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'       )

      UpperCAmelCase       :	int											= '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
      UpperCAmelCase       :	Tuple											= '\nif len_a > len_b:      result = a\nelse:      result = b'

      UpperCAmelCase       :	Dict											= tokenizer.encode(lowercase_       )
      UpperCAmelCase       :	Union[str, Any]											= ['^#', re.escape('<|endoftext|>'       ), '^\'\'\'', '^"""', '\n\n\n']
      UpperCAmelCase       :	List[str]											= tokenizer.decode(lowercase_     ,			truncate_before_pattern=lowercase_       )
      self.assertEqual(lowercase_     ,			lowercase_       )




     def    UpperCAmelCase_       (		self :     str       )						->	int:
      pass





"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__					       =	logging.get_logger(__name__)

lowercase__					       =	{
    'MIT/ast-finetuned-audioset-10-10-0.4593': (
        'https://huggingface.co/MIT/ast-finetuned-audioset-10-10-0.4593/resolve/main/config.json'
    ),
}




class    __snake_case       (     __lowerCAmelCase				):
      a__        =		"""audio-spectrogram-transformer"""



      def __init__(						self    ,			lowercase=7_68    ,			lowercase=12    ,			lowercase=12    ,			lowercase=30_72    ,			lowercase="gelu"    ,			lowercase=0.0    ,			lowercase=0.0    ,			lowercase=0.02    ,			lowercase=1e-12    ,			lowercase=16    ,			lowercase=True    ,			lowercase=10    ,			lowercase=10    ,			lowercase=10_24    ,			lowercase=1_28    ,			**lowercase    ,			)	->			str:





        '''simple docstring'''





        super().__init__(**lowercase)

        a__:     Any										=	hidden_size
        a__:     int										=	num_hidden_layers
        a__:     Union[str, Any]										=	num_attention_heads
        a__:     Any										=	intermediate_size
        a__:     Union[str, Any]										=	hidden_act
        a__:     int										=	hidden_dropout_prob
        a__:     str										=	attention_probs_dropout_prob
        a__:     str										=	initializer_range
        a__:     Tuple										=	layer_norm_eps
        a__:     Any										=	patch_size
        a__:     int										=	qkv_bias
        a__:     Optional[Any]										=	frequency_stride
        a__:     int										=	time_stride
        a__:     List[str]										=	max_length
        a__:     Tuple										=	num_mel_bins


