
import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
					from transformers import SpeechaTextFeatureExtractor

SCREAMING_SNAKE_CASE							:Any							 =				random.Random()


def 						UpperCAmelCase			(	a_		,   a_=1.0		,   a_=None		,   a_=None						)		->	Dict:







				"""simple docstring"""



				if rng is None:
								__A   			= global_rng

				__A   			= []
				for batch_idx in range(shape[0]						):
								values.append([]						)
								for _ in range(shape[1]						):
												values[-1].append(rng.random() * scale						)

				return values





@require_torch
@require_torchaudio
class 							UpperCAmelCase      (  unittest.TestCase  ):




	'''simple docstring'''





	def __init__(       self					: Dict							,A					: int							,A					: List[str]=7							,A					: Tuple=4_00							,A					: Union[str, Any]=20_00							,A					: Any=24							,A					: Union[str, Any]=24							,A					: Tuple=0.0							,A					: str=1_60_00							,A					: List[Any]=True							,A					: List[Any]=True							,):
					__A   			= parent
					__A   			= batch_size
					__A   			= min_seq_length
					__A   			= max_seq_length
					__A   			= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
					__A   			= feature_size
					__A   			= num_mel_bins
					__A   			= padding_value
					__A   			= sampling_rate
					__A   			= return_attention_mask
					__A   			= do_normalize






	def 				UpperCamelCase_  (       self					: Optional[int] ):
					return {
					    "feature_size": self.feature_size,
					    "num_mel_bins": self.num_mel_bins,
					    "padding_value": self.padding_value,
					    "sampling_rate": self.sampling_rate,
					    "return_attention_mask": self.return_attention_mask,
					    "do_normalize": self.do_normalize,
					}







	def 				UpperCamelCase_  (       self					: int							,A					: int=False							,A					: str=False ):




					def _flatten(A					: Any ):
									return list(itertools.chain(*lowerCAmelCase__ ) )

					if equal_length:
									__A   			= [floats_list((self.max_seq_length, self.feature_size) ) for _ in range(self.batch_size )]
					else:
									# make sure that inputs increase in size
									__A   			= [
									    floats_list((x, self.feature_size) )
									    for x in range(self.min_seq_length							,self.max_seq_length							,self.seq_length_diff )
									]
					if numpify:
									__A   			= [np.asarray(lowerCAmelCase__ ) for x in speech_inputs]
					return speech_inputs





@require_torch
@require_torchaudio
class 							UpperCAmelCase      (  UpperCamelCase__			,			unittest.TestCase  ):




	'''simple docstring'''





	snake_case_					    =     SpeechaTextFeatureExtractor if is_speech_available() else None






	def 				UpperCamelCase_  (       self					: Dict ):
					__A   			= SpeechaTextFeatureExtractionTester(self )






	def 				UpperCamelCase_  (       self					: List[str]							,A					: Union[str, Any] ):
					self.assertTrue(np.all(np.mean(lowerCAmelCase__							,axis=0 ) < 1E-3 ) )
					self.assertTrue(np.all(np.abs(np.var(lowerCAmelCase__							,axis=0 ) - 1 ) < 1E-3 ) )






	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					# create three inputs of length 800, 1000, and 1200
					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]
					__A   			= [np.asarray(lowerCAmelCase__ ) for speech_input in speech_inputs]

					# Test feature size
					__A   			= feature_extractor(lowerCAmelCase__							,padding=lowerCAmelCase__							,return_tensors="np" ).input_features
					self.assertTrue(input_features.ndim == 3 )
					self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size )

					# Test not batched input
					__A   			= feature_extractor(speech_inputs[0]							,return_tensors="np" ).input_features
					__A   			= feature_extractor(np_speech_inputs[0]							,return_tensors="np" ).input_features
					self.assertTrue(np.allclose(lowerCAmelCase__							,lowerCAmelCase__							,atol=1E-3 ) )

					# Test batched
					__A   			= feature_extractor(lowerCAmelCase__							,return_tensors="np" ).input_features
					__A   			= feature_extractor(lowerCAmelCase__							,return_tensors="np" ).input_features
					for enc_seq_a, enc_seq_a in zip(lowerCAmelCase__							,lowerCAmelCase__ ):
									self.assertTrue(np.allclose(lowerCAmelCase__							,lowerCAmelCase__							,atol=1E-3 ) )

					# Test 2-D numpy arrays are batched.
					__A   			= [floats_list((1, x) )[0] for x in (8_00, 8_00, 8_00)]
					__A   			= np.asarray(lowerCAmelCase__ )
					__A   			= feature_extractor(lowerCAmelCase__							,return_tensors="np" ).input_features
					__A   			= feature_extractor(lowerCAmelCase__							,return_tensors="np" ).input_features
					for enc_seq_a, enc_seq_a in zip(lowerCAmelCase__							,lowerCAmelCase__ ):
									self.assertTrue(np.allclose(lowerCAmelCase__							,lowerCAmelCase__							,atol=1E-3 ) )






	def 				UpperCamelCase_  (       self					: Any ):
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]

					__A   			= ["longest", "max_length", "do_not_pad"]
					__A   			= [None, 16, None]
					for max_length, padding in zip(lowerCAmelCase__							,lowerCAmelCase__ ):
									__A   			= feature_extractor(
									    lowerCAmelCase__							,padding=lowerCAmelCase__							,max_length=lowerCAmelCase__							,return_attention_mask=lowerCAmelCase__ )
									__A   			= inputs.input_features
									__A   			= inputs.attention_mask
									__A   			= [np.sum(lowerCAmelCase__ ) for x in attention_mask]

									self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]] )
									self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]] )
									self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]] )






	def 				UpperCamelCase_  (       self					: int ):
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]

					__A   			= ["longest", "max_length", "do_not_pad"]
					__A   			= [None, 16, None]
					for max_length, padding in zip(lowerCAmelCase__							,lowerCAmelCase__ ):
									__A   			= feature_extractor(
									    lowerCAmelCase__							,max_length=lowerCAmelCase__							,padding=lowerCAmelCase__							,return_tensors="np"							,return_attention_mask=lowerCAmelCase__ )
									__A   			= inputs.input_features
									__A   			= inputs.attention_mask
									__A   			= [np.sum(lowerCAmelCase__ ) for x in attention_mask]

									self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]] )
									self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1E-6 )
									self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]] )
									self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1E-6 )
									self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]] )






	def 				UpperCamelCase_  (       self					: int ):
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]
					__A   			= feature_extractor(
					    lowerCAmelCase__							,padding="max_length"							,max_length=4							,truncation=lowerCAmelCase__							,return_tensors="np"							,return_attention_mask=lowerCAmelCase__							,)
					__A   			= inputs.input_features
					__A   			= inputs.attention_mask
					__A   			= np.sum(attention_mask == 1							,axis=1 )

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]] )
					self._check_zero_mean_unit_variance(input_features[1] )
					self._check_zero_mean_unit_variance(input_features[2] )






	def 				UpperCamelCase_  (       self					: List[str] ):
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]
					__A   			= feature_extractor(
					    lowerCAmelCase__							,padding="longest"							,max_length=4							,truncation=lowerCAmelCase__							,return_tensors="np"							,return_attention_mask=lowerCAmelCase__							,)
					__A   			= inputs.input_features
					__A   			= inputs.attention_mask
					__A   			= np.sum(attention_mask == 1							,axis=1 )

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]] )
					self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]] )
					self._check_zero_mean_unit_variance(input_features[2] )

					# make sure that if max_length < longest -> then pad to max_length
					self.assertEqual(input_features.shape							,(3, 4, 24) )

					__A   			= [floats_list((1, x) )[0] for x in range(8_00							,14_00							,2_00 )]
					__A   			= feature_extractor(
					    lowerCAmelCase__							,padding="longest"							,max_length=16							,truncation=lowerCAmelCase__							,return_tensors="np"							,return_attention_mask=lowerCAmelCase__							,)
					__A   			= inputs.input_features
					__A   			= inputs.attention_mask
					__A   			= np.sum(attention_mask == 1							,axis=1 )

					self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]] )
					self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]] )
					self._check_zero_mean_unit_variance(input_features[2] )

					# make sure that if max_length < longest -> then pad to max_length
					self.assertEqual(input_features.shape							,(3, 6, 24) )






	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					import torch

					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= np.random.rand(1_00							,32 ).astype(np.floataa )
					__A   			= np_speech_inputs.tolist()

					for inputs in [py_speech_inputs, np_speech_inputs]:
									__A   			= feature_extractor.pad([{"input_features": inputs}]							,return_tensors="np" )
									self.assertTrue(np_processed.input_features.dtype == np.floataa )
									__A   			= feature_extractor.pad([{"input_features": inputs}]							,return_tensors="pt" )
									self.assertTrue(pt_processed.input_features.dtype == torch.floataa )






	def 				UpperCamelCase_  (       self					: str							,A					: Tuple ):
					from datasets import load_dataset

					__A   			= load_dataset("hf-internal-testing/librispeech_asr_dummy"							,"clean"							,split="validation" )
					# automatic decoding with librispeech
					__A   			= ds.sort("id" ).select(range(lowerCAmelCase__ ) )[:num_samples]["audio"]

					return [x["array"] for x in speech_samples]







	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					__A   			= np.array([
					    -1.57_45, -1.77_13, -1.70_20, -1.60_69, -1.22_50, -1.11_05, -0.90_72, -0.82_41,
					    -1.23_10, -0.80_98, -0.33_20, -0.41_01, -0.79_85, -0.49_96, -0.82_13, -0.91_28,
					    -1.04_20, -1.12_86, -1.04_40, -0.79_99, -0.84_05, -1.22_75, -1.54_43, -1.46_25,
					] )
					# fmt: on

					__A   			= self._load_datasamples(1 )
					__A   			= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict() )
					__A   			= feature_extractor(lowerCAmelCase__							,return_tensors="pt" ).input_features
					self.assertEquals(input_features.shape							,(1, 5_84, 24) )
					self.assertTrue(np.allclose(input_features[0, 0, :30]							,lowerCAmelCase__							,atol=1E-4 ) )







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase	:	int		     =       logging.get_logger(__name__)

UpperCAmelCase	:	Optional[Any]		     =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase	:	List[Any]		     =       [
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

UpperCAmelCase	:	Optional[int]		     =       {
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					str						=			VOCAB_FILES_NAMES
      _lowercase				:					List[Any]						=			PRETRAINED_VOCAB_FILES_MAP
      _lowercase				:					Dict						=			PRETRAINED_INIT_CONFIGURATION
      _lowercase				:					Union[str, Any]						=			FunnelTokenizer
      _lowercase				:					Any						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase				:					int						=			2







      def __init__(    self		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=True		,      lowerCAmelCase__="<unk>"		,      lowerCAmelCase__="<sep>"		,      lowerCAmelCase__="<pad>"		,      lowerCAmelCase__="<cls>"		,      lowerCAmelCase__="<mask>"		,      lowerCAmelCase__="<s>"		,      lowerCAmelCase__="</s>"		,      lowerCAmelCase__=True		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      lowerCAmelCase__="##"		,      **lowerCAmelCase__		,      )   ->						Optional[int]:



        '''simple docstring'''




        super().__init__(
            lowerCAmelCase__		,      tokenizer_file=lowerCAmelCase__		,      do_lower_case=lowerCAmelCase__		,      unk_token=lowerCAmelCase__		,      sep_token=lowerCAmelCase__		,      pad_token=lowerCAmelCase__		,      cls_token=lowerCAmelCase__		,      mask_token=lowerCAmelCase__		,      bos_token=lowerCAmelCase__		,      eos_token=lowerCAmelCase__		,      clean_text=lowerCAmelCase__		,      tokenize_chinese_chars=lowerCAmelCase__		,      strip_accents=lowerCAmelCase__		,      wordpieces_prefix=lowerCAmelCase__		,      **lowerCAmelCase__		,      )

        a__	:					Optional[Any]										=json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
        if (
            normalizer_state.get("lowercase"		,      lowerCAmelCase__							) != do_lower_case
            or normalizer_state.get("strip_accents"		,      lowerCAmelCase__							) != strip_accents
            or normalizer_state.get("handle_chinese_chars"		,      lowerCAmelCase__							) != tokenize_chinese_chars
        ):
          a__	:					List[str]										=getattr(lowerCAmelCase__		,      normalizer_state.pop("type"							)							)
          a__	:					Union[str, Any]										=do_lower_case
          a__	:					Any										=strip_accents
          a__	:					Optional[Any]										=tokenize_chinese_chars
          a__	:					Dict										=normalizer_class(**lowerCAmelCase__							)

        a__	:					Any										=do_lower_case







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__=None							)   ->						str:



        '''simple docstring'''




        a__	:					Dict										=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						List[int]:



        '''simple docstring'''




        a__	:					Optional[int]										=[self.sep_token_id]
        a__	:					Union[str, Any]										=[self.cls_token_id]
        if token_ids_a is None:
          return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0]
        return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						Tuple[str]:



        '''simple docstring'''




        a__	:					Tuple										=self._tokenizer.model.save(lowerCAmelCase__		,      name=lowerCAmelCase__							)
        return tuple(lowerCAmelCase__							)






