

'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_rembert import RemBertTokenizer
else:
  UpperCAmelCase :       str			   =     None

UpperCAmelCase :       Dict			   =     logging.get_logger(__name__)
UpperCAmelCase :       Optional[Any]			   =     {'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase :       List[str]			   =     {
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase :       Optional[int]			   =     {
    'google/rembert': 2_5_6,
}

UpperCAmelCase :       Any			   =     '▁'







class 						lowerCAmelCase__   (      a					):



 """simple docstring"""



 lowerCAmelCase__  					=      VOCAB_FILES_NAMES
 lowerCAmelCase__  					=      PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__  					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__  					=      RemBertTokenizer
 def __init__(		self					:       Any					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=None					,							__SCREAMING_SNAKE_CASE					:       int=None					,							__SCREAMING_SNAKE_CASE					:       int=True					,							__SCREAMING_SNAKE_CASE					:       List[Any]=True					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=False					,							__SCREAMING_SNAKE_CASE					:       List[str]="[CLS]"					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="[SEP]"					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="<unk>"					,							__SCREAMING_SNAKE_CASE					:       List[str]="[SEP]"					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="<pad>"					,							__SCREAMING_SNAKE_CASE					:       Any="[CLS]"					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="[MASK]"					,							**__SCREAMING_SNAKE_CASE					:       Union[str, Any]					,							)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else mask_token

      super().__init__(
          __SCREAMING_SNAKE_CASE					,							tokenizer_file=__SCREAMING_SNAKE_CASE					,							do_lower_case=__SCREAMING_SNAKE_CASE					,							remove_space=__SCREAMING_SNAKE_CASE					,							keep_accents=__SCREAMING_SNAKE_CASE					,							bos_token=__SCREAMING_SNAKE_CASE					,							eos_token=__SCREAMING_SNAKE_CASE					,							unk_token=__SCREAMING_SNAKE_CASE					,							sep_token=__SCREAMING_SNAKE_CASE					,							pad_token=__SCREAMING_SNAKE_CASE					,							cls_token=__SCREAMING_SNAKE_CASE					,							mask_token=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE					,							)

      __SCREAMING_SNAKE_CASE     =      do_lower_case
      __SCREAMING_SNAKE_CASE     =      remove_space
      __SCREAMING_SNAKE_CASE     =      keep_accents
      __SCREAMING_SNAKE_CASE     =      vocab_file
      __SCREAMING_SNAKE_CASE     =      False if not self.vocab_file else True
 def       UpperCAmelCase__		(		self					:       List[str]					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None			)  -> List[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
      __SCREAMING_SNAKE_CASE     =      [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep
 def       UpperCAmelCase__		(		self					:       Any					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None					,							__SCREAMING_SNAKE_CASE					:       bool = False			)  -> List[int]:


      """simple docstring"""




      if already_has_special_tokens:
           if token_ids_a is not None:
                raise ValueError(
                    """You should not supply a second sequence if the provided sequence of """
                    """ids is already formatted with special tokens for the model."""			)
           return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is not None:
           return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
      return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
 def       UpperCAmelCase__		(		self					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None			)  -> List[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
      __SCREAMING_SNAKE_CASE     =      [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]


 def       UpperCAmelCase__		(		self					:       Dict					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[str] = None			)  -> Tuple[str]:


      """simple docstring"""




      if not os.path.isdir(__SCREAMING_SNAKE_CASE			):
           logger.error("""Vocabulary path ({}) should be a directory""".format(__SCREAMING_SNAKE_CASE			)			)
           return
      __SCREAMING_SNAKE_CASE     =      os.path.join(
          __SCREAMING_SNAKE_CASE					,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

      if os.path.abspath(self.vocab_file			) != os.path.abspath(__SCREAMING_SNAKE_CASE			):
           copyfile(self.vocab_file					,							__SCREAMING_SNAKE_CASE			)

      return (out_vocab_file,)









'''simple docstring'''



import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
  from transformers import WhisperFeatureExtractor

if is_torch_available():
  import torch

UpperCAmelCase :       int			   =     random.Random()







def    a__       (  a__      ,    a__=1.0      ,    a__=None      ,    a__=None						):




     """simple docstring"""


     if rng is None:
          __SCREAMING_SNAKE_CASE     =      global_rng

     __SCREAMING_SNAKE_CASE     =      []
     for batch_idx in range(shape[0]						):
          values.append([]						)
          for _ in range(shape[1]						):
               values[-1].append(rng.random() * scale						)

     return values







@require_torch
@require_torchaudio
class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 def __init__(		self					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       str=7					,							__SCREAMING_SNAKE_CASE					:       List[str]=400					,							__SCREAMING_SNAKE_CASE					:       Any=2_000					,							__SCREAMING_SNAKE_CASE					:       List[str]=10					,							__SCREAMING_SNAKE_CASE					:       Optional[int]=160					,							__SCREAMING_SNAKE_CASE					:       List[str]=8					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=0.0					,							__SCREAMING_SNAKE_CASE					:       Dict=4_000					,							__SCREAMING_SNAKE_CASE					:       Optional[int]=False					,							__SCREAMING_SNAKE_CASE					:       List[Any]=True					,							)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      parent
      __SCREAMING_SNAKE_CASE     =      batch_size
      __SCREAMING_SNAKE_CASE     =      min_seq_length
      __SCREAMING_SNAKE_CASE     =      max_seq_length
      __SCREAMING_SNAKE_CASE     =      (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
      __SCREAMING_SNAKE_CASE     =      padding_value
      __SCREAMING_SNAKE_CASE     =      sampling_rate
      __SCREAMING_SNAKE_CASE     =      return_attention_mask
      __SCREAMING_SNAKE_CASE     =      do_normalize
      __SCREAMING_SNAKE_CASE     =      feature_size
      __SCREAMING_SNAKE_CASE     =      chunk_length
      __SCREAMING_SNAKE_CASE     =      hop_length
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Dict:


      """simple docstring"""




      return {
          "feature_size": self.feature_size,
          "hop_length": self.hop_length,
          "chunk_length": self.chunk_length,
          "padding_value": self.padding_value,
          "sampling_rate": self.sampling_rate,
          "return_attention_mask": self.return_attention_mask,
          "do_normalize": self.do_normalize,
      }


 def       UpperCAmelCase__		(		self					:       Union[str, Any]					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]=False					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]=False			)  -> Union[str, Any]:


      """simple docstring"""




      def _flatten(__SCREAMING_SNAKE_CASE					:       Dict			):
           return list(itertools.chain(*__SCREAMING_SNAKE_CASE			)			)

      if equal_length:
           __SCREAMING_SNAKE_CASE     =      [floats_list((self.max_seq_length, self.feature_size)			) for _ in range(self.batch_size			)]
      else:
           # make sure that inputs increase in size
           __SCREAMING_SNAKE_CASE     =      [
               floats_list((x, self.feature_size)			)
               for x in range(self.min_seq_length					,							self.max_seq_length					,							self.seq_length_diff			)
           ]
      if numpify:
           __SCREAMING_SNAKE_CASE     =      [np.asarray(__SCREAMING_SNAKE_CASE			) for x in speech_inputs]
      return speech_inputs







@require_torch
@require_torchaudio
class 						lowerCAmelCase__   (      a ,					unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      WhisperFeatureExtractor if is_speech_available() else None
 def       UpperCAmelCase__		(		self					:       str			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      WhisperFeatureExtractionTester(self			)
 def       UpperCAmelCase__		(		self					:       str			)  -> List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class(**self.feat_extract_dict			)

      with tempfile.TemporaryDirectory() as tmpdirname:
           __SCREAMING_SNAKE_CASE     =      feat_extract_first.save_pretrained(__SCREAMING_SNAKE_CASE			)[0]
           check_json_file_has_correct_format(__SCREAMING_SNAKE_CASE			)
           __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class.from_pretrained(__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      feat_extract_first.to_dict()
      __SCREAMING_SNAKE_CASE     =      feat_extract_second.to_dict()
      __SCREAMING_SNAKE_CASE     =      feat_extract_first.mel_filters
      __SCREAMING_SNAKE_CASE     =      feat_extract_second.mel_filters
      self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class(**self.feat_extract_dict			)

      with tempfile.TemporaryDirectory() as tmpdirname:
           __SCREAMING_SNAKE_CASE     =      os.path.join(__SCREAMING_SNAKE_CASE					,							"""feat_extract.json"""			)
           feat_extract_first.to_json_file(__SCREAMING_SNAKE_CASE			)
           __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class.from_json_file(__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      feat_extract_first.to_dict()
      __SCREAMING_SNAKE_CASE     =      feat_extract_second.to_dict()
      __SCREAMING_SNAKE_CASE     =      feat_extract_first.mel_filters
      __SCREAMING_SNAKE_CASE     =      feat_extract_second.mel_filters
      self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
      # create three inputs of length 800, 1000, and 1200
      __SCREAMING_SNAKE_CASE     =      [floats_list((1, x)			)[0] for x in range(800					,							1_400					,							200			)]
      __SCREAMING_SNAKE_CASE     =      [np.asarray(__SCREAMING_SNAKE_CASE			) for speech_input in speech_inputs]

      # Test feature size
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							padding="""max_length"""					,							return_tensors="""np"""			).input_features
      self.assertTrue(input_features.ndim == 3			)
      self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames			)
      self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size			)

      # Test not batched input
      __SCREAMING_SNAKE_CASE     =      feature_extractor(speech_inputs[0]					,							return_tensors="""np"""			).input_features
      __SCREAMING_SNAKE_CASE     =      feature_extractor(np_speech_inputs[0]					,							return_tensors="""np"""			).input_features
      self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							atol=1E-3			)			)

      # Test batched
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			):
           self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							atol=1E-3			)			)

      # Test 2-D numpy arrays are batched.
      __SCREAMING_SNAKE_CASE     =      [floats_list((1, x)			)[0] for x in (800, 800, 800)]
      __SCREAMING_SNAKE_CASE     =      np.asarray(__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			):
           self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							atol=1E-3			)			)

      # Test truncation required
      __SCREAMING_SNAKE_CASE     =      [floats_list((1, x)			)[0] for x in range(200					,							(feature_extractor.n_samples + 500)					,							200			)]
      __SCREAMING_SNAKE_CASE     =      [np.asarray(__SCREAMING_SNAKE_CASE			) for speech_input in speech_inputs]

      __SCREAMING_SNAKE_CASE     =      [x[: feature_extractor.n_samples] for x in speech_inputs]
      __SCREAMING_SNAKE_CASE     =      [np.asarray(__SCREAMING_SNAKE_CASE			) for speech_input in speech_inputs_truncated]

      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			).input_features
      for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			):
           self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							atol=1E-3			)			)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Optional[int]:


      """simple docstring"""




      import torch

      __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
      __SCREAMING_SNAKE_CASE     =      np.random.rand(100					,							32			).astype(np.floataa			)
      __SCREAMING_SNAKE_CASE     =      np_speech_inputs.tolist()

      for inputs in [py_speech_inputs, np_speech_inputs]:
           __SCREAMING_SNAKE_CASE     =      feature_extractor.pad([{"""input_features""": inputs}]					,							return_tensors="""np"""			)
           self.assertTrue(np_processed.input_features.dtype == np.floataa			)
           __SCREAMING_SNAKE_CASE     =      feature_extractor.pad([{"""input_features""": inputs}]					,							return_tensors="""pt"""			)
           self.assertTrue(pt_processed.input_features.dtype == torch.floataa			)
 def       UpperCAmelCase__		(		self					:       str					,							__SCREAMING_SNAKE_CASE					:       Tuple			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      load_dataset("""hf-internal-testing/librispeech_asr_dummy"""					,							"""clean"""					,							split="""validation"""			)
      # automatic decoding with librispeech
      __SCREAMING_SNAKE_CASE     =      ds.sort("""id"""			).select(range(__SCREAMING_SNAKE_CASE			)			)[:num_samples]["""audio"""]

      return [x["array"] for x in speech_samples]
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      torch.tensor(
          [
              0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
              0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
              0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
              -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
          ]			)
      # fmt: on

      __SCREAMING_SNAKE_CASE     =      self._load_datasamples(1			)
      __SCREAMING_SNAKE_CASE     =      WhisperFeatureExtractor()
      __SCREAMING_SNAKE_CASE     =      feature_extractor(__SCREAMING_SNAKE_CASE					,							return_tensors="""pt"""			).input_features
      self.assertEqual(input_features.shape					,							(1, 80, 3_000)			)
      self.assertTrue(torch.allclose(input_features[0, 0, :30]					,							__SCREAMING_SNAKE_CASE					,							atol=1E-4			)			)


 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
      __SCREAMING_SNAKE_CASE     =      self._load_datasamples(1			)[0]
      __SCREAMING_SNAKE_CASE     =      ((audio - audio.min()) / (audio.max() - audio.min())) * 65_535  # Rescale to [0, 65535] to show issue
      __SCREAMING_SNAKE_CASE     =      feat_extract.zero_mean_unit_var_norm([audio]					,							attention_mask=__SCREAMING_SNAKE_CASE			)[0]

      self.assertTrue(np.all(np.mean(__SCREAMING_SNAKE_CASE			) < 1E-3			)			)
      self.assertTrue(np.all(np.abs(np.var(__SCREAMING_SNAKE_CASE			) - 1			) < 1E-3			)			)






