
'''simple docstring'''



from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

__A								=			_symbol_database.Default()


__A								=			_descriptor_pool.Default().AddSerializedFile(
    B"\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"
)

__A								=			globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "sentencepiece_model_pb2", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
				__A								=			None
				__A								=			B"H\003"
				# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
				__A								=			45
				__A								=			1_581
				__A								=			1_517
				__A								=			1_570
				__A								=			1_584
				__A								=			1_793
				__A								=			1_795
				__A								=			1_916
				__A								=			1_864
				__A								=			1_905
				__A								=			1_919
				__A								=			2_429
				__A								=			2_208
				__A								=			2_418
				__A								=			2_323
				__A								=			2_407
# @@protoc_insertion_point(module_scope)







"""simple docstring"""
import itertools
import random
import unittest

import numpy as np

from transformers import WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST, WavaVecaConfig, WavaVecaFeatureExtractor
from transformers.testing_utils import require_torch, slow

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


lowercase__					       =	random.Random()







def 						__a   (	_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE=1.0		,     _SCREAMING_SNAKE_CASE=None		,     _SCREAMING_SNAKE_CASE=None   )	->Optional[int]:
  if rng is None:
    a__:     Any										=	global_rng

  a__:     int										=	[]
  for batch_idx in range(shape[0]   ):
    values.append([]   )
    for _ in range(shape[1]   ):
      values[-1].append(rng.random() * scale   )

  return values




class    __snake_case       (     unittest.TestCase				):



      def __init__(						self    ,			lowercase    ,			lowercase=7    ,			lowercase=4_00    ,			lowercase=20_00    ,			lowercase=1    ,			lowercase=0.0    ,			lowercase=1_60_00    ,			lowercase=True    ,			lowercase=True    ,			)	->			Union[str, Any]:





        '''simple docstring'''





        a__:     Tuple										=	parent
        a__:     Optional[int]										=	batch_size
        a__:     Optional[Any]										=	min_seq_length
        a__:     Optional[int]										=	max_seq_length
        a__:     Tuple										=	(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
        a__:     Dict										=	feature_size
        a__:     Any										=	padding_value
        a__:     Optional[Any]										=	sampling_rate
        a__:     Optional[Any]										=	return_attention_mask
        a__:     str										=	do_normalize



      def   lowerCamelCase_   (						self)	->			Dict:





        '''simple docstring'''





        return {
            "feature_size": self.feature_size,
            "padding_value": self.padding_value,
            "sampling_rate": self.sampling_rate,
            "return_attention_mask": self.return_attention_mask,
            "do_normalize": self.do_normalize,
        }



      def   lowerCamelCase_   (						self    ,			lowercase=False    ,			lowercase=False)	->			Tuple:





        '''simple docstring'''





        def _flatten(lowercase):
          return list(itertools.chain(*lowercase))

        if equal_length:
          a__:     Dict										=	floats_list((self.batch_size, self.max_seq_length))
        else:
          # make sure that inputs increase in size
          a__:     List[Any]										=	[
              _flatten(floats_list((x, self.feature_size)))
              for x in range(self.min_seq_length    ,			self.max_seq_length    ,			self.seq_length_diff)
          ]

        if numpify:
          a__:     str										=	[np.asarray(lowercase) for x in speech_inputs]

        return speech_inputs







class    __snake_case       (     __lowerCAmelCase       ,      unittest.TestCase				):
      a__        =		WavaVecaFeatureExtractor



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        a__:     Optional[int]										=	WavaVecaFeatureExtractionTester(self)



      def   lowerCamelCase_   (						self    ,			lowercase)	->			List[Any]:





        '''simple docstring'''





        self.assertTrue(np.all(np.mean(lowercase    ,			axis=0) < 1e-3))
        self.assertTrue(np.all(np.abs(np.var(lowercase    ,			axis=0) - 1) < 1e-3))



      def   lowerCamelCase_   (						self)	->			List[str]:





        '''simple docstring'''





        a__:     List[Any]										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        # create three inputs of length 800, 1000, and 1200
        a__:     Optional[Any]										=	[floats_list((1, x))[0] for x in range(8_00    ,			14_00    ,			2_00)]
        a__:     List[str]										=	[np.asarray(lowercase) for speech_input in speech_inputs]

        # Test not batched input
        a__:     Optional[Any]										=	feat_extract(speech_inputs[0]    ,			return_tensors='np').input_values
        a__:     Dict										=	feat_extract(np_speech_inputs[0]    ,			return_tensors='np').input_values
        self.assertTrue(np.allclose(lowercase    ,			lowercase    ,			atol=1e-3))

        # Test batched
        a__:     Dict										=	feat_extract(lowercase    ,			return_tensors='np').input_values
        a__:     int										=	feat_extract(lowercase    ,			return_tensors='np').input_values
        for enc_seq_a, enc_seq_a in zip(lowercase    ,			lowercase):
          self.assertTrue(np.allclose(lowercase    ,			lowercase    ,			atol=1e-3))

        # Test 2-D numpy arrays are batched.
        a__:     int										=	[floats_list((1, x))[0] for x in (8_00, 8_00, 8_00)]
        a__:     Union[str, Any]										=	np.asarray(lowercase)
        a__:     int										=	feat_extract(lowercase    ,			return_tensors='np').input_values
        a__:     Any										=	feat_extract(lowercase    ,			return_tensors='np').input_values
        for enc_seq_a, enc_seq_a in zip(lowercase    ,			lowercase):
          self.assertTrue(np.allclose(lowercase    ,			lowercase    ,			atol=1e-3))



      def   lowerCamelCase_   (						self)	->			List[Any]:





        '''simple docstring'''





        a__:     Tuple										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        a__:     List[Any]										=	[floats_list((1, x))[0] for x in range(8_00    ,			14_00    ,			2_00)]

        a__:     Optional[int]										=	['longest', 'max_length', 'do_not_pad']
        a__:     List[Any]										=	[None, 16_00, None]
        for max_length, padding in zip(lowercase    ,			lowercase):
          a__:     Dict										=	feat_extract(lowercase    ,			padding=lowercase    ,			max_length=lowercase    ,			return_tensors='np')
          a__:     Union[str, Any]										=	processed.input_values

          self._check_zero_mean_unit_variance(input_values[0][:8_00])
          self.assertTrue(input_values[0][8_00:].sum() < 1e-6)
          self._check_zero_mean_unit_variance(input_values[1][:10_00])
          self.assertTrue(input_values[0][10_00:].sum() < 1e-6)
          self._check_zero_mean_unit_variance(input_values[2][:12_00])



      def   lowerCamelCase_   (						self)	->			Optional[int]:





        '''simple docstring'''





        a__:     str										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        a__:     Optional[int]										=	range(8_00    ,			14_00    ,			2_00)
        a__:     List[str]										=	[floats_list((1, x))[0] for x in lengths]

        a__:     Tuple										=	['longest', 'max_length', 'do_not_pad']
        a__:     Dict										=	[None, 16_00, None]

        for max_length, padding in zip(lowercase    ,			lowercase):
          a__:     int										=	feat_extract(lowercase    ,			max_length=lowercase    ,			padding=lowercase)
          a__:     Any										=	processed.input_values

          self._check_zero_mean_unit_variance(input_values[0][:8_00])
          self._check_zero_mean_unit_variance(input_values[1][:10_00])
          self._check_zero_mean_unit_variance(input_values[2][:12_00])



      def   lowerCamelCase_   (						self)	->			List[str]:





        '''simple docstring'''





        a__:     Any										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        a__:     Any										=	[floats_list((1, x))[0] for x in range(8_00    ,			14_00    ,			2_00)]
        a__:     Dict										=	feat_extract(
            lowercase    ,			truncation=lowercase    ,			max_length=10_00    ,			padding='max_length'    ,			return_tensors='np')
        a__:     int										=	processed.input_values

        self._check_zero_mean_unit_variance(input_values[0, :8_00])
        self._check_zero_mean_unit_variance(input_values[1])
        self._check_zero_mean_unit_variance(input_values[2])



      def   lowerCamelCase_   (						self)	->			Dict:





        '''simple docstring'''





        a__:     Tuple										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        a__:     int										=	[floats_list((1, x))[0] for x in range(8_00    ,			14_00    ,			2_00)]
        a__:     str										=	feat_extract(
            lowercase    ,			truncation=lowercase    ,			max_length=10_00    ,			padding='longest'    ,			return_tensors='np')
        a__:     Any										=	processed.input_values

        self._check_zero_mean_unit_variance(input_values[0, :8_00])
        self._check_zero_mean_unit_variance(input_values[1, :10_00])
        self._check_zero_mean_unit_variance(input_values[2])

        # make sure that if max_length < longest -> then pad to max_length
        self.assertTrue(input_values.shape == (3, 10_00))

        a__:     Dict										=	[floats_list((1, x))[0] for x in range(8_00    ,			14_00    ,			2_00)]
        a__:     Tuple										=	feat_extract(
            lowercase    ,			truncation=lowercase    ,			max_length=20_00    ,			padding='longest'    ,			return_tensors='np')
        a__:     str										=	processed.input_values

        self._check_zero_mean_unit_variance(input_values[0, :8_00])
        self._check_zero_mean_unit_variance(input_values[1, :10_00])
        self._check_zero_mean_unit_variance(input_values[2])

        # make sure that if max_length > longest -> then pad to longest
        self.assertTrue(input_values.shape == (3, 12_00))



      @require_torch
      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        import torch

        a__:     Optional[int]										=	self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        a__:     Tuple										=	np.random.rand(1_00).astype(np.floataa)
        a__:     Tuple										=	np_speech_inputs.tolist()

        for inputs in [py_speech_inputs, np_speech_inputs]:
          a__:     Any										=	feature_extractor.pad([{'input_values': inputs}]    ,			return_tensors='np')
          self.assertTrue(np_processed.input_values.dtype == np.floataa)
          a__:     Optional[Any]										=	feature_extractor.pad([{'input_values': inputs}]    ,			return_tensors='pt')
          self.assertTrue(pt_processed.input_values.dtype == torch.floataa)



      @slow
      @require_torch
      def   lowerCamelCase_   (						self)	->			Optional[Any]:





        '''simple docstring'''





        for model_id in WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST:
          a__:     str										=	WavaVecaConfig.from_pretrained(lowercase)
          a__:     str										=	WavaVecaFeatureExtractor.from_pretrained(lowercase)

          # only "layer" feature extraction norm should make use of
          # attention_mask
          self.assertEqual(feat_extract.return_attention_mask    ,			config.feat_extract_norm == 'layer')


