codingutf8 2023 the fairseq s microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license speecht5 model configuration import functools import operator from configurationutils import pretrainedconfig from utils import logging logger logging getloggername speecht5pretrainedconfigarchivemap microsoftspeecht5asr https huggingface comicrosoftspeecht5asrresolvemainconfig json microsoftspeecht5tts https huggingface comicrosoftspeecht5ttsresolvemainconfig json microsoftspeecht5vc https huggingface comicrosoftspeecht5vcresolvemainconfig json speecht5pretrainedhifiganconfigarchivemap microsoftspeecht5hifigan https huggingface comicrosoftspeecht5hifiganresolvemainconfig json class speecht5configpretrainedconfig r this is the configuration class to store the configuration of a speecht5model it is used to instantiate a speecht5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the speecht5 microsoftspeecht5asrhttps huggingface comicrosoftspeecht5asr architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 81 vocabulary size of the speecht5 model defines the number of different tokens that can be represented by the inputsids passed to the forward method of speecht5model hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer encoderlayers int optional defaults to 12 number of hidden layers in the transformer encoder encoderattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder encoderffndim int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder encoderlayerdrop float optional defaults to 0 1 the layerdrop probability for the encoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details decoderlayers int optional defaults to 6 number of hidden layers in the transformer decoder decoderattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer decoder decoderffndim int optional defaults to 3072 dimensionality of the intermediate often named feedforward layer in the transformer decoder decoderlayerdrop float optional defaults to 0 1 the layerdrop probability for the decoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported positionaldropout float optional defaults to 0 1 the dropout probability for the text position encoding layers hiddendropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activationdropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers scaleembedding bool optional defaults to false scale embeddings by diving by sqrtdmodel featextractnorm str optional defaults to group the norm to be applied to 1d convolutional layers in the speech encoder prenet one of group for group normalization of only the first 1d convolutional layer or layer for layer normalization of all 1d convolutional layers featprojdropout float optional defaults to 0 0 the dropout probability for output of the speech encoder prenet featextractactivation str optional defaults to gelu the nonlinear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelunew are supported convdim tupleint or listint optional defaults to 512 512 512 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the speech encoder prenet the length of convdim defines the number of 1d convolutional layers convstride tupleint or listint optional defaults to 5 2 2 2 2 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the speech encoder prenet the length of convstride defines the number of convolutional layers and has to match the length of convdim convkernel tupleint or listint optional defaults to 10 3 3 3 3 3 3 a tuple of integers defining the kernel size of each 1d convolutional layer in the speech encoder prenet the length of convkernel defines the number of convolutional layers and has to match the length of convdim convbias bool optional defaults to false whether the 1d convolutional layers have a bias numconvposembeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer numconvposembeddinggroups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer applyspecaugment bool optional defaults to true whether to apply specaugment data augmentation to the outputs of the speech encoder prenet for reference see specaugment a simple data augmentation method for automatic speech recognitionhttps arxiv orgabs1904 08779 masktimeprob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates masktimeproblentimeaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked masktimeprob should be probvectorstartmasktimelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true masktimelength int optional defaults to 10 length of vector span along the time axis masktimeminmasks int optional defaults to 2 the minimum number of masks of length maskfeaturelength generated along the time axis each time step irrespectively of maskfeatureprob only relevant if masktimeproblentimeaxismasktimelength masktimeminmasks maskfeatureprob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates maskfeatureproblenfeatureaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked maskfeatureprob should be probvectorstartmaskfeaturelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true maskfeaturelength int optional defaults to 10 length of vector span along the feature axis maskfeatureminmasks int optional defaults to 0 the minimum number of masks of length maskfeaturelength generated along the feature axis each time step irrespectively of maskfeatureprob only relevant if maskfeatureproblenfeatureaxismaskfeaturelength maskfeatureminmasks nummelbins int optional defaults to 80 number of mel features used per input features used by the speech decoder prenet should correspond to the value used in the speecht5processor class speechdecoderprenetlayers int optional defaults to 2 number of layers in the speech decoder prenet speechdecoderprenetunits int optional defaults to 256 dimensionality of the layers in the speech decoder prenet speechdecoderprenetdropout float optional defaults to 0 5 the dropout probability for the speech decoder prenet layers speakerembeddingdim int optional defaults to 512 dimensionality of the xvector embedding vectors speechdecoderpostnetlayers int optional defaults to 5 number of layers in the speech decoder postnet speechdecoderpostnetunits int optional defaults to 256 dimensionality of the layers in the speech decoder postnet speechdecoderpostnetkernel int optional defaults to 5 number of convolutional filter channels in the speech decoder postnet speechdecoderpostnetdropout float optional defaults to 0 5 the dropout probability for the speech decoder postnet layers reductionfactor int optional defaults to 2 spectrogram length reduction factor for the speech decoder inputs maxspeechpositions int optional defaults to 4000 the maximum sequence length of speech features that this model might ever be used with maxtextpositions int optional defaults to 450 the maximum sequence length of text features that this model might ever be used with encodermaxrelativeposition int optional defaults to 160 maximum distance for relative position embedding in the encoder useguidedattentionloss bool optional defaults to true whether to apply guided attention loss while training the tts model guidedattentionlossnumheads int optional defaults to 2 number of attention heads the guided attention loss will be applied to use 1 to apply this loss to all attention heads guidedattentionlosssigma float optional defaults to 0 4 standard deviation for guided attention loss guidedattentionlossscale float optional defaults to 10 0 scaling coefficient for guided attention loss also known as lambda usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models example python from transformers import speecht5model speecht5config initializing a microsoftspeecht5asr style configuration configuration speecht5config initializing a model with random weights from the microsoftspeecht5asr style configuration model speecht5modelconfiguration accessing the model configuration configuration model config modeltype speecht5 attributemap numattentionheads encoderattentionheads numhiddenlayers encoderlayers def init self vocabsize81 hiddensize768 encoderlayers12 encoderattentionheads12 encoderffndim3072 encoderlayerdrop0 1 decoderlayers6 decoderffndim3072 decoderattentionheads12 decoderlayerdrop0 1 hiddenactgelu positionaldropout0 1 hiddendropout0 1 attentiondropout0 1 activationdropout0 1 initializerrange0 02 layernormeps1e5 scaleembeddingfalse featextractnormgroup featprojdropout0 0 featextractactivationgelu convdim512 512 512 512 512 512 512 convstride5 2 2 2 2 2 2 convkernel10 3 3 3 3 2 2 convbiasfalse numconvposembeddings128 numconvposembeddinggroups16 applyspecaugmenttrue masktimeprob0 05 masktimelength10 masktimeminmasks2 maskfeatureprob0 0 maskfeaturelength10 maskfeatureminmasks0 padtokenid1 bostokenid0 eostokenid2 decoderstarttokenid2 nummelbins80 speechdecoderprenetlayers2 speechdecoderprenetunits256 speechdecoderprenetdropout0 5 speakerembeddingdim512 speechdecoderpostnetlayers5 speechdecoderpostnetunits256 speechdecoderpostnetkernel5 speechdecoderpostnetdropout0 5 reductionfactor2 maxspeechpositions4000 maxtextpositions450 encodermaxrelativeposition160 useguidedattentionlosstrue guidedattentionlossnumheads2 guidedattentionlosssigma0 4 guidedattentionlossscale10 0 usecachetrue isencoderdecodertrue kwargs self vocabsize vocabsize self hiddensize hiddensize self encoderlayers encoderlayers self encoderffndim encoderffndim self encoderattentionheads encoderattentionheads self encoderlayerdrop encoderlayerdrop self decoderlayers decoderlayers self decoderffndim decoderffndim self decoderattentionheads decoderattentionheads self decoderlayerdrop decoderlayerdrop self hiddenact hiddenact self positionaldropout positionaldropout self hiddendropout hiddendropout self attentiondropout attentiondropout self activationdropout activationdropout self initializerrange initializerrange self layernormeps layernormeps self scaleembedding scaleembedding self featextractnorm featextractnorm self featprojdropout featprojdropout self featextractactivation featextractactivation self convdim listconvdim self convstride listconvstride self convkernel listconvkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self numfeatextractlayers lenself convdim if lenself convstride self numfeatextractlayers or lenself convkernel self numfeatextractlayers or lenself convdim self numfeatextractlayers raise valueerror configuration for convolutional layers is incorrect it is required that lenconfig convdim lenconfig convstride lenconfig convkernel but is lenconfig convdim f lenself convdim lenconfig convstride lenself convstride f lenconfig convkernel lenself convkernel finetuning config parameters for specaugment https arxiv orgabs1904 08779 self applyspecaugment applyspecaugment self masktimeprob masktimeprob self masktimelength masktimelength self masktimeminmasks masktimeminmasks self maskfeatureprob maskfeatureprob self maskfeaturelength maskfeaturelength self maskfeatureminmasks maskfeatureminmasks self nummelbins nummelbins self speechdecoderprenetlayers speechdecoderprenetlayers self speechdecoderprenetunits speechdecoderprenetunits self speechdecoderprenetdropout speechdecoderprenetdropout self speakerembeddingdim speakerembeddingdim self speechdecoderpostnetlayers speechdecoderpostnetlayers self speechdecoderpostnetunits speechdecoderpostnetunits self speechdecoderpostnetkernel speechdecoderpostnetkernel self speechdecoderpostnetdropout speechdecoderpostnetdropout self reductionfactor reductionfactor self maxspeechpositions maxspeechpositions self maxtextpositions maxtextpositions self encodermaxrelativeposition encodermaxrelativeposition self useguidedattentionloss useguidedattentionloss self guidedattentionlossnumheads guidedattentionlossnumheads self guidedattentionlosssigma guidedattentionlosssigma self guidedattentionlossscale guidedattentionlossscale self usecache usecache self isencoderdecoder isencoderdecoder super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid isencoderdecoderisencoderdecoder decoderstarttokeniddecoderstarttokenid kwargs def inputstologitsratioself return functools reduceoperator mul self convstride 1 class speecht5hifiganconfigpretrainedconfig r this is the configuration class to store the configuration of a speecht5hifiganmodel it is used to instantiate a speecht5 hifigan vocoder model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the speecht5 microsoftspeecht5hifiganhttps huggingface comicrosoftspeecht5hifigan architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args modelindim int optional defaults to 80 the number of frequency bins in the input logmel spectrogram samplingrate int optional defaults to 16000 the sampling rate at which the output audio will be generated expressed in hertz hz upsampleinitialchannel int optional defaults to 512 the number of input channels into the upsampling network upsamplerates tupleint or listint optional defaults to 4 4 4 4 a tuple of integers defining the stride of each 1d convolutional layer in the upsampling network the length of upsamplerates defines the number of convolutional layers and has to match the length of upsamplekernelsizes upsamplekernelsizes tupleint or listint optional defaults to 8 8 8 8 a tuple of integers defining the kernel size of each 1d convolutional layer in the upsampling network the length of upsamplekernelsizes defines the number of convolutional layers and has to match the length of upsamplerates resblockkernelsizes tupleint or listint optional defaults to 3 7 11 a tuple of integers defining the kernel sizes of the 1d convolutional layers in the multireceptive field fusion mrf module resblockdilationsizes tupletupleint or listlistint optional defaults to 1 3 5 1 3 5 1 3 5 a nested tuple of integers defining the dilation rates of the dilated 1d convolutional layers in the multireceptive field fusion mrf module initializerrange float optional defaults to 0 01 the standard deviation of the truncatednormalinitializer for initializing all weight matrices leakyreluslope float optional defaults to 0 1 the angle of the negative slope used by the leaky relu activation normalizebefore bool optional defaults to true whether or not to normalize the spectrogram before vocoding using the vocoder s learned mean and variance example python from transformers import speecht5hifigan speecht5hifiganconfig initializing a microsoftspeecht5hifigan style configuration configuration speecht5hifiganconfig initializing a model with random weights from the microsoftspeecht5hifigan style configuration model speecht5hifiganconfiguration accessing the model configuration configuration model config modeltype hifigan def init self modelindim80 samplingrate16000 upsampleinitialchannel512 upsamplerates4 4 4 4 upsamplekernelsizes8 8 8 8 resblockkernelsizes3 7 11 resblockdilationsizes1 3 5 1 3 5 1 3 5 initializerrange0 01 leakyreluslope0 1 normalizebeforetrue kwargs self modelindim modelindim self samplingrate samplingrate self upsampleinitialchannel upsampleinitialchannel self upsamplerates upsamplerates self upsamplekernelsizes upsamplekernelsizes self resblockkernelsizes resblockkernelsizes self resblockdilationsizes resblockdilationsizes self initializerrange initializerrange self leakyreluslope leakyreluslope self normalizebefore normalizebefore super initkwargs coding utf 8 2023 the fairseq s microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license speecht5 model configuration this is the configuration class to store the configuration of a speecht5model it is used to instantiate a speecht5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the speecht5 microsoft speecht5_asr https huggingface co microsoft speecht5_asr architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 81 vocabulary size of the speecht5 model defines the number of different tokens that can be represented by the inputs_ids passed to the forward method of speecht5model hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer encoder_layers int optional defaults to 12 number of hidden layers in the transformer encoder encoder_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder encoder_ffn_dim int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder encoder_layerdrop float optional defaults to 0 1 the layerdrop probability for the encoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details decoder_layers int optional defaults to 6 number of hidden layers in the transformer decoder decoder_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer decoder decoder_ffn_dim int optional defaults to 3072 dimensionality of the intermediate often named feed forward layer in the transformer decoder decoder_layerdrop float optional defaults to 0 1 the layerdrop probability for the decoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported positional_dropout float optional defaults to 0 1 the dropout probability for the text position encoding layers hidden_dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activation_dropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers scale_embedding bool optional defaults to false scale embeddings by diving by sqrt d_model feat_extract_norm str optional defaults to group the norm to be applied to 1d convolutional layers in the speech encoder pre net one of group for group normalization of only the first 1d convolutional layer or layer for layer normalization of all 1d convolutional layers feat_proj_dropout float optional defaults to 0 0 the dropout probability for output of the speech encoder pre net feat_extract_activation str optional defaults to gelu the non linear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelu_new are supported conv_dim tuple int or list int optional defaults to 512 512 512 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the speech encoder pre net the length of conv_dim defines the number of 1d convolutional layers conv_stride tuple int or list int optional defaults to 5 2 2 2 2 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the speech encoder pre net the length of conv_stride defines the number of convolutional layers and has to match the length of conv_dim conv_kernel tuple int or list int optional defaults to 10 3 3 3 3 3 3 a tuple of integers defining the kernel size of each 1d convolutional layer in the speech encoder pre net the length of conv_kernel defines the number of convolutional layers and has to match the length of conv_dim conv_bias bool optional defaults to false whether the 1d convolutional layers have a bias num_conv_pos_embeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer num_conv_pos_embedding_groups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer apply_spec_augment bool optional defaults to true whether to apply specaugment data augmentation to the outputs of the speech encoder pre net for reference see specaugment a simple data augmentation method for automatic speech recognition https arxiv org abs 1904 08779 mask_time_prob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates mask_time_prob len time_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_time_prob should be prob_vector_start mask_time_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_time_length int optional defaults to 10 length of vector span along the time axis mask_time_min_masks int optional defaults to 2 the minimum number of masks of length mask_feature_length generated along the time axis each time step irrespectively of mask_feature_prob only relevant if mask_time_prob len time_axis mask_time_length mask_time_min_masks mask_feature_prob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates mask_feature_prob len feature_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_feature_prob should be prob_vector_start mask_feature_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_feature_length int optional defaults to 10 length of vector span along the feature axis mask_feature_min_masks int optional defaults to 0 the minimum number of masks of length mask_feature_length generated along the feature axis each time step irrespectively of mask_feature_prob only relevant if mask_feature_prob len feature_axis mask_feature_length mask_feature_min_masks num_mel_bins int optional defaults to 80 number of mel features used per input features used by the speech decoder pre net should correspond to the value used in the speecht5processor class speech_decoder_prenet_layers int optional defaults to 2 number of layers in the speech decoder pre net speech_decoder_prenet_units int optional defaults to 256 dimensionality of the layers in the speech decoder pre net speech_decoder_prenet_dropout float optional defaults to 0 5 the dropout probability for the speech decoder pre net layers speaker_embedding_dim int optional defaults to 512 dimensionality of the xvector embedding vectors speech_decoder_postnet_layers int optional defaults to 5 number of layers in the speech decoder post net speech_decoder_postnet_units int optional defaults to 256 dimensionality of the layers in the speech decoder post net speech_decoder_postnet_kernel int optional defaults to 5 number of convolutional filter channels in the speech decoder post net speech_decoder_postnet_dropout float optional defaults to 0 5 the dropout probability for the speech decoder post net layers reduction_factor int optional defaults to 2 spectrogram length reduction factor for the speech decoder inputs max_speech_positions int optional defaults to 4000 the maximum sequence length of speech features that this model might ever be used with max_text_positions int optional defaults to 450 the maximum sequence length of text features that this model might ever be used with encoder_max_relative_position int optional defaults to 160 maximum distance for relative position embedding in the encoder use_guided_attention_loss bool optional defaults to true whether to apply guided attention loss while training the tts model guided_attention_loss_num_heads int optional defaults to 2 number of attention heads the guided attention loss will be applied to use 1 to apply this loss to all attention heads guided_attention_loss_sigma float optional defaults to 0 4 standard deviation for guided attention loss guided_attention_loss_scale float optional defaults to 10 0 scaling coefficient for guided attention loss also known as lambda use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models example python from transformers import speecht5model speecht5config initializing a microsoft speecht5_asr style configuration configuration speecht5config initializing a model with random weights from the microsoft speecht5_asr style configuration model speecht5model configuration accessing the model configuration configuration model config fine tuning config parameters for specaugment https arxiv org abs 1904 08779 this is the configuration class to store the configuration of a speecht5hifiganmodel it is used to instantiate a speecht5 hifi gan vocoder model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the speecht5 microsoft speecht5_hifigan https huggingface co microsoft speecht5_hifigan architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args model_in_dim int optional defaults to 80 the number of frequency bins in the input log mel spectrogram sampling_rate int optional defaults to 16000 the sampling rate at which the output audio will be generated expressed in hertz hz upsample_initial_channel int optional defaults to 512 the number of input channels into the upsampling network upsample_rates tuple int or list int optional defaults to 4 4 4 4 a tuple of integers defining the stride of each 1d convolutional layer in the upsampling network the length of upsample_rates defines the number of convolutional layers and has to match the length of upsample_kernel_sizes upsample_kernel_sizes tuple int or list int optional defaults to 8 8 8 8 a tuple of integers defining the kernel size of each 1d convolutional layer in the upsampling network the length of upsample_kernel_sizes defines the number of convolutional layers and has to match the length of upsample_rates resblock_kernel_sizes tuple int or list int optional defaults to 3 7 11 a tuple of integers defining the kernel sizes of the 1d convolutional layers in the multi receptive field fusion mrf module resblock_dilation_sizes tuple tuple int or list list int optional defaults to 1 3 5 1 3 5 1 3 5 a nested tuple of integers defining the dilation rates of the dilated 1d convolutional layers in the multi receptive field fusion mrf module initializer_range float optional defaults to 0 01 the standard deviation of the truncated_normal_initializer for initializing all weight matrices leaky_relu_slope float optional defaults to 0 1 the angle of the negative slope used by the leaky relu activation normalize_before bool optional defaults to true whether or not to normalize the spectrogram before vocoding using the vocoder s learned mean and variance example python from transformers import speecht5hifigan speecht5hifiganconfig initializing a microsoft speecht5_hifigan style configuration configuration speecht5hifiganconfig initializing a model with random weights from the microsoft speecht5_hifigan style configuration model speecht5hifigan configuration accessing the model configuration configuration model config
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

SPEECHT5_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "microsoft/speecht5_asr": "https://huggingface.co/microsoft/speecht5_asr/resolve/main/config.json",
    "microsoft/speecht5_tts": "https://huggingface.co/microsoft/speecht5_tts/resolve/main/config.json",
    "microsoft/speecht5_vc": "https://huggingface.co/microsoft/speecht5_vc/resolve/main/config.json",
}

SPEECHT5_PRETRAINED_HIFIGAN_CONFIG_ARCHIVE_MAP = {
    "microsoft/speecht5_hifigan": "https://huggingface.co/microsoft/speecht5_hifigan/resolve/main/config.json",
}


class SpeechT5Config(PretrainedConfig):
    r

    model_type = "speecht5"
    attribute_map = {"num_attention_heads": "encoder_attention_heads", "num_hidden_layers": "encoder_layers"}

    def __init__(
        self,
        vocab_size=81,
        hidden_size=768,
        encoder_layers=12,
        encoder_attention_heads=12,
        encoder_ffn_dim=3072,
        encoder_layerdrop=0.1,
        decoder_layers=6,
        decoder_ffn_dim=3072,
        decoder_attention_heads=12,
        decoder_layerdrop=0.1,
        hidden_act="gelu",
        positional_dropout=0.1,
        hidden_dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.1,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        scale_embedding=False,
        feat_extract_norm="group",
        feat_proj_dropout=0.0,
        feat_extract_activation="gelu",
        conv_dim=(512, 512, 512, 512, 512, 512, 512),
        conv_stride=(5, 2, 2, 2, 2, 2, 2),
        conv_kernel=(10, 3, 3, 3, 3, 2, 2),
        conv_bias=False,
        num_conv_pos_embeddings=128,
        num_conv_pos_embedding_groups=16,
        apply_spec_augment=True,
        mask_time_prob=0.05,
        mask_time_length=10,
        mask_time_min_masks=2,
        mask_feature_prob=0.0,
        mask_feature_length=10,
        mask_feature_min_masks=0,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        decoder_start_token_id=2,
        num_mel_bins=80,
        speech_decoder_prenet_layers=2,
        speech_decoder_prenet_units=256,
        speech_decoder_prenet_dropout=0.5,
        speaker_embedding_dim=512,
        speech_decoder_postnet_layers=5,
        speech_decoder_postnet_units=256,
        speech_decoder_postnet_kernel=5,
        speech_decoder_postnet_dropout=0.5,
        reduction_factor=2,
        max_speech_positions=4000,
        max_text_positions=450,
        encoder_max_relative_position=160,
        use_guided_attention_loss=True,
        guided_attention_loss_num_heads=2,
        guided_attention_loss_sigma=0.4,
        guided_attention_loss_scale=10.0,
        use_cache=True,
        is_encoder_decoder=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.encoder_layers = encoder_layers
        self.encoder_ffn_dim = encoder_ffn_dim
        self.encoder_attention_heads = encoder_attention_heads
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layers = decoder_layers
        self.decoder_ffn_dim = decoder_ffn_dim
        self.decoder_attention_heads = decoder_attention_heads
        self.decoder_layerdrop = decoder_layerdrop
        self.hidden_act = hidden_act
        self.positional_dropout = positional_dropout
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.scale_embedding = scale_embedding

        self.feat_extract_norm = feat_extract_norm
        self.feat_proj_dropout = feat_proj_dropout
        self.feat_extract_activation = feat_extract_activation
        self.conv_dim = list(conv_dim)
        self.conv_stride = list(conv_stride)
        self.conv_kernel = list(conv_kernel)
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.num_feat_extract_layers = len(self.conv_dim)

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                f" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"
                f" `len(config.conv_kernel) = {len(self.conv_kernel)}`."
            )

        
        self.apply_spec_augment = apply_spec_augment
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.mask_time_min_masks = mask_time_min_masks
        self.mask_feature_prob = mask_feature_prob
        self.mask_feature_length = mask_feature_length
        self.mask_feature_min_masks = mask_feature_min_masks

        self.num_mel_bins = num_mel_bins
        self.speech_decoder_prenet_layers = speech_decoder_prenet_layers
        self.speech_decoder_prenet_units = speech_decoder_prenet_units
        self.speech_decoder_prenet_dropout = speech_decoder_prenet_dropout
        self.speaker_embedding_dim = speaker_embedding_dim

        self.speech_decoder_postnet_layers = speech_decoder_postnet_layers
        self.speech_decoder_postnet_units = speech_decoder_postnet_units
        self.speech_decoder_postnet_kernel = speech_decoder_postnet_kernel
        self.speech_decoder_postnet_dropout = speech_decoder_postnet_dropout
        self.reduction_factor = reduction_factor

        self.max_speech_positions = max_speech_positions
        self.max_text_positions = max_text_positions
        self.encoder_max_relative_position = encoder_max_relative_position

        self.use_guided_attention_loss = use_guided_attention_loss
        self.guided_attention_loss_num_heads = guided_attention_loss_num_heads
        self.guided_attention_loss_sigma = guided_attention_loss_sigma
        self.guided_attention_loss_scale = guided_attention_loss_scale

        self.use_cache = use_cache
        self.is_encoder_decoder = is_encoder_decoder

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            decoder_start_token_id=decoder_start_token_id,
            **kwargs,
        )

    def inputs_to_logits_ratio(self):
        return functools.reduce(operator.mul, self.conv_stride, 1)


class SpeechT5HifiGanConfig(PretrainedConfig):
    r

    model_type = "hifigan"

    def __init__(
        self,
        model_in_dim=80,
        sampling_rate=16000,
        upsample_initial_channel=512,
        upsample_rates=[4, 4, 4, 4],
        upsample_kernel_sizes=[8, 8, 8, 8],
        resblock_kernel_sizes=[3, 7, 11],
        resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
        initializer_range=0.01,
        leaky_relu_slope=0.1,
        normalize_before=True,
        **kwargs,
    ):
        self.model_in_dim = model_in_dim
        self.sampling_rate = sampling_rate
        self.upsample_initial_channel = upsample_initial_channel
        self.upsample_rates = upsample_rates
        self.upsample_kernel_sizes = upsample_kernel_sizes
        self.resblock_kernel_sizes = resblock_kernel_sizes
        self.resblock_dilation_sizes = resblock_dilation_sizes
        self.initializer_range = initializer_range
        self.leaky_relu_slope = leaky_relu_slope
        self.normalize_before = normalize_before
        super().__init__(**kwargs)