codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license seamlessm4tv2 model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername seamlessm4tv2pretrainedconfigarchivemap https huggingface coresolvemainconfig json class seamlessm4tv2configpretrainedconfig r this is the configuration class to store the configuration of a seamlessm4tv2model it is used to instantiate an seamlessm4tv2 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the seamlessm4tv2 https huggingface co architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 256102 vocabulary size of the text modality of the seamlessm4tv2 model defines the number of different tokens that can be represented by the inputsids passed when calling seamlessm4tv2model seamlessm4tv2fortexttospeech or seamlessm4tv2fortexttotext t2uvocabsize int optional defaults to 10082 unit vocabulary size of the seamlessm4tv2 model defines the number of different unit tokens that can be represented by the inputsids passed when calling the texttounits submodel of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech charvocabsize int optional defaults to 10943 character vocabulary size of the seamlessm4tv2 model defines the number of different character tokens that can be represented by the charinputsids passed when calling the texttounits submodel of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech parameters shared across submodels hiddensize int optional defaults to 1024 dimensionality of the intermediate layers in the architecture initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e05 the epsilon used by the layer normalization layers usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models maxpositionembeddings int optional defaults to 4096 the maximum sequence length that this model text encoder and decoder might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 isencoderdecoder bool optional defaults to true whether the model is used as an encoderdecoder or not encoderlayerdrop float optional defaults to 0 05 the layerdrop probability for the encoders see the layerdrop papersee https arxiv orgabs1909 11556 for more details decoderlayerdrop float optional defaults to 0 05 the layerdrop probability for the decoders see the layerdrop papersee https arxiv orgabs1909 11556 for more details activationfunction str or function optional defaults to relu the nonlinear activation function function or string in the decoder and feedforward layers if string gelu relu selu swish and gelunew are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder decoder and pooler attentiondropout float optional defaults to 0 1 the dropout probability for all attention layers activationdropout float optional defaults to 0 0 the dropout probability for all activation layers in the model scaleembedding bool optional defaults to true scale embeddings by diving by sqrtdmodel text encoder and text decoder specific parameters encoderlayers int optional defaults to 24 number of hidden layers in the transformer text encoder encoderffndim int optional defaults to 8192 dimension of the intermediate i e feedforward layer in the transformer text encoder encoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer text encoder decoderlayers int optional defaults to 24 number of hidden layers in the transformer text decoder decoderffndim int optional defaults to 8192 dimension of the intermediate i e feedforward layer in the transformer text decoder decoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer text decoder decoderstarttokenid int optional defaults to 3 if an encoderdecoder model starts decoding with a different token than bos the id of that token only applied in the text decoder maxnewtokens int optional defaults to 256 the maximum numbers of text tokens to generate ignoring the number of tokens in the prompt padtokenid int optional defaults to 0 the id of the padding text token only applied to the textdecoder model bostokenid int optional defaults to 2 the id of the beginningofstream text token only applied to the textdecoder model eostokenid int optional defaults to 3 the id of the endofstream text token only applied to the textdecoder model speech encoder specific parameters speechencoderlayers int optional defaults to 24 number of hidden layers in the transformer speech encoder speechencoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer speech encoder speechencoderintermediatesize int optional defaults to 4096 dimension of the intermediate i e feedforward layer in the transformer speech encoder speechencoderhiddenact str or function optional defaults to swish the nonlinear activation function function or string in the speech encoder if string gelu relu selu swish and gelunew are supported speechencoderdropout float optional defaults to 0 0 the dropout probability for all layers in the speech encoder addadapter bool optional defaults to true add an adapter layer on top of the speech encoder speechencoderlayerdrop float optional defaults to 0 1 the layerdrop probability for the speech encoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details featureprojectioninputdim int optional defaults to 160 input dimension of the input feature projection of the speech encoder i e the dimension after processing input audios with seamlessm4tfeatureextractor adaptorkernelsize int optional defaults to 8 kernel size of the convolutional layers in the adapter network only relevant if addadapter is true adaptorstride int optional defaults to 8 stride of the convolutional layers in the adapter network only relevant if addadapter is true adaptordropout float optional defaults to 0 1 the dropout probability for all layers in the speech adapter numadapterlayers int optional defaults to 1 number of convolutional layers that should be used in the adapter network only relevant if addadapter is true positionembeddingstype str optional defaults to relativekey can be specified to relativekey if left to none no relative position embedding is applied only applied to the speech encoder for more information on relativekey please refer to selfattention with relative position representations shaw et al https arxiv orgabs1803 02155 convdepthwisekernelsize int optional defaults to 31 kernel size of convolutional depthwise 1d layer in conformer blocks only applied to the speech encoder leftmaxpositionembeddings int optional defaults to 64 the left clipping value for relative positions rightmaxpositionembeddings int optional defaults to 8 the right clipping value for relative positions speechencoderchunksize int optional defaults to 20000 the size of each attention chunk speechencoderleftchunknum int optional defaults to 128 number of chunks on the left up to which lookahead is allowed texttounit t2u model specific parameters t2ubostokenid int optional defaults to 0 the id of the beginningofstream unit token only applied to the texttounit seq2seq model t2upadtokenid int optional defaults to 1 the id of the padding unit token only applied to the texttounit seq2seq model t2ueostokenid int optional defaults to 2 the id of the endofstream unit token only applied to the texttounit seq2seq model t2uencoderlayers int optional defaults to 6 number of hidden layers in the transformer texttounit encoder t2uencoderffndim int optional defaults to 8192 dimension of the intermediate i e feedforward layer in the transformer texttounit encoder t2uencoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer texttounit encoder t2udecoderlayers int optional defaults to 6 number of hidden layers in the transformer texttounit decoder t2udecoderffndim int optional defaults to 8192 dimension of the intermediate i e feedforward layer in the transformer texttounit decoder t2udecoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer texttounit decoder t2umaxpositionembeddings int optional defaults to 4096 the maximum sequence length that this model texttounit component might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 t2uvariancepredictorembeddim int optional defaults to 1024 the projection dimension of the texttounit s duration predictor t2uvariancepredictorhiddendim int optional defaults to 256 internal dimension of the texttounit s duration predictor t2uvariancepredictorkernelsize int optional defaults to 3 kernel size of the convolutional layers of the texttounit s duration predictor t2uvariancepreddropout float optional defaults to 0 5 the dropout probabilitiy of the texttounit s duration predictor hifigan vocoder specific parameters samplingrate int optional defaults to 16000 the sampling rate at which the output audio will be generated expressed in hertz hz upsampleinitialchannel int optional defaults to 512 the number of input channels into the hifigan upsampling network applies to the vocoder only upsamplerates tupleint or listint optional defaults to 5 4 4 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the vocoder upsampling network the length of upsamplerates defines the number of convolutional layers and has to match the length of upsamplekernelsizes applies to the vocoder only upsamplekernelsizes tupleint or listint optional defaults to 11 8 8 4 4 a tuple of integers defining the kernel size of each 1d convolutional layer in the vocoder upsampling network the length of upsamplekernelsizes defines the number of convolutional layers and has to match the length of upsamplerates applies to the vocoder only resblockkernelsizes tupleint or listint optional defaults to 3 7 11 a tuple of integers defining the kernel sizes of the vocoder 1d convolutional layers in the multireceptive field fusion mrf module applies to the vocoder only resblockdilationsizes tupletupleint or listlistint optional defaults to 1 3 5 1 3 5 1 3 5 a nested tuple of integers defining the dilation rates of the vocoder dilated 1d convolutional layers in the multireceptive field fusion mrf module applies to the vocoder only leakyreluslope float optional defaults to 0 1 the angle of the negative slope used by the leaky relu activation in the vocoder applies to the vocoder only unithifiganvocabsize int optional defaults to 10000 vocabulary size of the seamlessm4tv2 vocoder defines the number of different unit tokens that can be represented by the inputsids passed when calling the vocoder of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech unitembeddim int optional defaults to 1280 the projection dimension of the input ids given to the hifigan vocoder applies to the vocoder only langembeddim int optional defaults to 256 the projection dimension of the target language given to the hifigan vocoder applies to the vocoder only spkrembeddim int optional defaults to 256 the projection dimension of the speaker id given to the hifigan vocoder applies to the vocoder only vocodernumlangs int optional defaults to 36 number of langs supported by the vocoder might be different from t2unumlangs vocodernumspkrs int optional defaults to 200 number of speakers supported by the vocoder variancepredictorkernelsize int optional defaults to 3 kernel size of the duration predictor applies to the vocoder only varpreddropout float optional defaults to 0 5 the dropout probabilitiy of the duration predictor applies to the vocoder only vocoderoffset int optional defaults to 4 offset the unit token ids by this number to account for symbol tokens applies to the vocoder only python from transformers import seamlessm4tv2model seamlessm4tv2config initializing a seamlessm4tv2 style configuration configuration seamlessm4tv2config initializing a model from the style configuration model seamlessm4tv2modelconfiguration accessing the model configuration configuration model config modeltype seamlessm4tv2 def init self vocabsize256102 t2uvocabsize10082 charvocabsize10943 shared config hiddensize1024 initializerrange0 02 layernormeps1e5 usecachetrue maxpositionembeddings4096 isencoderdecodertrue encoderlayerdrop0 05 decoderlayerdrop0 05 activationfunctionrelu dropout0 1 attentiondropout0 1 activationdropout0 0 scaleembeddingtrue text encoderdecoder encoderlayers24 encoderffndim8192 encoderattentionheads16 decoderlayers24 decoderffndim8192 decoderattentionheads16 decoderstarttokenid3 maxnewtokens256 padtokenid0 bostokenid2 eostokenid3 speechencoder speechencoderlayers24 speechencoderattentionheads16 speechencoderintermediatesize4096 speechencoderhiddenactswish speechencoderdropout0 0 addadaptertrue speechencoderlayerdrop0 1 featureprojectioninputdim160 adaptorkernelsize8 adaptorstride8 adaptordropout0 1 numadapterlayers1 positionembeddingstyperelativekey convdepthwisekernelsize31 leftmaxpositionembeddings64 rightmaxpositionembeddings8 speechencoderchunksize20000 speechencoderleftchunknum128 t2u config t2ubostokenid0 t2upadtokenid1 t2ueostokenid2 t2uencoderlayers6 t2uencoderffndim8192 t2uencoderattentionheads16 t2udecoderlayers6 t2udecoderffndim8192 t2udecoderattentionheads16 t2umaxpositionembeddings4096 t2uvariancepredictorembeddim1024 t2uvariancepredictorhiddendim256 t2uvariancepredictorkernelsize3 t2uvariancepreddropout0 5 hifigan vocoder config samplingrate16000 upsampleinitialchannel512 upsamplerates5 4 4 2 2 upsamplekernelsizes11 8 8 4 4 resblockkernelsizes3 7 11 resblockdilationsizes1 3 5 1 3 5 1 3 5 leakyreluslope0 1 specific to code hifigan unithifiganvocabsize10000 unitembeddim1280 langembeddim256 spkrembeddim256 vocodernumlangs36 vocodernumspkrs200 variancepredictorkernelsize3 varpreddropout0 5 vocoderoffset4 kwargs overallconfig self vocabsize vocabsize self t2uvocabsize t2uvocabsize self charvocabsize charvocabsize self hiddensize hiddensize self initializerrange initializerrange self layernormeps layernormeps self maxpositionembeddings maxpositionembeddings self usecache usecache self maxnewtokens maxnewtokens self encoderlayerdrop encoderlayerdrop self decoderlayerdrop decoderlayerdrop self activationfunction activationfunction self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self scaleembedding scaleembedding for proper config init self numattentionheads decoderattentionheads self numhiddenlayers decoderlayers textunit encoderdecoder self encoderlayers encoderlayers self encoderffndim encoderffndim self encoderattentionheads encoderattentionheads self decoderlayers decoderlayers self decoderffndim decoderffndim self decoderattentionheads decoderattentionheads speechencoder self speechencoderlayers speechencoderlayers self speechencoderhiddenact speechencoderhiddenact self speechencoderdropout speechencoderdropout self speechencoderattentionheads speechencoderattentionheads self speechencoderlayerdrop speechencoderlayerdrop self speechencoderintermediatesize speechencoderintermediatesize self featureprojectioninputdim featureprojectioninputdim self adaptorkernelsize adaptorkernelsize self adaptorstride adaptorstride self adaptordropout adaptordropout self numadapterlayers numadapterlayers self positionembeddingstype positionembeddingstype self convdepthwisekernelsize convdepthwisekernelsize self addadapter addadapter self leftmaxpositionembeddings leftmaxpositionembeddings self rightmaxpositionembeddings rightmaxpositionembeddings self speechencoderchunksize speechencoderchunksize self speechencoderleftchunknum speechencoderleftchunknum t2u config self t2ubostokenid t2ubostokenid self t2upadtokenid t2upadtokenid self t2ueostokenid t2ueostokenid self t2uencoderlayers t2uencoderlayers self t2uencoderffndim t2uencoderffndim self t2uencoderattentionheads t2uencoderattentionheads self t2udecoderlayers t2udecoderlayers self t2udecoderffndim t2udecoderffndim self t2udecoderattentionheads t2udecoderattentionheads self t2umaxpositionembeddings t2umaxpositionembeddings self t2uvariancepredictorembeddim t2uvariancepredictorembeddim todo add to docstrings self t2uvariancepredictorhiddendim t2uvariancepredictorhiddendim todo add to docstrings self t2uvariancepredictorkernelsize t2uvariancepredictorkernelsize todo add to docstrings self t2uvariancepreddropout t2uvariancepreddropout todo add to docstrings hifigan vocoder config original parameters specific to hifigan self samplingrate samplingrate self upsampleinitialchannel upsampleinitialchannel self upsamplerates upsamplerates self upsamplekernelsizes upsamplekernelsizes self resblockkernelsizes resblockkernelsizes self resblockdilationsizes resblockdilationsizes self leakyreluslope leakyreluslope specific to code hifigan self unithifiganvocabsize unithifiganvocabsize self unitembeddim unitembeddim self langembeddim langembeddim self spkrembeddim spkrembeddim self vocodernumlangs vocodernumlangs self vocodernumspkrs vocodernumspkrs self variancepredictorkernelsize variancepredictorkernelsize self varpreddropout varpreddropout self vocoderoffset vocoderoffset super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid decoderstarttokeniddecoderstarttokenid isencoderdecoderisencoderdecoder maxpositionembeddingsmaxpositionembeddings kwargs coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license seamlessm4tv2 model configuration this is the configuration class to store the configuration of a seamlessm4tv2model it is used to instantiate an seamlessm4tv2 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the seamlessm4tv2 https huggingface co architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 256102 vocabulary size of the text modality of the seamlessm4tv2 model defines the number of different tokens that can be represented by the inputs_ids passed when calling seamlessm4tv2model seamlessm4tv2fortexttospeech or seamlessm4tv2fortexttotext t2u_vocab_size int optional defaults to 10082 unit vocabulary size of the seamlessm4tv2 model defines the number of different unit tokens that can be represented by the inputs_ids passed when calling the text to units sub model of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech char_vocab_size int optional defaults to 10943 character vocabulary size of the seamlessm4tv2 model defines the number of different character tokens that can be represented by the char_inputs_ids passed when calling the text to units sub model of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech parameters shared across sub models hidden_size int optional defaults to 1024 dimensionality of the intermediate layers in the architecture initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 05 the epsilon used by the layer normalization layers use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models max_position_embeddings int optional defaults to 4096 the maximum sequence length that this model text encoder and decoder might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 is_encoder_decoder bool optional defaults to true whether the model is used as an encoder decoder or not encoder_layerdrop float optional defaults to 0 05 the layerdrop probability for the encoders see the layerdrop paper see https arxiv org abs 1909 11556 for more details decoder_layerdrop float optional defaults to 0 05 the layerdrop probability for the decoders see the layerdrop paper see https arxiv org abs 1909 11556 for more details activation_function str or function optional defaults to relu the non linear activation function function or string in the decoder and feed forward layers if string gelu relu selu swish and gelu_new are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder decoder and pooler attention_dropout float optional defaults to 0 1 the dropout probability for all attention layers activation_dropout float optional defaults to 0 0 the dropout probability for all activation layers in the model scale_embedding bool optional defaults to true scale embeddings by diving by sqrt d_model text encoder and text decoder specific parameters encoder_layers int optional defaults to 24 number of hidden layers in the transformer text encoder encoder_ffn_dim int optional defaults to 8192 dimension of the intermediate i e feed forward layer in the transformer text encoder encoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer text encoder decoder_layers int optional defaults to 24 number of hidden layers in the transformer text decoder decoder_ffn_dim int optional defaults to 8192 dimension of the intermediate i e feed forward layer in the transformer text decoder decoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer text decoder decoder_start_token_id int optional defaults to 3 if an encoder decoder model starts decoding with a different token than _bos_ the id of that token only applied in the text decoder max_new_tokens int optional defaults to 256 the maximum numbers of text tokens to generate ignoring the number of tokens in the prompt pad_token_id int optional defaults to 0 the id of the _padding_ text token only applied to the text decoder model bos_token_id int optional defaults to 2 the id of the _beginning of stream_ text token only applied to the text decoder model eos_token_id int optional defaults to 3 the id of the _end of stream_ text token only applied to the text decoder model speech encoder specific parameters speech_encoder_layers int optional defaults to 24 number of hidden layers in the transformer speech encoder speech_encoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer speech encoder speech_encoder_intermediate_size int optional defaults to 4096 dimension of the intermediate i e feed forward layer in the transformer speech encoder speech_encoder_hidden_act str or function optional defaults to swish the non linear activation function function or string in the speech encoder if string gelu relu selu swish and gelu_new are supported speech_encoder_dropout float optional defaults to 0 0 the dropout probability for all layers in the speech encoder add_adapter bool optional defaults to true add an adapter layer on top of the speech encoder speech_encoder_layerdrop float optional defaults to 0 1 the layerdrop probability for the speech encoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details feature_projection_input_dim int optional defaults to 160 input dimension of the input feature projection of the speech encoder i e the dimension after processing input audios with seamlessm4tfeatureextractor adaptor_kernel_size int optional defaults to 8 kernel size of the convolutional layers in the adapter network only relevant if add_adapter is true adaptor_stride int optional defaults to 8 stride of the convolutional layers in the adapter network only relevant if add_adapter is true adaptor_dropout float optional defaults to 0 1 the dropout probability for all layers in the speech adapter num_adapter_layers int optional defaults to 1 number of convolutional layers that should be used in the adapter network only relevant if add_adapter is true position_embeddings_type str optional defaults to relative_key can be specified to relative_key if left to none no relative position embedding is applied only applied to the speech encoder for more information on relative_key please refer to self attention with relative position representations shaw et al https arxiv org abs 1803 02155 conv_depthwise_kernel_size int optional defaults to 31 kernel size of convolutional depthwise 1d layer in conformer blocks only applied to the speech encoder left_max_position_embeddings int optional defaults to 64 the left clipping value for relative positions right_max_position_embeddings int optional defaults to 8 the right clipping value for relative positions speech_encoder_chunk_size int optional defaults to 20000 the size of each attention chunk speech_encoder_left_chunk_num int optional defaults to 128 number of chunks on the left up to which lookahead is allowed text to unit t2u model specific parameters t2u_bos_token_id int optional defaults to 0 the id of the _beginning of stream_ unit token only applied to the text to unit seq2seq model t2u_pad_token_id int optional defaults to 1 the id of the _padding_ unit token only applied to the text to unit seq2seq model t2u_eos_token_id int optional defaults to 2 the id of the _end of stream_ unit token only applied to the text to unit seq2seq model t2u_encoder_layers int optional defaults to 6 number of hidden layers in the transformer text to unit encoder t2u_encoder_ffn_dim int optional defaults to 8192 dimension of the intermediate i e feed forward layer in the transformer text to unit encoder t2u_encoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer text to unit encoder t2u_decoder_layers int optional defaults to 6 number of hidden layers in the transformer text to unit decoder t2u_decoder_ffn_dim int optional defaults to 8192 dimension of the intermediate i e feed forward layer in the transformer text to unit decoder t2u_decoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer text to unit decoder t2u_max_position_embeddings int optional defaults to 4096 the maximum sequence length that this model text to unit component might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 t2u_variance_predictor_embed_dim int optional defaults to 1024 the projection dimension of the text to unit s duration predictor t2u_variance_predictor_hidden_dim int optional defaults to 256 internal dimension of the text to unit s duration predictor t2u_variance_predictor_kernel_size int optional defaults to 3 kernel size of the convolutional layers of the text to unit s duration predictor t2u_variance_pred_dropout float optional defaults to 0 5 the dropout probabilitiy of the text to unit s duration predictor hifi gan vocoder specific parameters sampling_rate int optional defaults to 16000 the sampling rate at which the output audio will be generated expressed in hertz hz upsample_initial_channel int optional defaults to 512 the number of input channels into the hifi gan upsampling network applies to the vocoder only upsample_rates tuple int or list int optional defaults to 5 4 4 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the vocoder upsampling network the length of upsample_rates defines the number of convolutional layers and has to match the length of upsample_kernel_sizes applies to the vocoder only upsample_kernel_sizes tuple int or list int optional defaults to 11 8 8 4 4 a tuple of integers defining the kernel size of each 1d convolutional layer in the vocoder upsampling network the length of upsample_kernel_sizes defines the number of convolutional layers and has to match the length of upsample_rates applies to the vocoder only resblock_kernel_sizes tuple int or list int optional defaults to 3 7 11 a tuple of integers defining the kernel sizes of the vocoder 1d convolutional layers in the multi receptive field fusion mrf module applies to the vocoder only resblock_dilation_sizes tuple tuple int or list list int optional defaults to 1 3 5 1 3 5 1 3 5 a nested tuple of integers defining the dilation rates of the vocoder dilated 1d convolutional layers in the multi receptive field fusion mrf module applies to the vocoder only leaky_relu_slope float optional defaults to 0 1 the angle of the negative slope used by the leaky relu activation in the vocoder applies to the vocoder only unit_hifi_gan_vocab_size int optional defaults to 10000 vocabulary size of the seamlessm4tv2 vocoder defines the number of different unit tokens that can be represented by the inputs_ids passed when calling the vocoder of seamlessm4tv2model seamlessm4tv2forspeechtospeech or seamlessm4tv2fortexttospeech unit_embed_dim int optional defaults to 1280 the projection dimension of the input ids given to the hifi gan vocoder applies to the vocoder only lang_embed_dim int optional defaults to 256 the projection dimension of the target language given to the hifi gan vocoder applies to the vocoder only spkr_embed_dim int optional defaults to 256 the projection dimension of the speaker id given to the hifi gan vocoder applies to the vocoder only vocoder_num_langs int optional defaults to 36 number of langs supported by the vocoder might be different from t2u_num_langs vocoder_num_spkrs int optional defaults to 200 number of speakers supported by the vocoder variance_predictor_kernel_size int optional defaults to 3 kernel size of the duration predictor applies to the vocoder only var_pred_dropout float optional defaults to 0 5 the dropout probabilitiy of the duration predictor applies to the vocoder only vocoder_offset int optional defaults to 4 offset the unit token ids by this number to account for symbol tokens applies to the vocoder only python from transformers import seamlessm4tv2model seamlessm4tv2config initializing a seamlessm4tv2 style configuration configuration seamlessm4tv2config initializing a model from the style configuration model seamlessm4tv2model configuration accessing the model configuration configuration model config shared config text encoder decoder speech_encoder t2u config hifi gan vocoder config specific to code hifi gan overall_config for proper config init text unit encoder decoder speech_encoder t2u config todo add to docstrings todo add to docstrings todo add to docstrings todo add to docstrings hifi gan vocoder config original parameters specific to hifi gan specific to code hifi gan
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

SEAMLESS_M4T_V2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "": "https://huggingface.co//resolve/main/config.json",
}


class SeamlessM4Tv2Config(PretrainedConfig):
    r

    model_type = "seamless_m4t_v2"

    def __init__(
        self,
        vocab_size=256102,
        t2u_vocab_size=10082,
        char_vocab_size=10943,
        
        hidden_size=1024,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        use_cache=True,
        max_position_embeddings=4096,
        is_encoder_decoder=True,
        encoder_layerdrop=0.05,
        decoder_layerdrop=0.05,
        activation_function="relu",
        dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        scale_embedding=True,
        
        encoder_layers=24,
        encoder_ffn_dim=8192,
        encoder_attention_heads=16,
        decoder_layers=24,
        decoder_ffn_dim=8192,
        decoder_attention_heads=16,
        decoder_start_token_id=3,
        max_new_tokens=256,
        pad_token_id=0,
        bos_token_id=2,
        eos_token_id=3,
        
        speech_encoder_layers=24,
        speech_encoder_attention_heads=16,
        speech_encoder_intermediate_size=4096,
        speech_encoder_hidden_act="swish",
        speech_encoder_dropout=0.0,
        add_adapter=True,
        speech_encoder_layerdrop=0.1,
        feature_projection_input_dim=160,
        adaptor_kernel_size=8,
        adaptor_stride=8,
        adaptor_dropout=0.1,
        num_adapter_layers=1,
        position_embeddings_type="relative_key",
        conv_depthwise_kernel_size=31,
        left_max_position_embeddings=64,
        right_max_position_embeddings=8,
        speech_encoder_chunk_size=20000,
        speech_encoder_left_chunk_num=128,
        
        t2u_bos_token_id=0,
        t2u_pad_token_id=1,
        t2u_eos_token_id=2,
        t2u_encoder_layers=6,
        t2u_encoder_ffn_dim=8192,
        t2u_encoder_attention_heads=16,
        t2u_decoder_layers=6,
        t2u_decoder_ffn_dim=8192,
        t2u_decoder_attention_heads=16,
        t2u_max_position_embeddings=4096,
        t2u_variance_predictor_embed_dim=1024,
        t2u_variance_predictor_hidden_dim=256,
        t2u_variance_predictor_kernel_size=3,
        t2u_variance_pred_dropout=0.5,
        
        sampling_rate=16000,
        upsample_initial_channel=512,
        upsample_rates=[5, 4, 4, 2, 2],
        upsample_kernel_sizes=[11, 8, 8, 4, 4],
        resblock_kernel_sizes=[3, 7, 11],
        resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5], [1, 3, 5]],
        leaky_relu_slope=0.1,
        
        unit_hifi_gan_vocab_size=10000,
        unit_embed_dim=1280,
        lang_embed_dim=256,
        spkr_embed_dim=256,
        vocoder_num_langs=36,
        vocoder_num_spkrs=200,
        variance_predictor_kernel_size=3,
        var_pred_dropout=0.5,
        vocoder_offset=4,
        **kwargs,
    ):
        
        self.vocab_size = vocab_size
        self.t2u_vocab_size = t2u_vocab_size
        self.char_vocab_size = char_vocab_size
        self.hidden_size = hidden_size
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.max_position_embeddings = max_position_embeddings
        self.use_cache = use_cache
        self.max_new_tokens = max_new_tokens
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.activation_function = activation_function
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.scale_embedding = scale_embedding
        
        self.num_attention_heads = decoder_attention_heads
        self.num_hidden_layers = decoder_layers

        
        self.encoder_layers = encoder_layers
        self.encoder_ffn_dim = encoder_ffn_dim
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_layers = decoder_layers
        self.decoder_ffn_dim = decoder_ffn_dim
        self.decoder_attention_heads = decoder_attention_heads

        
        self.speech_encoder_layers = speech_encoder_layers
        self.speech_encoder_hidden_act = speech_encoder_hidden_act
        self.speech_encoder_dropout = speech_encoder_dropout
        self.speech_encoder_attention_heads = speech_encoder_attention_heads
        self.speech_encoder_layerdrop = speech_encoder_layerdrop
        self.speech_encoder_intermediate_size = speech_encoder_intermediate_size
        self.feature_projection_input_dim = feature_projection_input_dim
        self.adaptor_kernel_size = adaptor_kernel_size
        self.adaptor_stride = adaptor_stride
        self.adaptor_dropout = adaptor_dropout
        self.num_adapter_layers = num_adapter_layers
        self.position_embeddings_type = position_embeddings_type
        self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
        self.add_adapter = add_adapter
        self.left_max_position_embeddings = left_max_position_embeddings
        self.right_max_position_embeddings = right_max_position_embeddings
        self.speech_encoder_chunk_size = speech_encoder_chunk_size
        self.speech_encoder_left_chunk_num = speech_encoder_left_chunk_num

        
        self.t2u_bos_token_id = t2u_bos_token_id
        self.t2u_pad_token_id = t2u_pad_token_id
        self.t2u_eos_token_id = t2u_eos_token_id
        self.t2u_encoder_layers = t2u_encoder_layers
        self.t2u_encoder_ffn_dim = t2u_encoder_ffn_dim
        self.t2u_encoder_attention_heads = t2u_encoder_attention_heads
        self.t2u_decoder_layers = t2u_decoder_layers
        self.t2u_decoder_ffn_dim = t2u_decoder_ffn_dim
        self.t2u_decoder_attention_heads = t2u_decoder_attention_heads
        self.t2u_max_position_embeddings = t2u_max_position_embeddings
        self.t2u_variance_predictor_embed_dim = t2u_variance_predictor_embed_dim  
        self.t2u_variance_predictor_hidden_dim = t2u_variance_predictor_hidden_dim  
        self.t2u_variance_predictor_kernel_size = t2u_variance_predictor_kernel_size  
        self.t2u_variance_pred_dropout = t2u_variance_pred_dropout  

        
        
        self.sampling_rate = sampling_rate
        self.upsample_initial_channel = upsample_initial_channel
        self.upsample_rates = upsample_rates
        self.upsample_kernel_sizes = upsample_kernel_sizes
        self.resblock_kernel_sizes = resblock_kernel_sizes
        self.resblock_dilation_sizes = resblock_dilation_sizes
        self.leaky_relu_slope = leaky_relu_slope

        
        self.unit_hifi_gan_vocab_size = unit_hifi_gan_vocab_size
        self.unit_embed_dim = unit_embed_dim
        self.lang_embed_dim = lang_embed_dim
        self.spkr_embed_dim = spkr_embed_dim
        self.vocoder_num_langs = vocoder_num_langs
        self.vocoder_num_spkrs = vocoder_num_spkrs
        self.variance_predictor_kernel_size = variance_predictor_kernel_size
        self.var_pred_dropout = var_pred_dropout
        self.vocoder_offset = vocoder_offset

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            is_encoder_decoder=is_encoder_decoder,
            max_position_embeddings=max_position_embeddings,
            **kwargs,
        )