codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license data2vectext configuration import math from configurationutils import pretrainedconfig from utils import logging logger logging getloggername data2vecaudiopretrainedconfigarchivemap facebookdata2vecbase960h https huggingface cofacebookdata2vecaudiobase960hresolvemainconfig json see all data2vecaudio models at https huggingface comodels filterdata2vecaudio class data2vecaudioconfigpretrainedconfig r this is the configuration class to store the configuration of a data2vecaudiomodel it is used to instantiate an data2vecaudio model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the data2vecaudio facebookdata2vecaudiobase960hhttps huggingface cofacebookdata2vecaudiobase960h architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 32 vocabulary size of the data2vecaudio model defines the number of different tokens that can be represented by the inputsids passed when calling data2vecaudiomodel or tfdata2vecaudiomodel vocabulary size of the model defines the different tokens that can be represented by the inputsids passed to the forward method of data2vecaudiomodel hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler activationdropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities finaldropout float optional defaults to 0 1 the dropout probability for the final projection layer of data2vecaudioforctc layerdrop float optional defaults to 0 1 the layerdrop probability see the layerdrop papersee https arxiv orgabs1909 11556 for more details initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers featprojdropout float optional defaults to 0 0 the dropout probability for output of the feature encoder featextractactivation str optional defaults to gelu the nonlinear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelunew are supported convdim tupleint or listint optional defaults to 512 512 512 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the feature encoder the length of convdim defines the number of 1d convolutional layers convstride tupleint or listint optional defaults to 5 2 2 2 2 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the feature encoder the length of convstride defines the number of convolutional layers and has to match the length of convdim convkernel tupleint or listint optional defaults to 10 3 3 3 3 3 3 a tuple of integers defining the kernel size of each 1d convolutional layer in the feature encoder the length of convkernel defines the number of convolutional layers and has to match the length of convdim convbias bool optional defaults to false whether the 1d convolutional layers have a bias numconvposembeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer numconvposembeddinggroups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer masktimeprob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates masktimeproblentimeaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked masktimeprob should be probvectorstartmasktimelength note that overlap may decrease the masktimelength int optional defaults to 10 length of vector span along the time axis masktimeminmasks int optional defaults to 2 the minimum number of masks of length maskfeaturelength generated along the time axis each time step irrespectively of maskfeatureprob only relevant if masktimeproblentimeaxismasktimelength masktimeminmasks maskfeatureprob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates maskfeatureproblenfeatureaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked maskfeatureprob should be probvectorstartmaskfeaturelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true maskfeaturelength int optional defaults to 10 length of vector span along the feature axis maskfeatureminmasks int optional defaults to 0 the minimum number of masks of length maskfeaturelength generated along the feature axis each time step irrespectively of maskfeatureprob only relevant if maskfeatureproblenfeatureaxismaskfeaturelength maskfeatureminmasks ctclossreduction str optional defaults to sum specifies the reduction to apply to the output of torch nn ctcloss only relevant when training an instance of data2vecaudioforctc ctczeroinfinity bool optional defaults to false whether to zero infinite losses and the associated gradients of torch nn ctcloss infinite losses mainly occur when the inputs are too short to be aligned to the targets only relevant when training an instance of data2vecaudioforctc useweightedlayersum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of data2vecaudioforsequenceclassification classifierprojsize int optional defaults to 256 dimensionality of the projection before token meanpooling for classification tdnndim tupleint or listint optional defaults to 512 512 512 512 1500 a tuple of integers defining the number of output channels of each 1d convolutional layer in the tdnn module of the xvector model the length of tdnndim defines the number of tdnn layers tdnnkernel tupleint or listint optional defaults to 5 3 3 1 1 a tuple of integers defining the kernel size of each 1d convolutional layer in the tdnn module of the xvector model the length of tdnnkernel has to match the length of tdnndim tdnndilation tupleint or listint optional defaults to 1 2 3 1 1 a tuple of integers defining the dilation factor of each 1d convolutional layer in tdnn module of the xvector model the length of tdnndilation has to match the length of tdnndim xvectoroutputdim int optional defaults to 512 dimensionality of the xvector embedding vectors addadapter bool optional defaults to false whether a convolutional network should be stacked on top of the data2vecaudio encoder can be very useful for warmstarting data2vecaudio for speechencoderdecoder models adapterkernelsize int optional defaults to 3 kernel size of the convolutional layers in the adapter network only relevant if addadapter is true adapterstride int optional defaults to 2 stride of the convolutional layers in the adapter network only relevant if addadapter is true numadapterlayers int optional defaults to 3 number of convolutional layers that should be used in the adapter network only relevant if addadapter is true outputhiddensize int optional dimensionality of the encoder output layer if not defined this defaults to hiddensize only relevant if addadapter is true example python from transformers import data2vecaudioconfig data2vecaudiomodel initializing a data2vecaudio facebookdata2vecaudiobase960h style configuration configuration data2vecaudioconfig initializing a model with random weights from the facebookdata2vecaudiobase960h style configuration model data2vecaudiomodelconfiguration accessing the model configuration configuration model config modeltype data2vecaudio def init self vocabsize32 hiddensize768 numhiddenlayers12 numattentionheads12 intermediatesize3072 hiddenactgelu hiddendropout0 1 activationdropout0 1 attentiondropout0 1 featprojdropout0 0 finaldropout0 1 layerdrop0 1 initializerrange0 02 layernormeps1e5 featextractactivationgelu convdim512 512 512 512 512 512 512 convstride5 2 2 2 2 2 2 convkernel10 3 3 3 3 2 2 convbiasfalse numconvposembeddinggroups16 convposkernelsize19 numconvposembeddings5 masktimeprob0 05 masktimelength10 masktimeminmasks2 maskfeatureprob0 0 maskfeaturelength10 maskfeatureminmasks0 ctclossreductionsum ctczeroinfinityfalse useweightedlayersumfalse classifierprojsize256 tdnndim512 512 512 512 1500 tdnnkernel5 3 3 1 1 tdnndilation1 2 3 1 1 xvectoroutputdim512 padtokenid0 bostokenid1 eostokenid2 addadapterfalse adapterkernelsize3 adapterstride2 numadapterlayers3 outputhiddensizenone kwargs super initkwargs padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid self hiddensize hiddensize self featextractactivation featextractactivation self convdim listconvdim self convstride listconvstride self convkernel listconvkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self convposkernelsize convposkernelsize self numfeatextractlayers lenself convdim self numhiddenlayers numhiddenlayers self intermediatesize intermediatesize self hiddenact hiddenact self numattentionheads numattentionheads self hiddendropout hiddendropout self attentiondropout attentiondropout self activationdropout activationdropout self featprojdropout featprojdropout self finaldropout finaldropout self layerdrop layerdrop self layernormeps layernormeps self initializerrange initializerrange self vocabsize vocabsize self useweightedlayersum useweightedlayersum if lenself convstride self numfeatextractlayers or lenself convkernel self numfeatextractlayers or lenself convdim self numfeatextractlayers raise valueerror configuration for convolutional layers is incorrect it is required that lenconfig convdim lenconfig convstride lenconfig convkernel but is lenconfig convdim f lenself convdim lenconfig convstride lenself convstride f lenconfig convkernel lenself convkernel finetuning config parameters for specaugment https arxiv orgabs1904 08779 self masktimeprob masktimeprob self masktimelength masktimelength self masktimeminmasks masktimeminmasks self maskfeatureprob maskfeatureprob self maskfeaturelength maskfeaturelength self maskfeatureminmasks maskfeatureminmasks ctc loss self ctclossreduction ctclossreduction self ctczeroinfinity ctczeroinfinity adapter self addadapter addadapter self adapterkernelsize adapterkernelsize self adapterstride adapterstride self numadapterlayers numadapterlayers self outputhiddensize outputhiddensize or hiddensize sequenceclassificationspecific parameter feel free to ignore for other classes self classifierprojsize classifierprojsize xvectorspecific parameters feel free to ignore for other classes self tdnndim listtdnndim self tdnnkernel listtdnnkernel self tdnndilation listtdnndilation self xvectoroutputdim xvectoroutputdim property def inputstologitsratioself return math prodself convstride coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license data2vectext configuration see all data2vecaudio models at https huggingface co models filter data2vec audio this is the configuration class to store the configuration of a data2vecaudiomodel it is used to instantiate an data2vecaudio model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the data2vecaudio facebook data2vec audio base 960h https huggingface co facebook data2vec audio base 960h architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 32 vocabulary size of the data2vecaudio model defines the number of different tokens that can be represented by the inputs_ids passed when calling data2vecaudiomodel or tfdata2vecaudiomodel vocabulary size of the model defines the different tokens that can be represented by the inputs_ids passed to the forward method of data2vecaudiomodel hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler activation_dropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities final_dropout float optional defaults to 0 1 the dropout probability for the final projection layer of data2vecaudioforctc layerdrop float optional defaults to 0 1 the layerdrop probability see the layerdrop paper see https arxiv org abs 1909 11556 for more details initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers feat_proj_dropout float optional defaults to 0 0 the dropout probability for output of the feature encoder feat_extract_activation str optional defaults to gelu the non linear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelu_new are supported conv_dim tuple int or list int optional defaults to 512 512 512 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the feature encoder the length of conv_dim defines the number of 1d convolutional layers conv_stride tuple int or list int optional defaults to 5 2 2 2 2 2 2 a tuple of integers defining the stride of each 1d convolutional layer in the feature encoder the length of conv_stride defines the number of convolutional layers and has to match the length of conv_dim conv_kernel tuple int or list int optional defaults to 10 3 3 3 3 3 3 a tuple of integers defining the kernel size of each 1d convolutional layer in the feature encoder the length of conv_kernel defines the number of convolutional layers and has to match the length of conv_dim conv_bias bool optional defaults to false whether the 1d convolutional layers have a bias num_conv_pos_embeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer num_conv_pos_embedding_groups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer mask_time_prob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates mask_time_prob len time_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_time_prob should be prob_vector_start mask_time_length note that overlap may decrease the mask_time_length int optional defaults to 10 length of vector span along the time axis mask_time_min_masks int optional defaults to 2 the minimum number of masks of length mask_feature_length generated along the time axis each time step irrespectively of mask_feature_prob only relevant if mask_time_prob len time_axis mask_time_length mask_time_min_masks mask_feature_prob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates mask_feature_prob len feature_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_feature_prob should be prob_vector_start mask_feature_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_feature_length int optional defaults to 10 length of vector span along the feature axis mask_feature_min_masks int optional defaults to 0 the minimum number of masks of length mask_feature_length generated along the feature axis each time step irrespectively of mask_feature_prob only relevant if mask_feature_prob len feature_axis mask_feature_length mask_feature_min_masks ctc_loss_reduction str optional defaults to sum specifies the reduction to apply to the output of torch nn ctcloss only relevant when training an instance of data2vecaudioforctc ctc_zero_infinity bool optional defaults to false whether to zero infinite losses and the associated gradients of torch nn ctcloss infinite losses mainly occur when the inputs are too short to be aligned to the targets only relevant when training an instance of data2vecaudioforctc use_weighted_layer_sum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of data2vecaudioforsequenceclassification classifier_proj_size int optional defaults to 256 dimensionality of the projection before token mean pooling for classification tdnn_dim tuple int or list int optional defaults to 512 512 512 512 1500 a tuple of integers defining the number of output channels of each 1d convolutional layer in the tdnn module of the xvector model the length of tdnn_dim defines the number of tdnn layers tdnn_kernel tuple int or list int optional defaults to 5 3 3 1 1 a tuple of integers defining the kernel size of each 1d convolutional layer in the tdnn module of the xvector model the length of tdnn_kernel has to match the length of tdnn_dim tdnn_dilation tuple int or list int optional defaults to 1 2 3 1 1 a tuple of integers defining the dilation factor of each 1d convolutional layer in tdnn module of the xvector model the length of tdnn_dilation has to match the length of tdnn_dim xvector_output_dim int optional defaults to 512 dimensionality of the xvector embedding vectors add_adapter bool optional defaults to false whether a convolutional network should be stacked on top of the data2vecaudio encoder can be very useful for warm starting data2vecaudio for speechencoderdecoder models adapter_kernel_size int optional defaults to 3 kernel size of the convolutional layers in the adapter network only relevant if add_adapter is true adapter_stride int optional defaults to 2 stride of the convolutional layers in the adapter network only relevant if add_adapter is true num_adapter_layers int optional defaults to 3 number of convolutional layers that should be used in the adapter network only relevant if add_adapter is true output_hidden_size int optional dimensionality of the encoder output layer if not defined this defaults to hidden size only relevant if add_adapter is true example python from transformers import data2vecaudioconfig data2vecaudiomodel initializing a data2vecaudio facebook data2vec audio base 960h style configuration configuration data2vecaudioconfig initializing a model with random weights from the facebook data2vec audio base 960h style configuration model data2vecaudiomodel configuration accessing the model configuration configuration model config fine tuning config parameters for specaugment https arxiv org abs 1904 08779 ctc loss adapter sequenceclassification specific parameter feel free to ignore for other classes xvector specific parameters feel free to ignore for other classes
import math

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "facebook/data2vec-base-960h": "https://huggingface.co/facebook/data2vec-audio-base-960h/resolve/main/config.json",
    
}


class Data2VecAudioConfig(PretrainedConfig):
    r

    model_type = "data2vec-audio"

    def __init__(
        self,
        vocab_size=32,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout=0.1,
        activation_dropout=0.1,
        attention_dropout=0.1,
        feat_proj_dropout=0.0,
        final_dropout=0.1,
        layerdrop=0.1,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        feat_extract_activation="gelu",
        conv_dim=(512, 512, 512, 512, 512, 512, 512),
        conv_stride=(5, 2, 2, 2, 2, 2, 2),
        conv_kernel=(10, 3, 3, 3, 3, 2, 2),
        conv_bias=False,
        num_conv_pos_embedding_groups=16,
        conv_pos_kernel_size=19,
        num_conv_pos_embeddings=5,
        mask_time_prob=0.05,
        mask_time_length=10,
        mask_time_min_masks=2,
        mask_feature_prob=0.0,
        mask_feature_length=10,
        mask_feature_min_masks=0,
        ctc_loss_reduction="sum",
        ctc_zero_infinity=False,
        use_weighted_layer_sum=False,
        classifier_proj_size=256,
        tdnn_dim=(512, 512, 512, 512, 1500),
        tdnn_kernel=(5, 3, 3, 1, 1),
        tdnn_dilation=(1, 2, 3, 1, 1),
        xvector_output_dim=512,
        pad_token_id=0,
        bos_token_id=1,
        eos_token_id=2,
        add_adapter=False,
        adapter_kernel_size=3,
        adapter_stride=2,
        num_adapter_layers=3,
        output_hidden_size=None,
        **kwargs,
    ):
        super().__init__(**kwargs, pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id)
        self.hidden_size = hidden_size
        self.feat_extract_activation = feat_extract_activation
        self.conv_dim = list(conv_dim)
        self.conv_stride = list(conv_stride)
        self.conv_kernel = list(conv_kernel)
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.conv_pos_kernel_size = conv_pos_kernel_size
        self.num_feat_extract_layers = len(self.conv_dim)
        self.num_hidden_layers = num_hidden_layers
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.num_attention_heads = num_attention_heads
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.feat_proj_dropout = feat_proj_dropout
        self.final_dropout = final_dropout
        self.layerdrop = layerdrop
        self.layer_norm_eps = layer_norm_eps
        self.initializer_range = initializer_range
        self.vocab_size = vocab_size
        self.use_weighted_layer_sum = use_weighted_layer_sum

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                f" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"
                f" `len(config.conv_kernel) = {len(self.conv_kernel)}`."
            )

        
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.mask_time_min_masks = mask_time_min_masks
        self.mask_feature_prob = mask_feature_prob
        self.mask_feature_length = mask_feature_length
        self.mask_feature_min_masks = mask_feature_min_masks

        
        self.ctc_loss_reduction = ctc_loss_reduction
        self.ctc_zero_infinity = ctc_zero_infinity

        
        self.add_adapter = add_adapter
        self.adapter_kernel_size = adapter_kernel_size
        self.adapter_stride = adapter_stride
        self.num_adapter_layers = num_adapter_layers
        self.output_hidden_size = output_hidden_size or hidden_size

        
        self.classifier_proj_size = classifier_proj_size

        
        self.tdnn_dim = list(tdnn_dim)
        self.tdnn_kernel = list(tdnn_kernel)
        self.tdnn_dilation = list(tdnn_dilation)
        self.xvector_output_dim = xvector_output_dim

    @property
    def inputs_to_logits_ratio(self):
        return math.prod(self.conv_stride)