codingutf8 2021 asapp inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license sewd model configuration import functools import operator from configurationutils import pretrainedconfig from utils import logging logger logging getloggername sewdpretrainedconfigarchivemap asappsewdtiny100k https huggingface coasappsewdtiny100kresolvemainconfig json see all sewd models at https huggingface comodels filtersewd class sewdconfigpretrainedconfig r this is the configuration class to store the configuration of a sewdmodel it is used to instantiate a sewd model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the sewd asappsewdtiny100khttps huggingface coasappsewdtiny100k architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 32 vocabulary size of the sewd model defines the number of different tokens that can be represented by the inputsids passed when calling sewd hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder squeezefactor int optional defaults to 2 sequence length downsampling factor after the encoder and upsampling factor after the transformer maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 positionbuckets int optional defaults to 256 the maximum size of relative position embeddings shareattkey bool optional defaults to true whether to share attention key with c2p and p2c relativeattention bool optional defaults to true whether to use relative position encoding posatttype tuplestr optional defaults to p2c c2p the type of relative position attention it can be a combination of p2c c2p e g p2c p2c c2p p2c c2p normrelebd str optional defaults to layernorm whether to use layer norm in relative embedding layernorm if yes hiddenact str or function optional defaults to gelupython the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu gelupython and gelunew are supported hiddendropout float optional defaults to 0 1 deprecated not used by the model and will be removed in a future version activationdropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities finaldropout float optional defaults to 0 1 the dropout probability for the final projection layer of sewdforctc initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e7 the epsilon used by the layer normalization layers in the transformer encoder featurelayernormeps float optional defaults to 1e5 the epsilon used by the layer normalization after the feature encoder featextractnorm str optional defaults to group the norm to be applied to 1d convolutional layers in feature encoder one of group for group normalization of only the first 1d convolutional layer or layer for layer normalization of all 1d convolutional layers featprojdropout float optional defaults to 0 0 the dropout probability for output of the feature encoder featextractactivation str optional defaults to gelu the nonlinear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelunew are supported convdim tupleint or listint optional defaults to 64 128 128 128 128 256 256 256 256 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the feature encoder the length of convdim defines the number of 1d convolutional layers convstride tupleint or listint optional defaults to 5 2 1 2 1 2 1 2 1 2 1 2 1 a tuple of integers defining the stride of each 1d convolutional layer in the feature encoder the length of convstride defines the number of convolutional layers and has to match the length of convdim convkernel tupleint or listint optional defaults to 10 3 1 3 1 3 1 3 1 2 1 2 1 a tuple of integers defining the kernel size of each 1d convolutional layer in the feature encoder the length of convkernel defines the number of convolutional layers and has to match the length of convdim convbias bool optional defaults to false whether the 1d convolutional layers have a bias numconvposembeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer numconvposembeddinggroups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer applyspecaugment bool optional defaults to true whether to apply specaugment data augmentation to the outputs of the feature encoder for reference see specaugment a simple data augmentation method for automatic speech recognitionhttps arxiv orgabs1904 08779 masktimeprob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates masktimeproblentimeaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked masktimeprob should be probvectorstartmasktimelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true masktimelength int optional defaults to 10 length of vector span along the time axis masktimeminmasks int optional defaults to 2 the minimum number of masks of length maskfeaturelength generated along the time axis each time step irrespectively of maskfeatureprob only relevant if masktimeproblentimeaxismasktimelength masktimeminmasks maskfeatureprob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates maskfeatureproblenfeatureaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked maskfeatureprob should be probvectorstartmaskfeaturelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true maskfeaturelength int optional defaults to 10 length of vector span along the feature axis maskfeatureminmasks int optional defaults to 0 the minimum number of masks of length maskfeaturelength generated along the feature axis each time step irrespectively of maskfeatureprob only relevant if maskfeatureproblenfeatureaxismaskfeaturelength maskfeatureminmasks diversitylossweight int optional defaults to 0 1 the weight of the codebook diversity loss component ctclossreduction str optional defaults to sum specifies the reduction to apply to the output of torch nn ctcloss only relevant when training an instance of sewdforctc ctczeroinfinity bool optional defaults to false whether to zero infinite losses and the associated gradients of torch nn ctcloss infinite losses mainly occur when the inputs are too short to be aligned to the targets only relevant when training an instance of sewdforctc useweightedlayersum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of wav2vec2forsequenceclassification classifierprojsize int optional defaults to 256 dimensionality of the projection before token meanpooling for classification example python from transformers import sewdconfig sewdmodel initializing a sewd asappsewdtiny100k style configuration configuration sewdconfig initializing a model with random weights from the asappsewdtiny100k style configuration model sewdmodelconfiguration accessing the model configuration configuration model config modeltype sewd def init self vocabsize32 hiddensize768 numhiddenlayers12 numattentionheads12 intermediatesize3072 squeezefactor2 maxpositionembeddings512 positionbuckets256 shareattkeytrue relativeattentiontrue posatttypep2c c2p normrelebdlayernorm hiddenactgelupython hiddendropout0 1 activationdropout0 1 attentiondropout0 1 featprojdropout0 0 finaldropout0 1 initializerrange0 02 layernormeps1e7 featurelayernormeps1e5 featextractnormgroup featextractactivationgelu convdim64 128 128 128 128 256 256 256 256 512 512 512 512 convstride5 2 1 2 1 2 1 2 1 2 1 2 1 convkernel10 3 1 3 1 3 1 3 1 2 1 2 1 convbiasfalse numconvposembeddings128 numconvposembeddinggroups16 applyspecaugmenttrue masktimeprob0 05 masktimelength10 masktimeminmasks2 maskfeatureprob0 0 maskfeaturelength10 maskfeatureminmasks0 ctclossreductionmean ctczeroinfinityfalse useweightedlayersumfalse classifierprojsize256 padtokenid0 bostokenid1 eostokenid2 kwargs super initkwargs padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid self hiddensize hiddensize self featextractnorm featextractnorm self featextractactivation featextractactivation self convdim listconvdim self convstride listconvstride self convkernel listconvkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self numfeatextractlayers lenself convdim self numhiddenlayers numhiddenlayers self intermediatesize intermediatesize self squeezefactor squeezefactor self maxpositionembeddings maxpositionembeddings self positionbuckets positionbuckets self shareattkey shareattkey self relativeattention relativeattention self normrelebd normrelebd self posatttype listposatttype self hiddenact hiddenact self numattentionheads numattentionheads self hiddendropout hiddendropout self attentiondropout attentiondropout self activationdropout activationdropout self featprojdropout featprojdropout self finaldropout finaldropout self layernormeps layernormeps self featurelayernormeps featurelayernormeps self initializerrange initializerrange self vocabsize vocabsize if lenself convstride self numfeatextractlayers or lenself convkernel self numfeatextractlayers or lenself convdim self numfeatextractlayers raise valueerror configuration for convolutional layers is incorrect it is required that lenconfig convdim lenconfig convstride lenconfig convkernel fbut is lenconfig convdim lenself convdim lenconfig convstride f lenself convstride lenconfig convkernel lenself convkernel finetuning config parameters for specaugment https arxiv orgabs1904 08779 self applyspecaugment applyspecaugment self masktimeprob masktimeprob self masktimelength masktimelength self masktimeminmasks masktimeminmasks self maskfeatureprob maskfeatureprob self maskfeaturelength maskfeaturelength self maskfeatureminmasks maskfeatureminmasks ctc loss self ctclossreduction ctclossreduction self ctczeroinfinity ctczeroinfinity sequence classification self useweightedlayersum useweightedlayersum self classifierprojsize classifierprojsize property def inputstologitsratioself return functools reduceoperator mul self convstride 1 property def hiddendropoutself logger warningoncehiddendropout is not used by the model and will be removed as config attribute in v4 35 return self hiddendropout def todictself output super todict outputhiddendropout output pophiddendropout return output coding utf 8 2021 asapp inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license sew d model configuration see all sew d models at https huggingface co models filter sew d this is the configuration class to store the configuration of a sewdmodel it is used to instantiate a sew d model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the sew d asapp sew d tiny 100k https huggingface co asapp sew d tiny 100k architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 32 vocabulary size of the sew d model defines the number of different tokens that can be represented by the inputs_ids passed when calling sewd hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder squeeze_factor int optional defaults to 2 sequence length downsampling factor after the encoder and upsampling factor after the transformer max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 position_buckets int optional defaults to 256 the maximum size of relative position embeddings share_att_key bool optional defaults to true whether to share attention key with c2p and p2c relative_attention bool optional defaults to true whether to use relative position encoding pos_att_type tuple str optional defaults to p2c c2p the type of relative position attention it can be a combination of p2c c2p e g p2c p2c c2p p2c c2p norm_rel_ebd str optional defaults to layer_norm whether to use layer norm in relative embedding layer_norm if yes hidden_act str or function optional defaults to gelu_python the non linear activation function function or string in the encoder and pooler if string gelu relu selu gelu_python and gelu_new are supported hidden_dropout float optional defaults to 0 1 deprecated not used by the model and will be removed in a future version activation_dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities final_dropout float optional defaults to 0 1 the dropout probability for the final projection layer of sewdforctc initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 7 the epsilon used by the layer normalization layers in the transformer encoder feature_layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization after the feature encoder feat_extract_norm str optional defaults to group the norm to be applied to 1d convolutional layers in feature encoder one of group for group normalization of only the first 1d convolutional layer or layer for layer normalization of all 1d convolutional layers feat_proj_dropout float optional defaults to 0 0 the dropout probability for output of the feature encoder feat_extract_activation str optional defaults to gelu the non linear activation function function or string in the 1d convolutional layers of the feature extractor if string gelu relu selu and gelu_new are supported conv_dim tuple int or list int optional defaults to 64 128 128 128 128 256 256 256 256 512 512 512 512 a tuple of integers defining the number of input and output channels of each 1d convolutional layer in the feature encoder the length of conv_dim defines the number of 1d convolutional layers conv_stride tuple int or list int optional defaults to 5 2 1 2 1 2 1 2 1 2 1 2 1 a tuple of integers defining the stride of each 1d convolutional layer in the feature encoder the length of conv_stride defines the number of convolutional layers and has to match the length of conv_dim conv_kernel tuple int or list int optional defaults to 10 3 1 3 1 3 1 3 1 2 1 2 1 a tuple of integers defining the kernel size of each 1d convolutional layer in the feature encoder the length of conv_kernel defines the number of convolutional layers and has to match the length of conv_dim conv_bias bool optional defaults to false whether the 1d convolutional layers have a bias num_conv_pos_embeddings int optional defaults to 128 number of convolutional positional embeddings defines the kernel size of 1d convolutional positional embeddings layer num_conv_pos_embedding_groups int optional defaults to 16 number of groups of 1d convolutional positional embeddings layer apply_spec_augment bool optional defaults to true whether to apply specaugment data augmentation to the outputs of the feature encoder for reference see specaugment a simple data augmentation method for automatic speech recognition https arxiv org abs 1904 08779 mask_time_prob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates mask_time_prob len time_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_time_prob should be prob_vector_start mask_time_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_time_length int optional defaults to 10 length of vector span along the time axis mask_time_min_masks int optional defaults to 2 the minimum number of masks of length mask_feature_length generated along the time axis each time step irrespectively of mask_feature_prob only relevant if mask_time_prob len time_axis mask_time_length mask_time_min_masks mask_feature_prob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates mask_feature_prob len feature_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_feature_prob should be prob_vector_start mask_feature_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_feature_length int optional defaults to 10 length of vector span along the feature axis mask_feature_min_masks int optional defaults to 0 the minimum number of masks of length mask_feature_length generated along the feature axis each time step irrespectively of mask_feature_prob only relevant if mask_feature_prob len feature_axis mask_feature_length mask_feature_min_masks diversity_loss_weight int optional defaults to 0 1 the weight of the codebook diversity loss component ctc_loss_reduction str optional defaults to sum specifies the reduction to apply to the output of torch nn ctcloss only relevant when training an instance of sewdforctc ctc_zero_infinity bool optional defaults to false whether to zero infinite losses and the associated gradients of torch nn ctcloss infinite losses mainly occur when the inputs are too short to be aligned to the targets only relevant when training an instance of sewdforctc use_weighted_layer_sum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of wav2vec2forsequenceclassification classifier_proj_size int optional defaults to 256 dimensionality of the projection before token mean pooling for classification example python from transformers import sewdconfig sewdmodel initializing a sew d asapp sew d tiny 100k style configuration configuration sewdconfig initializing a model with random weights from the asapp sew d tiny 100k style configuration model sewdmodel configuration accessing the model configuration configuration model config fine tuning config parameters for specaugment https arxiv org abs 1904 08779 ctc loss sequence classification serializes this instance to a python dictionary
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

SEW_D_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "asapp/sew-d-tiny-100k": "https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json",
    
}


class SEWDConfig(PretrainedConfig):
    r

    model_type = "sew-d"

    def __init__(
        self,
        vocab_size=32,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        squeeze_factor=2,
        max_position_embeddings=512,
        position_buckets=256,
        share_att_key=True,
        relative_attention=True,
        pos_att_type=("p2c", "c2p"),
        norm_rel_ebd="layer_norm",
        hidden_act="gelu_python",
        hidden_dropout=0.1,
        activation_dropout=0.1,
        attention_dropout=0.1,
        feat_proj_dropout=0.0,
        final_dropout=0.1,
        initializer_range=0.02,
        layer_norm_eps=1e-7,
        feature_layer_norm_eps=1e-5,
        feat_extract_norm="group",
        feat_extract_activation="gelu",
        conv_dim=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512),
        conv_stride=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1),
        conv_kernel=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1),
        conv_bias=False,
        num_conv_pos_embeddings=128,
        num_conv_pos_embedding_groups=16,
        apply_spec_augment=True,
        mask_time_prob=0.05,
        mask_time_length=10,
        mask_time_min_masks=2,
        mask_feature_prob=0.0,
        mask_feature_length=10,
        mask_feature_min_masks=0,
        ctc_loss_reduction="mean",
        ctc_zero_infinity=False,
        use_weighted_layer_sum=False,
        classifier_proj_size=256,
        pad_token_id=0,
        bos_token_id=1,
        eos_token_id=2,
        **kwargs,
    ):
        super().__init__(**kwargs, pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id)
        self.hidden_size = hidden_size
        self.feat_extract_norm = feat_extract_norm
        self.feat_extract_activation = feat_extract_activation
        self.conv_dim = list(conv_dim)
        self.conv_stride = list(conv_stride)
        self.conv_kernel = list(conv_kernel)
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.num_feat_extract_layers = len(self.conv_dim)
        self.num_hidden_layers = num_hidden_layers
        self.intermediate_size = intermediate_size
        self.squeeze_factor = squeeze_factor
        self.max_position_embeddings = max_position_embeddings
        self.position_buckets = position_buckets
        self.share_att_key = share_att_key
        self.relative_attention = relative_attention
        self.norm_rel_ebd = norm_rel_ebd
        self.pos_att_type = list(pos_att_type)
        self.hidden_act = hidden_act
        self.num_attention_heads = num_attention_heads
        self._hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.feat_proj_dropout = feat_proj_dropout
        self.final_dropout = final_dropout
        self.layer_norm_eps = layer_norm_eps
        self.feature_layer_norm_eps = feature_layer_norm_eps
        self.initializer_range = initializer_range
        self.vocab_size = vocab_size

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                "Configuration for convolutional layers is incorrect. "
                "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`, "
                f"but is `len(config.conv_dim) = {len(self.conv_dim)}`, `len(config.conv_stride) "
                f"= {len(self.conv_stride)}`, `len(config.conv_kernel) = {len(self.conv_kernel)}`."
            )

        
        self.apply_spec_augment = apply_spec_augment
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.mask_time_min_masks = mask_time_min_masks
        self.mask_feature_prob = mask_feature_prob
        self.mask_feature_length = mask_feature_length
        self.mask_feature_min_masks = mask_feature_min_masks

        
        self.ctc_loss_reduction = ctc_loss_reduction
        self.ctc_zero_infinity = ctc_zero_infinity

        
        self.use_weighted_layer_sum = use_weighted_layer_sum
        self.classifier_proj_size = classifier_proj_size

    @property
    def inputs_to_logits_ratio(self):
        return functools.reduce(operator.mul, self.conv_stride, 1)

    @property
    def hidden_dropout(self):
        logger.warning_once("hidden_dropout is not used by the model and will be removed as config attribute in v4.35")
        return self._hidden_dropout

    def to_dict(self):
        
        output = super().to_dict()
        output["hidden_dropout"] = output.pop("_hidden_dropout")
        return output