codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license whisper model configuration from collections import ordereddict from typing import typechecking any mapping optional union from configurationutils import pretrainedconfig from onnx import onnxconfig onnxseq2seqconfigwithpast from utils import logging if typechecking from featureextractionutils import featureextractionmixin from tokenizationutilsbase import pretrainedtokenizerbase from utils import tensortype logger logging getloggername whisperpretrainedconfigarchivemap openaiwhisperbase https huggingface coopenaiwhisperbaseresolvemainconfig json fmt off nonspeechtokens 1 2 7 8 9 10 14 25 26 27 28 29 31 58 59 60 61 62 63 90 91 92 93 357 366 438 532 685 705 796 930 1058 1220 1267 1279 1303 1343 1377 1391 1635 1782 1875 2162 2361 2488 3467 4008 4211 4600 4808 5299 5855 6329 7203 9609 9959 10563 10786 11420 11709 11907 13163 13697 13700 14808 15306 16410 16791 17992 19203 19510 20724 22305 22935 27007 30109 30420 33409 34949 40283 40493 40549 47282 49146 50257 50359 50360 50361 nonspeechtokensmulti 1 2 7 8 9 10 14 25 26 27 28 29 31 58 59 60 61 62 63 90 91 92 93 359 503 522 542 873 893 902 918 922 931 1350 1853 1982 2460 2627 3246 3253 3268 3536 3846 3961 4183 4667 6585 6647 7273 9061 9383 10428 10929 11938 12033 12331 12562 13793 14157 14635 15265 15618 16553 16604 18362 18956 20075 21675 22520 26130 26161 26435 28279 29464 31650 32302 32470 36865 42863 47425 49870 50254 50258 50360 50361 50362 fmt on class whisperconfigpretrainedconfig r this is the configuration class to store the configuration of a whispermodel it is used to instantiate a whisper model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the whisper openaiwhispertinyhttps huggingface coopenaiwhispertiny architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 51865 vocabulary size of the whisper model defines the number of different tokens that can be represented by the decoderinputids passed when calling whispermodel nummelbins int optional defaults to 80 number of mel features used per input features should correspond to the value used in the whisperprocessor class encoderlayers int optional defaults to 4 number of encoder layers decoderlayers int optional defaults to 4 number of decoder layers encoderattentionheads int optional defaults to 6 number of attention heads for each attention layer in the transformer encoder decoderattentionheads int optional defaults to 6 number of attention heads for each attention layer in the transformer decoder encoderffndim int optional defaults to 1536 dimensionality of the intermediate often named feedforward layer in encoder decoderffndim int optional defaults to 1536 dimensionality of the intermediate often named feedforward layer in decoder encoderlayerdrop float optional defaults to 0 0 the layerdrop probability for the encoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details decoderlayerdrop float optional defaults to 0 0 the layerdrop probability for the decoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details decoderstarttokenid int optional defaults to 50257 corresponds to the startoftranscript token which is automatically used when no decoderinputids are provided to the generate function it is used to guide the models generation process depending on the task usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models isencoderdecoder bool optional defaults to true whether the model is used as an encoderdecoder or not activationfunction str optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported dmodel int optional defaults to 384 dimensionality of the layers dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities activationdropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer initstd float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices scaleembedding bool optional defaults to false scale embeddings by diving by sqrtdmodel maxsourcepositions int optional defaults to 1500 the maximum sequence length of logmel filterbank features that this model might ever be used with maxtargetpositions int optional defaults to 448 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 padtokenid int optional defaults to 50256 padding token id bostokenid int optional defaults to 50256 begin of stream token id eostokenid int optional defaults to 50256 end of stream token id suppresstokens listint optional a list containing the nonspeech tokens that will be used by the logit processor in the generate function nonspeechtokens and nonspeechtokensmulti each correspond to the englishonly and the multilingual model beginsuppresstokens listint optional defaults to 220 50256 a list containing tokens that will be supressed at the beginning of the sampling process initialized as the token for blanktokenid and the eostokenid useweightedlayersum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of whisperforaudioclassification classifierprojsize int optional defaults to 256 dimensionality of the projection before token meanpooling for classification only relevant when using an instance of whisperforaudioclassification applyspecaugment bool optional defaults to false whether to apply specaugment data augmentation to the outputs of the feature encoder for reference see specaugment a simple data augmentation method for automatic speech recognitionhttps arxiv orgabs1904 08779 masktimeprob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates masktimeproblentimeaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked masktimeprob should be probvectorstartmasktimelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment true masktimelength int optional defaults to 10 length of vector span along the time axis masktimeminmasks int optional defaults to 2 the minimum number of masks of length maskfeaturelength generated along the time axis each time step irrespectively of maskfeatureprob only relevant if masktimeproblentimeaxismasktimelength masktimeminmasks maskfeatureprob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates maskfeatureproblenfeatureaxismasktimelength independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked maskfeatureprob should be probvectorstartmaskfeaturelength note that overlap may decrease the actual percentage of masked vectors this is only relevant if applyspecaugment is true maskfeaturelength int optional defaults to 10 length of vector span along the feature axis maskfeatureminmasks int optional defaults to 0 the minimum number of masks of length maskfeaturelength generated along the feature axis each time step irrespectively of maskfeatureprob only relevant if maskfeatureproblenfeatureaxismaskfeaturelength maskfeatureminmasks medianfilterwidth int optional defaults to 7 width of the median filter used to smoothen to crossattention outputs when computing token timestamps should be an odd number example python from transformers import whisperconfig whispermodel initializing a whisper tiny style configuration configuration whisperconfig initializing a model with random weights from the tiny style configuration model whispermodelconfiguration accessing the model configuration configuration model config modeltype whisper keystoignoreatinference pastkeyvalues attributemap numattentionheads encoderattentionheads hiddensize dmodel def init self vocabsize51865 nummelbins80 encoderlayers4 encoderattentionheads6 decoderlayers4 decoderattentionheads6 decoderffndim1536 encoderffndim1536 encoderlayerdrop0 0 decoderlayerdrop0 0 decoderstarttokenid50257 usecachetrue isencoderdecodertrue activationfunctiongelu dmodel384 dropout0 0 attentiondropout0 0 activationdropout0 0 initstd0 02 scaleembeddingfalse maxsourcepositions1500 maxtargetpositions448 padtokenid50256 bostokenid50256 eostokenid50256 suppresstokensnone beginsuppresstokens220 50256 useweightedlayersumfalse classifierprojsize256 applyspecaugmentfalse masktimeprob0 05 masktimelength10 masktimeminmasks2 maskfeatureprob0 0 maskfeaturelength10 maskfeatureminmasks0 medianfilterwidth7 kwargs self vocabsize vocabsize self nummelbins nummelbins self dmodel dmodel self encoderlayers encoderlayers self encoderattentionheads encoderattentionheads self decoderlayers decoderlayers self decoderattentionheads decoderattentionheads self decoderffndim decoderffndim self encoderffndim encoderffndim self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self activationfunction activationfunction self initstd initstd self encoderlayerdrop encoderlayerdrop self decoderlayerdrop decoderlayerdrop self usecache usecache self numhiddenlayers encoderlayers self scaleembedding scaleembedding scale factor will be sqrtdmodel if true self maxsourcepositions maxsourcepositions self maxtargetpositions maxtargetpositions audio classificationspecific parameters feel free to ignore for other classes self classifierprojsize classifierprojsize self useweightedlayersum useweightedlayersum finetuning config parameters for specaugment https arxiv orgabs1904 08779 self applyspecaugment applyspecaugment self masktimeprob masktimeprob self masktimelength masktimelength self masktimeminmasks masktimeminmasks self maskfeatureprob maskfeatureprob self maskfeaturelength maskfeaturelength self maskfeatureminmasks maskfeatureminmasks self medianfilterwidth medianfilterwidth super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid isencoderdecoderisencoderdecoder decoderstarttokeniddecoderstarttokenid suppresstokenssuppresstokens beginsuppresstokensbeginsuppresstokens kwargs class whisperonnxconfigonnxseq2seqconfigwithpast property def inputsself mappingstr mappingint str commoninputs ordereddict inputfeatures 0 batch 1 featuresize 2 encodersequence if self usepast commoninputsdecoderinputids 0 batch else commoninputsdecoderinputids 0 batch 1 decodersequence if self usepast self fillwithpastkeyvaluescommoninputs directioninputs return commoninputs def generatedummyinputs self preprocessor unionpretrainedtokenizerbase featureextractionmixin batchsize int 1 seqlength int 1 ispair bool false framework optionaltensortype none samplingrate int 22050 timeduration float 5 0 frequency int 220 mappingstr any dummyinputs ordereddict encoderinputs onnxconfig generatedummyinputs self preprocessorpreprocessor featureextractor batchsizebatchsize frameworkframework samplingratesamplingrate timedurationtimeduration frequencyfrequency encodersequencelength encoderinputsinputfeatures shape2 seqlength encodersequencelength 2 if self usepast else seqlength decoderinputs super generatedummyinputs preprocessor tokenizer batchsize seqlength ispair framework dummyinputsinputfeatures encoderinputs popinputfeatures dummyinputsdecoderinputids decoderinputs popdecoderinputids if pastkeyvalues in decoderinputs dummyinputspastkeyvalues decoderinputs poppastkeyvalues return dummyinputs property def atolforvalidationself float return 1e3 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license whisper model configuration fmt off fmt on this is the configuration class to store the configuration of a whispermodel it is used to instantiate a whisper model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the whisper openai whisper tiny https huggingface co openai whisper tiny architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 51865 vocabulary size of the whisper model defines the number of different tokens that can be represented by the decoder_input_ids passed when calling whispermodel num_mel_bins int optional defaults to 80 number of mel features used per input features should correspond to the value used in the whisperprocessor class encoder_layers int optional defaults to 4 number of encoder layers decoder_layers int optional defaults to 4 number of decoder layers encoder_attention_heads int optional defaults to 6 number of attention heads for each attention layer in the transformer encoder decoder_attention_heads int optional defaults to 6 number of attention heads for each attention layer in the transformer decoder encoder_ffn_dim int optional defaults to 1536 dimensionality of the intermediate often named feed forward layer in encoder decoder_ffn_dim int optional defaults to 1536 dimensionality of the intermediate often named feed forward layer in decoder encoder_layerdrop float optional defaults to 0 0 the layerdrop probability for the encoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details decoder_layerdrop float optional defaults to 0 0 the layerdrop probability for the decoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details decoder_start_token_id int optional defaults to 50257 corresponds to the startoftranscript token which is automatically used when no decoder_input_ids are provided to the generate function it is used to guide the model s generation process depending on the task use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models is_encoder_decoder bool optional defaults to true whether the model is used as an encoder decoder or not activation_function str optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported d_model int optional defaults to 384 dimensionality of the layers dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities activation_dropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer init_std float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices scale_embedding bool optional defaults to false scale embeddings by diving by sqrt d_model max_source_positions int optional defaults to 1500 the maximum sequence length of log mel filter bank features that this model might ever be used with max_target_positions int optional defaults to 448 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 pad_token_id int optional defaults to 50256 padding token id bos_token_id int optional defaults to 50256 begin of stream token id eos_token_id int optional defaults to 50256 end of stream token id suppress_tokens list int optional a list containing the non speech tokens that will be used by the logit processor in the generate function non_speech_tokens and non_speech_tokens_multi each correspond to the english only and the multilingual model begin_suppress_tokens list int optional defaults to 220 50256 a list containing tokens that will be supressed at the beginning of the sampling process initialized as the token for blank_token_id and the eos_token_id use_weighted_layer_sum bool optional defaults to false whether to use a weighted average of layer outputs with learned weights only relevant when using an instance of whisperforaudioclassification classifier_proj_size int optional defaults to 256 dimensionality of the projection before token mean pooling for classification only relevant when using an instance of whisperforaudioclassification apply_spec_augment bool optional defaults to false whether to apply specaugment data augmentation to the outputs of the feature encoder for reference see specaugment a simple data augmentation method for automatic speech recognition https arxiv org abs 1904 08779 mask_time_prob float optional defaults to 0 05 percentage between 0 and 1 of all feature vectors along the time axis which will be masked the masking procecure generates mask_time_prob len time_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_time_prob should be prob_vector_start mask_time_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment true mask_time_length int optional defaults to 10 length of vector span along the time axis mask_time_min_masks int optional defaults to 2 the minimum number of masks of length mask_feature_length generated along the time axis each time step irrespectively of mask_feature_prob only relevant if mask_time_prob len time_axis mask_time_length mask_time_min_masks mask_feature_prob float optional defaults to 0 0 percentage between 0 and 1 of all feature vectors along the feature axis which will be masked the masking procecure generates mask_feature_prob len feature_axis mask_time_length independent masks over the axis if reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked mask_feature_prob should be prob_vector_start mask_feature_length note that overlap may decrease the actual percentage of masked vectors this is only relevant if apply_spec_augment is true mask_feature_length int optional defaults to 10 length of vector span along the feature axis mask_feature_min_masks int optional defaults to 0 the minimum number of masks of length mask_feature_length generated along the feature axis each time step irrespectively of mask_feature_prob only relevant if mask_feature_prob len feature_axis mask_feature_length mask_feature_min_masks median_filter_width int optional defaults to 7 width of the median filter used to smoothen to cross attention outputs when computing token timestamps should be an odd number example python from transformers import whisperconfig whispermodel initializing a whisper tiny style configuration configuration whisperconfig initializing a model with random weights from the tiny style configuration model whispermodel configuration accessing the model configuration configuration model config scale factor will be sqrt d_model if true audio classification specific parameters feel free to ignore for other classes fine tuning config parameters for specaugment https arxiv org abs 1904 08779
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeq2SeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
    from ...feature_extraction_utils import FeatureExtractionMixin
    from ...tokenization_utils_base import PreTrainedTokenizerBase
    from ...utils import TensorType

logger = logging.get_logger(__name__)

WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}


NON_SPEECH_TOKENS = [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
NON_SPEECH_TOKENS_MULTI = [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]



class WhisperConfig(PretrainedConfig):
    r

    model_type = "whisper"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

    def __init__(
        self,
        vocab_size=51865,
        num_mel_bins=80,
        encoder_layers=4,
        encoder_attention_heads=6,
        decoder_layers=4,
        decoder_attention_heads=6,
        decoder_ffn_dim=1536,
        encoder_ffn_dim=1536,
        encoder_layerdrop=0.0,
        decoder_layerdrop=0.0,
        decoder_start_token_id=50257,
        use_cache=True,
        is_encoder_decoder=True,
        activation_function="gelu",
        d_model=384,
        dropout=0.0,
        attention_dropout=0.0,
        activation_dropout=0.0,
        init_std=0.02,
        scale_embedding=False,
        max_source_positions=1500,
        max_target_positions=448,
        pad_token_id=50256,
        bos_token_id=50256,
        eos_token_id=50256,
        suppress_tokens=None,
        begin_suppress_tokens=[220, 50256],
        use_weighted_layer_sum=False,
        classifier_proj_size=256,
        apply_spec_augment=False,
        mask_time_prob=0.05,
        mask_time_length=10,
        mask_time_min_masks=2,
        mask_feature_prob=0.0,
        mask_feature_length=10,
        mask_feature_min_masks=0,
        median_filter_width=7,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.num_mel_bins = num_mel_bins
        self.d_model = d_model
        self.encoder_layers = encoder_layers
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_layers = decoder_layers
        self.decoder_attention_heads = decoder_attention_heads
        self.decoder_ffn_dim = decoder_ffn_dim
        self.encoder_ffn_dim = encoder_ffn_dim
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.activation_function = activation_function
        self.init_std = init_std
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.use_cache = use_cache
        self.num_hidden_layers = encoder_layers
        self.scale_embedding = scale_embedding  
        self.max_source_positions = max_source_positions
        self.max_target_positions = max_target_positions

        
        self.classifier_proj_size = classifier_proj_size
        self.use_weighted_layer_sum = use_weighted_layer_sum

        
        self.apply_spec_augment = apply_spec_augment
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.mask_time_min_masks = mask_time_min_masks
        self.mask_feature_prob = mask_feature_prob
        self.mask_feature_length = mask_feature_length
        self.mask_feature_min_masks = mask_feature_min_masks

        self.median_filter_width = median_filter_width

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            decoder_start_token_id=decoder_start_token_id,
            suppress_tokens=suppress_tokens,
            begin_suppress_tokens=begin_suppress_tokens,
            **kwargs,
        )


class WhisperOnnxConfig(OnnxSeq2SeqConfigWithPast):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        common_inputs = OrderedDict(
            [
                ("input_features", {0: "batch", 1: "feature_size", 2: "encoder_sequence"}),
            ]
        )
        if self.use_past:
            common_inputs["decoder_input_ids"] = {0: "batch"}
        else:
            common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"}

        if self.use_past:
            self.fill_with_past_key_values_(common_inputs, direction="inputs")

        return common_inputs

    def generate_dummy_inputs(
        self,
        preprocessor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"],
        batch_size: int = -1,
        seq_length: int = -1,
        is_pair: bool = False,
        framework: Optional["TensorType"] = None,
        sampling_rate: int = 22050,
        time_duration: float = 5.0,
        frequency: int = 220,
    ) -> Mapping[str, Any]:
        dummy_inputs = OrderedDict()
        encoder_inputs = OnnxConfig.generate_dummy_inputs(
            self,
            preprocessor=preprocessor.feature_extractor,
            batch_size=batch_size,
            framework=framework,
            sampling_rate=sampling_rate,
            time_duration=time_duration,
            frequency=frequency,
        )
        encoder_sequence_length = encoder_inputs["input_features"].shape[2]
        seq_length = encoder_sequence_length // 2 if self.use_past else seq_length

        decoder_inputs = super().generate_dummy_inputs(
            preprocessor.tokenizer, batch_size, seq_length, is_pair, framework
        )

        dummy_inputs["input_features"] = encoder_inputs.pop("input_features")
        dummy_inputs["decoder_input_ids"] = decoder_inputs.pop("decoder_input_ids")

        if "past_key_values" in decoder_inputs:
            dummy_inputs["past_key_values"] = decoder_inputs.pop("past_key_values")

        return dummy_inputs

    @property
    def atol_for_validation(self) -> float:
        return 1e-3