codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license speech processor class for wav2vec2 featureextractorclass wav2vec2featureextractor tokenizerclass autotokenizer def initself featureextractor tokenizer super initfeatureextractor tokenizer self currentprocessor self featureextractor self intargetcontextmanager false classmethod def frompretrainedcls pretrainedmodelnameorpath kwargs try return super frompretrainedpretrainedmodelnameorpath kwargs except oserror warnings warn floading a tokenizer inside cls name from a config that does not include a tokenizerclass attribute is deprecated and will be removed in v5 please add tokenizerclass wav2vec2ctctokenizer attribute to either your config json or tokenizerconfig json file to suppress this warning futurewarning featureextractor wav2vec2featureextractor frompretrainedpretrainedmodelnameorpath kwargs tokenizer wav2vec2ctctokenizer frompretrainedpretrainedmodelnameorpath kwargs return clsfeatureextractorfeatureextractor tokenizertokenizer def callself args kwargs for backward compatibility if self intargetcontextmanager return self currentprocessorargs kwargs if rawspeech in kwargs warnings warnusing rawspeech as a keyword argument is deprecated use audio instead audio kwargs poprawspeech else audio kwargs popaudio none samplingrate kwargs popsamplingrate none text kwargs poptext none if lenargs 0 audio args0 args args1 if audio is none and text is none raise valueerroryou need to specify either an audio or text input to process if audio is not none inputs self featureextractoraudio args samplingratesamplingrate kwargs if text is not none encodings self tokenizertext kwargs if text is none return inputs elif audio is none return encodings else inputslabels encodingsinputids return inputs def padself args kwargs for backward compatibility if self intargetcontextmanager return self currentprocessor padargs kwargs inputfeatures kwargs popinputfeatures none labels kwargs poplabels none if lenargs 0 inputfeatures args0 args args1 if inputfeatures is not none inputfeatures self featureextractor padinputfeatures args kwargs if labels is not none labels self tokenizer padlabels kwargs if labels is none return inputfeatures elif inputfeatures is none return labels else inputfeatureslabels labelsinputids return inputfeatures def batchdecodeself args kwargs return self tokenizer batchdecodeargs kwargs def decodeself args kwargs return self tokenizer decodeargs kwargs contextmanager def astargetprocessorself warnings warn astargetprocessor is deprecated and will be removed in v5 of transformers you can process your labels by using the argument text of the regular call method either in the same call as your audio inputs or in a separate call self intargetcontextmanager true self currentprocessor self tokenizer yield self currentprocessor self featureextractor self intargetcontextmanager false coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license speech processor class for wav2vec2 constructs a wav2vec2 processor which wraps a wav2vec2 feature extractor and a wav2vec2 ctc tokenizer into a single processor wav2vec2processor offers all the functionalities of wav2vec2featureextractor and pretrainedtokenizer see the docstring of wav2vec2processor __call__ and wav2vec2processor decode for more information args feature_extractor wav2vec2featureextractor an instance of wav2vec2featureextractor the feature extractor is a required input tokenizer pretrainedtokenizer an instance of pretrainedtokenizer the tokenizer is a required input when used in normal mode this method forwards all its arguments to wav2vec2featureextractor s wav2vec2featureextractor __call__ and returns its output if used in the context wav2vec2processor as_target_processor this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer __call__ please refer to the docstring of the above two methods for more information for backward compatibility when used in normal mode this method forwards all its arguments to wav2vec2featureextractor s wav2vec2featureextractor pad and returns its output if used in the context wav2vec2processor as_target_processor this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer pad please refer to the docstring of the above two methods for more information for backward compatibility this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer batch_decode please refer to the docstring of this method for more information this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer decode please refer to the docstring of this method for more information temporarily sets the tokenizer for processing the input useful for encoding the labels when fine tuning wav2vec2
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wav2vec2 import Wav2Vec2FeatureExtractor
from .tokenization_wav2vec2 import Wav2Vec2CTCTokenizer


class Wav2Vec2Processor(ProcessorMixin):
    r

    feature_extractor_class = "Wav2Vec2FeatureExtractor"
    tokenizer_class = "AutoTokenizer"

    def __init__(self, feature_extractor, tokenizer):
        super().__init__(feature_extractor, tokenizer)
        self.current_processor = self.feature_extractor
        self._in_target_context_manager = False

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
        try:
            return super().from_pretrained(pretrained_model_name_or_path, **kwargs)
        except OSError:
            warnings.warn(
                f"Loading a tokenizer inside {cls.__name__} from a config that does not"
                " include a `tokenizer_class` attribute is deprecated and will be "
                "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"
                " attribute to either your `config.json` or `tokenizer_config.json` "
                "file to suppress this warning: ",
                FutureWarning,
            )

            feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(pretrained_model_name_or_path, **kwargs)
            tokenizer = Wav2Vec2CTCTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)

            return cls(feature_extractor=feature_extractor, tokenizer=tokenizer)

    def __call__(self, *args, **kwargs):
        
        
        if self._in_target_context_manager:
            return self.current_processor(*args, **kwargs)

        if "raw_speech" in kwargs:
            warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.")
            audio = kwargs.pop("raw_speech")
        else:
            audio = kwargs.pop("audio", None)
        sampling_rate = kwargs.pop("sampling_rate", None)
        text = kwargs.pop("text", None)
        if len(args) > 0:
            audio = args[0]
            args = args[1:]

        if audio is None and text is None:
            raise ValueError("You need to specify either an `audio` or `text` input to process.")

        if audio is not None:
            inputs = self.feature_extractor(audio, *args, sampling_rate=sampling_rate, **kwargs)
        if text is not None:
            encodings = self.tokenizer(text, **kwargs)

        if text is None:
            return inputs
        elif audio is None:
            return encodings
        else:
            inputs["labels"] = encodings["input_ids"]
            return inputs

    def pad(self, *args, **kwargs):
        
        
        if self._in_target_context_manager:
            return self.current_processor.pad(*args, **kwargs)

        input_features = kwargs.pop("input_features", None)
        labels = kwargs.pop("labels", None)
        if len(args) > 0:
            input_features = args[0]
            args = args[1:]

        if input_features is not None:
            input_features = self.feature_extractor.pad(input_features, *args, **kwargs)
        if labels is not None:
            labels = self.tokenizer.pad(labels, **kwargs)

        if labels is None:
            return input_features
        elif input_features is None:
            return labels
        else:
            input_features["labels"] = labels["input_ids"]
            return input_features

    def batch_decode(self, *args, **kwargs):
        
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        
        return self.tokenizer.decode(*args, **kwargs)

    @contextmanager
    def as_target_processor(self):
        
        warnings.warn(
            "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
            "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
            "your audio inputs, or in a separate call."
        )
        self._in_target_context_manager = True
        self.current_processor = self.tokenizer
        yield
        self.current_processor = self.feature_extractor
        self._in_target_context_manager = False