codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license zero shot audio classification pipeline using clapmodel this pipeline predicts the class of an audio when you provide an audio and a set of candidatelabels example python from transformers import pipeline from datasets import loaddataset dataset loaddatasetashraqesc50 audio nextiterdatasettrainaudioarray classifier pipelinetaskzeroshotaudioclassification modellaionclaphtsatunfused classifieraudio candidatelabelssound of a dog sound of vaccum cleaner score 0 9996 label sound of a dog score 0 0004 label sound of vaccum cleaner learn more about the basics of using a pipeline in the pipeline tutorial pipelinetutorial this audio classification pipeline can currently be loaded from pipeline using the following task identifier zeroshotaudioclassification see the list of available models on huggingface comodelshttps huggingface comodels filterzeroshotaudioclassification no specific forxxx available yet assign labels to the audios passed as inputs args audios str liststr np array or listnp array the pipeline handles three types of inputs a string containing a http link pointing to an audio a string containing a local path to an audio an audio loaded in numpy candidatelabels liststr the candidate labels for this audio hypothesistemplate str optional defaults to this is a sound of the sentence used in cunjunction with candidatelabels to attempt the audio classification by replacing the placeholder with the candidatelabels then likelihood is estimated by using logitsperaudio return a list of dictionaries containing result one dictionary per proposed label the dictionaries contain the following keys label str the label identified by the model it is one of the suggested candidatelabel score float the score attributed by the model for that label between 0 and 1 we need to actually check for a real protocol otherwise it s impossible to use a local file like httphuggingfaceco png batching case coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license zero shot audio classification pipeline using clapmodel this pipeline predicts the class of an audio when you provide an audio and a set of candidate_labels example python from transformers import pipeline from datasets import load_dataset dataset load_dataset ashraq esc50 audio next iter dataset train audio array classifier pipeline task zero shot audio classification model laion clap htsat unfused classifier audio candidate_labels sound of a dog sound of vaccum cleaner score 0 9996 label sound of a dog score 0 0004 label sound of vaccum cleaner learn more about the basics of using a pipeline in the pipeline tutorial pipeline_tutorial this audio classification pipeline can currently be loaded from pipeline using the following task identifier zero shot audio classification see the list of available models on huggingface co models https huggingface co models filter zero shot audio classification no specific for_xxx available yet assign labels to the audio s passed as inputs args audios str list str np array or list np array the pipeline handles three types of inputs a string containing a http link pointing to an audio a string containing a local path to an audio an audio loaded in numpy candidate_labels list str the candidate labels for this audio hypothesis_template str optional defaults to this is a sound of the sentence used in cunjunction with candidate_labels to attempt the audio classification by replacing the placeholder with the candidate_labels then likelihood is estimated by using logits_per_audio return a list of dictionaries containing result one dictionary per proposed label the dictionaries contain the following keys label str the label identified by the model it is one of the suggested candidate_label score float the score attributed by the model for that label between 0 and 1 we need to actually check for a real protocol otherwise it s impossible to use a local file like http_huggingface_co png batching case
from collections import UserDict
from typing import Union

import numpy as np
import requests

from ..utils import (
    add_end_docstrings,
    logging,
)
from .audio_classification import ffmpeg_read
from .base import PIPELINE_INIT_ARGS, Pipeline


logger = logging.get_logger(__name__)


@add_end_docstrings(PIPELINE_INIT_ARGS)
class ZeroShotAudioClassificationPipeline(Pipeline):
    

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        if self.framework != "pt":
            raise ValueError(f"The {self.__class__} is only available in PyTorch.")
        

    def __call__(self, audios: Union[np.ndarray, bytes, str], **kwargs):
        
        return super().__call__(audios, **kwargs)

    def _sanitize_parameters(self, **kwargs):
        preprocess_params = {}
        if "candidate_labels" in kwargs:
            preprocess_params["candidate_labels"] = kwargs["candidate_labels"]
        if "hypothesis_template" in kwargs:
            preprocess_params["hypothesis_template"] = kwargs["hypothesis_template"]

        return preprocess_params, {}, {}

    def preprocess(self, audio, candidate_labels=None, hypothesis_template="This is a sound of {}."):
        if isinstance(audio, str):
            if audio.startswith("http://") or audio.startswith("https://"):
                
                
                audio = requests.get(audio).content
            else:
                with open(audio, "rb") as f:
                    audio = f.read()

        if isinstance(audio, bytes):
            audio = ffmpeg_read(audio, self.feature_extractor.sampling_rate)

        if not isinstance(audio, np.ndarray):
            raise ValueError("We expect a numpy ndarray as input")
        if len(audio.shape) != 1:
            raise ValueError("We expect a single channel audio input for ZeroShotAudioClassificationPipeline")

        inputs = self.feature_extractor(
            [audio], sampling_rate=self.feature_extractor.sampling_rate, return_tensors="pt"
        )
        inputs["candidate_labels"] = candidate_labels
        sequences = [hypothesis_template.format(x) for x in candidate_labels]
        text_inputs = self.tokenizer(sequences, return_tensors=self.framework, padding=True)
        inputs["text_inputs"] = [text_inputs]
        return inputs

    def _forward(self, model_inputs):
        candidate_labels = model_inputs.pop("candidate_labels")
        text_inputs = model_inputs.pop("text_inputs")
        if isinstance(text_inputs[0], UserDict):
            text_inputs = text_inputs[0]
        else:
            
            text_inputs = text_inputs[0][0]

        outputs = self.model(**text_inputs, **model_inputs)

        model_outputs = {
            "candidate_labels": candidate_labels,
            "logits": outputs.logits_per_audio,
        }
        return model_outputs

    def postprocess(self, model_outputs):
        candidate_labels = model_outputs.pop("candidate_labels")
        logits = model_outputs["logits"][0]

        if self.framework == "pt":
            probs = logits.softmax(dim=0)
            scores = probs.tolist()
        else:
            raise ValueError("`tf` framework not supported.")

        result = [
            {"score": score, "label": candidate_label}
            for score, candidate_label in sorted(zip(scores, candidate_labels), key=lambda x: -x[0])
        ]
        return result