zero shot image classification pipeline using clipmodel this pipeline predicts the class of an image when you provide an image and a set of candidatelabels example python from transformers import pipeline classifier pipelinemodelopenaiclipvitlargepatch14 classifier https huggingface codatasetsnarsilimagedummyrawmainparrots png candidatelabelsanimals humans landscape score 0 965 label animals score 0 03 label humans score 0 005 label landscape classifier https huggingface codatasetsnarsilimagedummyrawmainparrots png candidatelabelsblack and white photorealist painting score 0 996 label black and white score 0 003 label photorealist score 0 0 label painting learn more about the basics of using a pipeline in the pipeline tutorial pipelinetutorial this image classification pipeline can currently be loaded from pipeline using the following task identifier zeroshotimageclassification see the list of available models on huggingface comodelshttps huggingface comodels filterzeroshotimageclassification assign labels to the images passed as inputs args images str liststr pil image or listpil image the pipeline handles three types of images a string containing a http link pointing to an image a string containing a local path to an image an image loaded in pil directly candidatelabels liststr the candidate labels for this image hypothesistemplate str optional defaults to this is a photo of the sentence used in cunjunction with candidatelabels to attempt the image classification by replacing the placeholder with the candidatelabels then likelihood is estimated by using logitsperimage timeout float optional defaults to none the maximum time in seconds to wait for fetching images from the web if none no timeout is set and the call may block forever return a list of dictionaries containing result one dictionary per proposed label the dictionaries contain the following keys label str the label identified by the model it is one of the suggested candidatelabel score float the score attributed by the model for that label between 0 and 1 batching case zero shot image classification pipeline using clipmodel this pipeline predicts the class of an image when you provide an image and a set of candidate_labels example python from transformers import pipeline classifier pipeline model openai clip vit large patch14 classifier https huggingface co datasets narsil image_dummy raw main parrots png candidate_labels animals humans landscape score 0 965 label animals score 0 03 label humans score 0 005 label landscape classifier https huggingface co datasets narsil image_dummy raw main parrots png candidate_labels black and white photorealist painting score 0 996 label black and white score 0 003 label photorealist score 0 0 label painting learn more about the basics of using a pipeline in the pipeline tutorial pipeline_tutorial this image classification pipeline can currently be loaded from pipeline using the following task identifier zero shot image classification see the list of available models on huggingface co models https huggingface co models filter zero shot image classification assign labels to the image s passed as inputs args images str list str pil image or list pil image the pipeline handles three types of images a string containing a http link pointing to an image a string containing a local path to an image an image loaded in pil directly candidate_labels list str the candidate labels for this image hypothesis_template str optional defaults to this is a photo of the sentence used in cunjunction with candidate_labels to attempt the image classification by replacing the placeholder with the candidate_labels then likelihood is estimated by using logits_per_image timeout float optional defaults to none the maximum time in seconds to wait for fetching images from the web if none no timeout is set and the call may block forever return a list of dictionaries containing result one dictionary per proposed label the dictionaries contain the following keys label str the label identified by the model it is one of the suggested candidate_label score float the score attributed by the model for that label between 0 and 1 batching case
from collections import UserDict
from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
    from PIL import Image

    from ..image_utils import load_image

if is_torch_available():
    from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES

if is_tf_available():
    from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES
    from ..tf_utils import stable_softmax

logger = logging.get_logger(__name__)


@add_end_docstrings(PIPELINE_INIT_ARGS)
class ZeroShotImageClassificationPipeline(Pipeline):
    

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        requires_backends(self, "vision")
        self.check_model_type(
            TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES
            if self.framework == "tf"
            else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES
        )

    def __call__(self, images: Union[str, List[str], "Image", List["Image"]], **kwargs):
        
        return super().__call__(images, **kwargs)

    def _sanitize_parameters(self, **kwargs):
        preprocess_params = {}
        if "candidate_labels" in kwargs:
            preprocess_params["candidate_labels"] = kwargs["candidate_labels"]
        if "timeout" in kwargs:
            preprocess_params["timeout"] = kwargs["timeout"]
        if "hypothesis_template" in kwargs:
            preprocess_params["hypothesis_template"] = kwargs["hypothesis_template"]

        return preprocess_params, {}, {}

    def preprocess(self, image, candidate_labels=None, hypothesis_template="This is a photo of {}.", timeout=None):
        image = load_image(image, timeout=timeout)
        inputs = self.image_processor(images=[image], return_tensors=self.framework)
        inputs["candidate_labels"] = candidate_labels
        sequences = [hypothesis_template.format(x) for x in candidate_labels]
        text_inputs = self.tokenizer(sequences, return_tensors=self.framework, padding=True)
        inputs["text_inputs"] = [text_inputs]
        return inputs

    def _forward(self, model_inputs):
        candidate_labels = model_inputs.pop("candidate_labels")
        text_inputs = model_inputs.pop("text_inputs")
        if isinstance(text_inputs[0], UserDict):
            text_inputs = text_inputs[0]
        else:
            
            text_inputs = text_inputs[0][0]

        outputs = self.model(**text_inputs, **model_inputs)

        model_outputs = {
            "candidate_labels": candidate_labels,
            "logits": outputs.logits_per_image,
        }
        return model_outputs

    def postprocess(self, model_outputs):
        candidate_labels = model_outputs.pop("candidate_labels")
        logits = model_outputs["logits"][0]
        if self.framework == "pt":
            probs = logits.softmax(dim=-1).squeeze(-1)
            scores = probs.tolist()
            if not isinstance(scores, list):
                scores = [scores]
        elif self.framework == "tf":
            probs = stable_softmax(logits, axis=-1)
            scores = probs.numpy().tolist()
        else:
            raise ValueError(f"Unsupported framework: {self.framework}")

        result = [
            {"score": score, "label": candidate_label}
            for score, candidate_label in sorted(zip(scores, candidate_labels), key=lambda x: -x[0])
        ]
        return result