can t use addenddocstringspipelineinitargs here because this one does not accept binaryoutput feature extraction pipeline using no model head this pipeline extracts the hidden states from the base transformer which can be used as features in downstream tasks example python from transformers import pipeline extractor pipelinemodelbertbaseuncased taskfeatureextraction result extractorthis is a simple test returntensorstrue result shape this is a tensor of shape 1 sequencelenth hiddendimension representing the input string torch size1 8 768 learn more about the basics of using a pipeline in the pipeline tutorial pipelinetutorial this feature extraction pipeline can currently be loaded from pipeline using the task identifier featureextraction all models may be used for this pipeline see a list of all models including communitycontributed models on huggingface comodelshttps huggingface comodels arguments model pretrainedmodel or tfpretrainedmodel the model that will be used by the pipeline to make predictions this needs to be a model inheriting from pretrainedmodel for pytorch and tfpretrainedmodel for tensorflow tokenizer pretrainedtokenizer the tokenizer that will be used by the pipeline to encode data for the model this object inherits from pretrainedtokenizer modelcard str or modelcard optional model card attributed to the model for this pipeline framework str optional the framework to use either pt for pytorch or tf for tensorflow the specified framework must be installed if no framework is specified will default to the one currently installed if no framework is specified and both frameworks are installed will default to the framework of the model or to pytorch if no model is provided returntensors bool optional if true returns a tensor according to the specified framework otherwise returns a list task str defaults to a taskidentifier for the pipeline argsparser pipelines argumenthandler optional reference to the object in charge of parsing supplied pipeline parameters device int optional defaults to 1 device ordinal for cpugpu supports setting this to 1 will leverage cpu a positive will run the model on the associated cuda device id tokenizekwargs dict optional additional dictionary of keyword arguments passed along to the tokenizer 0 is the first available tensor logits or lasthiddenstate extract the features of the inputs args args str or liststr one or several texts or one list of texts to get the features of return a nested list of float the features computed by the model can t use add_end_docstrings pipeline_init_args here because this one does not accept binary_output feature extraction pipeline using no model head this pipeline extracts the hidden states from the base transformer which can be used as features in downstream tasks example python from transformers import pipeline extractor pipeline model bert base uncased task feature extraction result extractor this is a simple test return_tensors true result shape this is a tensor of shape 1 sequence_lenth hidden_dimension representing the input string torch size 1 8 768 learn more about the basics of using a pipeline in the pipeline tutorial pipeline_tutorial this feature extraction pipeline can currently be loaded from pipeline using the task identifier feature extraction all models may be used for this pipeline see a list of all models including community contributed models on huggingface co models https huggingface co models arguments model pretrainedmodel or tfpretrainedmodel the model that will be used by the pipeline to make predictions this needs to be a model inheriting from pretrainedmodel for pytorch and tfpretrainedmodel for tensorflow tokenizer pretrainedtokenizer the tokenizer that will be used by the pipeline to encode data for the model this object inherits from pretrainedtokenizer modelcard str or modelcard optional model card attributed to the model for this pipeline framework str optional the framework to use either pt for pytorch or tf for tensorflow the specified framework must be installed if no framework is specified will default to the one currently installed if no framework is specified and both frameworks are installed will default to the framework of the model or to pytorch if no model is provided return_tensors bool optional if true returns a tensor according to the specified framework otherwise returns a list task str defaults to a task identifier for the pipeline args_parser pipelines argumenthandler optional reference to the object in charge of parsing supplied pipeline parameters device int optional defaults to 1 device ordinal for cpu gpu supports setting this to 1 will leverage cpu a positive will run the model on the associated cuda device id tokenize_kwargs dict optional additional dictionary of keyword arguments passed along to the tokenizer 0 is the first available tensor logits or last_hidden_state extract the features of the input s args args str or list str one or several texts or one list of texts to get the features of return a nested list of float the features computed by the model
from typing import Dict

from .base import GenericTensor, Pipeline



class FeatureExtractionPipeline(Pipeline):
    

    def _sanitize_parameters(self, truncation=None, tokenize_kwargs=None, return_tensors=None, **kwargs):
        if tokenize_kwargs is None:
            tokenize_kwargs = {}

        if truncation is not None:
            if "truncation" in tokenize_kwargs:
                raise ValueError(
                    "truncation parameter defined twice (given as keyword argument as well as in tokenize_kwargs)"
                )
            tokenize_kwargs["truncation"] = truncation

        preprocess_params = tokenize_kwargs

        postprocess_params = {}
        if return_tensors is not None:
            postprocess_params["return_tensors"] = return_tensors

        return preprocess_params, {}, postprocess_params

    def preprocess(self, inputs, **tokenize_kwargs) -> Dict[str, GenericTensor]:
        model_inputs = self.tokenizer(inputs, return_tensors=self.framework, **tokenize_kwargs)
        return model_inputs

    def _forward(self, model_inputs):
        model_outputs = self.model(**model_inputs)
        return model_outputs

    def postprocess(self, model_outputs, return_tensors=False):
        
        if return_tensors:
            return model_outputs[0]
        if self.framework == "pt":
            return model_outputs[0].tolist()
        elif self.framework == "tf":
            return model_outputs[0].numpy().tolist()

    def __call__(self, *args, **kwargs):
        
        return super().__call__(*args, **kwargs)