codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extraction savingloading class for common feature extractors def initself data optionaldictstr any none tensortype unionnone str tensortype none super initdata self converttotensorstensortypetensortype def getitemself item str unionany if isinstanceitem str return self dataitem else raise keyerrorindexing with integers is not available when using python based feature extractors def getattrself item str try return self dataitem except keyerror raise attributeerror def getstateself return data self data def setstateself state if data in state self data statedata copied from transformers tokenizationutilsbase batchencoding keys def keysself return self data keys copied from transformers tokenizationutilsbase batchencoding values def valuesself return self data values copied from transformers tokenizationutilsbase batchencoding items def itemsself return self data items def getisastensorfnsself tensortype optionalunionstr tensortype none if tensortype is none return none none convert to tensortype if not isinstancetensortype tensortype tensortype tensortypetensortype get a function reference for the correct framework if tensortype tensortype tensorflow if not istfavailable raise importerror unable to convert output to tensorflow tensors format tensorflow is not installed import tensorflow as tf astensor tf constant istensor tf istensor elif tensortype tensortype pytorch if not istorchavailable raise importerrorunable to convert output to pytorch tensors format pytorch is not installed import torch noqa def astensorvalue if isinstancevalue list tuple and lenvalue 0 and isinstancevalue0 np ndarray value np arrayvalue return torch tensorvalue istensor torch istensor elif tensortype tensortype jax if not isflaxavailable raise importerrorunable to convert output to jax tensors format jax is not installed import jax numpy as jnp noqa f811 astensor jnp array istensor isjaxtensor else def astensorvalue dtypenone if isinstancevalue list tuple and isinstancevalue0 list tuple np ndarray valuelens lenval for val in value if lensetvaluelens 1 and dtype is none we have a ragged list so handle explicitly value astensornp asarrayval for val in value dtypeobject return np asarrayvalue dtypedtype istensor isnumpyarray return istensor astensor def converttotensorsself tensortype optionalunionstr tensortype none if tensortype is none return self istensor astensor self getisastensorfnstensortype do the tensor conversion in batch for key value in self items try if not istensorvalue tensor astensorvalue selfkey tensor except noqa e722 if key overflowingvalues raise valueerrorunable to create tensor returning overflowing values of different lengths raise valueerror unable to create tensor you should probably activate padding with paddingtrue to have batched tensors with the same length return self def toself args kwargs batchfeature requiresbackendsself torch import torch noqa newdata device kwargs getdevice check if the args are a device or a dtype if device is none and lenargs 0 device should be always the first argument arg args0 if istorchdtypearg the first argument is a dtype pass elif isinstancearg str or istorchdevicearg or isinstancearg int device arg else it s something else raise valueerrorfattempting to cast a batchfeature to type strarg this is not supported we cast only floating point tensors to avoid issues with tokenizers casting longtensor to floattensor for k v in self items check if v is a floating point if torch isfloatingpointv cast and send to device newdatak v toargs kwargs elif device is not none newdatak v todevicedevice else newdatak v self data newdata return self class featureextractionmixinpushtohubmixin autoclass none def initself kwargs pop processorclass as it should be saved as private attribute additional attributes without default values sets processor class as an attribute self processorclass processorclass classmethod def frompretrained cls pretrainedmodelnameorpath unionstr os pathlike cachedir optionalunionstr os pathlike none forcedownload bool false localfilesonly bool false token optionalunionstr bool none revision str main kwargs r instantiate a type of featureextractionutils featureextractionmixin from a feature extractor e g a derived class of sequencefeatureextractor args pretrainedmodelnameorpath str or os pathlike this can be either a string the model id of a pretrained featureextractor hosted inside a model repo on huggingface co valid model ids can be located at the rootlevel like bertbaseuncased or namespaced under a user or organization name like dbmdzbertbasegermancased a path to a directory containing a feature extractor file saved using the featureextractionutils featureextractionmixin savepretrained method e g mymodeldirectory a path or url to a saved feature extractor json file e g mymodeldirectorypreprocessorconfig json cachedir str or os pathlike optional path to a directory in which a downloaded pretrained model feature extractor should be cached if the standard cache should not be used forcedownload bool optional defaults to false whether or not to force to redownload the feature extractor files and override the cached versions if they exist resumedownload bool optional defaults to false whether or not to delete incompletely received file attempts to resume the download if such a file exists proxies dictstr str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request token str or bool optional the token to use as http bearer ization for remote files if true or not specified will use the token generated when running huggingfacecli login stored in huggingface revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a gitbased system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revisionrefsprprnumber tip returnunusedkwargs bool optional defaults to false if false then this function returns just the final feature extractor object if true then this functions returns a tuplefeatureextractor unusedkwargs where unusedkwargs is a dictionary consisting of the keyvalue pairs whose keys are not feature extractor attributes i e the part of kwargs which has not been used to update featureextractor and is otherwise ignored kwargs dictstr any optional the values in kwargs of any keys which are feature extractor attributes will be used to override the loaded values behavior concerning keyvalue pairs whose keys are not feature extractor attributes is controlled by the returnunusedkwargs keyword parameter returns a feature extractor of type featureextractionutils featureextractionmixin examples python we can t instantiate directly the base class featureextractionmixin nor sequencefeatureextractor so let s show the examples on a derived class wav2vec2featureextractor featureextractor wav2vec2featureextractor frompretrained facebookwav2vec2base960h download featureextractionconfig from huggingface co and cache featureextractor wav2vec2featureextractor frompretrained testsavedmodel e g featureextractor or model was saved using savepretrained testsavedmodel featureextractor wav2vec2featureextractor frompretrained testsavedmodelpreprocessorconfig json featureextractor wav2vec2featureextractor frompretrained facebookwav2vec2base960h returnattentionmaskfalse foofalse assert featureextractor returnattentionmask is false featureextractor unusedkwargs wav2vec2featureextractor frompretrained facebookwav2vec2base960h returnattentionmaskfalse foofalse returnunusedkwargstrue assert featureextractor returnattentionmask is false assert unusedkwargs foo false kwargscachedir cachedir kwargsforcedownload forcedownload kwargslocalfilesonly localfilesonly kwargsrevision revision useauthtoken kwargs popuseauthtoken none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if token is not none raise valueerror token and useauthtoken are both specified please set only the argument token token useauthtoken if token is not none kwargstoken token featureextractordict kwargs cls getfeatureextractordictpretrainedmodelnameorpath kwargs return cls fromdictfeatureextractordict kwargs def savepretrainedself savedirectory unionstr os pathlike pushtohub bool false kwargs useauthtoken kwargs popuseauthtoken none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if kwargs gettoken none is not none raise valueerror token and useauthtoken are both specified please set only the argument token kwargstoken useauthtoken if os path isfilesavedirectory raise assertionerrorfprovided path savedirectory should be a directory not a file os makedirssavedirectory existoktrue if pushtohub commitmessage kwargs popcommitmessage none repoid kwargs poprepoid savedirectory splitos path sep1 repoid self createreporepoid kwargs filestimestamps self getfilestimestampssavedirectory if we have a custom config we copy the file defining it in the folder and set the attributes so it can be loaded from the hub if self autoclass is not none customobjectsaveself savedirectory configself if we save using the predefined names we can load using frompretrained outputfeatureextractorfile os path joinsavedirectory featureextractorname self tojsonfileoutputfeatureextractorfile logger infoffeature extractor saved in outputfeatureextractorfile if pushtohub self uploadmodifiedfiles savedirectory repoid filestimestamps commitmessagecommitmessage tokenkwargs gettoken return outputfeatureextractorfile classmethod def getfeatureextractordict cls pretrainedmodelnameorpath unionstr os pathlike kwargs tupledictstr any dictstr any cachedir kwargs popcachedir none forcedownload kwargs popforcedownload false resumedownload kwargs popresumedownload false proxies kwargs popproxies none token kwargs poptoken none useauthtoken kwargs popuseauthtoken none localfilesonly kwargs poplocalfilesonly false revision kwargs poprevision none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if token is not none raise valueerror token and useauthtoken are both specified please set only the argument token token useauthtoken frompipeline kwargs popfrompipeline none fromautoclass kwargs popfromauto false useragent filetype feature extractor fromautoclass fromautoclass if frompipeline is not none useragentusingpipeline frompipeline if isofflinemode and not localfilesonly logger infooffline mode forcing localfilesonlytrue localfilesonly true pretrainedmodelnameorpath strpretrainedmodelnameorpath islocal os path isdirpretrainedmodelnameorpath if os path isdirpretrainedmodelnameorpath featureextractorfile os path joinpretrainedmodelnameorpath featureextractorname if os path isfilepretrainedmodelnameorpath resolvedfeatureextractorfile pretrainedmodelnameorpath islocal true elif isremoteurlpretrainedmodelnameorpath featureextractorfile pretrainedmodelnameorpath resolvedfeatureextractorfile downloadurlpretrainedmodelnameorpath else featureextractorfile featureextractorname try load from local folder or from cache or download from model hub and cache resolvedfeatureextractorfile cachedfile pretrainedmodelnameorpath featureextractorfile cachedircachedir forcedownloadforcedownload proxiesproxies resumedownloadresumedownload localfilesonlylocalfilesonly tokentoken useragentuseragent revisionrevision except environmenterror raise any environment error raise by cachedfile it will have a helpful error message adapted to the original exception raise except exception for any other exception we throw a generic error raise environmenterror fcan t load feature extractor for pretrainedmodelnameorpath if you were trying to load it from https huggingface comodels make sure you don t have a local directory with the f same name otherwise make sure pretrainedmodelnameorpath is the correct path to a f directory containing a featureextractorname file try load featureextractor dict with openresolvedfeatureextractorfile r encodingutf8 as reader text reader read featureextractordict json loadstext except json jsondecodeerror raise environmenterror fit looks like the config file at resolvedfeatureextractorfile is not a valid json file if islocal logger infofloading configuration file resolvedfeatureextractorfile else logger info floading configuration file featureextractorfile from cache at resolvedfeatureextractorfile if automap in featureextractordict and not islocal featureextractordictautomap addmodelinfotoautomap featureextractordictautomap pretrainedmodelnameorpath return featureextractordict kwargs classmethod def fromdictcls featureextractordict dictstr any kwargs pretrainedfeatureextractor returnunusedkwargs kwargs popreturnunusedkwargs false featureextractor clsfeatureextractordict update featureextractor with kwargs if needed toremove for key value in kwargs items if hasattrfeatureextractor key setattrfeatureextractor key value toremove appendkey for key in toremove kwargs popkey none logger infoffeature extractor featureextractor if returnunusedkwargs return featureextractor kwargs else return featureextractor def todictself dictstr any output copy deepcopyself dict outputfeatureextractortype self class name if melfilters in output del outputmelfilters if window in output del outputwindow return output classmethod def fromjsonfilecls jsonfile unionstr os pathlike pretrainedfeatureextractor with openjsonfile r encodingutf8 as reader text reader read featureextractordict json loadstext return clsfeatureextractordict def tojsonstringself str dictionary self todict for key value in dictionary items if isinstancevalue np ndarray dictionarykey value tolist make sure private name processorclass is correctly saved as processorclass processorclass dictionary popprocessorclass none if processorclass is not none dictionaryprocessorclass processorclass return json dumpsdictionary indent2 sortkeystrue n def tojsonfileself jsonfilepath unionstr os pathlike with openjsonfilepath w encodingutf8 as writer writer writeself tojsonstring def reprself return fself class name self tojsonstring classmethod def registerforautoclasscls autoclassautofeatureextractor if not isinstanceautoclass str autoclass autoclass name import transformers models auto as automodule if not hasattrautomodule autoclass raise valueerrorfautoclass is not a valid auto class cls autoclass autoclass featureextractionmixin pushtohub copyfuncfeatureextractionmixin pushtohub if featureextractionmixin pushtohub doc is not none featureextractionmixin pushtohub doc featureextractionmixin pushtohub doc format objectfeature extractor objectclassautofeatureextractor objectfilesfeature extractor file coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extraction saving loading class for common feature extractors noqa noqa f821 holds the output of the sequencefeatureextractor pad and feature extractor specific __call__ methods this class is derived from a python dictionary and can be used as a dictionary args data dict optional dictionary of lists arrays tensors returned by the __call__ pad methods input_values attention_mask etc tensor_type union none str tensortype optional you can give a tensor_type here to convert the lists of integers in pytorch tensorflow numpy tensors at initialization if the key is a string returns the value of the dict associated to key input_values attention_mask etc copied from transformers tokenization_utils_base batchencoding keys copied from transformers tokenization_utils_base batchencoding values copied from transformers tokenization_utils_base batchencoding items convert to tensortype get a function reference for the correct framework noqa noqa f811 we have a ragged list so handle explicitly convert the inner content to tensors args tensor_type str or utils tensortype optional the type of tensors to use if str should be one of the values of the enum utils tensortype if none no modification is done do the tensor conversion in batch noqa e722 send all values to device by calling v to args kwargs pytorch only this should support casting in different dtypes and sending the batchfeature to a different device args args tuple will be passed to the to function of the tensors kwargs dict optional will be passed to the to function of the tensors returns batchfeature the same instance after modification noqa check if the args are a device or a dtype device should be always the first argument the first argument is a dtype it s something else we cast only floating point tensors to avoid issues with tokenizers casting longtensor to floattensor check if v is a floating point cast and send to device this is a feature extraction mixin used to provide saving loading functionality for sequential and image feature extractors set elements of kwargs as attributes pop processor_class as it should be saved as private attribute additional attributes without default values sets processor class as an attribute instantiate a type of feature_extraction_utils featureextractionmixin from a feature extractor e g a derived class of sequencefeatureextractor args pretrained_model_name_or_path str or os pathlike this can be either a string the model id of a pretrained feature_extractor hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing a feature extractor file saved using the feature_extraction_utils featureextractionmixin save_pretrained method e g my_model_directory a path or url to a saved feature extractor json file e g my_model_directory preprocessor_config json cache_dir str or os pathlike optional path to a directory in which a downloaded pretrained model feature extractor should be cached if the standard cache should not be used force_download bool optional defaults to false whether or not to force to re download the feature extractor files and override the cached versions if they exist resume_download bool optional defaults to false whether or not to delete incompletely received file attempts to resume the download if such a file exists proxies dict str str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request token str or bool optional the token to use as http bearer ization for remote files if true or not specified will use the token generated when running huggingface cli login stored in huggingface revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a git based system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revision refs pr pr_number tip return_unused_kwargs bool optional defaults to false if false then this function returns just the final feature extractor object if true then this functions returns a tuple feature_extractor unused_kwargs where unused_kwargs is a dictionary consisting of the key value pairs whose keys are not feature extractor attributes i e the part of kwargs which has not been used to update feature_extractor and is otherwise ignored kwargs dict str any optional the values in kwargs of any keys which are feature extractor attributes will be used to override the loaded values behavior concerning key value pairs whose keys are not feature extractor attributes is controlled by the return_unused_kwargs keyword parameter returns a feature extractor of type feature_extraction_utils featureextractionmixin examples python we can t instantiate directly the base class featureextractionmixin nor sequencefeatureextractor so let s show the examples on a derived class wav2vec2featureextractor feature_extractor wav2vec2featureextractor from_pretrained facebook wav2vec2 base 960h download feature_extraction_config from huggingface co and cache feature_extractor wav2vec2featureextractor from_pretrained test saved_model e g feature_extractor or model was saved using save_pretrained test saved_model feature_extractor wav2vec2featureextractor from_pretrained test saved_model preprocessor_config json feature_extractor wav2vec2featureextractor from_pretrained facebook wav2vec2 base 960h return_attention_mask false foo false assert feature_extractor return_attention_mask is false feature_extractor unused_kwargs wav2vec2featureextractor from_pretrained facebook wav2vec2 base 960h return_attention_mask false foo false return_unused_kwargs true assert feature_extractor return_attention_mask is false assert unused_kwargs foo false save a feature_extractor object to the directory save_directory so that it can be re loaded using the feature_extraction_utils featureextractionmixin from_pretrained class method args save_directory str or os pathlike directory where the feature extractor json file will be saved will be created if it does not exist push_to_hub bool optional defaults to false whether or not to push your model to the hugging face model hub after saving it you can specify the repository you want to push to with repo_id will default to the name of save_directory in your namespace kwargs dict str any optional additional key word arguments passed along to the utils pushtohubmixin push_to_hub method if we have a custom config we copy the file defining it in the folder and set the attributes so it can be loaded from the hub if we save using the predefined names we can load using from_pretrained from a pretrained_model_name_or_path resolve to a dictionary of parameters to be used for instantiating a feature extractor of type feature_extraction_utils featureextractionmixin using from_dict parameters pretrained_model_name_or_path str or os pathlike the identifier of the pre trained checkpoint from which we want the dictionary of parameters returns tuple dict dict the dictionary ies that will be used to instantiate the feature extractor object load from local folder or from cache or download from model hub and cache raise any environment error raise by cached_file it will have a helpful error message adapted to the original exception for any other exception we throw a generic error load feature_extractor dict instantiates a type of feature_extraction_utils featureextractionmixin from a python dictionary of parameters args feature_extractor_dict dict str any dictionary that will be used to instantiate the feature extractor object such a dictionary can be retrieved from a pretrained checkpoint by leveraging the feature_extraction_utils featureextractionmixin to_dict method kwargs dict str any additional parameters from which to initialize the feature extractor object returns feature_extraction_utils featureextractionmixin the feature extractor object instantiated from those parameters update feature_extractor with kwargs if needed serializes this instance to a python dictionary returns dict str any dictionary of all the attributes that make up this configuration instance instantiates a feature extractor of type feature_extraction_utils featureextractionmixin from the path to a json file of parameters args json_file str or os pathlike path to the json file containing the parameters returns a feature extractor of type feature_extraction_utils featureextractionmixin the feature_extractor object instantiated from that json file serializes this instance to a json string returns str string containing all the attributes that make up this feature_extractor instance in json format make sure private name _processor_class is correctly saved as processor_class save this instance to a json file args json_file_path str or os pathlike path to the json file in which this feature_extractor instance s parameters will be saved register this class with a given auto class this should only be used for custom feature extractors as the ones in the library are already mapped with autofeatureextractor tip warning true this api is experimental and may have some slight breaking changes in the next releases tip args auto_class str or type optional defaults to autofeatureextractor the auto class to register this new feature extractor with
import copy
import json
import os
import warnings
from collections import UserDict
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union

import numpy as np

from .dynamic_module_utils import custom_object_save
from .utils import (
    FEATURE_EXTRACTOR_NAME,
    PushToHubMixin,
    TensorType,
    add_model_info_to_auto_map,
    cached_file,
    copy_func,
    download_url,
    is_flax_available,
    is_jax_tensor,
    is_numpy_array,
    is_offline_mode,
    is_remote_url,
    is_tf_available,
    is_torch_available,
    is_torch_device,
    is_torch_dtype,
    logging,
    requires_backends,
)


if TYPE_CHECKING:
    if is_torch_available():
        import torch  


logger = logging.get_logger(__name__)

PreTrainedFeatureExtractor = Union["SequenceFeatureExtractor"]  


class BatchFeature(UserDict):
    r

    def __init__(self, data: Optional[Dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None):
        super().__init__(data)
        self.convert_to_tensors(tensor_type=tensor_type)

    def __getitem__(self, item: str) -> Union[Any]:
        
        if isinstance(item, str):
            return self.data[item]
        else:
            raise KeyError("Indexing with integers is not available when using Python based feature extractors")

    def __getattr__(self, item: str):
        try:
            return self.data[item]
        except KeyError:
            raise AttributeError

    def __getstate__(self):
        return {"data": self.data}

    def __setstate__(self, state):
        if "data" in state:
            self.data = state["data"]

    
    def keys(self):
        return self.data.keys()

    
    def values(self):
        return self.data.values()

    
    def items(self):
        return self.data.items()

    def _get_is_as_tensor_fns(self, tensor_type: Optional[Union[str, TensorType]] = None):
        if tensor_type is None:
            return None, None

        
        if not isinstance(tensor_type, TensorType):
            tensor_type = TensorType(tensor_type)

        
        if tensor_type == TensorType.TENSORFLOW:
            if not is_tf_available():
                raise ImportError(
                    "Unable to convert output to TensorFlow tensors format, TensorFlow is not installed."
                )
            import tensorflow as tf

            as_tensor = tf.constant
            is_tensor = tf.is_tensor
        elif tensor_type == TensorType.PYTORCH:
            if not is_torch_available():
                raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.")
            import torch  

            def as_tensor(value):
                if isinstance(value, (list, tuple)) and len(value) > 0 and isinstance(value[0], np.ndarray):
                    value = np.array(value)
                return torch.tensor(value)

            is_tensor = torch.is_tensor
        elif tensor_type == TensorType.JAX:
            if not is_flax_available():
                raise ImportError("Unable to convert output to JAX tensors format, JAX is not installed.")
            import jax.numpy as jnp  

            as_tensor = jnp.array
            is_tensor = is_jax_tensor
        else:

            def as_tensor(value, dtype=None):
                if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)):
                    value_lens = [len(val) for val in value]
                    if len(set(value_lens)) > 1 and dtype is None:
                        
                        value = as_tensor([np.asarray(val) for val in value], dtype=object)
                return np.asarray(value, dtype=dtype)

            is_tensor = is_numpy_array
        return is_tensor, as_tensor

    def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None):
        
        if tensor_type is None:
            return self

        is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)

        
        for key, value in self.items():
            try:
                if not is_tensor(value):
                    tensor = as_tensor(value)

                    self[key] = tensor
            except:  
                if key == "overflowing_values":
                    raise ValueError("Unable to create tensor returning overflowing values of different lengths. ")
                raise ValueError(
                    "Unable to create tensor, you should probably activate padding "
                    "with 'padding=True' to have batched tensors with the same length."
                )

        return self

    def to(self, *args, **kwargs) -> "BatchFeature":
        
        requires_backends(self, ["torch"])
        import torch  

        new_data = {}
        device = kwargs.get("device")
        
        if device is None and len(args) > 0:
            
            arg = args[0]
            if is_torch_dtype(arg):
                
                pass
            elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
                device = arg
            else:
                
                raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
        
        for k, v in self.items():
            
            if torch.is_floating_point(v):
                
                new_data[k] = v.to(*args, **kwargs)
            elif device is not None:
                new_data[k] = v.to(device=device)
            else:
                new_data[k] = v
        self.data = new_data
        return self


class FeatureExtractionMixin(PushToHubMixin):
    

    _auto_class = None

    def __init__(self, **kwargs):
        
        
        self._processor_class = kwargs.pop("processor_class", None)
        
        for key, value in kwargs.items():
            try:
                setattr(self, key, value)
            except AttributeError as err:
                logger.error(f"Can't set {key} with value {value} for {self}")
                raise err

    def _set_processor_class(self, processor_class: str):
        
        self._processor_class = processor_class

    @classmethod
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Union[str, os.PathLike],
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        **kwargs,
    ):
        r
        kwargs["cache_dir"] = cache_dir
        kwargs["force_download"] = force_download
        kwargs["local_files_only"] = local_files_only
        kwargs["revision"] = revision

        use_auth_token = kwargs.pop("use_auth_token", None)
        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if token is not None:
            kwargs["token"] = token

        feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)

        return cls.from_dict(feature_extractor_dict, **kwargs)

    def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
        
        use_auth_token = kwargs.pop("use_auth_token", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if kwargs.get("token", None) is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            kwargs["token"] = use_auth_token

        if os.path.isfile(save_directory):
            raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")

        os.makedirs(save_directory, exist_ok=True)

        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
            repo_id = self._create_repo(repo_id, **kwargs)
            files_timestamps = self._get_files_timestamps(save_directory)

        
        
        if self._auto_class is not None:
            custom_object_save(self, save_directory, config=self)

        
        output_feature_extractor_file = os.path.join(save_directory, FEATURE_EXTRACTOR_NAME)

        self.to_json_file(output_feature_extractor_file)
        logger.info(f"Feature extractor saved in {output_feature_extractor_file}")

        if push_to_hub:
            self._upload_modified_files(
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
                token=kwargs.get("token"),
            )

        return [output_feature_extractor_file]

    @classmethod
    def get_feature_extractor_dict(
        cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
        
        cache_dir = kwargs.pop("cache_dir", None)
        force_download = kwargs.pop("force_download", False)
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        token = kwargs.pop("token", None)
        use_auth_token = kwargs.pop("use_auth_token", None)
        local_files_only = kwargs.pop("local_files_only", False)
        revision = kwargs.pop("revision", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        from_pipeline = kwargs.pop("_from_pipeline", None)
        from_auto_class = kwargs.pop("_from_auto", False)

        user_agent = {"file_type": "feature extractor", "from_auto_class": from_auto_class}
        if from_pipeline is not None:
            user_agent["using_pipeline"] = from_pipeline

        if is_offline_mode() and not local_files_only:
            logger.info("Offline mode: forcing local_files_only=True")
            local_files_only = True

        pretrained_model_name_or_path = str(pretrained_model_name_or_path)
        is_local = os.path.isdir(pretrained_model_name_or_path)
        if os.path.isdir(pretrained_model_name_or_path):
            feature_extractor_file = os.path.join(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME)
        if os.path.isfile(pretrained_model_name_or_path):
            resolved_feature_extractor_file = pretrained_model_name_or_path
            is_local = True
        elif is_remote_url(pretrained_model_name_or_path):
            feature_extractor_file = pretrained_model_name_or_path
            resolved_feature_extractor_file = download_url(pretrained_model_name_or_path)
        else:
            feature_extractor_file = FEATURE_EXTRACTOR_NAME
            try:
                
                resolved_feature_extractor_file = cached_file(
                    pretrained_model_name_or_path,
                    feature_extractor_file,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    local_files_only=local_files_only,
                    token=token,
                    user_agent=user_agent,
                    revision=revision,
                )
            except EnvironmentError:
                
                
                raise
            except Exception:
                
                raise EnvironmentError(
                    f"Can't load feature extractor for '{pretrained_model_name_or_path}'. If you were trying to load"
                    " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
                    f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
                    f" directory containing a {FEATURE_EXTRACTOR_NAME} file"
                )

        try:
            
            with open(resolved_feature_extractor_file, "r", encoding="utf-8") as reader:
                text = reader.read()
            feature_extractor_dict = json.loads(text)

        except json.JSONDecodeError:
            raise EnvironmentError(
                f"It looks like the config file at '{resolved_feature_extractor_file}' is not a valid JSON file."
            )

        if is_local:
            logger.info(f"loading configuration file {resolved_feature_extractor_file}")
        else:
            logger.info(
                f"loading configuration file {feature_extractor_file} from cache at {resolved_feature_extractor_file}"
            )

        if "auto_map" in feature_extractor_dict and not is_local:
            feature_extractor_dict["auto_map"] = add_model_info_to_auto_map(
                feature_extractor_dict["auto_map"], pretrained_model_name_or_path
            )

        return feature_extractor_dict, kwargs

    @classmethod
    def from_dict(cls, feature_extractor_dict: Dict[str, Any], **kwargs) -> PreTrainedFeatureExtractor:
        
        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)

        feature_extractor = cls(**feature_extractor_dict)

        
        to_remove = []
        for key, value in kwargs.items():
            if hasattr(feature_extractor, key):
                setattr(feature_extractor, key, value)
                to_remove.append(key)
        for key in to_remove:
            kwargs.pop(key, None)

        logger.info(f"Feature extractor {feature_extractor}")
        if return_unused_kwargs:
            return feature_extractor, kwargs
        else:
            return feature_extractor

    def to_dict(self) -> Dict[str, Any]:
        
        output = copy.deepcopy(self.__dict__)
        output["feature_extractor_type"] = self.__class__.__name__
        if "mel_filters" in output:
            del output["mel_filters"]
        if "window" in output:
            del output["window"]
        return output

    @classmethod
    def from_json_file(cls, json_file: Union[str, os.PathLike]) -> PreTrainedFeatureExtractor:
        
        with open(json_file, "r", encoding="utf-8") as reader:
            text = reader.read()
        feature_extractor_dict = json.loads(text)
        return cls(**feature_extractor_dict)

    def to_json_string(self) -> str:
        
        dictionary = self.to_dict()

        for key, value in dictionary.items():
            if isinstance(value, np.ndarray):
                dictionary[key] = value.tolist()

        
        
        _processor_class = dictionary.pop("_processor_class", None)
        if _processor_class is not None:
            dictionary["processor_class"] = _processor_class

        return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"

    def to_json_file(self, json_file_path: Union[str, os.PathLike]):
        
        with open(json_file_path, "w", encoding="utf-8") as writer:
            writer.write(self.to_json_string())

    def __repr__(self):
        return f"{self.__class__.__name__} {self.to_json_string()}"

    @classmethod
    def register_for_auto_class(cls, auto_class="AutoFeatureExtractor"):
        
        if not isinstance(auto_class, str):
            auto_class = auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(auto_module, auto_class):
            raise ValueError(f"{auto_class} is not a valid auto class.")

        cls._auto_class = auto_class


FeatureExtractionMixin.push_to_hub = copy_func(FeatureExtractionMixin.push_to_hub)
if FeatureExtractionMixin.push_to_hub.__doc__ is not None:
    FeatureExtractionMixin.push_to_hub.__doc__ = FeatureExtractionMixin.push_to_hub.__doc__.format(
        object="feature extractor", object_class="AutoFeatureExtractor", object_files="feature extractor file"
    )