codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processing savingloading class for common processors dynamically import the transformers module to grab the attribute classes of the processor form their names this is a mixin used to provide savingloading functionality for all processor classes names need to be attrclass for attr in attributes args have to match the attributes class attribute sanitize args and kwargs check each arg is of the proper class this will also catch a user initializing in the wrong order nothing is ever going to be an instance of autoxxx in that case we check the base class saves the attributes of this processor feature extractor tokenizer in the specified directory so that it can be reloaded using the processormixin frompretrained method tip this class method is simply calling featureextractionutils featureextractionmixin savepretrained and tokenizationutilsbase pretrainedtokenizerbase savepretrained please refer to the docstrings of the methods above for more information tip args savedirectory str or os pathlike directory where the feature extractor json file and the tokenizer files will be saved directory will be created if it does not exist pushtohub bool optional defaults to false whether or not to push your model to the hugging face model hub after saving it you can specify the repository you want to push to with repoid will default to the name of savedirectory in your namespace kwargs dictstr any optional additional key word arguments passed along to the utils pushtohubmixin pushtohub method if we have a custom config we copy the file defining it in the folder and set the attributes so it can be loaded from the hub include the processor class in the attribute config so this processor can then be reloaded with the autoprocessor api we added an attribute to the initkwargs of the tokenizers which needs to be cleaned up kwargscachedir cachedir kwargsforcedownload forcedownload kwargslocalfilesonly localfilesonly kwargsrevision revision useauthtoken kwargs popuseauthtoken none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if token is not none raise valueerror token and useauthtoken are both specified please set only the argument token token useauthtoken if token is not none kwargstoken token args cls getargumentsfrompretrainedpretrainedmodelnameorpath kwargs return clsargs classmethod def registerforautoclasscls autoclassautoprocessor if not isinstanceautoclass str autoclass autoclass name import transformers models auto as automodule if not hasattrautomodule autoclass raise valueerrorfautoclass is not a valid auto class cls autoclass autoclass classmethod def getargumentsfrompretrainedcls pretrainedmodelnameorpath kwargs args for attributename in cls attributes classname getattrcls fattributenameclass if isinstanceclassname tuple classes tuplegetattrtransformersmodule n if n is not none else none for n in classname usefast kwargs getusefast true if usefast and classes1 is not none attributeclass classes1 else attributeclass classes0 else attributeclass getattrtransformersmodule classname args appendattributeclass frompretrainedpretrainedmodelnameorpath kwargs return args property def modelinputnamesself firstattribute getattrself self attributes0 return getattrfirstattribute modelinputnames none processormixin pushtohub copyfuncprocessormixin pushtohub if processormixin pushtohub doc is not none processormixin pushtohub doc processormixin pushtohub doc format objectprocessor objectclassautoprocessor objectfilesprocessor files coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processing saving loading class for common processors dynamically import the transformers module to grab the attribute classes of the processor form their names this is a mixin used to provide saving loading functionality for all processor classes names need to be attr_class for attr in attributes args have to match the attributes class attribute sanitize args and kwargs check each arg is of the proper class this will also catch a user initializing in the wrong order nothing is ever going to be an instance of autoxxx in that case we check the base class saves the attributes of this processor feature extractor tokenizer in the specified directory so that it can be reloaded using the processormixin from_pretrained method tip this class method is simply calling feature_extraction_utils featureextractionmixin save_pretrained and tokenization_utils_base pretrainedtokenizerbase save_pretrained please refer to the docstrings of the methods above for more information tip args save_directory str or os pathlike directory where the feature extractor json file and the tokenizer files will be saved directory will be created if it does not exist push_to_hub bool optional defaults to false whether or not to push your model to the hugging face model hub after saving it you can specify the repository you want to push to with repo_id will default to the name of save_directory in your namespace kwargs dict str any optional additional key word arguments passed along to the utils pushtohubmixin push_to_hub method if we have a custom config we copy the file defining it in the folder and set the attributes so it can be loaded from the hub include the processor class in the attribute config so this processor can then be reloaded with the autoprocessor api we added an attribute to the init_kwargs of the tokenizers which needs to be cleaned up instantiate a processor associated with a pretrained model tip this class method is simply calling the feature extractor feature_extraction_utils featureextractionmixin from_pretrained image processor image_processing_utils imageprocessingmixin and the tokenizer tokenization_utils_base pretrainedtokenizer from_pretrained methods please refer to the docstrings of the methods above for more information tip args pretrained_model_name_or_path str or os pathlike this can be either a string the model id of a pretrained feature_extractor hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing a feature extractor file saved using the sequencefeatureextractor save_pretrained method e g my_model_directory a path or url to a saved feature extractor json file e g my_model_directory preprocessor_config json kwargs additional keyword arguments passed along to both feature_extraction_utils featureextractionmixin from_pretrained and tokenization_utils_base pretrainedtokenizer from_pretrained register this class with a given auto class this should only be used for custom feature extractors as the ones in the library are already mapped with autoprocessor tip warning true this api is experimental and may have some slight breaking changes in the next releases tip args auto_class str or type optional defaults to autoprocessor the auto class to register this new feature extractor with
import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


logger = logging.get_logger(__name__)


transformers_module = direct_transformers_import(Path(__file__).parent)


AUTO_TO_BASE_CLASS_MAPPING = {
    "AutoTokenizer": "PreTrainedTokenizerBase",
    "AutoFeatureExtractor": "FeatureExtractionMixin",
    "AutoImageProcessor": "ImageProcessingMixin",
}


class ProcessorMixin(PushToHubMixin):
    

    attributes = ["feature_extractor", "tokenizer"]
    
    feature_extractor_class = None
    tokenizer_class = None
    _auto_class = None

    
    def __init__(self, *args, **kwargs):
        
        for key in kwargs:
            if key not in self.attributes:
                raise TypeError(f"Unexpected keyword argument {key}.")
        for arg, attribute_name in zip(args, self.attributes):
            if attribute_name in kwargs:
                raise TypeError(f"Got multiple values for argument {attribute_name}.")
            else:
                kwargs[attribute_name] = arg

        if len(kwargs) != len(self.attributes):
            raise ValueError(
                f"This processor requires {len(self.attributes)} arguments: {', '.join(self.attributes)}. Got "
                f"{len(args)} arguments instead."
            )

        
        for attribute_name, arg in kwargs.items():
            class_name = getattr(self, f"{attribute_name}_class")
            
            class_name = AUTO_TO_BASE_CLASS_MAPPING.get(class_name, class_name)
            if isinstance(class_name, tuple):
                proper_class = tuple(getattr(transformers_module, n) for n in class_name if n is not None)
            else:
                proper_class = getattr(transformers_module, class_name)

            if not isinstance(arg, proper_class):
                raise ValueError(
                    f"Received a {type(arg).__name__} for argument {attribute_name}, but a {class_name} was expected."
                )

            setattr(self, attribute_name, arg)

    def __repr__(self):
        attributes_repr = [f"- {name}: {repr(getattr(self, name))}" for name in self.attributes]
        attributes_repr = "\n".join(attributes_repr)
        return f"{self.__class__.__name__}:\n{attributes_repr}"

    def save_pretrained(self, save_directory, push_to_hub: bool = False, **kwargs):
        
        use_auth_token = kwargs.pop("use_auth_token", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if kwargs.get("token", None) is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            kwargs["token"] = use_auth_token

        os.makedirs(save_directory, exist_ok=True)

        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
            repo_id = self._create_repo(repo_id, **kwargs)
            files_timestamps = self._get_files_timestamps(save_directory)
        
        
        if self._auto_class is not None:
            attrs = [getattr(self, attribute_name) for attribute_name in self.attributes]
            configs = [(a.init_kwargs if isinstance(a, PreTrainedTokenizerBase) else a) for a in attrs]
            custom_object_save(self, save_directory, config=configs)

        for attribute_name in self.attributes:
            attribute = getattr(self, attribute_name)
            
            
            if hasattr(attribute, "_set_processor_class"):
                attribute._set_processor_class(self.__class__.__name__)
            attribute.save_pretrained(save_directory)

        if self._auto_class is not None:
            
            for attribute_name in self.attributes:
                attribute = getattr(self, attribute_name)
                if isinstance(attribute, PreTrainedTokenizerBase):
                    del attribute.init_kwargs["auto_map"]

        if push_to_hub:
            self._upload_modified_files(
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
                token=kwargs.get("token"),
            )

    @classmethod
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Union[str, os.PathLike],
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        **kwargs,
    ):
        r
        kwargs["cache_dir"] = cache_dir
        kwargs["force_download"] = force_download
        kwargs["local_files_only"] = local_files_only
        kwargs["revision"] = revision

        use_auth_token = kwargs.pop("use_auth_token", None)
        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if token is not None:
            kwargs["token"] = token

        args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs)
        return cls(*args)

    @classmethod
    def register_for_auto_class(cls, auto_class="AutoProcessor"):
        
        if not isinstance(auto_class, str):
            auto_class = auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(auto_module, auto_class):
            raise ValueError(f"{auto_class} is not a valid auto class.")

        cls._auto_class = auto_class

    @classmethod
    def _get_arguments_from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
        args = []
        for attribute_name in cls.attributes:
            class_name = getattr(cls, f"{attribute_name}_class")
            if isinstance(class_name, tuple):
                classes = tuple(getattr(transformers_module, n) if n is not None else None for n in class_name)
                use_fast = kwargs.get("use_fast", True)
                if use_fast and classes[1] is not None:
                    attribute_class = classes[1]
                else:
                    attribute_class = classes[0]
            else:
                attribute_class = getattr(transformers_module, class_name)

            args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs))
        return args

    @property
    def model_input_names(self):
        first_attribute = getattr(self, self.attributes[0])
        return getattr(first_attribute, "model_input_names", None)


ProcessorMixin.push_to_hub = copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
    ProcessorMixin.push_to_hub.__doc__ = ProcessorMixin.push_to_hub.__doc__.format(
        object="processor", object_class="AutoProcessor", object_files="processor files"
    )