2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license number of byte required for this data type returns integer 0 args dimension fixeddimension numtokentoadd returns 0 is possible if using a dynamic axis compute the size taken by all the parameters in the given the storage format when serializing the model args numparameters number of parameters to be saved dtype the data format each parameter will be saved returns size in byte taken to save all the parameters gets a preprocessor tokenizer feature extractor or processor that is available for modelname args modelname str name of the model for which a preprocessor are loaded returns optionalunionautotokenizer autofeatureextractor autoprocessor if a processor is found it is returned otherwise if a tokenizer or a feature extractor exists it is returned if both a tokenizer and a feature extractor exist an error is raised the function returns none if no preprocessor is found avoid circular imports by only importing this here 2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tests_ignore number of byte required for this data type returns integer 0 args dimension fixed_dimension num_token_to_add returns 0 is possible if using a dynamic axis compute the size taken by all the parameters in the given the storage format when serializing the model args num_parameters number of parameters to be saved dtype the data format each parameter will be saved returns size in byte taken to save all the parameters gets a preprocessor tokenizer feature extractor or processor that is available for model_name args model_name str name of the model for which a preprocessor are loaded returns optional union autotokenizer autofeatureextractor autoprocessor if a processor is found it is returned otherwise if a tokenizer or a feature extractor exists it is returned if both a tokenizer and a feature extractor exist an error is raised the function returns none if no preprocessor is found avoid circular imports by only importing this here tests_ignore
from ctypes import c_float, sizeof
from enum import Enum
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
    from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  


class ParameterFormat(Enum):
    Float = c_float

    @property
    def size(self) -> int:
        
        return sizeof(self.value)


def compute_effective_axis_dimension(dimension: int, fixed_dimension: int, num_token_to_add: int = 0) -> int:
    
    
    if dimension <= 0:
        dimension = fixed_dimension

    dimension -= num_token_to_add
    return dimension


def compute_serialized_parameters_size(num_parameters: int, dtype: ParameterFormat) -> int:
    
    return num_parameters * dtype.size


def get_preprocessor(model_name: str) -> Optional[Union["AutoTokenizer", "AutoFeatureExtractor", "AutoProcessor"]]:
    
    
    from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  

    try:
        return AutoProcessor.from_pretrained(model_name)
    except (ValueError, OSError, KeyError):
        tokenizer, feature_extractor = None, None
        try:
            tokenizer = AutoTokenizer.from_pretrained(model_name)
        except (OSError, KeyError):
            pass
        try:
            feature_extractor = AutoFeatureExtractor.from_pretrained(model_name)
        except (OSError, KeyError):
            pass

        if tokenizer is not None and feature_extractor is not None:
            raise ValueError(
                f"Couldn't auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor."
            )
        elif tokenizer is None and feature_extractor is None:
            return None
        elif tokenizer is not None:
            return tokenizer
        else:
            return feature_extractor