codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license all paths are set with the intent you should run this script from the root of the repo with the command python utilscheckconfigdocstrings py this is to make sure the transformers module imported is the one in the repo used to compute the property self chunklength used as self bertmodel bertmodelconfig not used in modeling files but it s an important information used internally in the configuration class file used internally in the configuration class file used during training despite we don t have training script for these models yet ignorevalue used during training despite we don t have training script for these models yet norm used in conversion script despite not using in the modeling file used during preprocessing and collation see collatinggraphormer py used internally in the configuration class file used internally in the configuration class file tokenizerclass get default value t5tokenizer intentionally used internally in the configuration class file used internally in the configuration class file used internally in the configuration class file having default values other than 1e5 we can t fix them without breaking having default values other than 1e5 we can t fix them without breaking having default values other than 1e5 we can t fix them without breaking having default values other than 1e5 we can t fix them without breaking having default values other than 1e5 we can t fix them without breaking used internally to calculate the feature size used internally to calculate the feature size used internally to calculate the feature size used internally to calculate mlpdim for head training but so far not implemented not used but providing useful information to users actually used in the config or generation config in that case necessary for the subcomponents generation actually used in the config or generation config in that case necessary for the subcomponents generation todo ydshieh check the failing cases try to fix them or move some cases to the above block once we are sure for backward compatibility with trust remote code models todo arthur for alignmenthead and alignmentlayer todo younes for isdecoder check if any name in attributes is used in one of the strings in sourcestrings args configclass type the configuration class for which the arguments in its init will be checked attributes liststr the name of an argument or attribute and its variant names if any defaultvalue any a default value for the attribute in attributes assigned in the init of configclass sourcestrings liststr the python source code strings in the same modeling directory where configclass is defined the file containing the definition of configclass should be excluded check if we can find config xxx getattrconfig xxx or getattrself config xxx deal with multiline cases sequencesummary is called with sequencesummaryconfig common and important attributes even if they do not always appear in the modeling files special cases to be allowed allow if the default value in the configuration class is different from the one in pretrainedconfig allow cases without checking the default value in the configuration class configuration class specific cases check the arguments in init of configclass are used in the modeling files in the same directory args configclass type the configuration class for which the arguments in its init will be checked get the parameters in init of the configuration class and the default values if any if attributemap exists an attribute can have different names to be used in the modeling files and as long as one variant is used the test should pass get the path to modeling source files let s check against all frameworks as long as one framework uses an attribute we are good get the source code strings attributes here is all the variant names for configparam some configuration classes have nonempty attributemap and both names could be used in the corresponding modeling files as long as one of them appears it is fine check the arguments in init of all configuration classes are used in python files configswithunusedattributes for configclass in listconfigmapping values skip deprecated models if models deprecated in configclass module continue some config classes are not in configmapping e g clipvisionconfig blip2visionconfig etc configclassesinmodule cls for name cls in inspect getmembers inspect getmoduleconfigclass lambda x inspect isclassx and issubclassx pretrainedconfig and inspect getmodulex inspect getmoduleconfigclass for configclass in configclassesinmodule unusedattributes checkconfigattributesbeingusedconfigclass if lenunusedattributes 0 configswithunusedattributesconfigclass name unusedattributes if lenconfigswithunusedattributes 0 error the following configuration classes contain unused attributes in the corresponding modeling files n for name attributes in configswithunusedattributes items error fname attributesn raise valueerrorerror if name main checkconfigattributes coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license all paths are set with the intent you should run this script from the root of the repo with the command python utils check_config_docstrings py this is to make sure the transformers module imported is the one in the repo used to compute the property self chunk_length used as self bert_model bertmodel config not used in modeling files but it s an important information used internally in the configuration class file used internally in the configuration class file used during training despite we don t have training script for these models yet ignore_value used during training despite we don t have training script for these models yet norm used in conversion script despite not using in the modeling file used during preprocessing and collation see collating_graphormer py used internally in the configuration class file used internally in the configuration class file tokenizer_class get default value t5tokenizer intentionally used internally in the configuration class file used internally in the configuration class file used internally in the configuration class file having default values other than 1e 5 we can t fix them without breaking having default values other than 1e 5 we can t fix them without breaking having default values other than 1e 5 we can t fix them without breaking having default values other than 1e 5 we can t fix them without breaking having default values other than 1e 5 we can t fix them without breaking used internally to calculate the feature size used internally to calculate the feature size used internally to calculate the feature size used internally to calculate mlp_dim for head training but so far not implemented not used but providing useful information to users actually used in the config or generation config in that case necessary for the sub components generation actually used in the config or generation config in that case necessary for the sub components generation todo ydshieh check the failing cases try to fix them or move some cases to the above block once we are sure for backward compatibility with trust remote code models todo arthur for alignment_head and alignment_layer todo younes for is_decoder check if any name in attributes is used in one of the strings in source_strings args config_class type the configuration class for which the arguments in its __init__ will be checked attributes list str the name of an argument or attribute and its variant names if any default_value any a default value for the attribute in attributes assigned in the __init__ of config_class source_strings list str the python source code strings in the same modeling directory where config_class is defined the file containing the definition of config_class should be excluded check if we can find config xxx getattr config xxx or getattr self config xxx deal with multi line cases sequencesummary is called with sequencesummary config common and important attributes even if they do not always appear in the modeling files special cases to be allowed allow if the default value in the configuration class is different from the one in pretrainedconfig allow cases without checking the default value in the configuration class configuration class specific cases check the arguments in __init__ of config_class are used in the modeling files in the same directory args config_class type the configuration class for which the arguments in its __init__ will be checked get the parameters in __init__ of the configuration class and the default values if any if attribute_map exists an attribute can have different names to be used in the modeling files and as long as one variant is used the test should pass get the path to modeling source files let s check against all frameworks as long as one framework uses an attribute we are good get the source code strings attributes here is all the variant names for config_param some configuration classes have non empty attribute_map and both names could be used in the corresponding modeling files as long as one of them appears it is fine check the arguments in __init__ of all configuration classes are used in python files skip deprecated models some config classes are not in config_mapping e g clipvisionconfig blip2visionconfig etc
import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import




PATH_TO_TRANSFORMERS = "src/transformers"



transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)

CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

SPECIAL_CASES_TO_ALLOW = {
    
    "EncodecConfig": ["overlap"],
    
    "DPRConfig": True,
    "FuyuConfig": True,
    
    "FSMTConfig": ["langs"],
    
    "GPTNeoConfig": ["attention_types"],
    
    "EsmConfig": ["is_folding_model"],
    
    "Mask2FormerConfig": ["ignore_value"],
    
    
    "OneFormerConfig": ["ignore_value", "norm"],
    
    "GraphormerConfig": ["spatial_pos_max"],
    
    "T5Config": ["feed_forward_proj"],
    
    
    "MT5Config": ["feed_forward_proj", "tokenizer_class"],
    "UMT5Config": ["feed_forward_proj", "tokenizer_class"],
    
    "LongT5Config": ["feed_forward_proj"],
    
    "Pop2PianoConfig": ["feed_forward_proj"],
    
    "SwitchTransformersConfig": ["feed_forward_proj"],
    
    "BioGptConfig": ["layer_norm_eps"],
    
    "GLPNConfig": ["layer_norm_eps"],
    
    "SegformerConfig": ["layer_norm_eps"],
    
    "CvtConfig": ["layer_norm_eps"],
    
    "PerceiverConfig": ["layer_norm_eps"],
    
    "InformerConfig": ["num_static_real_features", "num_time_features"],
    
    "TimeSeriesTransformerConfig": ["num_static_real_features", "num_time_features"],
    
    "AutoformerConfig": ["num_static_real_features", "num_time_features"],
    
    "SamVisionConfig": ["mlp_ratio"],
    
    "ClapAudioConfig": ["num_classes"],
    
    "SpeechT5HifiGanConfig": ["sampling_rate"],
    
    "SeamlessM4TConfig": [
        "max_new_tokens",
        "t2u_max_new_tokens",
        "t2u_decoder_attention_heads",
        "t2u_decoder_ffn_dim",
        "t2u_decoder_layers",
        "t2u_encoder_attention_heads",
        "t2u_encoder_ffn_dim",
        "t2u_encoder_layers",
        "t2u_max_position_embeddings",
    ],
    
    "SeamlessM4Tv2Config": [
        "max_new_tokens",
        "t2u_decoder_attention_heads",
        "t2u_decoder_ffn_dim",
        "t2u_decoder_layers",
        "t2u_encoder_attention_heads",
        "t2u_encoder_ffn_dim",
        "t2u_encoder_layers",
        "t2u_max_position_embeddings",
        "t2u_variance_pred_dropout",
        "t2u_variance_predictor_embed_dim",
        "t2u_variance_predictor_hidden_dim",
        "t2u_variance_predictor_kernel_size",
    ],
}



SPECIAL_CASES_TO_ALLOW.update(
    {
        "CLIPSegConfig": True,
        "DeformableDetrConfig": True,
        "DetaConfig": True,
        "DinatConfig": True,
        "DonutSwinConfig": True,
        "EfficientFormerConfig": True,
        "FSMTConfig": True,
        "JukeboxConfig": True,
        "LayoutLMv2Config": True,
        "MaskFormerSwinConfig": True,
        "MT5Config": True,
        
        "MptConfig": True,
        "MptAttentionConfig": True,
        "NatConfig": True,
        "OneFormerConfig": True,
        "PerceiverConfig": True,
        "RagConfig": True,
        "SpeechT5Config": True,
        "SwinConfig": True,
        "Swin2SRConfig": True,
        "Swinv2Config": True,
        "SwitchTransformersConfig": True,
        "TableTransformerConfig": True,
        "TapasConfig": True,
        "UniSpeechConfig": True,
        "UniSpeechSatConfig": True,
        "WavLMConfig": True,
        "WhisperConfig": True,
        
        "JukeboxPriorConfig": True,
        
        "Pix2StructTextConfig": True,
        "IdeficsConfig": True,
        "IdeficsVisionConfig": True,
        "IdeficsPerceiverConfig": True,
    }
)


def check_attribute_being_used(config_class, attributes, default_value, source_strings):
    
    attribute_used = False
    for attribute in attributes:
        for modeling_source in source_strings:
            
            if (
                f"config.{attribute}" in modeling_source
                or f'getattr(config, "{attribute}"' in modeling_source
                or f'getattr(self.config, "{attribute}"' in modeling_source
            ):
                attribute_used = True
            
            elif (
                re.search(
                    rf'getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*"{attribute}"',
                    modeling_source,
                )
                is not None
            ):
                attribute_used = True
            
            elif attribute in [
                "summary_type",
                "summary_use_proj",
                "summary_activation",
                "summary_last_dropout",
                "summary_proj_to_labels",
                "summary_first_dropout",
            ]:
                if "SequenceSummary" in modeling_source:
                    attribute_used = True
            if attribute_used:
                break
        if attribute_used:
            break

    
    attributes_to_allow = [
        "bos_index",
        "eos_index",
        "pad_index",
        "unk_index",
        "mask_index",
        "image_size",
        "use_cache",
        "out_features",
        "out_indices",
        "sampling_rate",
    ]
    attributes_used_in_generation = ["encoder_no_repeat_ngram_size"]

    
    case_allowed = True
    if not attribute_used:
        case_allowed = False
        for attribute in attributes:
            
            if attribute in ["is_encoder_decoder"] and default_value is True:
                case_allowed = True
            elif attribute in ["tie_word_embeddings"] and default_value is False:
                case_allowed = True

            
            elif attribute in attributes_to_allow + attributes_used_in_generation:
                case_allowed = True
            elif attribute.endswith("_token_id"):
                case_allowed = True

            
            if not case_allowed:
                allowed_cases = SPECIAL_CASES_TO_ALLOW.get(config_class.__name__, [])
                case_allowed = allowed_cases is True or attribute in allowed_cases

    return attribute_used or case_allowed


def check_config_attributes_being_used(config_class):
    
    
    signature = dict(inspect.signature(config_class.__init__).parameters)
    parameter_names = [x for x in list(signature.keys()) if x not in ["self", "kwargs"]]
    parameter_defaults = [signature[param].default for param in parameter_names]

    
    
    reversed_attribute_map = {}
    if len(config_class.attribute_map) > 0:
        reversed_attribute_map = {v: k for k, v in config_class.attribute_map.items()}

    
    config_source_file = inspect.getsourcefile(config_class)
    model_dir = os.path.dirname(config_source_file)
    
    modeling_paths = [os.path.join(model_dir, fn) for fn in os.listdir(model_dir) if fn.startswith("modeling_")]

    
    modeling_sources = []
    for path in modeling_paths:
        if os.path.isfile(path):
            with open(path, encoding="utf8") as fp:
                modeling_sources.append(fp.read())

    unused_attributes = []
    for config_param, default_value in zip(parameter_names, parameter_defaults):
        
        attributes = [config_param]
        
        
        if config_param in reversed_attribute_map:
            attributes.append(reversed_attribute_map[config_param])

        if not check_attribute_being_used(config_class, attributes, default_value, modeling_sources):
            unused_attributes.append(attributes[0])

    return sorted(unused_attributes)


def check_config_attributes():
    
    configs_with_unused_attributes = {}
    for _config_class in list(CONFIG_MAPPING.values()):
        
        if "models.deprecated" in _config_class.__module__:
            continue
        
        config_classes_in_module = [
            cls
            for name, cls in inspect.getmembers(
                inspect.getmodule(_config_class),
                lambda x: inspect.isclass(x)
                and issubclass(x, PretrainedConfig)
                and inspect.getmodule(x) == inspect.getmodule(_config_class),
            )
        ]
        for config_class in config_classes_in_module:
            unused_attributes = check_config_attributes_being_used(config_class)
            if len(unused_attributes) > 0:
                configs_with_unused_attributes[config_class.__name__] = unused_attributes

    if len(configs_with_unused_attributes) > 0:
        error = "The following configuration classes contain unused attributes in the corresponding modeling files:\n"
        for name, attributes in configs_with_unused_attributes.items():
            error += f"{name}: {attributes}\n"

        raise ValueError(error)


if __name__ == "__main__":
    check_config_attributes()