codingutf8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that checks the big table in the file docssourceenindex md and potentially updates it use from the root of the repo with bash python utilscheckinits py for a check that will error in case of inconsistencies used by make repoconsistency to autofix issues run bash python utilscheckinits py fixandoverwrite which is used by make fixcopies all paths are set with the intent you should run this script from the root of the repo with the command python utilschecktable py find the text in filename between two prompts args filename str the file to search into startprompt str a string to look for at the start of the content searched endprompt str a string that will mark the end of the content to look for returns str the content between the prompts find the start prompt now go until the end prompt regexes that match tfflaxpt model names add here suffixes that are used to identify models separated by will match any tf or flax model too so need to be in an else branch after the two previous regexes this is to make sure the transformers module imported is the one in the repo split a camelcased name into words args identifier str the camelcased name to parse returns liststr the list of words in the identifier as seprated by capital letters example py camelcasesplitcamelcasedclass camel cased class regex thanks to https stackoverflow comquestions29916065howtodocamelcasesplitinpython utility that will add spaces on the left and right of a text to make it centered for a given width args text str the text to center width int the desired length of the result returns str a text of length width with the original text in the middle generates an uptodate model table from the content of the auto modules dictionary model names to config dictionaries flagging if each model prefix has a backend in pttfflax let s lookup through all transformers object once try again after removing the last word in the name let s build that table model name to doc link mapping update mapping with special model names maskformerswin and timmbackbone are backbones and so not meant to be loaded and used on their own instead they define architectures which can be loaded using the autobackbone api we ll need widths to properly display everything in the center 2 is to leave one extra space on each side build the table per se use format to centeraligned table cell texts check the model table in the index md is consistent with the state of the lib and potentially fix it args overwrite bool optional defaults to false whether or not to overwrite the table when it s not up to date coding utf 8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that checks the big table in the file docs source en index md and potentially updates it use from the root of the repo with bash python utils check_inits py for a check that will error in case of inconsistencies used by make repo consistency to auto fix issues run bash python utils check_inits py fix_and_overwrite which is used by make fix copies all paths are set with the intent you should run this script from the root of the repo with the command python utils check_table py find the text in filename between two prompts args filename str the file to search into start_prompt str a string to look for at the start of the content searched end_prompt str a string that will mark the end of the content to look for returns str the content between the prompts find the start prompt now go until the end prompt regexes that match tf flax pt model names add here suffixes that are used to identify models separated by will match any tf or flax model too so need to be in an else branch after the two previous regexes this is to make sure the transformers module imported is the one in the repo split a camel cased name into words args identifier str the camel cased name to parse returns list str the list of words in the identifier as seprated by capital letters example py camel_case_split camelcasedclass camel cased class regex thanks to https stackoverflow com questions 29916065 how to do camelcase split in python utility that will add spaces on the left and right of a text to make it centered for a given width args text str the text to center width int the desired length of the result returns str a text of length width with the original text in the middle generates an up to date model table from the content of the auto modules dictionary model names to config dictionaries flagging if each model prefix has a backend in pt tf flax let s lookup through all transformers object once try again after removing the last word in the name let s build that table model name to doc link mapping update mapping with special model names maskformerswin and timmbackbone are backbones and so not meant to be loaded and used on their own instead they define architectures which can be loaded using the autobackbone api we ll need widths to properly display everything in the center 2 is to leave one extra space on each side build the table per se use format to center aligned table cell texts check the model table in the index md is consistent with the state of the lib and potentially fix it args overwrite bool optional defaults to false whether or not to overwrite the table when it s not up to date
import argparse
import collections
import os
import re
from typing import List

from transformers.utils import direct_transformers_import




TRANSFORMERS_PATH = "src/transformers"
PATH_TO_DOCS = "docs/source/en"
REPO_PATH = "."


def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> str:
    
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()

    
    start_index = 0
    while not lines[start_index].startswith(start_prompt):
        start_index += 1
    start_index += 1

    
    end_index = start_index
    while not lines[end_index].startswith(end_prompt):
        end_index += 1
    end_index -= 1

    while len(lines[start_index]) <= 1:
        start_index += 1
    while len(lines[end_index]) <= 1:
        end_index -= 1
    end_index += 1
    return "".join(lines[start_index:end_index]), start_index, end_index, lines



_re_tf_models = re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
_re_flax_models = re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")

_re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")



transformers_module = direct_transformers_import(TRANSFORMERS_PATH)


def camel_case_split(identifier: str) -> List[str]:
    
    
    matches = re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier)
    return [m.group(0) for m in matches]


def _center_text(text: str, width: int) -> str:
    
    text_length = 2 if text == "✅" or text == "❌" else len(text)
    left_indent = (width - text_length) // 2
    right_indent = width - text_length - left_indent
    return " " * left_indent + text + " " * right_indent


SPECIAL_MODEL_NAME_LINK_MAPPING = {
    "Data2VecAudio": "[Data2VecAudio](model_doc/data2vec)",
    "Data2VecText": "[Data2VecText](model_doc/data2vec)",
    "Data2VecVision": "[Data2VecVision](model_doc/data2vec)",
    "DonutSwin": "[DonutSwin](model_doc/donut)",
}

MODEL_NAMES_WITH_SAME_CONFIG = {
    "BARThez": "BART",
    "BARTpho": "BART",
    "BertJapanese": "BERT",
    "BERTweet": "BERT",
    "BORT": "BERT",
    "ByT5": "T5",
    "CPM": "OpenAI GPT-2",
    "DePlot": "Pix2Struct",
    "DialoGPT": "OpenAI GPT-2",
    "DiT": "BEiT",
    "FLAN-T5": "T5",
    "FLAN-UL2": "T5",
    "HerBERT": "BERT",
    "LayoutXLM": "LayoutLMv2",
    "Llama2": "LLaMA",
    "MADLAD-400": "T5",
    "MatCha": "Pix2Struct",
    "mBART-50": "mBART",
    "Megatron-GPT2": "OpenAI GPT-2",
    "mLUKE": "LUKE",
    "MMS": "Wav2Vec2",
    "NLLB": "M2M100",
    "PhoBERT": "BERT",
    "T5v1.1": "T5",
    "TAPEX": "BART",
    "UL2": "T5",
    "Wav2Vec2Phoneme": "Wav2Vec2",
    "XLM-V": "XLM-RoBERTa",
    "XLS-R": "Wav2Vec2",
    "XLSR-Wav2Vec2": "Wav2Vec2",
}


def get_model_table_from_auto_modules() -> str:
    
    
    config_maping_names = transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    model_name_to_config = {
        name: config_maping_names[code]
        for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
        if code in config_maping_names
    }
    model_name_to_prefix = {name: config.replace("Config", "") for name, config in model_name_to_config.items()}

    
    pt_models = collections.defaultdict(bool)
    tf_models = collections.defaultdict(bool)
    flax_models = collections.defaultdict(bool)

    
    for attr_name in dir(transformers_module):
        lookup_dict = None
        if _re_tf_models.match(attr_name) is not None:
            lookup_dict = tf_models
            attr_name = _re_tf_models.match(attr_name).groups()[0]
        elif _re_flax_models.match(attr_name) is not None:
            lookup_dict = flax_models
            attr_name = _re_flax_models.match(attr_name).groups()[0]
        elif _re_pt_models.match(attr_name) is not None:
            lookup_dict = pt_models
            attr_name = _re_pt_models.match(attr_name).groups()[0]

        if lookup_dict is not None:
            while len(attr_name) > 0:
                if attr_name in model_name_to_prefix.values():
                    lookup_dict[attr_name] = True
                    break
                
                attr_name = "".join(camel_case_split(attr_name)[:-1])

    
    model_names = list(model_name_to_config.keys()) + list(MODEL_NAMES_WITH_SAME_CONFIG.keys())

    
    model_names_mapping = transformers_module.models.auto.configuration_auto.MODEL_NAMES_MAPPING
    model_name_to_link_mapping = {value: f"[{value}](model_doc/{key})" for key, value in model_names_mapping.items()}
    
    model_name_to_link_mapping = {
        k: SPECIAL_MODEL_NAME_LINK_MAPPING[k] if k in SPECIAL_MODEL_NAME_LINK_MAPPING else v
        for k, v in model_name_to_link_mapping.items()
    }

    
    names_to_exclude = ["MaskFormerSwin", "TimmBackbone", "Speech2Text2"]
    model_names = [name for name in model_names if name not in names_to_exclude]
    model_names.sort(key=str.lower)

    columns = ["Model", "PyTorch support", "TensorFlow support", "Flax Support"]
    

    widths = [len(c) + 2 for c in columns]
    widths[0] = max([len(doc_link) for doc_link in model_name_to_link_mapping.values()]) + 2

    
    table = "|" + "|".join([_center_text(c, w) for c, w in zip(columns, widths)]) + "|\n"
    
    table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]) + "|\n"

    check = {True: "✅", False: "❌"}

    for name in model_names:
        if name in MODEL_NAMES_WITH_SAME_CONFIG.keys():
            prefix = model_name_to_prefix[MODEL_NAMES_WITH_SAME_CONFIG[name]]
        else:
            prefix = model_name_to_prefix[name]
        line = [
            model_name_to_link_mapping[name],
            check[pt_models[prefix]],
            check[tf_models[prefix]],
            check[flax_models[prefix]],
        ]
        table += "|" + "|".join([_center_text(l, w) for l, w in zip(line, widths)]) + "|\n"
    return table


def check_model_table(overwrite=False):
    
    current_table, start_index, end_index, lines = _find_text_in_file(
        filename=os.path.join(PATH_TO_DOCS, "index.md"),
        start_prompt="<!--This table is updated automatically from the auto modules",
        end_prompt="<!-- End table-->",
    )
    new_table = get_model_table_from_auto_modules()

    if current_table != new_table:
        if overwrite:
            with open(os.path.join(PATH_TO_DOCS, "index.md"), "w", encoding="utf-8", newline="\n") as f:
                f.writelines(lines[:start_index] + [new_table] + lines[end_index:])
        else:
            raise ValueError(
                "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."
            )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()

    check_model_table(args.fix_and_overwrite)