codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that sorts the names in the auto mappings defines in the auto modules in alphabetical order use from the root of the repo with bash python utilssortautomappings py to autofix all the auto mappings used in make style to only check if the mappings are properly sorted as used in make quality do bash python utilssortautomappings py checkonly path are set with the intent you should run this script from the root of the repo re pattern that matches mapping introductions supermodelmappingnames ordereddict or supermodelmapping ordereddict re pattern that matches identifiers in mappings sort all auto mappings in a file args fname str the name of the file where we want to sort automappings overwrite bool optional defaults to false whether or not to fix and overwrite the file returns optionalbool returns none if overwritetrue otherwise returns true if the file has an automapping improperly sorted false if the file is okay start of a new mapping blocks either fit in one line or not sort blocks by their identifiers sort all auto mappings in the library args overwrite bool optional defaults to false whether or not to fix and overwrite the file coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that sorts the names in the auto mappings defines in the auto modules in alphabetical order use from the root of the repo with bash python utils sort_auto_mappings py to auto fix all the auto mappings used in make style to only check if the mappings are properly sorted as used in make quality do bash python utils sort_auto_mappings py check_only path are set with the intent you should run this script from the root of the repo re pattern that matches mapping introductions super_model_mapping_names ordereddict or super_model_mapping ordereddict re pattern that matches identifiers in mappings sort all auto mappings in a file args fname str the name of the file where we want to sort auto mappings overwrite bool optional defaults to false whether or not to fix and overwrite the file returns optional bool returns none if overwrite true otherwise returns true if the file has an auto mapping improperly sorted false if the file is okay start of a new mapping blocks either fit in one line or not sort blocks by their identifiers sort all auto mappings in the library args overwrite bool optional defaults to false whether or not to fix and overwrite the file
import argparse
import os
import re
from typing import Optional



PATH_TO_AUTO_MODULE = "src/transformers/models/auto"




_re_intro_mapping = re.compile(r"[A-Z_]+_MAPPING(\s+|_[A-Z_]+\s+)=\s+OrderedDict")

_re_identifier = re.compile(r'\s*\(\s*"(\S[^"]+)"')


def sort_auto_mapping(fname: str, overwrite: bool = False) -> Optional[bool]:
    
    with open(fname, "r", encoding="utf-8") as f:
        content = f.read()

    lines = content.split("\n")
    new_lines = []
    line_idx = 0
    while line_idx < len(lines):
        if _re_intro_mapping.search(lines[line_idx]) is not None:
            
            indent = len(re.search(r"^(\s*)\S", lines[line_idx]).groups()[0]) + 8
            while not lines[line_idx].startswith(" " * indent + "("):
                new_lines.append(lines[line_idx])
                line_idx += 1

            blocks = []
            while lines[line_idx].strip() != "]":
                
                if lines[line_idx].strip() == "(":
                    start_idx = line_idx
                    while not lines[line_idx].startswith(" " * indent + ")"):
                        line_idx += 1
                    blocks.append("\n".join(lines[start_idx : line_idx + 1]))
                else:
                    blocks.append(lines[line_idx])
                line_idx += 1

            
            blocks = sorted(blocks, key=lambda x: _re_identifier.search(x).groups()[0])
            new_lines += blocks
        else:
            new_lines.append(lines[line_idx])
            line_idx += 1

    if overwrite:
        with open(fname, "w", encoding="utf-8") as f:
            f.write("\n".join(new_lines))
    else:
        return "\n".join(new_lines) != content


def sort_all_auto_mappings(overwrite: bool = False):
    
    fnames = [os.path.join(PATH_TO_AUTO_MODULE, f) for f in os.listdir(PATH_TO_AUTO_MODULE) if f.endswith(".py")]
    diffs = [sort_auto_mapping(fname, overwrite=overwrite) for fname in fnames]

    if not overwrite and any(diffs):
        failures = [f for f, d in zip(fnames, diffs) if d]
        raise ValueError(
            f"The following files have auto mappings that need sorting: {', '.join(failures)}. Run `make style` to fix"
            " this."
        )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--check_only", action="store_true", help="Whether to only check or fix style.")
    args = parser.parse_args()

    sort_all_auto_mappings(not args.check_only)