codingutf8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that checks the custom inits of transformers are welldefined transformers uses init files that delay the import of an object to when it s actually needed this is to avoid the main init importing all models which would make the line import transformers very slow when the user has all optional dependencies installed the inits with delayed imports have two halves one definining a dictionary importstructure which maps modules to the name of the objects in each module and one in typechecking which looks like a normal init for typecheckers the goal of this script is to check the objects defined in both halves are the same this also checks the main init properly references all submodules even if it doesn t import anything from them every submodule should be defined as a key of importstructure with an empty list as value potentially or the submodule won t be importable use from the root of the repo with bash python utilscheckinits py for a check that will error in case of inconsistencies used by make repoconsistency there is no autofix possible here sadly path is set with the intent you should run this script from the root of the repo matches isxxxavailable catches a oneline importstruct xxx catches a line with a keyvalues pattern bla foo bar catches a line if not isfooavailable catches a line importstructbla appendfoo catches a line importstructbla extendfoo bar or importstructbla foo bar catches a line with an object between quotes and a comma mymodel catches a line with objects between brackets only foo bar catches a line with from foo import bar bla boo catches a line with try catches a line with else find one or multiple backend in a code line of the init args line str a code line of the main init returns optionalstr if one or several backend is found returns it in the case of multiple backends the line contains if isxxxavailable and isyyyavailable returns all backends joined on and so xxxandyyy for instance read an initfile and parse per backend the importstructure objects defined and the typechecking objects defined args initfile str path to the init file to inspect returns optionaltupledictstr liststr dictstr liststr a tuple of two dictionaries mapping backends to list of imported objects one for the importstructure part of the init and one for the typechecking part of the init returns none if the init is not a custom init get the to importstructure definition if this is a traditional init just return first grab the objects without a specific backend in importstructure if we have everything on a single line let s deal with it those are stored with the key none let s continue with backendspecific objects in importstructure if the line is an if not isbackendavailable we grab all objects associated check if the backend declaration is inside a try block scroll until we hit the else block of tryexceptelse until we unindent add backend objects to the list at this stage we are in the typechecking part first grab the objects without a specific backend let s continue with backendspecific objects if the line is an if isbackendavailable we grab all objects associated check if the backend declaration is inside a try block scroll until we hit the else block of tryexceptelse until we unindent add backend objects to the list analyze the differences between importstructure objects and typechecking objects found in an init args importdictobjects dictstr liststr a dictionary mapping backend names none for the objects independent of any specific backend to list of imported objects typehintobjects dictstr liststr a dictionary mapping backend names none for the objects independent of any specific backend to list of imported objects returns liststr the list of errors corresponding to mismatches if one backend is missing from the other part of the init error early find all errors duplicate imports in any half missing imports in either part of the init check all inits in the transformers repo and raise an error if at least one does not define the same objects in both halves returns the list of transformers submodules ignore private modules ignore leftovers from branches empty folders apart from pycache check all submodules of transformers are properly registered in the main init error otherwise this is to make sure the transformers module imported is the one in the repo this contains all the base keys of the importstructure object defined in the init but if the user is missing some optional dependencies they may not have all of them thus we read the init to read all additions and potentiall re add them coding utf 8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license utility that checks the custom inits of transformers are well defined transformers uses init files that delay the import of an object to when it s actually needed this is to avoid the main init importing all models which would make the line import transformers very slow when the user has all optional dependencies installed the inits with delayed imports have two halves one definining a dictionary _import_structure which maps modules to the name of the objects in each module and one in type_checking which looks like a normal init for type checkers the goal of this script is to check the objects defined in both halves are the same this also checks the main init properly references all submodules even if it doesn t import anything from them every submodule should be defined as a key of _import_structure with an empty list as value potentially or the submodule won t be importable use from the root of the repo with bash python utils check_inits py for a check that will error in case of inconsistencies used by make repo consistency there is no auto fix possible here sadly path is set with the intent you should run this script from the root of the repo matches is_xxx_available catches a one line _import_struct xxx catches a line with a key values pattern bla foo bar catches a line if not is_foo_available catches a line _import_struct bla append foo catches a line _import_struct bla extend foo bar or _import_struct bla foo bar catches a line with an object between quotes and a comma mymodel catches a line with objects between brackets only foo bar catches a line with from foo import bar bla boo catches a line with try catches a line with else find one or multiple backend in a code line of the init args line str a code line of the main init returns optional str if one or several backend is found returns it in the case of multiple backends the line contains if is_xxx_available and is_yyy_available returns all backends joined on _and_ so xxx_and_yyy for instance read an init_file and parse per backend the _import_structure objects defined and the type_checking objects defined args init_file str path to the init file to inspect returns optional tuple dict str list str dict str list str a tuple of two dictionaries mapping backends to list of imported objects one for the _import_structure part of the init and one for the type_checking part of the init returns none if the init is not a custom init get the to _import_structure definition if this is a traditional init just return first grab the objects without a specific backend in _import_structure if we have everything on a single line let s deal with it those are stored with the key none let s continue with backend specific objects in _import_structure if the line is an if not is_backend_available we grab all objects associated check if the backend declaration is inside a try block scroll until we hit the else block of try except else until we unindent add backend objects to the list at this stage we are in the type_checking part first grab the objects without a specific backend let s continue with backend specific objects if the line is an if is_backend_available we grab all objects associated check if the backend declaration is inside a try block scroll until we hit the else block of try except else until we unindent add backend objects to the list analyze the differences between _import_structure objects and type_checking objects found in an init args import_dict_objects dict str list str a dictionary mapping backend names none for the objects independent of any specific backend to list of imported objects type_hint_objects dict str list str a dictionary mapping backend names none for the objects independent of any specific backend to list of imported objects returns list str the list of errors corresponding to mismatches if one backend is missing from the other part of the init error early find all errors duplicate imports in any half missing imports in either part of the init check all inits in the transformers repo and raise an error if at least one does not define the same objects in both halves returns the list of transformers submodules ignore private modules ignore leftovers from branches empty folders apart from pycache check all submodules of transformers are properly registered in the main init error otherwise this is to make sure the transformers module imported is the one in the repo this contains all the base keys of the _import_structure object defined in the init but if the user is missing some optional dependencies they may not have all of them thus we read the init to read all additions and potentiall re add them
import collections
import os
import re
from pathlib import Path
from typing import Dict, List, Optional, Tuple



PATH_TO_TRANSFORMERS = "src/transformers"



_re_backend = re.compile(r"is\_([a-z_]*)_available()")

_re_one_line_import_struct = re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")

_re_import_struct_key_value = re.compile(r'\s+"\S*":\s+\[([^\]]*)\]')

_re_test_backend = re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")

_re_import_struct_add_one = re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')

_re_import_struct_add_many = re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")

_re_quote_object = re.compile(r'^\s+"([^"]+)",')

_re_between_brackets = re.compile(r"^\s+\[([^\]]+)\]")

_re_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")

_re_try = re.compile(r"^\s*try:")

_re_else = re.compile(r"^\s*else:")


def find_backend(line: str) -> Optional[str]:
    
    if _re_test_backend.search(line) is None:
        return None
    backends = [b[0] for b in _re_backend.findall(line)]
    backends.sort()
    return "_and_".join(backends)


def parse_init(init_file) -> Optional[Tuple[Dict[str, List[str]], Dict[str, List[str]]]]:
    
    with open(init_file, "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()

    
    line_index = 0
    while line_index < len(lines) and not lines[line_index].startswith("_import_structure = {"):
        line_index += 1

    
    if line_index >= len(lines):
        return None

    
    objects = []
    while not lines[line_index].startswith("if TYPE_CHECKING") and find_backend(lines[line_index]) is None:
        line = lines[line_index]
        
        if _re_one_line_import_struct.search(line):
            content = _re_one_line_import_struct.search(line).groups()[0]
            imports = re.findall(r"\[([^\]]+)\]", content)
            for imp in imports:
                objects.extend([obj[1:-1] for obj in imp.split(", ")])
            line_index += 1
            continue
        single_line_import_search = _re_import_struct_key_value.search(line)
        if single_line_import_search is not None:
            imports = [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", ") if len(obj) > 0]
            objects.extend(imports)
        elif line.startswith(" " * 8 + '"'):
            objects.append(line[9:-3])
        line_index += 1

    
    import_dict_objects = {"none": objects}

    
    while not lines[line_index].startswith("if TYPE_CHECKING"):
        
        backend = find_backend(lines[line_index])
        
        if _re_try.search(lines[line_index - 1]) is None:
            backend = None

        if backend is not None:
            line_index += 1

            
            while _re_else.search(lines[line_index]) is None:
                line_index += 1

            line_index += 1

            objects = []
            
            while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 4):
                line = lines[line_index]
                if _re_import_struct_add_one.search(line) is not None:
                    objects.append(_re_import_struct_add_one.search(line).groups()[0])
                elif _re_import_struct_add_many.search(line) is not None:
                    imports = _re_import_struct_add_many.search(line).groups()[0].split(", ")
                    imports = [obj[1:-1] for obj in imports if len(obj) > 0]
                    objects.extend(imports)
                elif _re_between_brackets.search(line) is not None:
                    imports = _re_between_brackets.search(line).groups()[0].split(", ")
                    imports = [obj[1:-1] for obj in imports if len(obj) > 0]
                    objects.extend(imports)
                elif _re_quote_object.search(line) is not None:
                    objects.append(_re_quote_object.search(line).groups()[0])
                elif line.startswith(" " * 8 + '"'):
                    objects.append(line[9:-3])
                elif line.startswith(" " * 12 + '"'):
                    objects.append(line[13:-3])
                line_index += 1

            import_dict_objects[backend] = objects
        else:
            line_index += 1

    
    objects = []
    while (
        line_index < len(lines)
        and find_backend(lines[line_index]) is None
        and not lines[line_index].startswith("else")
    ):
        line = lines[line_index]
        single_line_import_search = _re_import.search(line)
        if single_line_import_search is not None:
            objects.extend(single_line_import_search.groups()[0].split(", "))
        elif line.startswith(" " * 8):
            objects.append(line[8:-2])
        line_index += 1

    type_hint_objects = {"none": objects}

    
    while line_index < len(lines):
        
        backend = find_backend(lines[line_index])
        
        if _re_try.search(lines[line_index - 1]) is None:
            backend = None

        if backend is not None:
            line_index += 1

            
            while _re_else.search(lines[line_index]) is None:
                line_index += 1

            line_index += 1

            objects = []
            
            while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8):
                line = lines[line_index]
                single_line_import_search = _re_import.search(line)
                if single_line_import_search is not None:
                    objects.extend(single_line_import_search.groups()[0].split(", "))
                elif line.startswith(" " * 12):
                    objects.append(line[12:-2])
                line_index += 1

            type_hint_objects[backend] = objects
        else:
            line_index += 1

    return import_dict_objects, type_hint_objects


def analyze_results(import_dict_objects: Dict[str, List[str]], type_hint_objects: Dict[str, List[str]]) -> List[str]:
    

    def find_duplicates(seq):
        return [k for k, v in collections.Counter(seq).items() if v > 1]

    
    if list(import_dict_objects.keys()) != list(type_hint_objects.keys()):
        return ["Both sides of the init do not have the same backends!"]

    errors = []
    
    for key in import_dict_objects.keys():
        
        duplicate_imports = find_duplicates(import_dict_objects[key])
        if duplicate_imports:
            errors.append(f"Duplicate _import_structure definitions for: {duplicate_imports}")
        duplicate_type_hints = find_duplicates(type_hint_objects[key])
        if duplicate_type_hints:
            errors.append(f"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}")

        
        if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])):
            name = "base imports" if key == "none" else f"{key} backend"
            errors.append(f"Differences for {name}:")
            for a in type_hint_objects[key]:
                if a not in import_dict_objects[key]:
                    errors.append(f"  {a} in TYPE_HINT but not in _import_structure.")
            for a in import_dict_objects[key]:
                if a not in type_hint_objects[key]:
                    errors.append(f"  {a} in _import_structure but not in TYPE_HINT.")
    return errors


def check_all_inits():
    
    failures = []
    for root, _, files in os.walk(PATH_TO_TRANSFORMERS):
        if "__init__.py" in files:
            fname = os.path.join(root, "__init__.py")
            objects = parse_init(fname)
            if objects is not None:
                errors = analyze_results(*objects)
                if len(errors) > 0:
                    errors[0] = f"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"
                    failures.append("\n".join(errors))
    if len(failures) > 0:
        raise ValueError("\n\n".join(failures))


def get_transformers_submodules() -> List[str]:
    
    submodules = []
    for path, directories, files in os.walk(PATH_TO_TRANSFORMERS):
        for folder in directories:
            
            if folder.startswith("_"):
                directories.remove(folder)
                continue
            
            if len(list((Path(path) / folder).glob("*.py"))) == 0:
                continue
            short_path = str((Path(path) / folder).relative_to(PATH_TO_TRANSFORMERS))
            submodule = short_path.replace(os.path.sep, ".")
            submodules.append(submodule)
        for fname in files:
            if fname == "__init__.py":
                continue
            short_path = str((Path(path) / fname).relative_to(PATH_TO_TRANSFORMERS))
            submodule = short_path.replace(".py", "").replace(os.path.sep, ".")
            if len(submodule.split(".")) == 1:
                submodules.append(submodule)
    return submodules


IGNORE_SUBMODULES = [
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
    "models.esm.openfold_utils",
    "modeling_attn_mask_utils",
    "safetensors_conversion",
]


def check_submodules():
    
    
    from transformers.utils import direct_transformers_import

    transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)

    import_structure_keys = set(transformers._import_structure.keys())
    
    
    
    with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r") as f:
        init_content = f.read()
    import_structure_keys.update(set(re.findall(r"import_structure\[\"([^\"]*)\"\]", init_content)))

    module_not_registered = [
        module
        for module in get_transformers_submodules()
        if module not in IGNORE_SUBMODULES and module not in import_structure_keys
    ]

    if len(module_not_registered) > 0:
        list_of_modules = "\n".join(f"- {module}" for module in module_not_registered)
        raise ValueError(
            "The following submodules are not properly registed in the main init of Transformers:\n"
            f"{list_of_modules}\n"
            "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."
        )


if __name__ == "__main__":
    check_all_inits()
    check_submodules()