codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license this script is responsible for cleaning the list of doctests by making sure the entries all exist and are in alphabetical order usage from the root of the repo check that the doctest list is properly sorted and all files exist used in make repoconsistency bash python utilscheckdoctestlist py autosort the doctest list if it is not properly sorted used in make fixcopies bash python utilscheckdoctestlist py fixandoverwrite all paths are set with the intent you should run this script from the root of the repo with the command python utilscheckdoctestlist py cleans the doctest in a given file args doctestfile str the path to the doctest file to check or clean overwrite bool optional defaults to false whether or not to fix problems if false will error when the file is not clean coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license this script is responsible for cleaning the list of doctests by making sure the entries all exist and are in alphabetical order usage from the root of the repo check that the doctest list is properly sorted and all files exist used in make repo consistency bash python utils check_doctest_list py auto sort the doctest list if it is not properly sorted used in make fix copies bash python utils check_doctest_list py fix_and_overwrite all paths are set with the intent you should run this script from the root of the repo with the command python utils check_doctest_list py cleans the doctest in a given file args doctest_file str the path to the doctest file to check or clean overwrite bool optional defaults to false whether or not to fix problems if false will error when the file is not clean
import argparse
import os




REPO_PATH = "."
DOCTEST_FILE_PATHS = ["not_doctested.txt", "slow_documentation_tests.txt"]


def clean_doctest_list(doctest_file: str, overwrite: bool = False):
    
    non_existent_paths = []
    all_paths = []
    with open(doctest_file, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip().split(" ")[0]
            path = os.path.join(REPO_PATH, line)
            if not (os.path.isfile(path) or os.path.isdir(path)):
                non_existent_paths.append(line)
            all_paths.append(line)

    if len(non_existent_paths) > 0:
        non_existent_paths = "\n".join([f"- {f}" for f in non_existent_paths])
        raise ValueError(f"`{doctest_file}` contains non-existent paths:\n{non_existent_paths}")

    sorted_paths = sorted(all_paths)
    if all_paths != sorted_paths:
        if not overwrite:
            raise ValueError(
                f"Files in `{doctest_file}` are not in alphabetical order, run `make fix-copies` to fix "
                "this automatically."
            )
        with open(doctest_file, "w", encoding="utf-8") as f:
            f.write("\n".join(sorted_paths) + "\n")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()

    for doctest_file in DOCTEST_FILE_PATHS:
        doctest_file = os.path.join(REPO_PATH, "utils", doctest_file)
        clean_doctest_list(doctest_file, args.fix_and_overwrite)