<commit_before>import re
from pathlib import Path


def test_build_dependencies(en_vocab):
    # Check that library requirements are pinned exactly the same across different setup files.
    libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"]
    libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]

    # check requirements.txt
    req_dict = {}
    try:
        # for CLI usage
        root_dir = Path(__file__).parent.parent
        req_file = root_dir / "requirements.txt"
        with req_file.open() as f:
            lines = f.readlines()
    except FileNotFoundError as e:
        try:
            # for local usage
            root_dir = Path(__file__).parent.parent.parent
            req_file = root_dir / "requirements.txt"
            with req_file.open() as f:
                lines = f.readlines()
        except FileNotFoundError as e:
            # where areth thou ?
            root_dir = Path(__file__).parent.parent.parent.parent
            req_file = root_dir / "requirements.txt"
            with req_file.open() as f:
                lines = f.readlines()

    for line in lines:
        line = line.strip()
        if not line.startswith("#"):
            lib, v = _parse_req(line)
            if lib and lib not in libs_ignore_requirements:
                req_dict[lib] = v

    # check setup.cfg and compare to requirements.txt
    # also fails when there are missing or additional libs
    setup_file = root_dir / "setup.cfg"
    with setup_file.open() as f:
        lines = f.readlines()
    setup_keys = set()
    for line in lines:
        line = line.strip()
        if not line.startswith("#"):
            lib, v = _parse_req(line)
            if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
                req_v = req_dict.get(lib, None)
                assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib)
                assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \
                                               "{} and {} respectively".format(lib, v, req_v)
                setup_keys.add(lib)<commit_msg>avoid try except<commit_after>import re
from pathlib import Path


def test_build_dependencies(en_vocab):
    # Check that library requirements are pinned exactly the same across different setup files.
    libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"]
    libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]

    # check requirements.txt
    req_dict = {}

    root_dir = None
    # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up.
    roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent]  # or whatever
    for r in roots:
        req_file = root_dir / "requirements.txt"
        if req_file.exists():
            root_dir = r
            with req_file.open() as f:
                lines = f.readlines()
                for line in lines:
                    line = line.strip()
                    if not line.startswith("#"):
                        lib, v = _parse_req(line)
                        if lib and lib not in libs_ignore_requirements:
                            req_dict[lib] = v

    # check setup.cfg and compare to requirements.txt
    # also fails when there are missing or additional libs
    setup_file = root_dir / "setup.cfg"
    with setup_file.open() as f:
        lines = f.readlines()
    setup_keys = set()
    for line in lines:
        line = line.strip()
        if not line.startswith("#"):
            lib, v = _parse_req(line)
            if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
                req_v = req_dict.get(lib, None)
                assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib)
                assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \
                                               "{} and {} respectively".format(lib, v, req_v)
                setup_keys.add(lib)
{'fraction_non_alphanumeric': 0.09833293065957961, 'fraction_numerical': 0.0009664170089393573, 'mean_word_length': 1.9741379310344827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4130159', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1182, 'n_words': 360}