<commit_before>import os, sys
import pytest
import logging
from subprocess import check_output

from . import errors
from .jam.compiler import compile, compileRun

TESTS_PATH = os.path.join("compiler", "tests")
BUILD_PATH = os.path.join("build", "tests")

for file in os.listdir(TESTS_PATH):
    file = os.path.join(TESTS_PATH, file)

    # Ignore non-jam files
    if os.path.splitext(file)[1] != ".jm": continue

    def test(verbosity, file=file):
        logging.basicConfig(level=logging.WARNING - verbosity*10, stream=sys.stdout)

        # Get the path to the built file
        build = os.path.join(BUILD_PATH, os.path.basename(file).replace(".jm", ".ll"))

        # Make the testing directories, if needed
        os.makedirs(BUILD_PATH, exist_ok=True)

        # Get output data
        with open(file, "r") as f_in, open(build, "w") as f_out:
            # The first character is the test type
            type = f_in.read(1)
            # Or specific test attributes (ignore for now)
            if type == "?":
                type = f_in.read(1)

            # The first line is the output
            output = f_in.readline()[:-1].encode("UTF-8").decode("unicode-escape")

            # Check if the output was correct
            if type == "#":
                compile(f_in, f_out)
                f_out.close()
                assert output == check_output(["lli-3.6", build]).decode("UTF-8")<commit_msg>Changed tests to<commit_after>import os, sys
import pytest
import logging
from subprocess import check_output

from . import errors
from .jam.compiler import compile, compileRun

TESTS_PATH = os.path.join("compiler", "tests")
BUILD_PATH = os.path.join("build", "tests")

for file in os.listdir(TESTS_PATH):
    file = os.path.join(TESTS_PATH, file)

    # Ignore non-jam files
    if os.path.splitext(file)[1] != ".jm": continue

    def test(verbosity, file=file):
        logging.basicConfig(level=logging.WARNING - verbosity*10, stream=sys.stdout)

        # Get the path to the built file
        build = os.path.join(BUILD_PATH, os.path.basename(file).replace(".jm", ".ll"))

        # Make the testing directories, if needed
        os.makedirs(BUILD_PATH, exist_ok=True)

        # Get output data
        with open(file, "r") as f_in, open(build, "w") as f_out:
            # The first character is the test type
            type = f_in.read(1)
            # Or specific test attributes (ignore for now)
            if type == "?":
                type = f_in.read(1)

            # The first line is the output
            output = f_in.readline()[:-1].encode("UTF-8").decode("unicode-escape")

            # Check if the output was correct
            if type == "#":
                compile(f_in, f_out)
                f_out.close()
                assert output == check_output(["lli", build]).decode("UTF-8")
{'fraction_non_alphanumeric': 0.12372583479789104, 'fraction_numerical': 0.00632688927943761, 'mean_word_length': 2.3680473372781066, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3576741', 'n_tokens_mistral': 887, 'n_tokens_neox': 852, 'n_words': 239}