<commit_before>"""Run all tests: look for gold files in given directory,
run the corresponding test, then do a simple line-by-line 
comparison."""

import sys
import os
from subprocess import Popen, STDOUT, PIPE

for entry in sorted(os.listdir(sys.argv[1])):

    # Indentify the file -- whether it is gold.
    # Gold files are *.go for exact gold content, or 
    # *.gor if "random" (to be sorted before comparing.
    gold_fname = os.path.join(sys.argv[1], entry)
    suffix = None
    for choice in ('.go', '.gor'):
        if gold_fname[-len(choice):] == choice:
            suffix = choice
    if not suffix:
        continue

    # Read the gold file.
    f = open(gold_fname)
    gold_lines = f.readlines()
    f.close()

    # Run the test.
    test_fname = gold_fname[:-len(suffix):] + '.py'
    command = '%s %s'%(sys.executable, test_fname)
    print(command)
    p = Popen(command, shell=True, stdout=PIPE, stderr=STDOUT)
    test_lines = p.stdout.readlines()

    # If test "randomizes" output, sort lines before comparing.
    if suffix == '.gor':
        gold_lines = sorted(gold_lines)
        test_lines = sorted(test_lines)

    # Compare line by line.
    failed = False
    for gline, tline in zip(gold_lines, test_lines):
        gline = gline.strip()
        tline = tline.strip().decode()<commit_msg>Operate on current directory (no need for command-line argument.)<commit_after>"""Run all tests: look for gold files in given directory,
run the corresponding test, then do a simple line-by-line 
comparison."""

import sys
import os
import inspect
from subprocess import Popen, STDOUT, PIPE

# Compute the absolute path to directory of this file.
this_dir = os.path.dirname(
    os.path.abspath(
        inspect.getfile(
            inspect.currentframe())))

for entry in sorted(os.listdir(this_dir)):

    # Identify the file -- whether it is gold.
    # Gold files are *.go for exact gold content, or 
    # *.gor if "random" (to be sorted before comparing.
    gold_fname = os.path.join(this_dir, entry)
    suffix = None
    for choice in ('.go', '.gor'):
        if gold_fname[-len(choice):] == choice:
            suffix = choice
    if not suffix:
        continue

    # Read the gold file.
    f = open(gold_fname)
    gold_lines = f.readlines()
    f.close()

    # Run the test.
    test_fname = gold_fname[:-len(suffix):] + '.py'
    command = '%s %s'%(sys.executable, test_fname)
    print(command)
    p = Popen(command, shell=True, stdout=PIPE, stderr=STDOUT)
    test_lines = p.stdout.readlines()

    # If test "randomizes" output, sort lines before comparing.
    if suffix == '.gor':
        gold_lines = sorted(gold_lines)
        test_lines = sorted(test_lines)

    # Compare line by line.
    failed = False
    for gline, tline in zip(gold_lines, test_lines):
        gline = gline.strip()
        tline = tline.strip().decode()
{'fraction_non_alphanumeric': 0.11640027797081307, 'fraction_numerical': 0.0006949270326615705, 'mean_word_length': 2.8132450331125827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3544529', 'n_tokens_mistral': 933, 'n_tokens_neox': 907, 'n_words': 293}