<commit_before>import os
from os.path import join as pjoin
import sys
import platform
import textwrap
import re
import subprocess
import difflib
import time
import types
import shutil

import SCons.Errors

class DefineDict(object):
    """
    A dictionary-like object which generates appropriate preprocessor
    define statements from its dict of variable / value
    pairs. Variables whose value is None or that are not in the dict
    are left undefined.
    """
    def __init__(self, data):
        self.data = data
        self.undefined = set()

    def __getitem__(self, key):
        if key not in self.data:
            self.undefined.add(key)
            return '/* #undef %s */' % key
        elif self.data[key] is None:
            return '/* #undef %s */' % key
        else:
            return '#define %s %s' % (key, self.data[key])


class ConfigBuilder(object):
    """
    Used along with DefineDict to generate a customized config.h file
    from a config.h.in file using the variables given in 'defines'.
    """
    def __init__(self, defines):
        self.defines = DefineDict(defines)

    def __call__(self, source, target, env):
        for s, t in zip(source, target):
            config_h_in = file(str(s), "r")
            config_h = file(str(t), "w")

            config_h.write(config_h_in.read() % self.defines)
            config_h_in.close()
            config_h.close()
            self.print_config(str(t))

    def print_config(self, filename):
        print 'Generating %s with the following settings:' % filename
        for key, val in sorted(self.defines.data.iteritems()):
            if val is not None:
                print "    %-35s %s" % (key, val)
        for key in sorted(self.defines.undefined):
            print "    %-35s %s" % (key, '*undefined*')


class TestResults(object):
    """
    A class that stores information about all the regression tests
    that are defined and which ones have passed / failed in order to
    print a summary at the end of the build process.
    """
    def __init__(self):
        self.tests = {}
        self.passed = {}
        self.failed = {}

    def printReport(self, target, source, env):
        total = len(self.passed) + len(self.failed)
        print """
**********************************
*** Regression Testing Summary ***
**********************************

Tests passed: %(passed)s
Tests failed: %(failed)s
Up-to-date tests skipped: %(skipped)s

**********************************""" % dict(
            passed=len(self.passed),
            failed=len(self.failed),
            skipped=len(self.tests))

        if self.failed:
            raise SCons.Errors.BuildError(self, 'One or more tests failed.')


testResults = TestResults()


def regression_test(target, source, env):
    """
    Run a regression test comparing the output of a test program with
    existing "blessed" output files.

    target - The name of an output file that will be generated if
    the test is successful.

    source - A list containing the name of the program to run and
    (optionally) a list of file names to be passed as command line
    arguments.

    The test also relies on several parameters that are passed in via
    variables in the SCons environment:

    env['test_command_options'] - non-file command line options
    to be passed to to the test program
    """
    # unpack:
    program = source[0]
    if len(source) > 1:
        clargs = [s.name for s in source[1:]]
    else:
        clargs = []

    # Name to use for the output file
    blessedName = env['test_blessed_file']
    if blessedName is not None and 'blessed' in blessedName:
        outputName = blessedName.replace('blessed', 'output')
    else:
        outputName = 'test_output.txt'

    # command line options
    clopts = env['test_command_options'].split()

    # Run the test program
    dir = str(target[0].dir.abspath)
    with open(pjoin(dir,outputName), 'w') as outfile:
        code = subprocess.call([program.abspath] + clopts + clargs,
                               stdout=outfile, stderr=outfile,
                               cwd=dir)

    diff = 0
    # Compare output files
    comparisons = env['test_comparisons']
    if blessedName is not None:
        comparisons.append((blessedName,outputName))

    for blessed,output in comparisons:
        print """Comparing '%s' with '%s'""" % (blessed, output)
        diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output))

    del testResults.tests[env['active_test_name']]

    if diff or code:
        print 'FAILED'

        if os.path.exists(target[0].abspath):
            os.path.unlink(target[0].abspath)

        testResults.failed[env['active_test_name']] = 1
    else:
        print 'PASSED'
        open(target[0].path, 'w').write(time.asctime()+'\n')
        testResults.passed[env['active_test_name']] = 1


def compareFiles(env, file1, file2):
    """
    Compare the contents of two files, using a method chosen based on
    their file extensions.
    """
    if file1.endswith('csv') and file2.endswith('csv'):
        return compareCsvFiles(env, file1, file2)
    else:
        return compareTextFiles(env, file1, file2)


def compareTextFiles(env, file1, file2):
    """
    Compare the contents of two text files while:
       - ignoring trailing whitespace
       - ignoring any lines starting with strings specified in the
         variable env['test_ignoreLines'].
       - comparing floating point numbers only up to the printed precision
    """
    text1 = [line.rstrip() for line in open(file1).readlines()
             if not line.startswith(tuple(env['test_ignoreLines']))]
    text2 = [line.rstrip() for line in open(file2).readlines()
             if not line.startswith(tuple(env['test_ignoreLines']))]

    # Try to compare the files without testing the floating point numbers
    diff = list(difflib.unified_diff(text1, text2))
    if not diff:
        return 0

    # Replace nearly-equal floating point numbers with exactly equivalent
    # representations to avoid confusing difflib
    reFloat = re.compile(r'(\s*)([+-]{0,1}\d+\.\d+[eE]{0,1}[+-]{0,1}\d*)')
    for i in range(min(len(text1), len(text2))):
        line1 = text1[i]
        line2 = text2[i]
        if line1 == line2:
            continue

        # group(1) is the left space padding
        # group(2) is the number
        floats1 = [(m.group(1),m.group(2))
                   for m in list(reFloat.finditer(line1))]
        floats2 = [(m.group(1),m.group(2))
                   for m in list(reFloat.finditer(line2))]

        # If the lines don't contain the same number of numbers,
        # we're not going to pass the diff comparison no matter what
        if len(floats1) != len(floats2):
            continue

        for j in range(len(floats1)):
            if floats1[j] == floats2[j]:
                # String representations match, so replacement is unnecessary
                continue

            delta = max(getPrecision(floats1[j][1]), getPrecision(floats2[j][1]))
            num1 = float(floats1[j][1])
            num2 = float(floats2[j][1])
            if num1 - 1.1*delta < num2 < num1 + 1.1*delta:
                # update the string with a matching string
                line2 = line2.replace(''.join(floats2[j]),
                                      ''.join(floats1[j]))
                text2[i] = line2

    # Try the comparison again
    diff = list(difflib.unified_diff(text1, text2))
    if diff:
        'Found differences between %s and %s:' % (file1, file2)
        print '>>>'
        print '\n'.join(diff)
        print '<<<'
        return 1

    return 0


def getPrecision(x):
    """
    Return the number corresponding to the least significant digit of
    the number represented by the string 'x'.
    """
    x = x.lower()
    if x.find('e') != -1:
        precision = x.find('.') - x.find('e') + 1
        precision += int(x[x.find('e')+1:])
    else:
        precision =  x.find('.') - len(x) + 1

    return 10**precision


def compareCsvFiles(env, file1, file2):
    """
    Compare the contents of two .csv file to see if they are
    similar. Similarity is defined according to tolerances stored in
    the environment as:

        env['test_csv_threshold']
        env['test_csv_tolerance']

    The comparison for each variable is:

        |a-b|/(max(|a|,|b|) + threshold) < tolerance

    Returns 0 if all variables in the files are similar and 1 if the
    files are dissimilar. Lines containing non-numeric data are<commit_msg>Fixed comparison of numbers of the form 2e-008 in the regression tests<commit_after>import os
from os.path import join as pjoin
import sys
import platform
import textwrap
import re
import subprocess
import difflib
import time
import types
import shutil

import SCons.Errors

class DefineDict(object):
    """
    A dictionary-like object which generates appropriate preprocessor
    define statements from its dict of variable / value
    pairs. Variables whose value is None or that are not in the dict
    are left undefined.
    """
    def __init__(self, data):
        self.data = data
        self.undefined = set()

    def __getitem__(self, key):
        if key not in self.data:
            self.undefined.add(key)
            return '/* #undef %s */' % key
        elif self.data[key] is None:
            return '/* #undef %s */' % key
        else:
            return '#define %s %s' % (key, self.data[key])


class ConfigBuilder(object):
    """
    Used along with DefineDict to generate a customized config.h file
    from a config.h.in file using the variables given in 'defines'.
    """
    def __init__(self, defines):
        self.defines = DefineDict(defines)

    def __call__(self, source, target, env):
        for s, t in zip(source, target):
            config_h_in = file(str(s), "r")
            config_h = file(str(t), "w")

            config_h.write(config_h_in.read() % self.defines)
            config_h_in.close()
            config_h.close()
            self.print_config(str(t))

    def print_config(self, filename):
        print 'Generating %s with the following settings:' % filename
        for key, val in sorted(self.defines.data.iteritems()):
            if val is not None:
                print "    %-35s %s" % (key, val)
        for key in sorted(self.defines.undefined):
            print "    %-35s %s" % (key, '*undefined*')


class TestResults(object):
    """
    A class that stores information about all the regression tests
    that are defined and which ones have passed / failed in order to
    print a summary at the end of the build process.
    """
    def __init__(self):
        self.tests = {}
        self.passed = {}
        self.failed = {}

    def printReport(self, target, source, env):
        total = len(self.passed) + len(self.failed)
        print """
**********************************
*** Regression Testing Summary ***
**********************************

Tests passed: %(passed)s
Tests failed: %(failed)s
Up-to-date tests skipped: %(skipped)s

**********************************""" % dict(
            passed=len(self.passed),
            failed=len(self.failed),
            skipped=len(self.tests))

        if self.failed:
            raise SCons.Errors.BuildError(self, 'One or more tests failed.')


testResults = TestResults()


def regression_test(target, source, env):
    """
    Run a regression test comparing the output of a test program with
    existing "blessed" output files.

    target - The name of an output file that will be generated if
    the test is successful.

    source - A list containing the name of the program to run and
    (optionally) a list of file names to be passed as command line
    arguments.

    The test also relies on several parameters that are passed in via
    variables in the SCons environment:

    env['test_command_options'] - non-file command line options
    to be passed to to the test program
    """
    # unpack:
    program = source[0]
    if len(source) > 1:
        clargs = [s.name for s in source[1:]]
    else:
        clargs = []

    # Name to use for the output file
    blessedName = env['test_blessed_file']
    if blessedName is not None and 'blessed' in blessedName:
        outputName = blessedName.replace('blessed', 'output')
    else:
        outputName = 'test_output.txt'

    # command line options
    clopts = env['test_command_options'].split()

    # Run the test program
    dir = str(target[0].dir.abspath)
    with open(pjoin(dir,outputName), 'w') as outfile:
        code = subprocess.call([program.abspath] + clopts + clargs,
                               stdout=outfile, stderr=outfile,
                               cwd=dir)

    diff = 0
    # Compare output files
    comparisons = env['test_comparisons']
    if blessedName is not None:
        comparisons.append((blessedName,outputName))

    for blessed,output in comparisons:
        print """Comparing '%s' with '%s'""" % (blessed, output)
        diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output))

    del testResults.tests[env['active_test_name']]

    if diff or code:
        print 'FAILED'

        if os.path.exists(target[0].abspath):
            os.path.unlink(target[0].abspath)

        testResults.failed[env['active_test_name']] = 1
    else:
        print 'PASSED'
        open(target[0].path, 'w').write(time.asctime()+'\n')
        testResults.passed[env['active_test_name']] = 1


def compareFiles(env, file1, file2):
    """
    Compare the contents of two files, using a method chosen based on
    their file extensions.
    """
    if file1.endswith('csv') and file2.endswith('csv'):
        return compareCsvFiles(env, file1, file2)
    else:
        return compareTextFiles(env, file1, file2)


def compareTextFiles(env, file1, file2):
    """
    Compare the contents of two text files while:
       - ignoring trailing whitespace
       - ignoring any lines starting with strings specified in the
         variable env['test_ignoreLines'].
       - comparing floating point numbers only up to the printed precision
    """
    text1 = [line.rstrip() for line in open(file1).readlines()
             if not line.startswith(tuple(env['test_ignoreLines']))]
    text2 = [line.rstrip() for line in open(file2).readlines()
             if not line.startswith(tuple(env['test_ignoreLines']))]

    # Try to compare the files without testing the floating point numbers
    diff = list(difflib.unified_diff(text1, text2))
    if not diff:
        return 0

    # Replace nearly-equal floating point numbers with exactly equivalent
    # representations to avoid confusing difflib
    reFloat = re.compile(r'(\s*)([+-]{0,1}\d+\.{0,1}\d*[eE]{0,1}[+-]{0,1}\d*)')
    for i in range(min(len(text1), len(text2))):
        line1 = text1[i]
        line2 = text2[i]
        if line1 == line2:
            continue

        # group(1) is the left space padding
        # group(2) is the number
        floats1 = [(m.group(1),m.group(2))
                   for m in list(reFloat.finditer(line1))]
        floats2 = [(m.group(1),m.group(2))
                   for m in list(reFloat.finditer(line2))]

        # If the lines don't contain the same number of numbers,
        # we're not going to pass the diff comparison no matter what
        if len(floats1) != len(floats2):
            continue

        for j in range(len(floats1)):
            if floats1[j] == floats2[j]:
                # String representations match, so replacement is unnecessary
                continue

            delta = max(getPrecision(floats1[j][1]), getPrecision(floats2[j][1]))
            num1 = float(floats1[j][1])
            num2 = float(floats2[j][1])
            if num1 - 1.1*delta < num2 < num1 + 1.1*delta:
                # update the string with a matching string
                line2 = line2.replace(''.join(floats2[j]),
                                      ''.join(floats1[j]))
                text2[i] = line2

    # Try the comparison again
    diff = list(difflib.unified_diff(text1, text2))
    if diff:
        'Found differences between %s and %s:' % (file1, file2)
        print '>>>'
        print '\n'.join(diff)
        print '<<<'
        return 1

    return 0


def getPrecision(x):
    """
    Return the number corresponding to the least significant digit of
    the number represented by the string 'x'.
    """
    x = x.lower()
    decimalPt = x.find('.')
    if decimalPt == -1:
        decimalPt = 0

    if x.find('e') != -1:
        precision = decimalPt - x.find('e') + 1
        precision += int(x[x.find('e')+1:])
    else:
        precision = decimalPt - len(x) + 1

    return 10**precision


def compareCsvFiles(env, file1, file2):
    """
    Compare the contents of two .csv file to see if they are
    similar. Similarity is defined according to tolerances stored in
    the environment as:

        env['test_csv_threshold']
        env['test_csv_tolerance']

    The comparison for each variable is:

        |a-b|/(max(|a|,|b|) + threshold) < tolerance

    Returns 0 if all variables in the files are similar and 1 if the
    files are dissimilar. Lines containing non-numeric data are
{'fraction_non_alphanumeric': 0.12001858952015801, 'fraction_numerical': 0.01196700360171953, 'mean_word_length': 2.400829711576452, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357733', 'n_tokens_mistral': 5198, 'n_tokens_neox': 4986, 'n_words': 1619}