<commit_before>import os
import glob
import pickle
import sys
from subprocess import Popen, PIPE
from multiprocessing import Pool, cpu_count
import time




#
# testRoot
#

testRoot = os.path.dirname(os.path.abspath(__file__))




#
# getClayPlatform, fileForPlatform
#

def getClayPlatform():
    if sys.platform == "win32" or sys.platform == "cygwin":
        return "windows"
    if sys.platform == "darwin":
        return "macosx"
    if sys.platform.startswith("linux"):
        return "linux"

clayPlatform = getClayPlatform()

def fileForPlatform(folder, name, ext):
    platformName = os.path.join(folder, "%s.%s.%s" % (name, clayPlatform, ext))
    if os.path.isfile(platformName):
        return platformName
    else:
        return os.path.join(folder, "%s.%s" % (name, ext))



#
# getCompilerPath
#

def getCompilerPath() :
    buildPath = ["..", "build", "compiler", "src"]
    if sys.platform == "win32" :
        compiler = os.path.join(testRoot, *(buildPath + ["Release", "clay.exe"]))
        compiler2 = os.path.join(testRoot, "..", "clay.exe") # for binary distributions
    else :
        compiler = os.path.join(testRoot, *(buildPath + ["clay"]))
        compiler2 = os.path.join(testRoot, "..", "clay") # for binary distributions

    if not os.path.exists(compiler) :
        compiler = compiler2
        if not os.path.exists(compiler) :
            print "could not find the clay compiler"
            sys.exit(-1)
    return compiler

compiler = getCompilerPath()




#
# TestCase
#

class TestCase(object):
    allCases = []
    def __init__(self, folder, testfile, base = None):
        entries = os.listdir(folder)
        self.testfile = os.path.basename(testfile)
        self.path = folder
        if os.path.isfile(testfile):
            self.loadTest(testfile)

        runscript = fileForPlatform(folder, "run", "py")
        if os.path.isfile(runscript):
            self.runscript = runscript
        else:
            self.runscript = None

        buildflags = fileForPlatform(folder, "buildflags", "txt")
        if os.path.isfile(buildflags):
            self.buildflags = open(buildflags).read().split()
        else:
            self.buildflags = []

        for entry in entries:
            fullpath = os.path.join(folder, entry)
            if not os.path.isdir(fullpath):
                continue
            findTestCase(fullpath, self)
    
    def loadTest(self, testfile):
        TestCase.allCases.append(self)
        self.testfile = testfile

    def cmdline(self, clay) :
        return [clay] + self.buildflags + [self.testfile]

    def pre_build(self) :
        pass

    def post_build(self) :
        pass

    def post_run(self) :
        [os.unlink(f) for f in glob.glob("temp*")]
        [os.unlink(f) for f in glob.glob("*.data")]

    def match(self, resultout, resulterr, returncode) :
        outfile = fileForPlatform(".", "out", "txt")
        errfile = fileForPlatform(".", "err", "txt")
        refout = open(outfile).read()
        referr = ""
        if os.path.isfile(errfile):
            referr = open(errfile).read()
        resultout = resultout.replace('\r', '')
        refout    = refout.replace('\r', '')<commit_msg>test/runtests.py: handle missing out.txt gracefully.<commit_after>import os
import glob
import pickle
import sys
from subprocess import Popen, PIPE
from multiprocessing import Pool, cpu_count
import time




#
# testRoot
#

testRoot = os.path.dirname(os.path.abspath(__file__))




#
# getClayPlatform, fileForPlatform
#

def getClayPlatform():
    if sys.platform == "win32" or sys.platform == "cygwin":
        return "windows"
    if sys.platform == "darwin":
        return "macosx"
    if sys.platform.startswith("linux"):
        return "linux"

clayPlatform = getClayPlatform()

def fileForPlatform(folder, name, ext):
    platformName = os.path.join(folder, "%s.%s.%s" % (name, clayPlatform, ext))
    if os.path.isfile(platformName):
        return platformName
    else:
        return os.path.join(folder, "%s.%s" % (name, ext))



#
# getCompilerPath
#

def getCompilerPath() :
    buildPath = ["..", "build", "compiler", "src"]
    if sys.platform == "win32" :
        compiler = os.path.join(testRoot, *(buildPath + ["Release", "clay.exe"]))
        compiler2 = os.path.join(testRoot, "..", "clay.exe") # for binary distributions
    else :
        compiler = os.path.join(testRoot, *(buildPath + ["clay"]))
        compiler2 = os.path.join(testRoot, "..", "clay") # for binary distributions

    if not os.path.exists(compiler) :
        compiler = compiler2
        if not os.path.exists(compiler) :
            print "could not find the clay compiler"
            sys.exit(-1)
    return compiler

compiler = getCompilerPath()




#
# TestCase
#

class TestCase(object):
    allCases = []
    def __init__(self, folder, testfile, base = None):
        entries = os.listdir(folder)
        self.testfile = os.path.basename(testfile)
        self.path = folder
        if os.path.isfile(testfile):
            self.loadTest(testfile)

        runscript = fileForPlatform(folder, "run", "py")
        if os.path.isfile(runscript):
            self.runscript = runscript
        else:
            self.runscript = None

        buildflags = fileForPlatform(folder, "buildflags", "txt")
        if os.path.isfile(buildflags):
            self.buildflags = open(buildflags).read().split()
        else:
            self.buildflags = []

        for entry in entries:
            fullpath = os.path.join(folder, entry)
            if not os.path.isdir(fullpath):
                continue
            findTestCase(fullpath, self)
    
    def loadTest(self, testfile):
        TestCase.allCases.append(self)
        self.testfile = testfile

    def cmdline(self, clay) :
        return [clay] + self.buildflags + [self.testfile]

    def pre_build(self) :
        pass

    def post_build(self) :
        pass

    def post_run(self) :
        [os.unlink(f) for f in glob.glob("temp*")]
        [os.unlink(f) for f in glob.glob("*.data")]

    def match(self, resultout, resulterr, returncode) :
        outfile = fileForPlatform(".", "out", "txt")
        errfile = fileForPlatform(".", "err", "txt")
        if not os.path.isfile(outfile) :
            return False
        refout = open(outfile).read()
        referr = ""
        if os.path.isfile(errfile):
            referr = open(errfile).read()
        resultout = resultout.replace('\r', '')
        refout    = refout.replace('\r', '')
{'fraction_non_alphanumeric': 0.13462425080682341, 'fraction_numerical': 0.0024588904256954047, 'mean_word_length': 2.427066877303844, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1498914', 'n_tokens_mistral': 2081, 'n_tokens_neox': 2025, 'n_words': 461}