<commit_before>from mparts.manager import Task
from mparts.host import HostInfo, CHECKED, UNCHECKED
from mparts.util import Progress
from support import BenchmarkRunner, SetCPUs, PrefetchList, FileSystem

import os

__all__ = []

__all__.append("Mkdb")
class Mkdb(Task, BenchmarkRunner):
    MODE_THREAD = intern("thread")
    MODE_PROCESS = intern("process")

    ORDER_SEQ = intern("seq")
    ORDER_RR = intern("rr")

    __config__ = ["host", "psearchyPath", "filesPath", "dbPath",
                  "mode", "order", "mem"]

    def __init__(self, host, psearchyPath, filesPath, dbPath, cores,
                 mode = MODE_THREAD, order = ORDER_SEQ,
                 mem = 256, trials = 3):
        assert mode in [Mkdb.MODE_THREAD, Mkdb.MODE_PROCESS], \
            "Invalid mode %s" % mode
        assert order in [Mkdb.ORDER_SEQ, Mkdb.ORDER_RR], \
            "Invalid order %s" % order

        Task.__init__(self, host = host)
        BenchmarkRunner.__init__(self, cores, trials)
        self.host = host
        self.psearchyPath = psearchyPath
        self.filesPath = filesPath
        self.dbPath = dbPath
        self.mode = mode
        self.order = order
        self.mem = mem

    def runTrial(self, m):
        # Construct command
        cmd = [os.path.join(self.psearchyPath, "mkdb", "pedsort"),
               "-t", self.dbPath,
               "-c", str(self.cores),
               "-m", str(self.mem)]
        if self.mode == Mkdb.MODE_PROCESS:
            cmd.append("-p")
        if self.order == Mkdb.ORDER_RR:
            cmd.extend(["-s", "1"])

        # Run
        logPath = self.host.getLogPath(self)
        self.host.r.run(cmd, stdin = self.filesPath, stdout = logPath,
                        wait = CHECKED)

        # Get result
        log = self.host.r.readFile(logPath)
        last = log.strip().splitlines()[-1]
        last = last.split("throughput:", 1)[1].strip()
        return float(last.split()[0]), last.split()[1]

__all__.append("Mkfiles")
class Mkfiles(Task):
    __config__ = ["host", "psearchyPath", "baseDir"]<commit_msg>Separate out results parser and robustify<commit_after>from mparts.manager import Task
from mparts.host import HostInfo, CHECKED, UNCHECKED
from mparts.util import Progress
from support import BenchmarkRunner, SetCPUs, PrefetchList, FileSystem

import os, re

__all__ = []

__all__.append("Mkdb")
class Mkdb(Task, BenchmarkRunner):
    MODE_THREAD = intern("thread")
    MODE_PROCESS = intern("process")

    ORDER_SEQ = intern("seq")
    ORDER_RR = intern("rr")

    __config__ = ["host", "psearchyPath", "filesPath", "dbPath",
                  "mode", "order", "mem"]

    def __init__(self, host, psearchyPath, filesPath, dbPath, cores,
                 mode = MODE_THREAD, order = ORDER_SEQ,
                 mem = 256, trials = 3):
        assert mode in [Mkdb.MODE_THREAD, Mkdb.MODE_PROCESS], \
            "Invalid mode %s" % mode
        assert order in [Mkdb.ORDER_SEQ, Mkdb.ORDER_RR], \
            "Invalid order %s" % order

        Task.__init__(self, host = host)
        BenchmarkRunner.__init__(self, cores, trials)
        self.host = host
        self.psearchyPath = psearchyPath
        self.filesPath = filesPath
        self.dbPath = dbPath
        self.mode = mode
        self.order = order
        self.mem = mem

    def runTrial(self, m):
        # Construct command
        cmd = [os.path.join(self.psearchyPath, "mkdb", "pedsort"),
               "-t", self.dbPath,
               "-c", str(self.cores),
               "-m", str(self.mem)]
        if self.mode == Mkdb.MODE_PROCESS:
            cmd.append("-p")
        if self.order == Mkdb.ORDER_RR:
            cmd.extend(["-s", "1"])

        # Run
        logPath = self.host.getLogPath(self)
        self.host.r.run(cmd, stdin = self.filesPath, stdout = logPath,
                        wait = CHECKED)

        # Get result
        log = self.host.r.readFile(logPath)
        # XXX Take the trial number so we can be sure to get the right result
        return parseResults(log)[-1]

__all__.append("parseResults")
def parseResults(log):
    out = []
    tputRe = re.compile(r"[0-9]+:.*\bthroughput:\s*([0-9]+\.?[0-9]*)\s+(\S+)\s*$")
    for line in log.splitlines():
        m = tputRe.match(line)
        if m:
            out.append((float(m.group(1)), m.group(2)))
    if not out:
        raise ValueError("Failed to parse results log")
    return out

__all__.append("Mkfiles")
class Mkfiles(Task):
    __config__ = ["host", "psearchyPath", "baseDir"]
{'fraction_non_alphanumeric': 0.13302346170872067, 'fraction_numerical': 0.005312084993359893, 'mean_word_length': 2.2817719680464776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1316274', 'n_tokens_mistral': 1492, 'n_tokens_neox': 1431, 'n_words': 341}