<commit_before>import os
import re
import sys
import json
import math
import shutil
import getopt
import logging
import tempfile
import subprocess
import collections


STATS_FULL_TIME_MEAN    = "full time mean"
STATS_FULL_TIME_STD     = "full time std"
STATS_GC_TIME_MEAN      = "gc time mean"
STATS_GC_TIME_STD       = "gc time std"
STATS_STW_TIME_MEAN     = "stw time mean"
STATS_STW_TIME_STD      = "stw time std"
STATS_STW_TIME_MAX      = "stw time max"
STATS_GC_COUNT          = "gc count"


def mean(xs):
    s = 0.0
    for x in xs:
        s += x
    return s / len(xs)


def std(xs):
    return math.sqrt(mean([x**2 for x in xs]) - mean(xs)**2)


def stat_mean(arr, default=float('NaN'), ndigits=None):
    return round(mean(arr), ndigits) if len(arr) > 0 else default


def stat_std(arr, default=float('NaN'), ndigits=None):
    return round(std(arr), ndigits) if len(arr) > 0 else default


def stat_max(arr, default=float('NaN'), ndigits=None):
    return round(max(arr), ndigits) if len(arr) > 0 else default


def call_with_cwd(args, cwd):
    proc = subprocess.Popen(args, cwd=cwd)
    proc.wait()
    assert proc.returncode == 0


def call_output(args, cwd=None, timeout=None):
    proc = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    try:
        out, err = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        proc.kill()
        raise
    return proc.returncode, out.decode("utf-8")


class Build:

    def __init__(self, name, prj_dir, runnable):
        self._name = name
        self._prj_dir = prj_dir
        self._runnable = runnable
        self._tmpdir = tempfile.TemporaryDirectory()

    def __enter__(self):
        self._tmpdir.__enter__()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._tmpdir.__exit__(exc_type, exc_val, exc_tb)

    def name(self):
        return self._name

    def build_dir(self):
        return self._tmpdir.name

    def project_dir(self):
        return self._prj_dir

    def runnable(self):
        return self._runnable


class CMakeBuild(Build):

    _tmpdir = None

    def __init__(self, prj_dir, target, runnable, cmake_ops, *args):
        super(CMakeBuild, self).__init__("cmake", prj_dir, runnable)
        if "run_cmake" in args:
            cmake_cmd = ["cmake", "-DCMAKE_BUILD_TYPE=Release", self._prj_dir] + self._parse_cmake_options(cmake_ops)
            call_with_cwd(cmake_cmd, self.build_dir())
        else:
            self._build_dir = "."
        make_cmd = ["make", target]
        call_with_cwd(make_cmd, self.build_dir())

    def run(self, args):
        runnable = os.path.join(self.build_dir(), self.runnable())
        call_args = [runnable] + args
        return call_output(call_args, cwd=self.build_dir(), timeout=30)

    @staticmethod
    def _parse_cmake_options(flags):
        res = []
        if isinstance(flags, str):
            res.append("-D{}=ON".format(flags))
        elif isinstance(flags, dict):
            for k, v in flags.items():
                if v is None:
                    res.append("-D{}=ON".format(k))
                else:
                    res.append("-D{}={}".format(k, v))
        return res


class MonoBuild(Build):

    def __init__(self, prj_dir, target, runnable, options):
        super(MonoBuild, self).__init__("mono", prj_dir, runnable)<commit_msg>fix build dir<commit_after>import os
import re
import sys
import json
import math
import shutil
import getopt
import logging
import tempfile
import subprocess
import collections


STATS_FULL_TIME_MEAN    = "full time mean"
STATS_FULL_TIME_STD     = "full time std"
STATS_GC_TIME_MEAN      = "gc time mean"
STATS_GC_TIME_STD       = "gc time std"
STATS_STW_TIME_MEAN     = "stw time mean"
STATS_STW_TIME_STD      = "stw time std"
STATS_STW_TIME_MAX      = "stw time max"
STATS_GC_COUNT          = "gc count"


def mean(xs):
    s = 0.0
    for x in xs:
        s += x
    return s / len(xs)


def std(xs):
    return math.sqrt(mean([x**2 for x in xs]) - mean(xs)**2)


def stat_mean(arr, default=float('NaN'), ndigits=None):
    return round(mean(arr), ndigits) if len(arr) > 0 else default


def stat_std(arr, default=float('NaN'), ndigits=None):
    return round(std(arr), ndigits) if len(arr) > 0 else default


def stat_max(arr, default=float('NaN'), ndigits=None):
    return round(max(arr), ndigits) if len(arr) > 0 else default


def call_with_cwd(args, cwd):
    proc = subprocess.Popen(args, cwd=cwd)
    proc.wait()
    assert proc.returncode == 0


def call_output(args, cwd=None, timeout=None):
    proc = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    try:
        out, err = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        proc.kill()
        raise
    return proc.returncode, out.decode("utf-8")


class Build:

    def __init__(self, name, prj_dir, runnable):
        self._name = name
        self._prj_dir = prj_dir
        self._runnable = runnable
        self._tmpdir = tempfile.TemporaryDirectory()

    def __enter__(self):
        self._tmpdir.__enter__()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._tmpdir.__exit__(exc_type, exc_val, exc_tb)

    def name(self):
        return self._name

    def build_dir(self):
        return self._tmpdir.name

    def project_dir(self):
        return self._prj_dir

    def runnable(self):
        return self._runnable


class CMakeBuild(Build):

    _tmpdir = None

    def __init__(self, prj_dir, target, runnable, cmake_ops, *args):
        super(CMakeBuild, self).__init__("cmake", prj_dir, runnable)
        if "run_cmake" in args:
            cmake_cmd = ["cmake", "-DCMAKE_BUILD_TYPE=Release", self._prj_dir] + self._parse_cmake_options(cmake_ops)
            call_with_cwd(cmake_cmd, self.build_dir())
        else:
            self._build_dir = "../"
        make_cmd = ["make", target]
        call_with_cwd(make_cmd, self.build_dir())

    def run(self, args):
        runnable = os.path.join(self.build_dir(), self.runnable())
        call_args = [runnable] + args
        return call_output(call_args, cwd=self.build_dir(), timeout=30)

    @staticmethod
    def _parse_cmake_options(flags):
        res = []
        if isinstance(flags, str):
            res.append("-D{}=ON".format(flags))
        elif isinstance(flags, dict):
            for k, v in flags.items():
                if v is None:
                    res.append("-D{}=ON".format(k))
                else:
                    res.append("-D{}={}".format(k, v))
        return res


class MonoBuild(Build):

    def __init__(self, prj_dir, target, runnable, options):
        super(MonoBuild, self).__init__("mono", prj_dir, runnable)
{'fraction_non_alphanumeric': 0.11740412979351032, 'fraction_numerical': 0.0032448377581120944, 'mean_word_length': 2.777715877437326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160765', 'n_tokens_mistral': 2393, 'n_tokens_neox': 2306, 'n_words': 539}