<commit_before># BenchExec is a framework for reliable benchmarking.
# This file is part of BenchExec.
#
# Copyright (C) 2007-2015  <NAME>
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contians the tool benchexec for executing a whole benchmark (suite).
To use it, instantiate the "benchexec.benchexec.BenchExec"
and either call "instance.start()" or "benchexec.benchexec.main(instance)".
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import logging
import os
import signal
import sys
import time

from benchexec import __version__
from benchexec.model import Benchmark
from benchexec.outputhandler import OutputHandler
from benchexec import util


class BenchExec(object):
    """
    The main class of BenchExec.
    It is designed to be extended by inheritance, and for example
    allows configuration options to be added and the executor to be replaced.
    By default, it uses an executor that executes all runs on the local machine.
    """

    DEFAULT_OUTPUT_PATH = "results/"

    def __init__(self):
        self.executor = None
        self.stopped_by_interrupt = False

    def start(self, argv):
        """
        Start BenchExec.
        @param argv: command-line options for BenchExec
        """
        parser = self.create_argument_parser()
        self.config = parser.parse_args(argv[1:])

        for arg in self.config.files:
            if not os.path.exists(arg) or not os.path.isfile(arg):
                parser.error("File {0} does not exist.".format(repr(arg)))

        if os.path.isdir(self.config.output_path):
            self.config.output_path = os.path.normpath(self.config.output_path) + os.sep

        self.setup_logging()

        self.executor = self.load_executor()

        returnCode = 0
        for arg in self.config.files:
            if self.stopped_by_interrupt:
                break
            logging.debug("Benchmark %r is started.", arg)
            rc = self.execute_benchmark(arg)
            returnCode = returnCode or rc
            logging.debug("Benchmark %r is done.", arg)

        logging.debug("I think my job is done. Have a nice day!")
        return returnCode


    def create_argument_parser(self):
        """
        Create a parser for the command-line options.
        May be overwritten for adding more configuration options.
        @return: an argparse.ArgumentParser instance
        """
        parser = argparse.ArgumentParser(
            fromfile_prefix_chars='@',
            description=
            """Execute benchmarks for a given tool with a set of input files.
               Benchmarks are defined in an XML file given as input.
               Command-line parameters can additionally be read from a file if file name prefixed with '@' is given as argument.
               The tool table-generator can be used to create tables for the results.
               Part of BenchExec: https://github.com/sosy-lab/benchexec/""")

        parser.add_argument("files", nargs='+', metavar="FILE",
                          help="XML file with benchmark definition")
        parser.add_argument("-d", "--debug",
                          action="store_true",
                          help="Enable debug output")

        parser.add_argument("-r", "--rundefinition", dest="selected_run_definitions",
                          action="append",
                          help="Run only the specified RUN_DEFINITION from the benchmark definition file. "
                                + "This option can be specified several times.",
                          metavar="RUN_DEFINITION")

        parser.add_argument("-t", "--tasks", dest="selected_sourcefile_sets",
                          action="append",
                          help="Run only the tasks from the tasks tag with TASKS as name. "
                                + "This option can be specified several times.",
                          metavar="TASKS")

        parser.add_argument("-n", "--name",
                          dest="name", default=None,
                          help="Set name of benchmark execution to NAME",
                          metavar="NAME")

        parser.add_argument("-o", "--outputpath",
                          dest="output_path", type=str,
                          default=self.DEFAULT_OUTPUT_PATH,
                          help="Output prefix for the generated results. "
                                + "If the path is a folder files are put into it,"
                                + "otherwise it is used as a prefix for the resulting files.")

        parser.add_argument("-T", "--timelimit",
                          dest="timelimit", default=None,
                          help="Time limit in seconds for each run (-1 to disable)",
                          metavar="SECONDS")

        parser.add_argument("-M", "--memorylimit",
                          dest="memorylimit", default=None,
                          help="Memory limit in MB (-1 to disable)",
                          metavar="MB")

        parser.add_argument("-N", "--numOfThreads",
                          dest="num_of_threads", default=None, type=int,
                          help="Run n benchmarks in parallel",
                          metavar="n")

        parser.add_argument("-c", "--limitCores", dest="corelimit",
                          type=int, default=None,
                          metavar="N",
                          help="Limit each run of the tool to N CPU cores (-1 to disable).")

        parser.add_argument("--user",
                            dest="users",
                            action="append",
                            metavar="USER",
                            help="Execute benchmarks under given user account(s) (needs password-less sudo setup).")

        parser.add_argument("--maxLogfileSize",
                          dest="maxLogfileSize", type=int, default=20,
                          metavar="MB",
                          help="Shrink logfiles to given size in MB, if they are too big. (-1 to disable, default value: 20 MB).")

        parser.add_argument("--commit", dest="commit",
                          action="store_true",
                          help="If the output path is a git repository without local changes, "<commit_msg>Fix specifying core limit on benchexec command line.<commit_after># BenchExec is a framework for reliable benchmarking.
# This file is part of BenchExec.
#
# Copyright (C) 2007-2015  <NAME>
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contians the tool benchexec for executing a whole benchmark (suite).
To use it, instantiate the "benchexec.benchexec.BenchExec"
and either call "instance.start()" or "benchexec.benchexec.main(instance)".
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import logging
import os
import signal
import sys
import time

from benchexec import __version__
from benchexec.model import Benchmark
from benchexec.outputhandler import OutputHandler
from benchexec import util


class BenchExec(object):
    """
    The main class of BenchExec.
    It is designed to be extended by inheritance, and for example
    allows configuration options to be added and the executor to be replaced.
    By default, it uses an executor that executes all runs on the local machine.
    """

    DEFAULT_OUTPUT_PATH = "results/"

    def __init__(self):
        self.executor = None
        self.stopped_by_interrupt = False

    def start(self, argv):
        """
        Start BenchExec.
        @param argv: command-line options for BenchExec
        """
        parser = self.create_argument_parser()
        self.config = parser.parse_args(argv[1:])

        for arg in self.config.files:
            if not os.path.exists(arg) or not os.path.isfile(arg):
                parser.error("File {0} does not exist.".format(repr(arg)))

        if os.path.isdir(self.config.output_path):
            self.config.output_path = os.path.normpath(self.config.output_path) + os.sep

        self.setup_logging()

        self.executor = self.load_executor()

        returnCode = 0
        for arg in self.config.files:
            if self.stopped_by_interrupt:
                break
            logging.debug("Benchmark %r is started.", arg)
            rc = self.execute_benchmark(arg)
            returnCode = returnCode or rc
            logging.debug("Benchmark %r is done.", arg)

        logging.debug("I think my job is done. Have a nice day!")
        return returnCode


    def create_argument_parser(self):
        """
        Create a parser for the command-line options.
        May be overwritten for adding more configuration options.
        @return: an argparse.ArgumentParser instance
        """
        parser = argparse.ArgumentParser(
            fromfile_prefix_chars='@',
            description=
            """Execute benchmarks for a given tool with a set of input files.
               Benchmarks are defined in an XML file given as input.
               Command-line parameters can additionally be read from a file if file name prefixed with '@' is given as argument.
               The tool table-generator can be used to create tables for the results.
               Part of BenchExec: https://github.com/sosy-lab/benchexec/""")

        parser.add_argument("files", nargs='+', metavar="FILE",
                          help="XML file with benchmark definition")
        parser.add_argument("-d", "--debug",
                          action="store_true",
                          help="Enable debug output")

        parser.add_argument("-r", "--rundefinition", dest="selected_run_definitions",
                          action="append",
                          help="Run only the specified RUN_DEFINITION from the benchmark definition file. "
                                + "This option can be specified several times.",
                          metavar="RUN_DEFINITION")

        parser.add_argument("-t", "--tasks", dest="selected_sourcefile_sets",
                          action="append",
                          help="Run only the tasks from the tasks tag with TASKS as name. "
                                + "This option can be specified several times.",
                          metavar="TASKS")

        parser.add_argument("-n", "--name",
                          dest="name", default=None,
                          help="Set name of benchmark execution to NAME",
                          metavar="NAME")

        parser.add_argument("-o", "--outputpath",
                          dest="output_path", type=str,
                          default=self.DEFAULT_OUTPUT_PATH,
                          help="Output prefix for the generated results. "
                                + "If the path is a folder files are put into it,"
                                + "otherwise it is used as a prefix for the resulting files.")

        parser.add_argument("-T", "--timelimit",
                          dest="timelimit", default=None,
                          help="Time limit in seconds for each run (-1 to disable)",
                          metavar="SECONDS")

        parser.add_argument("-M", "--memorylimit",
                          dest="memorylimit", default=None,
                          help="Memory limit in MB (-1 to disable)",
                          metavar="MB")

        parser.add_argument("-N", "--numOfThreads",
                          dest="num_of_threads", default=None, type=int,
                          help="Run n benchmarks in parallel",
                          metavar="n")

        parser.add_argument("-c", "--limitCores", dest="corelimit",
                          default=None,
                          metavar="N",
                          help="Limit each run of the tool to N CPU cores (-1 to disable).")

        parser.add_argument("--user",
                            dest="users",
                            action="append",
                            metavar="USER",
                            help="Execute benchmarks under given user account(s) (needs password-less sudo setup).")

        parser.add_argument("--maxLogfileSize",
                          dest="maxLogfileSize", type=int, default=20,
                          metavar="MB",
                          help="Shrink logfiles to given size in MB, if they are too big. (-1 to disable, default value: 20 MB).")

        parser.add_argument("--commit", dest="commit",
                          action="store_true",
                          help="If the output path is a git repository without local changes, "
{'fraction_non_alphanumeric': 0.08996013048205871, 'fraction_numerical': 0.0034795215657847044, 'mean_word_length': 1.8515915667631253, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5248863', 'n_tokens_mistral': 3493, 'n_tokens_neox': 3361, 'n_words': 1174}