<commit_before>#!/usr/bin/env python3
"""
Created on Sep 28, 2014

@author: <NAME>

This script assumes that the required dependencies are available from either 
the local Maven repository or the Maven Central Repository.
"""
import subprocess
import argparse

import yaml
import util

def build(config, formats, skip_tests):
    profiles = {"core"}
    profiles = profiles.union(formats)
    profiles = profiles.union(config["tools"])

    profiles_arg = ",".join(profiles)

    cmd = ["mvn", "clean", "install", "-P", profiles_arg]
    if skip_tests:
        cmd.append("-DskipTests")
    subprocess.check_call(cmd)


def build_ci():
    cmd_ci = ["mvn", "clean", "install", "-P", "ci"]
    subprocess.check_call(cmd_ci)
    # skip the tests for tools with third-party dependencies
    cmd_notest = ["mvn", "clean", "install", "-P", "notest", "-DskipTests"]
    subprocess.check_call(cmd_notest)


def generate(config, formats):
    for format in formats:
        for scenario in config["scenarios"]:
            args = [""]
            if format in config["generator_optional_arguments"]:
                args.append("-" + config["generator_optional_arguments"]["rdf"][0])

            for arg in args:
                path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format)
                util.set_working_directory(path)
                target = util.get_generator_jar(format)
                for size in config["sizes"]:
                    cmd = ["java", "-Xmx" + config["java_opts"]["xmx"],
                         "-jar", target,
                         "-scenario", scenario,
                         "-size", str(size),
                       arg]
                    try:
                        subprocess.check_call(cmd)
                    except subprocess.CalledProcessError:
                        print("An error occured during model generation, skipping larger sizes for this scenario/format.")
                        break
                util.set_working_directory("..")


def measure(config):
    for tool in config["tools"]:
        args = [""]
        if tool in config["benchmark_optional_arguments"]:
            args.append("-" + config["benchmark_optional_arguments"]["rdf"][0])

        for arg in args:
            path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
            util.set_working_directory(path)
            target = util.get_tool_jar(tool)

            for scenario in config["scenarios"]:
                for query in config["queries"]:
                    for size in config["sizes"]:
                        print("Run benchmark -- " +
                              "runs: " + str(config["runs"]) +
                              ", tool: " + tool +
                              ", scenario: " + scenario +
                              ", query: " + query +
                              ", size: " + str(size) +
                              (", arg: " + arg if arg != "" else ""))
                        cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target,
                               "-runs", str(config["runs"]),
                               "-scenario", scenario,<commit_msg>Fixed run script<commit_after>#!/usr/bin/env python3
"""
Created on Sep 28, 2014

@author: <NAME>

This script assumes that the required dependencies are available from either 
the local Maven repository or the Maven Central Repository.
"""
import subprocess
import argparse

import yaml
import util

def build(config, formats, skip_tests):
    profiles = {"core"}
    profiles = profiles.union(formats)
    profiles = profiles.union(config["tools"])

    profiles_arg = ",".join(profiles)

    cmd = ["mvn", "clean", "install", "-P", profiles_arg]
    if skip_tests:
        cmd.append("-DskipTests")
    subprocess.check_call(cmd)


def build_ci():
    cmd_ci = ["mvn", "clean", "install", "-P", "ci"]
    subprocess.check_call(cmd_ci)
    # skip the tests for tools with third-party dependencies
    cmd_notest = ["mvn", "clean", "install", "-P", "notest", "-DskipTests"]
    subprocess.check_call(cmd_notest)


def generate(config, formats):
    for format in formats:
        for scenario in config["scenarios"]:
            args = [""]
            if format in config["generator_optional_arguments"]:
                args.append("-" + config["generator_optional_arguments"]["rdf"][0])

            for arg in args:
                path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format)
                util.set_working_directory(path)
                target = util.get_generator_jar(format)
                for size in config["sizes"]:
                    cmd = ["java", "-Xmx" + config["java_opts"]["xmx"],
                         "-jar", target,
                         "-scenario", scenario,
                         "-size", str(size),
                       arg]
                    try:
                        subprocess.check_call(cmd)
                    except subprocess.CalledProcessError:
                        print("An error occured during model generation, skipping larger sizes for this scenario/format.")
                        break
                util.set_working_directory("..")


def measure(config):
    for tool in config["tools"]:
        args = [""]
        if tool in config["benchmark_optional_arguments"]:
            args.append("-" + config["benchmark_optional_arguments"][tool][0])

        for arg in args:
            path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
            util.set_working_directory(path)
            target = util.get_tool_jar(tool)

            for scenario in config["scenarios"]:
                for query in config["queries"]:
                    for size in config["sizes"]:
                        print("Run benchmark -- " +
                              "runs: " + str(config["runs"]) +
                              ", tool: " + tool +
                              ", scenario: " + scenario +
                              ", query: " + query +
                              ", size: " + str(size) +
                              (", arg: " + arg if arg != "" else ""))
                        cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target,
                               "-runs", str(config["runs"]),
                               "-scenario", scenario,
{'fraction_non_alphanumeric': 0.13081486106774898, 'fraction_numerical': 0.00280986575085857, 'mean_word_length': 1.7205944798301487, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '602104', 'n_tokens_mistral': 1736, 'n_tokens_neox': 1679, 'n_words': 411}