<commit_before>import collections
import csv
import openml
import os
import subprocess
import sys
import matplotlib.pyplot as plt


def _determine_eligibility(strategy, include_pattern, exclude_pattern):
    if include_pattern is not None and not isinstance(include_pattern, list):
        raise TypeError()
    if exclude_pattern is not None and not isinstance(exclude_pattern, list):
        raise TypeError()

    # exlude strategies
    if exclude_pattern is not None:
        for pattern in exclude_pattern:
            if pattern in strategy:
                return False
    # include strategies
    if include_pattern is not None:
        for pattern in include_pattern:
            if pattern in strategy:
                return True

    # if include_pattern is None, all that was not excluded can be used.
    if include_pattern is None:
        return True
    else:
        return False


def _determine_name(strategy):
    strategy_splitted = strategy.split('__')
    if len(strategy_splitted) < 3:
        return strategy
    return strategy_splitted[0] + '__' + strategy_splitted[3]


def to_csv_file(ranks_dict, location):
    hyperparameters = None
    for task_id, params in ranks_dict.items():
        hyperparameters = set(params)

    with open(location, 'w') as csvfile:
        fieldnames = ['task_id']
        fieldnames.extend(hyperparameters)
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        for task_id, param_values in ranks_dict.items():
            result = {}
            result.update(param_values)
            result['task_id'] = 'Task %d' %task_id
            writer.writerow(result)
    pass


def to_csv_unpivot(ranks_dict, location):
    with open(location, 'w') as csvfile:
        fieldnames = ['task_id', 'param_id', 'param_name', 'variance_contribution']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        for task_id, param_values in ranks_dict.items():

            for param_name, variance_contribution in param_values.items():
                result = {'task_id' : task_id,
                          'param_id': param_name,
                          'param_name': param_name,
                          'variance_contribution': variance_contribution}
                writer.writerow(result)
    pass


def plot_task(plotting_virtual_env, plotting_scripts_dir, strategy_directory, plot_directory, task_id, include_pattern=None, exclude_pattern=None, wildcard_depth=1):
    # make regret plot:
    script = "%s %s/plot_test_performance_from_csv.py " % (plotting_virtual_env, plotting_scripts_dir)
    cmd = [script]
    for strategy, directory in strategy_directory.items():
        if _determine_eligibility(strategy, include_pattern, exclude_pattern) is False:
            continue
        strategy_name = _determine_name(strategy)

        cmd.append(strategy_name)
        cmd.append(directory + '/' + str(task_id) + ('/*' * wildcard_depth) + '.csv')
    try:
        os.makedirs(plot_directory)
    except FileExistsError:
        pass

    cmd.append('--save %s ' % os.path.join(plot_directory, 'validation_regret%s.png' %str(task_id)))
    cmd.append('--ylabel "Accuracy Loss"')
    subprocess.run(' '.join(cmd), shell=True)
    print('CMD: ', ' '.join(cmd))


def boxplot_traces(strategy_traces, save_directory, name):
    try:
        os.makedirs(save_directory)
    except FileExistsError:
        pass

    data = []
    label_names = []

    for strategy, trace in strategy_traces.items():
        current = [trace_item.evaluation for trace_item in trace.trace_iterations.values()]
        data.append(current)
        label_names.append(strategy)

    plt.figure()
    plt.boxplot(data)
    plt.xticks(list(range(1, len(label_names) + 1)), label_names)
    plt.savefig(os.path.join(save_directory, name))
    plt.close()


def average_rank(plotting_virtual_env, plotting_scripts_dir, output_directory, curves_directory, include_pattern=None, exclude_pattern=None):
    script = "%s %s/plot_ranks_from_csv.py " % (plotting_virtual_env, plotting_scripts_dir)
    cmd = [script]

    results = collections.defaultdict(dict)
    for strategy in os.listdir(curves_directory):<commit_msg>matplotlib import moved<commit_after>import collections
import csv
import openml
import os
import subprocess
import sys


def _determine_eligibility(strategy, include_pattern, exclude_pattern):
    if include_pattern is not None and not isinstance(include_pattern, list):
        raise TypeError()
    if exclude_pattern is not None and not isinstance(exclude_pattern, list):
        raise TypeError()

    # exlude strategies
    if exclude_pattern is not None:
        for pattern in exclude_pattern:
            if pattern in strategy:
                return False
    # include strategies
    if include_pattern is not None:
        for pattern in include_pattern:
            if pattern in strategy:
                return True

    # if include_pattern is None, all that was not excluded can be used.
    if include_pattern is None:
        return True
    else:
        return False


def _determine_name(strategy):
    strategy_splitted = strategy.split('__')
    if len(strategy_splitted) < 3:
        return strategy
    return strategy_splitted[0] + '__' + strategy_splitted[3]


def to_csv_file(ranks_dict, location):
    hyperparameters = None
    for task_id, params in ranks_dict.items():
        hyperparameters = set(params)

    with open(location, 'w') as csvfile:
        fieldnames = ['task_id']
        fieldnames.extend(hyperparameters)
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        for task_id, param_values in ranks_dict.items():
            result = {}
            result.update(param_values)
            result['task_id'] = 'Task %d' %task_id
            writer.writerow(result)
    pass


def to_csv_unpivot(ranks_dict, location):
    with open(location, 'w') as csvfile:
        fieldnames = ['task_id', 'param_id', 'param_name', 'variance_contribution']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        for task_id, param_values in ranks_dict.items():

            for param_name, variance_contribution in param_values.items():
                result = {'task_id' : task_id,
                          'param_id': param_name,
                          'param_name': param_name,
                          'variance_contribution': variance_contribution}
                writer.writerow(result)
    pass


def plot_task(plotting_virtual_env, plotting_scripts_dir, strategy_directory, plot_directory, task_id, include_pattern=None, exclude_pattern=None, wildcard_depth=1):
    # make regret plot:
    script = "%s %s/plot_test_performance_from_csv.py " % (plotting_virtual_env, plotting_scripts_dir)
    cmd = [script]
    for strategy, directory in strategy_directory.items():
        if _determine_eligibility(strategy, include_pattern, exclude_pattern) is False:
            continue
        strategy_name = _determine_name(strategy)

        cmd.append(strategy_name)
        cmd.append(directory + '/' + str(task_id) + ('/*' * wildcard_depth) + '.csv')
    try:
        os.makedirs(plot_directory)
    except FileExistsError:
        pass

    cmd.append('--save %s ' % os.path.join(plot_directory, 'validation_regret%s.png' %str(task_id)))
    cmd.append('--ylabel "Accuracy Loss"')
    subprocess.run(' '.join(cmd), shell=True)
    print('CMD: ', ' '.join(cmd))


def boxplot_traces(strategy_traces, save_directory, name):
    # only import matplotlib for this function, as meta cluster doesn't like it
    import matplotlib.pyplot as plt
    try:
        os.makedirs(save_directory)
    except FileExistsError:
        pass

    data = []
    label_names = []

    for strategy, trace in strategy_traces.items():
        current = [trace_item.evaluation for trace_item in trace.trace_iterations.values()]
        data.append(current)
        label_names.append(strategy)

    plt.figure()
    plt.boxplot(data)
    plt.xticks(list(range(1, len(label_names) + 1)), label_names)
    plt.savefig(os.path.join(save_directory, name))
    plt.close()


def average_rank(plotting_virtual_env, plotting_scripts_dir, output_directory, curves_directory, include_pattern=None, exclude_pattern=None):
    script = "%s %s/plot_ranks_from_csv.py " % (plotting_virtual_env, plotting_scripts_dir)
    cmd = [script]

    results = collections.defaultdict(dict)
    for strategy in os.listdir(curves_directory):
{'fraction_non_alphanumeric': 0.09293376865671642, 'fraction_numerical': 0.0013992537313432835, 'mean_word_length': 3.045754716981132, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '323218', 'n_tokens_mistral': 2578, 'n_tokens_neox': 2500, 'n_words': 599}