<jupyter_start><jupyter_text>Marginal Contribution<jupyter_code># helper functions
def rank_dict(dictionary, reverse=False):
    dictionary = copy.copy(dictionary)
    if reverse:
        for key in dictionary.keys():
            dictionary[key] = 1 - dictionary[key]
    sortdict = collections.OrderedDict(sorted(dictionary.items()))
    ranks = scipy.stats.rankdata(list(sortdict.values()))
    result = {}
    for idx, (key, value) in enumerate(sortdict.items()):
        result[key] = ranks[idx]
    return result


def sum_dict_values(a, b, allow_subsets=False):
    result = {}
    a_total = sum(a.values())
    b_total = sum(b.values())
    a_min_b = set(a.keys()) - set(b.keys())
    b_min_a = set(b.keys()) - set(a.keys())
    if len(b_min_a) > 0:
        raise ValueError('dict b got illegal keys: %s' %str(b_min_a))
    if not allow_subsets and len(a_min_b):
        raise ValueError('keys not the same')
    for idx in a.keys():
        if idx in b:
            result[idx] = a[idx] + b[idx]
        else:
            result[idx] = a[idx]
    if sum(result.values()) != a_total + b_total:
        raise ValueError()
    return result


def divide_dict_values(d, denominator):
    result = {}
    for idx in d.keys():
        result[idx] = d[idx] / denominator
    return result


def format_name(name):
    mapping_plain = {
        'strategy': 'imputation',
        'max_features': 'max. features',
        'min_samples_leaf': 'min. samples leaf',
        'min_samples_split': 'min. samples split',
        'criterion': 'split criterion',
        'learning_rate': 'learning rate',
        'max_depth': 'max. depth',
        'n_estimators': 'iterations',
        'algorithm': 'algorithm',
    }
    mapping_short = {
        'strategy': 'imputation',
        'max_features': 'max. feat.',
        'min_samples_leaf': 'samples leaf',
        'min_samples_split': 'samples split',
        'criterion': 'split criterion',
        'learning_rate': 'learning r.',
        'max_depth': 'max. depth',
        'n_estimators': 'iterations',
        'algorithm': 'algo.',
    }

    parts = name.split('__')
    for idx, part in enumerate(parts):
        if part in mapping_plain:
            if len(parts) < 3:
                parts[idx] = mapping_plain[part]
            else:
                parts[idx] = mapping_short[part]

    return ' / '.join(parts)


def marginal_plots(sorted_values, keys, fig_title):
    plt.figure()
    plt.violinplot(list(sorted_values), list(range(len(sorted_values))))
    plt.plot([-0.5, len(sorted_values) - 0.5], [0, 0], 'k-', linestyle='--', lw=1)
    keys = [format_name(key) for key in keys]
    plt.xticks(list(range(len(sorted_values))), list(keys), rotation=45, ha='right')
    plt.ylabel('marginal contribution')
    # plt.title(fig_title)
    print(fig_title)
    plt.show()
    plt.close()


def determine_relevant(data, max_items=None, max_interactions=None):
    from statistics import median
    
    sorted_values = []
    keys = []
    interactions_seen = 0
    for key in sorted(data, key=lambda k: median(data[k]), reverse=True):
        if '__' in key:
            interactions_seen += 1
            if interactions_seen > max_interactions:
                continue

        sorted_values.append(data[key])
        keys.append(key)

    if max_items is not None:
        sorted_values = sorted_values[:max_items]
        keys = keys[:max_items]

    return sorted_values, keys

def obtain_marginal_contributions(result_directory):
    all_ranks = dict()
    all_tasks = list()
    total_ranks = None
    num_tasks = 0
    marginal_contribution = collections.defaultdict(list)

    for task_id in os.listdir(result_directory):
        task_dir = os.path.join(result_directory, task_id)
        if os.path.isdir(task_dir):
            pimp_file = os.path.join(task_dir, 'pimp_values_fanova.json')
            interaction_file = os.path.join(task_dir, 'pimp_values_fanova_interaction.json')

            if os.path.isfile(pimp_file) and os.path.isfile(interaction_file):
                hyperparameters = json.loads(open(pimp_file).read())
                hyperparameters.update(json.loads(open(interaction_file).read()))

                for hyperparameter, value in hyperparameters.items():
                    parts = hyperparameter.split('__')
                    if sorted(parts) != parts: continue

                    marginal_contribution[hyperparameter].append(value)
                all_tasks.append(task_id)

                all_ranks[task_id] = hyperparameters
                ranks = rank_dict(hyperparameters, reverse=True)
                if total_ranks is None:
                    total_ranks = ranks
                else:
                    total_ranks = sum_dict_values(total_ranks, ranks, allow_subsets=False)
                    num_tasks += 1
    total_ranks = divide_dict_values(total_ranks, num_tasks)
    return total_ranks, marginal_contribution, all_tasks

for classifier, directory_suffix in classifiers.items():
    total_ranks, marginal_contribution, _ = obtain_marginal_contributions('data/fanova/' + directory_suffix)
    sorted_values, keys = determine_relevant(marginal_contribution, max_interactions=3)
    marginal_plots(sorted_values, keys, classifier)<jupyter_output><empty_output><jupyter_text>Most important hyperparameter per dataset<jupyter_code>x_axis_feature = 'NumberOfInstances'
y_axis_feature = 'NumberOfFeatures'

# consistent colors with the other plot
colors = {
    '6969/vanilla': {'min_samples_leaf': '#ff61c3', 'max_features': '#db72fb', 'criterion': '#d19100', 'bootstrap': '#619cff', 'max_features__min_samples_leaf': '#00c19f'},
    '6970/vanilla': {'max_depth': '#ff61cc', 'algorithm': '#7cae00', 'learning_rate': '#c77cff', 'learning_rate__max_depth': '#00a9ff', 'algorithm__max_depth': '#00bfc4'},
    '7707/kernel_rbf': {'gamma': '#ff61cc', 'gamma_tol': '#00bfc4', 'C': '#c77cff', 'C_gamma': '#00a9ff'},
    '7707/kernel_sigmoid': {'gamma': '#ff61c3', 'gamma_tol': '#00c19f', 'tol': '#93aa00', 'coef0':'#00ba38', 'C_gamma':'#db72fb'},
}


for classifier, directory_suffix in classifiers.items():
    x_vals = {}
    y_vals = {}
    area = {}
    
    directory = 'data/fanova/' + directory_suffix
    task_qualities = json.load(open('data/fanova/task_qualities.json', 'r'))
    for task_id in os.listdir(directory):
        task_dir = os.path.join(directory, task_id)
        if not os.path.isdir(task_dir):
            continue
        pimp_file = os.path.join(task_dir, 'pimp_values_fanova.json')
        interaction_file = os.path.join(task_dir, 'pimp_values_fanova_interaction.json')
        
        if not (os.path.isfile(pimp_file) and os.path.isfile(interaction_file)):
            continue
        
        hyperparameters = json.load(open(pimp_file, 'r'))
        hyperparameters.update(json.load(open(interaction_file, 'r')))
        
        most_important = max(hyperparameters, key=hyperparameters.get) 
        value = hyperparameters[most_important]
        
        if most_important not in x_vals:
            x_vals[most_important] = []
            y_vals[most_important] = []
            area[most_important] = []
        x_vals[most_important].append(float(task_qualities[task_id][x_axis_feature]))
        y_vals[most_important].append(float(task_qualities[task_id][y_axis_feature]))
        area[most_important].append(float(value) * 50)
    
    plt.figure(figsize=(8, 6))
    plotted_items = []
    legend_keys = []
    for param in x_vals.keys():
        occurances = len(x_vals[param])
        if param in colors[directory_suffix]:
            current = plt.scatter(x_vals[param], y_vals[param], c=colors[directory_suffix][param], s=area[param], alpha=0.9)
        else:
            current = plt.scatter(x_vals[param], y_vals[param], s=area[param], alpha=1.0)
        plotted_items.append(current)
        legend_keys.append(format_name(param))
    

    legend = plt.legend(plotted_items, legend_keys, scatterpoints=1, loc='upper right')
    for idx in range(len(plotted_items)):
        legend.legendHandles[idx]._sizes = [50]
    
    print(classifier)
    # dimensions of the datasets
    plt.axis((450,100000,3,2100))
    plt.xscale("log")
    plt.yscale("log")

    plt.xlabel(x_axis_feature, fontsize='xx-large')
    plt.ylabel(y_axis_feature, fontsize='xx-large')
    plt.show()
    # plt.savefig('result_' + classifier + '.pdf', bbox_inches='tight')
    plt.close()<jupyter_output><empty_output><jupyter_text>VerificationThe plots in the submission are generated with an external plotting library. To keep the notebook understandable, we will just display the dataframes with results.<jupyter_code>for classifier, directory_suffix in classifiers.items():
    
    file = 'data/verification/' + directory_suffix + '/random_search.pkl'
    if os.path.isfile(file):
        dataframe = pickle.load(open(file, 'rb'))
        print(classifier)
        print(dataframe)<jupyter_output><empty_output><jupyter_text>Prior experiments<jupyter_code>def plot_violin(results):
    data = []
    kde_wins = 0
    uni_wins = 0
    draws = 0
    dataframe = pd.DataFrame(columns=['task_id', 'mean_result_kde', 'mean_result_uniform', 'difference'])
    
    for task_id in results:
        if len(results[task_id]) == 2 and 'uniform' in results[task_id] and 'kde' in results[task_id] and len(results[task_id]['kde']) > 0 and len(results[task_id]['uniform']) > 0:
            scores_kde = sum(results[task_id]['kde'].values()) / len(results[task_id]['kde'])
            scores_uniform = sum(results[task_id]['uniform'].values()) / len(results[task_id]['uniform'])
            current_difference = scores_kde - scores_uniform
            data.append(current_difference)
            current_row = {'task_id': task_id, 'mean_result_kde': scores_kde, 'mean_result_uniform': scores_uniform, 'difference': current_difference}
            dataframe = dataframe.append(current_row, ignore_index=True)
    dataframe = dataframe.set_index('task_id')
    
    plt.figure(figsize=(2, 6))
    plt.tick_params(axis='x', which='both', bottom='off', top='off', labelbottom='off')
    plt.plot([0.5, 1.5], [0, 0], 'k-', linestyle='--', lw=1)
    plt.violinplot(data)
    plt.show()
    plt.close()


for classifier, directory_suffix in classifiers.items():
    directory = 'data/priors/' + directory_suffix
    if not os.path.isfile(directory + '/cache_test.pkl'):
        raise ValueError('Could not find cache file:', directory + '/cache_test.pkl')
    cache_results_test = pickle.load(open(directory + '/cache_test.pkl', 'rb'))
    print(classifier)
    plot_violin(cache_results_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13278164378408877, 'fraction_numerical': 0.012318976866654128, 'mean_word_length': 3.228628230616302, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '461809', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3559, 'n_tokens_neox': 3420, 'n_words': 653, 'path': 'KDD2018/results.ipynb', 'repo_name': 'mjbahmani/openml-pimp', 'size': '10634'}