<jupyter_start><jupyter_text>Use this to create graphs from results generated by the adversarial search scripts.<jupyter_code>%load_ext  autoreload
%autoreload 2
import sys
sys.path.append('..')
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
from collections import defaultdict

import pickle
import plot_params<jupyter_output><empty_output><jupyter_text>Define helper functions.<jupyter_code>def load_results(datafile):
    with open(datafile, 'rb') as f:
        results = pickle.load(f)
        # Remove unused data.
        results = [{
            'search_results': x['search_results'], 
            'baseline_results': x['baseline_results'],
            'bins': x['bins'],
            'p_norm': x['p_norm'],
            'q_norm': x['q_norm'],
            'epsilon': x['epsilon']
        } for x in results]
        return results
    
def group_by_key(data, key):
    obj = defaultdict(list)
    for d in data:
        value = d[key]
        obj[value].append(d)        
    return obj
def get_improvement(ast_results, bfs_results):
    assert ast_results['index'].equals(bfs_results['index'])
    assert ast_results['found'].equals(bfs_results['found'])
    
    # Get the expansion arrays.
    bfs_expansions = bfs_results['nodes_expanded']
    ast_expansions = ast_results['nodes_expanded']
    
    # Obtain indexes that would sort the BFS expansions array.
    idxs = np.argsort(bfs_expansions.values)
    
    # Use indexes to sort BFS and A* expansions arrays and compute improvement. 
    bfs_expansions_sorted = bfs_expansions.values[idxs]
    ast_expansions_sorted = ast_expansions.values[idxs]
    
    improvement = ((bfs_expansions_sorted - ast_expansions_sorted) / ast_expansions_sorted) * 100
    return improvement, ast_expansions_sorted, bfs_expansions_sorted
def plot_comparison(bfs_sort, ast_sorts, output_file):
    # Display the "A* search vs BFS search" plot with different epsilons.
    plt.figure(figsize=(12,4))
    plt.plot(bfs_sort, color='blue', label='BFS')
    
    for epsilon, ast_sort in ast_sorts.items():    
        plt.plot(ast_sort, label='A* (epsilon = {})'.format(epsilon))
        
    plt.xlabel('Example (sorted by BFS expansions)', fontsize=20)
    plt.ylabel('Nodes expanded', fontsize=20)
    plt.legend(loc='best', fontsize=12)
    plt.title('A* vs. BFS')
    plt.tight_layout()
    plt.savefig(output_file)
    plt.show()
    
def plot_improvement(improvements, output_file):
    # Display the "A* over BFS search improvement" plot with different epsilons.
    plt.figure(figsize=(12,4))
    
    for epsilon, improvement in improvements.items():
        plt.plot(improvement, label='Epsilon = {}'.format(epsilon))
#         print('Average A* (eps = {}) improvement over BFS: {:.2f}%.'.format(epsilon, np.sum(improvement) / len(improvement)))
        
    plt.xlabel('Example (sorted by expanded BFS nodes)', fontsize=20)
    plt.ylabel('Relative Improvement (\%)', fontsize=20)
    plt.title('A* vs. BFS')
    plt.legend(loc='best', fontsize=12)
    plt.tight_layout()
    plt.savefig(output_file)
    plt.show()<jupyter_output><empty_output><jupyter_text>Now we plot the required graphs.<jupyter_code># Load results from running adversarial search on credit dataset.
results = load_results('../out/credit_3.pickle')

# Group the results by the required key.
results_by_key = group_by_key(results, 'epsilon')

# Pick the number of bins to examine (this can be chosen in different ways).
bins = 75

# Get the required improvement data.
improvements, ast_sorts = {}, {}
for epsilon, items in results_by_key.items():
    
    for item in items:
        
        if item['bins'] != bins:
            continue
        
        ast_results, bfs_results = item['search_results'], item['baseline_results']
        improvement = get_improvement(ast_results=ast_results, bfs_results=bfs_results)
        
        improvements[epsilon] = improvement[0]
        ast_sorts[epsilon] = improvement[1]

        
bfs_sort = improvement[2] # changing epslion does not change bfs results

# Now plot the graphs.
plot_comparison(bfs_sort, ast_sorts, output_file='figures/comparison.pdf')
plot_improvement(improvements, output_file='figures/improvement.pdf')<jupyter_output><empty_output><jupyter_text>This can be used to establish the best number of bins given different parameters.<jupyter_code># Load results from running adversarial search on credit dataset.
results = load_results('../out/credit_3.pickle')
results_by_key = group_by_key(results, 'epsilon')

for epsilon, items in results_by_key.items():
    
    improvements, binss = [], []
    for item in items:
        
        ast_results, bfs_results = item['search_results'], item['baseline_results']
        improvement = get_improvement(ast_results=ast_results, bfs_results=bfs_results)
    
        improvements.append(improvement)
        binss.append(item['bins'])
        
    arg_max = np.argmax([x[0].mean() for x in improvements])
    arg_max_bins = binss[arg_max]
    print('For epsilon: {} best number of bins is: {}.'.format(epsilon, arg_max_bins))<jupyter_output>For epsilon: 1 best number of bins is: 45.
For epsilon: 2 best number of bins is: 40.
For epsilon: 3 best number of bins is: 20.
For epsilon: 5 best number of bins is: 75.
For epsilon: 10 best number of bins is: 45.
{'fraction_non_alphanumeric': 0.11077675380853864, 'fraction_numerical': 0.00883957118675945, 'mean_word_length': 3.6937334510150044, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '396937', 'license': 'MIT', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1675, 'n_words': 432, 'path': 'notebooks/legacy/visualizer.ipynb', 'repo_name': 'spring-epfl/trickster', 'size': '5317'}