<jupyter_start><jupyter_text>Parameter space exploration analysis<jupyter_code>import json
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import math
from config import EXPLORATION_SPACE
%matplotlib inline
# loading results of parameter space exploration
with open('results-space.json', 'r') as fp:
    data = json.load(fp)
# check keys of stored values
data.keys()
# nr of evaluations
print('Nr of evaluations performed:', len(data['trials']))
# configuration used for hyperparameter space exploration
config = EXPLORATION_SPACE
# parsing all results
allResults = [res['loss'] for res in data['results']]

# extract results
results = {}
for trial in data['trials']:
    res = trial['misc']['vals']
    loss = trial['result']['loss']
    for k in config.keys():
        results.setdefault(k, {})
        results[k].setdefault(res[k][0], [])
        results[k][res[k][0]].append(loss)
results.keys()
# plot loss value of all evaluations
fig, ax = plt.subplots(figsize=(15, 6))
_ = ax.scatter(np.arange(len(allResults)), allResults, s=1.5)
_ = ax.set_title('Loss on all {} evaluations'.format(len(data['trials'])), fontweight='bold')
_ = ax.set_ylabel('Loss')
fig.savefig('001-overall-loss.pdf', bbox_inches='tight')
# check loss using different window sizes
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(15, 6))
# batch size
orderId = [config['crossoverPB'][i] for i in results['crossoverPB'].keys()]
_ = ax1.boxplot(results['crossoverPB'].values())
_ = ax1.set_xticklabels(np.round(orderId, 2))
_ = ax1.set_ylabel('Loss')
_ = ax1.set_xlabel('Crossover probabilities')
_ = ax1.set_title('Loss with different crossover prob.', fontweight='bold')
# loss with two vs one layers
orderId = [config['nrGenerations'][i] for i in results['nrGenerations'].keys()]
_ = ax2.boxplot(results['nrGenerations'].values())
_ = ax2.set_xticklabels(orderId)
_ = ax2.set_ylabel('Loss')
_ = ax2.set_xlabel('Generation numbers')
_ = ax2.set_title('Loss with different generation numbers', fontweight='bold')
#fig.savefig('002-loss-crossgen.pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Best trialParameter value for the best overall trial<jupyter_code>print('Best trial is achieved using:')
for k in data['best'].keys():
    print('{}: {}'.format(k, config[k][data['best'][k]]))
print()
print('Best trial raw:')
data['best_trial']<jupyter_output>Best trial is achieved using:
crossoverPB: 0.75
mutationPB: 0.5000000000000001
notImprovingLimit: 10
nrGenerations: 450
populationSize: 350

Best trial raw:<jupyter_text>Check statistics on 15 best trialsCheck parameter values in the first 15 best results to check if there is some consistency or pattern<jupyter_code># create dataframe with parameter values of the top 15 trials
sortedResults = sorted(allResults)
order = np.argsort(allResults)
topResults = [res['misc']['vals'] for res in data['trials'] if res['result']['loss'] in sortedResults[:15]]
# convert lists to values
for e in topResults:
    for k,v in e.items():
        if type(v) == type(list()):
            e[k] = v[0]
df = pd.DataFrame(topResults)
df['points'] = [data['stepPoints'][i] for i in order][:15]
df.head()
# isolate each parameter 
def convertValue(row):
    row['Real value'] = config[c][row.name]
    return row
for c in df.columns:
    print('## Top 15 values for parameter: {}'.format(c))
    dfs = df[c].value_counts().to_frame()
    dfs.columns = ['Counts']
    dfs.index.name = 'Raw value'
    if c != 'points':
        dfs = dfs.apply(convertValue, axis=1)
    print(dfs)
    print()<jupyter_output>## Top 15 values for parameter: crossoverPB
           Counts  Real value
Raw value                    
10            5.0        0.90
3             4.0        0.55
7             3.0        0.75
5             2.0        0.65
4             1.0        0.60

## Top 15 values for parameter: mutationPB
           Counts  Real value
Raw value                    
4             7.0        0.30
3             4.0        0.25
5             2.0        0.35
8             1.0        0.50
6             1.0        0.40

## Top 15 values for parameter: notImprovingLimit
           Counts  Real value
Raw value                    
4              11          25
5               2          30
1               2          10

## Top 15 values for parameter: nrGenerations
           Counts  Real value
Raw value                    
6               4         350
4               4         250
8               2         450
7               2         400
2               2         150
9               1         500

## [...]<jupyter_text>Lets isolate each point distribution to see the mean value of each parameter. The goal would be to find a single value for each parameter which works well for all point distributions. This assumption might be unreasonable and can be investidated through appropriate plots.<jupyter_code># function to convert raw values from hyperopt to real values from config file
def convert(row):
    for col in [col for col in row.index if col not in ['point', 'fitness']]:
        row[col] = config[col][int(row[col])]
    return row

# number of best results to consider to compute the statistics
topPoints = 15
uniquePoints = sorted(set(data['stepPoints']))
dfData = []
for point in uniquePoints:
    # find all points matching
    pointIds = [i for i, el in enumerate(data['stepPoints']) if el==point]
    pointResults = [allResults[i] for i in pointIds]
    pointResultsOrdered = sorted(pointResults)
    
    order = np.argsort(pointResults)
    pointTrials = [data['trials'][i] for i in pointIds]
    # for each point distribution, extract the topPoints in terms of loss
    pointTrials = [pointTrials[i] for i in order][:topPoints]
    topResults = [res['misc']['vals'] for res in pointTrials]
    
    # convert lists to values
    for i,e in enumerate(topResults):
        singleResult = {}
        for k,v in e.items():
            if type(v) == type(list()):
                singleResult[k] = v[0]
            else:
                singleResult[k] = v
        singleResult['point']=point
        singleResult['fitness']=pointResultsOrdered[i]
        dfData.append(singleResult)
df = pd.DataFrame(dfData, dtype='float')
df = df.apply(convert, axis=1)
df.head()
# group above dataframe by point in order to show specific statistics
dfGrouped = df.groupby(['point']).agg(['mean','std'])
dfGrouped
# Helper function to return plot row and column
def currentRowCol(c,cols,rows):
    if rows == 1 or cols == 1:
        return c,0
    row = math.floor(c/cols)
    col = c-row*cols
    return row,col
cols = 2
rows = 3
fig, ax = plt.subplots(rows,cols,figsize=(15,12))
fig.suptitle('LOSSES ON ALL PARAMETERS AND POINT DISTRIBUTIONS', fontweight='bold')
for i,groupName in enumerate(set(dfGrouped.columns.get_level_values(0))):
    if groupName == 'fitness':
        continue
    r,c = currentRowCol(i,cols,rows)
    group = dfGrouped[groupName]
    x = group.index
    y = group['mean']
    e = group['std']/2
    ax[r,c].errorbar(x,y,e, linestyle='None', marker='o', color='k', capsize=5, ecolor='r', elinewidth=0.5, label='Mean/stdev')
    ax[r,c].axhline(y=np.mean(y), linewidth=1, color='k', linestyle='--', label='Global mean')
    ax[r,c].axhline(y=config[groupName][0], linewidth=1, color='r', linestyle='--', label='Limits')
    ax[r,c].axhline(y=config[groupName][-1], linewidth=1, color='r', linestyle='--')
    ax[r,c].legend(loc='upper right')
    ax[r,c].set_title('Top 15 {} values statistics'.format(groupName))
    ax[r,c].set_xlabel('Points')
plt.tight_layout()
plt.subplots_adjust(top=0.92)
fig.savefig('003-parameters-points-loss.pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>As it was expected, the optimal values for each parameter varies significantly over different point distributions.<jupyter_code># The mean values shown above in the black dotted line, which will be used
# as optimal parameters for the final genetic algorithm
np.round(dfGrouped.mean(),2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1278110944527736, 'fraction_numerical': 0.02711144427786107, 'mean_word_length': 3.289924973204716, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '509784', 'license': 'MIT', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2514, 'n_words': 737, 'path': '005-parameter-exploration-analysis.ipynb', 'repo_name': 'luigifilippochiara/travelling-salesman-problem-opl-genetic-python', 'size': '8004'}