<jupyter_start><jupyter_text>Initialization<jupyter_code>min_cluster_size = 2
max_prop_clust_incorrect = 0.01
metadata_filename_ident = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'metadata',
    f'massivekb_ids_{config.massivekb_task_id}.parquet')

metadata_filename_val = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'embed_{config.massivekb_task_id}_val.parquet')
embed_filename_val = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'embed_{config.massivekb_task_id}_val.npy')
cluster_filename_val = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'clusters_{config.massivekb_task_id}_val.npy')
dist_filename_val = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'dist_{config.massivekb_task_id}_val.npz')

metadata_filename_test = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'embed_{config.massivekb_task_id}_test.parquet')
embed_filename_test = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'embed_{config.massivekb_task_id}_test.npy')
cluster_filename_test = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'clusters_{config.massivekb_task_id}_test.npy')
dist_filename_test = os.path.join(
    os.environ['GLEAMS_HOME'], 'data', 'cluster',
    f'dist_{config.massivekb_task_id}_test.npz')
# Make sure the pairwise distances are precomputed.
cluster.compute_pairwise_distances(embed_filename_val, metadata_filename_val)
cluster.compute_pairwise_distances(embed_filename_test, metadata_filename_test)
def get_metadata(metadata_filename, metadata_filename_ident):
    metadata = pd.merge(pd.read_parquet(metadata_filename),
                        pd.read_parquet(metadata_filename_ident)
                        [['dataset', 'filename', 'scan', 'sequence']],
                        'left', ['dataset', 'filename', 'scan'], copy=False)
    # Don't disambiguate between I/L.
    metadata['sequence'] = (metadata['sequence'].str.replace('I', 'L')
                            + '/' + metadata['charge'].astype(str))
    return metadata
# Get the metadata including all spectrum identifications.
metadata_val = get_metadata(metadata_filename_val, metadata_filename_ident)
metadata_test = get_metadata(metadata_filename_test, metadata_filename_ident)
def run_cluster(eps, min_samples, cluster_filename, dist_filename,
                metadata_filename, sequences, min_cluster_size):
    if os.path.isfile(cluster_filename):
        os.remove(cluster_filename)
    config.eps, config.min_samples = eps, min_samples
    cluster.cluster(dist_filename, metadata_filename)
    return evaluate_clusters(
        pd.DataFrame({'sequence': sequences,
                      'cluster': np.load(cluster_filename)}),
        min_cluster_size)
run_cluster_val = functools.partial(
    run_cluster,
    cluster_filename=cluster_filename_val,
    metadata_filename=metadata_filename_val,
    sequences=metadata_val['sequence'],
    dist_filename=dist_filename_val,
    min_cluster_size=min_cluster_size)


def cluster_optim(args):
    _, _, prop_clust, prop_clust_incorrect, _, _ = run_cluster_val(*args)
    props_clust.append(prop_clust)
    props_clust_incorrect.append(prop_clust_incorrect)
    if prop_clust_incorrect > max_prop_clust_incorrect:
        return 1
    else:
        return 1 - prop_clust<jupyter_output><empty_output><jupyter_text>Clustering hyperparameter optimization<jupyter_code>props_clust, props_clust_incorrect = [], []
optim = skopt.gp_minimize(cluster_optim,
                          [skopt.space.Real(0.01, config.margin / 3, name='eps'),
                           skopt.space.Integer(2, 5, name='min_samples')],
                          n_jobs=-1)
# Remove final (potentially suboptimal) clustering.
os.remove(cluster_filename_val)
cluster_hyperparameter = pd.DataFrame(
    {'eps': np.asarray(optim.x_iters)[:, 0],
     'min_samples': np.asarray(optim.x_iters)[:, 1],
     'prop_clustered': props_clust,
     'prop_clustered_incorrect': props_clust_incorrect})
acceptable_hyperparam = cluster_hyperparameter[
    cluster_hyperparameter['prop_clustered_incorrect']
    < max_prop_clust_incorrect]
best_hyperparam = acceptable_hyperparam.loc[
    acceptable_hyperparam['prop_clustered'].idxmax()]
print(f'Optimal clustering hyperparameters:\n'
      f'  - eps = {best_hyperparam["eps"]:.4f}\n'
      f'  - min_samples = {best_hyperparam["min_samples"]:.0f}\n'
      f'-> {best_hyperparam["prop_clustered"]:.2%} clustered, '
      f'{best_hyperparam["prop_clustered_incorrect"]:.2%} clustered incorrectly')<jupyter_output><empty_output><jupyter_text>Validate clustering hyperparameters<jupyter_code># Cluster on the test set with the optimal hyperparameters.
prop_clust_test, prop_clust_incorrect_test, _, _ =  run_cluster(
    best_hyperparam['eps'], best_hyperparam['min_samples'],
    cluster_filename_test, dist_filename_test, metadata_filename_test,
    metadata_test['sequence'], min_cluster_size)
prop_clust_test, prop_clust_incorrect_test
print(f'Cluster the test dataset with optimal hyperparameters\n'
      f'-> {prop_clust_test:.2%} clustered, '
      f'{prop_clust_incorrect_test:.2%} clustered incorrectly')
joblib.dump(
    (cluster_hyperparameter, (prop_clust_test, prop_clust_incorrect_test)),
    'cluster_hyperparameter.joblib')<jupyter_output><empty_output><jupyter_text>Plot clustering performance<jupyter_code># cluster_hyperparameter, (prop_clust_test, prop_clust_incorrect_test) = \
#     joblib.load('cluster_hyperparameter.joblib')
def get_pareto_frontier(arr):
    # Sort by the first column.
    arr_sorted = arr[arr[:, 0].argsort()]
    # Iteratively add points to the Pareto frontier.
    pareto_idx = [0]
    for i in range(1, arr_sorted.shape[0]):
        if (arr_sorted[i, 0] > arr_sorted[pareto_idx[-1], 0] and
                arr_sorted[i, 1] > arr_sorted[pareto_idx[-1], 1]):
            pareto_idx.append(i)
    return arr_sorted[pareto_idx]
width = 7
height = width / 1.618    # golden ratio
fig, ax = plt.subplots(figsize=(width, height))

# Hyperparameter optimization.
clustering_pareto = get_pareto_frontier(np.column_stack(
    [cluster_hyperparameter['prop_clustered_incorrect'],
     cluster_hyperparameter['prop_clustered']]))
ax.plot(clustering_pareto[:, 0], clustering_pareto[:, 1], marker='o',
        markersize=7, label='validation')
ax.scatter(cluster_hyperparameter['prop_clustered_incorrect'],
           cluster_hyperparameter['prop_clustered'], marker='.', s=49)
ax.axvline(max_prop_clust_incorrect, c='darkgray', ls='--')

# Performance of optimal hyperparameters on validation set.
ax.scatter(prop_clust_incorrect_test, prop_clust_test, marker='s', s=49,
           label='test', zorder=10)

ax.legend(loc='lower right')

ax.set_xlim(0, 0.05)
ax.set_ylim(0, 1)

ax.set_xlabel('Incorrectly clustered spectra')
ax.set_ylabel('Clustered spectra')

ax.xaxis.set_major_formatter(mticker.PercentFormatter(1))
ax.yaxis.set_major_formatter(mticker.PercentFormatter(1))

sns.despine()

plt.savefig('cluster_hyperparameter.png', dpi=300, bbox_inches='tight')
plt.show()
plt.close()
logging.shutdown()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12403967034502025, 'fraction_numerical': 0.007822321553289565, 'mean_word_length': 5.0575296108291035, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '248867', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2502, 'n_tokens_neox': 2426, 'n_words': 356, 'path': 'cluster_hyperparameter.ipynb', 'repo_name': 'bittremieux/GLEAMS_notebooks', 'size': '7159'}