<jupyter_start><jupyter_text>Go directly to the page end and see an explanation  When to run this scriptThis script is meant to be run after test_similarity.py is used to test different cases for the validation and/or test sets. It reads results from results.txt<jupyter_code>from __future__ import print_function

from rdkit import RDLogger
lg = RDLogger.logger()
lg.setLevel(4)

from rdkit.Chem.Draw import IPythonConsole, ReactionToImage, MolToImage, MolsToGridImage
import rdkit.Chem as Chem
import rdkit.Chem.AllChem as AllChem
from rdkit import DataStructs
import pandas as pd
import numpy as np
from tqdm import tqdm
import json
import sys
sys.path.append('../../')
from retrosim.utils.generate_retro_templates import process_an_example
from retrosim.data.get_data import get_data_df, split_data_df
from retrosim.utils.draw import ReactionStringToImage, TransformStringToImage
import os
if os.path.isfile(os.path.join('out', 'results.txt')):
    results = pd.read_csv(os.path.join('out', 'results.txt'), sep='\t')

# Get rid of whitespace
results.columns = [txt.strip() for txt in results.columns]
for fieldstrip in ['dataset', 'getfp_label', 'similarity_label', 'class_']:
    results[fieldstrip] = [txt.strip() for txt in results[fieldstrip]]
results['class_'] = [int(x) if x != 'all' else 'all' for x in results['class_']]
val_results = results.loc[results['dataset'] == 'val']
val_results
val_results.loc[val_results['getfp_label']=='Morgan2noFeat'].loc[val_results['similarity_label']=='Tanimoto']
val_performance = np.zeros((
    len(np.unique(val_results['similarity_label'])),
    len(np.unique(val_results['getfp_label'])),
    6,        
))
ns = [1, 3, 5, 10, 20, 50]
for i, similarity_label in enumerate(np.unique(val_results['similarity_label'])):
    for j, getfp_label in enumerate(np.unique(val_results['getfp_label'])):
        this_data = val_results.loc[val_results['similarity_label'] == similarity_label].\
             loc[val_results['getfp_label'] == getfp_label].loc[val_results['class_'] != 'all']
        for k, n in enumerate(ns):
            acc_label = 'top-{} acc'.format(n)
            accs = np.array(this_data[acc_label])
            n_in_class = np.array(this_data['n_in_class'])
            overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
            val_performance[i, j, k] = overall_acc
import matplotlib.pyplot as plt
%matplotlib inline
from mpl_toolkits.axes_grid1 import make_axes_locatable
fig, axes = plt.subplots(2,3)
axes = axes.flatten()
for k, n in enumerate(ns):
    ax = axes[k]
    im = ax.imshow(val_performance[:,:,k], cmap='RdYlGn', interpolation='nearest')
    ax.set_xticklabels(['', 0,10,20,30,40])
    ax.set_title('Top-{} Acc.'.format(n))
    #cbar = fig.colorbar(im, ax=ax, ticks=np.arange(0.0, 1.02, 0.002))
    
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax, ticks = np.arange(0.0, 1.02, 0.002))
    
    # from http://matplotlib.org/examples/ticks_and_spines/spines_demo_dropped.html
    # Move left and bottom spines outward by 10 points
    ax.spines['left'].set_position(('outward', 10))
    ax.spines['bottom'].set_position(('outward', 10))
    # Hide the right and top spines
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    # Only show ticks on the left and bottom spines
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    
    ax.xaxis.set_ticks(range(val_performance.shape[0]))
    if k >= 3:
        ax.xaxis.set_ticklabels(np.unique(val_results['similarity_label']), rotation='vertical')
    else:
        ax.xaxis.set_ticklabels([])
        
    ax.yaxis.set_ticks(range(val_performance.shape[1]))
    if k % 3 == 0:
        ax.yaxis.set_ticklabels(np.unique(val_results['getfp_label']))
    else:
        ax.yaxis.set_ticklabels([])
fig.set_size_inches(17.1/2.54, 4)
plt.tight_layout()

fig.savefig(os.path.join('out', 'validation_results.png'))<jupyter_output><empty_output><jupyter_text>Now when model information not provided<jupyter_code>val_performance = np.zeros((
    len(np.unique(val_results['similarity_label'])),
    len(np.unique(val_results['getfp_label'])),
    6,        
))
ns = [1, 3, 5, 10, 20, 50]
for i, similarity_label in enumerate(np.unique(val_results['similarity_label'])):
    for j, getfp_label in enumerate(np.unique(val_results['getfp_label'])):
        this_data = val_results.loc[val_results['similarity_label'] == similarity_label].\
             loc[val_results['getfp_label'] == getfp_label].loc[val_results['class_'] == 'all']
        for k, n in enumerate(ns):
            acc_label = 'top-{} acc'.format(n)
            accs = np.array(this_data[acc_label])
            n_in_class = np.array(this_data['n_in_class'])
            overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
            val_performance[i, j, k] = overall_acc
fig, axes = plt.subplots(2,3)
axes = axes.flatten()
for k, n in enumerate(ns):
    ax = axes[k]
    im = ax.imshow(val_performance[:,:,k], cmap='RdYlGn', interpolation='nearest')
    ax.set_xticklabels(['', 0,10,20,30,40])
    ax.set_title('Top-{} Acc.'.format(n))
    #cbar = fig.colorbar(im, ax=ax, ticks=np.arange(0.0, 1.02, 0.002))
    
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax, ticks = np.arange(0.0, 1.02, 0.002))
    
    # from http://matplotlib.org/examples/ticks_and_spines/spines_demo_dropped.html
    # Move left and bottom spines outward by 10 points
    ax.spines['left'].set_position(('outward', 10))
    ax.spines['bottom'].set_position(('outward', 10))
    # Hide the right and top spines
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    # Only show ticks on the left and bottom spines
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    
    ax.xaxis.set_ticks(range(val_performance.shape[0]))
    if k >= 3:
        ax.xaxis.set_ticklabels(np.unique(val_results['similarity_label']), rotation='vertical')
    else:
        ax.xaxis.set_ticklabels([])
        
    ax.yaxis.set_ticks(range(val_performance.shape[1]))
    if k % 3 == 0:
        ax.yaxis.set_ticklabels(np.unique(val_results['getfp_label']))
    else:
        ax.yaxis.set_ticklabels([])
fig.set_size_inches(17.1/2.54, 4)
plt.tight_layout()
fig.savefig(os.path.join('out', 'validation_results_noclassinfo.png'))<jupyter_output><empty_output><jupyter_text>Look at test performance<jupyter_code>test_results = results.loc[results['dataset'] == 'test'].loc[results['class_'] != 'all']
test_results.sort_values(['class_'])
ns = [1, 3, 5, 10, 20, 50]
overall_accs = []
for k, n in enumerate(ns):
    acc_label = 'top-{} acc'.format(n)
    accs = np.array(test_results.loc[test_results['class_'] != 'all'][acc_label])
    n_in_class = np.array(test_results.loc[test_results['class_'] != 'all']['n_in_class'])
    overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
    overall_accs.append(overall_acc)
    print('Top-{} acc on test --> {}'.format(n, overall_acc))<jupyter_output>Top-1 acc on test --> 0.529458757739242
Top-3 acc on test --> 0.737567405632197
Top-5 acc on test --> 0.8122628320353683
Top-10 acc on test --> 0.8813660874776328
Top-20 acc on test --> 0.9183143598962845
Top-50 acc on test --> 0.928500099860109<jupyter_text>...and test performance when class not given<jupyter_code>test_results = results.loc[results['dataset'] == 'test'].loc[results['class_'] == 'all']
test_results
ns = [1, 3, 5, 10, 20, 50]
overall_accs = []
for k, n in enumerate(ns):
    acc_label = 'top-{} acc'.format(n)
    accs = np.array(test_results[acc_label])
    n_in_class = np.array(test_results['n_in_class'])
    overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
    overall_accs.append(overall_acc)
    print('Top-{} acc on test --> {}'.format(n, overall_acc))<jupyter_output>Top-1 acc on test --> 0.373277411624
Top-3 acc on test --> 0.546834431795
Top-5 acc on test --> 0.633113640903
Top-10 acc on test --> 0.7407629318950001
Top-20 acc on test --> 0.8204513680850001
Top-50 acc on test --> 0.8532055122830001<jupyter_text>Explanation<jupyter_code>print('Each colored piece is a unique combination of fingerprint algorithm and similarity algorithm')
flag = True
ns = [1, 3, 5, 10, 20, 50]
for i, similarity_label in enumerate(np.unique(val_results['similarity_label'])):
    for j, getfp_label in enumerate(np.unique(val_results['getfp_label'])):
        this_data = val_results.loc[val_results['similarity_label'] == similarity_label].\
             loc[val_results['getfp_label'] == getfp_label].loc[val_results['class_'] == 'all'] # ALL
        for k, n in enumerate(ns):
            acc_label = 'top-{} acc'.format(n)
            accs = np.array(this_data[acc_label])
            n_in_class = np.array(this_data['n_in_class'])
            overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
            val_performance[i, j, k] = overall_acc 
            if flag:
                print('SA:', similarity_label, ', FA:', getfp_label,',', overall_acc, 'of chance in Top', n) 
                print('\nThere is {} of chance to locate the actual reactants (Top 1) of an abitrary class of reaction if all reactions of ALL class are used to conduct product similarity comparison (Algorithms: Dice, Morgan2Feat), which is the first step of retrosim method.\n'.format(overall_acc))
            flag = False
ax = plt.axes()
array = val_performance[:,:,0]
print(array)
im = ax.imshow(array, cmap='RdYlGn', interpolation='nearest') 
ax.set_title('ALL')
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im, cax=cax, ticks = np.arange(0.0, 1.02, 0.002)) 
print('Each colored piece is a unique combination of fingerprint algorithm and similarity algorithm')
flag = True
ns = [1, 3, 5, 10, 20, 50]
for i, similarity_label in enumerate(np.unique(val_results['similarity_label'])):
    for j, getfp_label in enumerate(np.unique(val_results['getfp_label'])):
        this_data = val_results.loc[val_results['similarity_label'] == similarity_label].\
             loc[val_results['getfp_label'] == getfp_label].loc[val_results['class_'] != 'all'] # SAME
        for k, n in enumerate(ns):
            acc_label = 'top-{} acc'.format(n)
            accs = np.array(this_data[acc_label])
            n_in_class = np.array(this_data['n_in_class'])
            overall_acc = sum([accs[z] * n_in_class[z] for z in range(len(accs))]) / np.sum(n_in_class)
            val_performance[i, j, k] = overall_acc 
            if flag:
                print('SA:', similarity_label, ', FA:', getfp_label,',', overall_acc, 'of chance in Top', n) 
                print('\nThere is {} of chance to locate the actual reactants (Top 1) of an abitrary class of reaction if all reactions of ALL class are used to conduct product similarity comparison (Algorithms: Dice, Morgan2Feat), which is the first step of retrosim method.\n'.format(overall_acc))
            flag = False
ax = plt.axes()
array = val_performance[:,:,0]
print(array)
im = ax.imshow(array, cmap='RdYlGn', interpolation='nearest') 
ax.set_title('SAME')
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im, cax=cax, ticks = np.arange(0.0, 1.02, 0.002))<jupyter_output>Each colored piece is a unique combination of fingerprint algorithm and similarity algorithm
SA: Dice , FA: Morgan2Feat , 0.528294341132 of chance in Top 1

There is 0.5282943411318984 of chance to locate the actual reactants (Top 1) of an abitrary class of reaction if all reactions of ALL class are used to conduct product similarity comparison (Algorithms: Dice, Morgan2Feat), which is the first step of retrosim method.

[[ 0.52829434  0.5354929   0.5314937   0.53109378]
 [ 0.52769446  0.5344931   0.53069386  0.5304939 ]
 [ 0.53169366  0.53409318  0.53629274  0.5314937 ]
 [ 0.52169566  0.52969406  0.52969406  0.52689462]]
{'fraction_non_alphanumeric': 0.14513216220653424, 'fraction_numerical': 0.04703661139385979, 'mean_word_length': 4.101758793969849, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '197671', 'license': 'MIT', 'n_tokens_mistral': 4790, 'n_tokens_neox': 4323, 'n_words': 958, 'path': 'retrosim/scripts/process_results.ipynb', 'repo_name': 'Furuidemu/retrosim', 'size': '12182'}