<jupyter_start><jupyter_text>Import pre-processed dataset "AllPositive"<jupyter_code>import time
import pickle

#loading data as pickled object goes a lot quicker
# outfile = os.path.join(path_data, 'gnps_positive_ionmode_cleaned_by_matchms_and_lookups.pickle')

# print(outfile)
# start = time.time()
# with open(outfile, 'rb') as inf:
#         spectrums = pickle.load(inf)
# end = time.time()
# print('loading took {:.2f} s'.format(end-start))<jupyter_output><empty_output><jupyter_text>Import spec2vec matches of 1,000 spectra where their inchikeys occur >=5 times in the library<jupyter_code>import pickle
outfile = os.path.join(path_data, 'found_matches_s2v.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        found_matches_s2v = pickle.load(inf)
else:
    print('error')

#contains info on the query/library
outfile = os.path.join(path_data, 'documents_query_s2v.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        documents_query_s2v = pickle.load(inf)

outfile = os.path.join(path_data, 'documents_library_s2v.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        documents_library_s2v = pickle.load(inf)
#and another set of 1,000 different spectra
#contains info on the query/library
outfile = os.path.join(path_data, 'new_documents_query_s2v.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        new_documents_query_s2v = pickle.load(inf)

outfile = os.path.join(path_data, 'new_documents_library_s2v.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        new_documents_library_s2v = pickle.load(inf)
outfile = os.path.join(path_data, 'rfreg_all_match_perc_pmass.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        rfreg_all_match_perc_pmass = pickle.load(inf)

outfile = os.path.join(path_data, 'found_matches_s2v_match_perc_pmass_sim.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        found_matches_s2v_match_perc_pmass_sim = pickle.load(inf)<jupyter_output>C:\Users\joris\Documents\eScience_data\data\rfreg_all_match_perc_pmass.pickle
C:\Users\joris\Documents\eScience_data\data\found_matches_s2v_match_perc_pmass_sim.pickle<jupyter_text>Calculate pairwise tanimoto similarity between top 20 results for each query<jupyter_code>import pickle
from rdkit import Chem, DataStructs

outfile = os.path.join(path_data, 'similarity_matrices_test_set1.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        similarity_matrices = pickle.load(inf)
else:
    #for each matches df make a similarity matrix based on tanimoto
    similarity_matrices = []
    topn = 20

    for ID in range(len(found_matches_s2v)):
        df = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
        library_ids = df.index.values
        rows = []
        for lib_id_i in library_ids:
            smiles_i = documents_library_s2v[lib_id_i]._obj.get("smiles")

            if not smiles_i or smiles_i == "None": # check that lib_id_i smiles exist
                rows.append([0] * len(library_ids)) #default to all 0 if it doesnt exist
                continue
            ms_i = Chem.MolFromSmiles(smiles_i)
            if not ms_i: #in case something is wrong with smiles
                rows.append([0] * len(library_ids)) #default to all 0 if it doesnt exist
                continue
            fp_i = Chem.RDKFingerprint(ms_i)

            row = []
            for lib_id_j in library_ids:
                smiles_j = documents_library_s2v[lib_id_j]._obj.get("smiles")

                if smiles_j and smiles_j != "None":
                    ms_j = Chem.MolFromSmiles(smiles_j)
                    if ms_j:
                        fp_j = Chem.RDKFingerprint(ms_j)
                        score = DataStructs.FingerprintSimilarity(fp_i, fp_j)
                    else: #in case something is wrong with smiles
                        score = 0
                else: #in case it doesnt have smiles
                    score = 0
                row.append(score)

            rows.append(row)
        similarity_matrices.append(pd.DataFrame(rows))
    #write to file    
    with open(outfile, 'wb') as outf:
        pickle.dump(similarity_matrices, outf)
#corrected version where true matches get a slightly higher value in the diagonal
import pickle
cor_val = 1.001
outfile = os.path.join(path_data, 'similarity_matrices_corrected_test_set1.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        similarity_matrices_cor = pickle.load(inf)
else:
    #for each matches df make a similarity matrix based on tanimoto
    similarity_matrices_cor = []
    topn = 20

    for ID in range(len(found_matches_s2v)):
        df = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
        library_ids = df.index.values
        rows = []
        for lib_id_i in library_ids:
            smiles_i = documents_library_s2v[lib_id_i]._obj.get("smiles")

            if not smiles_i or smiles_i == "None": # check that lib_id_i smiles exist
                rows.append([0] * len(library_ids)) #default to all 0 if it doesnt exist
                continue
            ms_i = Chem.MolFromSmiles(smiles_i)
            if not ms_i: #in case something is wrong with smiles
                rows.append([0] * len(library_ids)) #default to all 0 if it doesnt exist
                continue
            fp_i = Chem.RDKFingerprint(ms_i)

            row = []
            for lib_id_j in library_ids:
                if lib_id_i == lib_id_j: #if diagonal see if its a true match and give it a slightly higher value for plotting
                    df_select = found_matches_s2v_match_perc_pmass_sim[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
                    if df_select.loc[lib_id_j]['label'] == 1:
                        score = cor_val
                    else:
                        score = 1
                    
                else:
                    smiles_j = documents_library_s2v[lib_id_j]._obj.get("smiles")

                    if smiles_j and smiles_j != "None":
                        ms_j = Chem.MolFromSmiles(smiles_j)
                        if ms_j:
                            fp_j = Chem.RDKFingerprint(ms_j)
                            score = DataStructs.FingerprintSimilarity(fp_i, fp_j)
                        else: #in case something is wrong with smiles
                            score = 0
                    else: #in case it doesnt have smiles
                        score = 0
                row.append(score)

            rows.append(row)
        similarity_matrices_cor.append(pd.DataFrame(rows))
    #write to file
    with open(outfile, 'wb') as outf:
        pickle.dump(similarity_matrices_cor, outf)
df_rows = similarity_matrices_cor[9]
df_rows
from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap

x = 100000
inferno = cm.get_cmap('inferno', x)
green = np.array([50/256, 205/256, 50/256, 1])
new_colours = np.vstack((inferno(np.linspace(0, 1, x)), green))
new_cmap = ListedColormap(new_colours, name='inferno-green')
plt.imshow(df_rows, cmap=new_cmap, vmin=0, vmax=cor_val)
plt.colorbar()
plt.show()
fig, axs = plt.subplots(4, 5, figsize=(14,10))
for i, ax in enumerate(axs.flat):
    im = ax.imshow(similarity_matrices[i], cmap='inferno', vmin = 0, vmax = 1)

fig.subplots_adjust(right=0.95)
cbar_ax = fig.add_axes([0.97, 0.15, 0.02, 0.69])
fig.colorbar(im, cax=cbar_ax)

plt.show()
fig, axs = plt.subplots(4, 5, figsize=(14,10))
for i, ax in enumerate(axs.flat):
    im = ax.imshow(similarity_matrices_cor[i], cmap=new_cmap, vmax=cor_val, vmin=0)

fig.subplots_adjust(right=0.95)
cbar_ax = fig.add_axes([0.97, 0.15, 0.02, 0.69])
fig.colorbar(im, cax=cbar_ax)

plt.show()
plt.close()
#for each best true hit, how many connections > 0.6 does it have?
topn = 20
ID=0
cutoff = 0.6
amount_sims = []
for ID in range(len(found_matches_s2v_match_perc_pmass_sim)):
    df_select = found_matches_s2v_match_perc_pmass_sim[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    true_positives = [i for i, lab in enumerate(df_select['label']) if lab == 1]
    try:
        true_pos = true_positives[0] #take top hit
    except IndexError:
        amount_sim = -1 #true positive can't be found in topn
    else:
        amount_sim = len(similarity_matrices[ID][similarity_matrices[ID][true_pos] > cutoff])
    #report max cases
    #if amount_sim == 20:
        #print(ID)
    amount_sims.append(amount_sim)
plt.hist(amount_sims, bins = range(-1, max(amount_sims)+1))
plt.xlabel('Connections above 0.6')
plt.title('Amount of connections above 0.6 for each best true match')
plt.show()
#if top hit is not true match, what is the average similarity of top hit above a threshold?
topn = 20
threshold = 0.4
false_m_simils_thresh = []
ids_top_false_hits_thresh = []
for ID in range(len(found_matches_s2v_match_perc_pmass_sim)):
    df_select = found_matches_s2v_match_perc_pmass_sim[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    df_select = df_select[(df_select['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        top_hit = df_select.iloc[0]
        if top_hit['label'] == 0:
            false_m_simil = top_hit['similarity']
            print(ID,false_m_simil)
            ids_top_false_hits_thresh.append(ID)
            false_m_simils_thresh.append(false_m_simil)
print(len(false_m_simils_thresh))
plt.figure(figsize=(2,2))
plt.boxplot(false_m_simils_thresh)
plt.show()
plt.close()
pd.DataFrame(false_m_simils_thresh, columns = ['avg_similarity_false_top_hit']).describe()
#same plot but only the ones where the top hit is not the true hit
plt.style.use('default')
fig, axs = plt.subplots(4, 5, figsize=(13,9))
for i, ax in enumerate(axs.flat):
    im = ax.imshow(similarity_matrices_cor[ids_top_false_hits_thresh[i]], cmap=new_cmap, vmax=cor_val, vmin=0)

fig.subplots_adjust(right=0.95)
cbar_ax = fig.add_axes([0.97, 0.15, 0.02, 0.69])
fig.colorbar(im, cax=cbar_ax)

plt.show()
found_matches_s2v_match_perc_pmass_sim[8].sort_values(by=['s2v_score'], ascending = False).iloc[:40]
#visualise tanimoto score of query vs match per position
topn = 20
all_similarities = []
for ID in range(len(found_matches_s2v_match_perc_pmass_sim)):
    df_select = found_matches_s2v_match_perc_pmass_sim[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    simil = df_select['similarity']
    all_similarities.append(list(simil))
per_pos = list(zip(*all_similarities))
#print(per_pos)
means = list(map(np.mean, per_pos))
stdevs = list(map(np.std, per_pos))
plt.bar(range(len(means)), means, yerr = stdevs)
found_matches_s2v_info_connections = []
cutoff = 0.6
connections = [] #list of lists, just the connections column for each query
for ID in range(len(found_matches_s2v_match_perc_pmass_sim)):
    df_select = found_matches_s2v_match_perc_pmass_sim[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    sim_matrix_select = similarity_matrices[ID]
    amount_sims = []
    for inner_ID in range(df_select.shape[0]):
        amount_sim = len(sim_matrix_select[sim_matrix_select[inner_ID] > cutoff])
        amount_sims.append(amount_sim)
    new_df = df_select.copy()
    new_df['connections'] = amount_sims
    connections.append(amount_sims)
    found_matches_s2v_info_connections.append(new_df)
found_matches_s2v_info_connections[13]
#make new score which is s2v_score * factor^connections
factor = 1.04
cor_scores = []
found_matches_s2v_info_new_score = []
for ID in range(len(found_matches_s2v_info_connections)):
    cur_connections = connections[ID]
    cur_df = found_matches_s2v_info_connections[ID].copy()
    s2v_scores = cur_df['s2v_score']
    cor_score = []
    for con, s2v_sc in zip(cur_connections, s2v_scores):
        new_score = s2v_sc * (factor ** con)
        cor_score.append(new_score)
    cor_scores.append(cor_score)
    cur_df['new_score'] = cor_score
    found_matches_s2v_info_new_score.append(cur_df)
found_matches_s2v_info_new_score[3]
#testing new_score + mass
#look at top20
test_m_s2v_top20 = []
test_m_new_score_top20 = []
threshold = 0.4
topn = 20 #only look at top 20

for ID in range(len(documents_query_s2v)):
    inchikey_query = documents_query_s2v[ID]._obj.get("inchikey")[:14]
    #select top20
    current_match = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending=False).iloc[:topn]

    # Scenario 2: mass + Spec2Vec + threshold
    df_select = current_match[(current_match['mass_match'] == 1)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        best_bet = 1 * (inchikey_match == inchikey_query)
    else:
        best_bet = -1 # meaning: not found
    test_m_s2v_top20.append(best_bet)
    
    # Scenario 2: mass + new_score + threshold
    current_match_new = found_matches_s2v_info_new_score[ID].sort_values(by=['new_score'], ascending=False).iloc[:topn]
    df_select = current_match_new[(current_match_new['mass_match'] == 1)
                                  & (current_match_new['new_score'] > threshold)]
    if df_select.shape[0] > 0:
        best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        best_bet = 1 * (inchikey_match == inchikey_query)
    else:
        best_bet = -1 # meaning: not found
    test_m_new_score_top20.append(best_bet)


#make arrays from lists
test_m_s2v_arr_top20 = np.array(test_m_s2v_top20)
test_m_new_score_arr_top20 = np.array(test_m_new_score_top20)
plt.style.use('ggplot')

ylabels = ['mass + Spec2Vec',
           'mass + new_score',]
xlabels = ['true matches', 'false matches', 'no matches found']
          #'mass + mod.cosine \n + as backup: Spec2Vec + mod.cosine']
data = np.array([[np.sum(test_m_s2v_arr_top20 == 1), np.sum(test_m_s2v_arr_top20 == 0), np.sum(test_m_s2v_arr_top20 == -1)],
                 [np.sum(test_m_new_score_arr_top20 == 1), np.sum(test_m_new_score_arr_top20 == 0), np.sum(test_m_new_score_arr_top20 == -1)]])

dataframe = pd.DataFrame(data, columns=xlabels, index=ylabels)

ax = dataframe.sort_values(by=['false matches'], ascending=True).plot.barh(stacked=True, edgecolor='none')

horiz_offset = 1.03
vert_offset = 1.
ax.legend(bbox_to_anchor=(horiz_offset, vert_offset))
plt.xlabel('Number of spectra')
plt.title('Comparing new_score vs spec2vec looking at top 20\n(>5 matching inchikey in library)')
#plt.figure(figsize=(5,5))
plt.show()<jupyter_output><empty_output><jupyter_text>Prep data for NN<jupyter_code>#visualise mod_cosine_matches for all top20 results
mod_cosine_matches = [elem for ID in range(len(found_matches_s2v_info_new_score)) for elem in list(found_matches_s2v_info_new_score[ID]['mod_cosine_matches'])]
plt.hist(mod_cosine_matches, bins = max(mod_cosine_matches)+1)
plt.show()
#visualise cosine_matches for all top20 results
cosine_matches = [elem for ID in range(len(found_matches_s2v_info_new_score)) for elem in list(found_matches_s2v_info_new_score[ID]['cosine_matches'])]
plt.hist(cosine_matches, bins = max(cosine_matches)+1)
plt.show()
#testing scaling
x = 15
base = 0.8
dist = [(base**i) for i in range(x)]
plt.scatter(range(x), dist)
def transform_num_matches(input_df, exp = 0.93):
    '''Transform the cosine_matches and mod_cosine_matches to between 0-1
    
    input_df: pandas DataFrame, spec2vec matches for one query
    exp: int, the base for the exponential, default: 0.93
    
    Both matches are transformed to between 0-1 by doing 1-0.93^num_matches
    '''
    df = input_df.copy() #otherwise it edits the df outside the function
    df['cosine_matches'] = [(1-0.93**i) for i in df['cosine_matches']]
    df['mod_cosine_matches'] = [(1-0.93**i) for i in df['mod_cosine_matches']]
    return df
def find_mass_similarity(matches, documents_library, query_mass, base_num = 0.8):
    '''
    To each match in matches df, add a scaled value for how similar the parent_mass is to the query

    matches: pandas DataFrame, library matching result of 1 query on library
    documents_library: list of SpectrumDocuments, spectra in library
    query_mass: float, parent mass of query
    base_num: float, the base for the exponent
    df: pandas DataFrame, library matching result of 1 query on library with mass sims
    
    The similarity in dalton is calculated and transformed into a value 0 - 1 by doing
    1 - base_num^diff_in_dalton
    '''
    df = matches.copy()
    library_ids = df.index.values
    scaled_mass_sims = []
    for lib_id in library_ids:
        lib_mass = documents_library[lib_id]._obj.get("parent_mass")
        mass_diff = abs(lib_mass - query_mass)
        scaled_mass_sim = base_num ** mass_diff
        scaled_mass_sims.append(scaled_mass_sim)

    #add to df
    df['mass_sim'] = scaled_mass_sims
    return df
#get max parent mass to make it relative
all_parent_masses = [elem for ID in range(len(found_matches_s2v_match_perc_pmass_sim)) for elem in list(found_matches_s2v_match_perc_pmass_sim[ID]['parent_mass'])]
max_parent_mass = max(all_parent_masses)
max_parent_mass
#preprocess for neural network, remove mass_match and perc_mass_change, make number of matches relative
#add new transform of mass difference
nn_prep_found_matches_s2v_match_perc_pmass_sim = []
for ID in range(len(found_matches_s2v_match_perc_pmass_sim)):
    current = found_matches_s2v_match_perc_pmass_sim[ID].copy()
    current = transform_num_matches(current)
    current['parent_mass'] = [cur_pm/max_parent_mass for cur_pm in current['parent_mass']]
    current.drop(['mass_match', 'perc_mass_change'], axis=1, inplace=True)
    q_mass = documents_query_s2v[ID]._obj.get("parent_mass")
    current = find_mass_similarity(current, documents_library_s2v, q_mass, base_num = 0.8)
    nn_prep_found_matches_s2v_match_perc_pmass_sim.append(current)
#finalising data for nn, add label 1 if similarity is above cutoff
sim_cutoff = 0.6
nn_found_matches_s2v_match_perc_pmass_sim = nn_prep_found_matches_s2v_match_perc_pmass_sim[0]\
    .append(nn_prep_found_matches_s2v_match_perc_pmass_sim[1:])\
    .drop(['label'], axis = 1)
simlabs = [1 if sim >= sim_cutoff else 0 for sim in nn_found_matches_s2v_match_perc_pmass_sim['similarity']]
nn_found_matches_s2v_match_perc_pmass_sim['sim_label'] = simlabs
nn_found_matches_s2v_match_perc_pmass_sim.drop(['similarity'], axis = 1, inplace = True)
nn_found_matches_s2v_match_perc_pmass_sim.sort_values(by='s2v_score', ascending = False)
nn_prep_found_matches_s2v_match_perc_pmass_sim[0].sort_values(by=['s2v_score'], ascending=False).iloc[:20]
#loading second test set to test nn output
outfile = os.path.join(path_data, 'new_found_matches_s2v_match_perc_pmass_sim.pickle')
print(outfile)
if os.path.exists(outfile):
    with open(outfile, 'rb') as inf:
        new_found_matches_s2v_match_perc_pmass_sim = pickle.load(inf)

#preprocess for neural network, remove mass_match, make number of matches and parent_mass relative and let s2v_score start at 0
nn_prep_new_found_matches_s2v_match_perc_pmass_sim = []
for ID in range(len(new_found_matches_s2v_match_perc_pmass_sim)):
    current = new_found_matches_s2v_match_perc_pmass_sim[ID].copy()
    current = transform_num_matches(current)
    current['parent_mass'] = [cur_pm/max_parent_mass for cur_pm in current['parent_mass']]
    current.drop(['mass_match', 'perc_mass_change'], axis=1, inplace=True)
    q_mass = new_documents_query_s2v[ID]._obj.get("parent_mass")
    current = find_mass_similarity(current, new_documents_library_s2v, q_mass, base_num = 0.8)
    nn_prep_new_found_matches_s2v_match_perc_pmass_sim.append(current)

#finalising data for nn, add label 1 if similarity is above cutoff
sim_cutoff = 0.6
nn_new_found_matches_s2v_match_perc_pmass_sim = nn_prep_new_found_matches_s2v_match_perc_pmass_sim[0]\
    .append(nn_prep_new_found_matches_s2v_match_perc_pmass_sim[1:])\
    .drop(['label'], axis = 1)
simlabs = [1 if sim >= sim_cutoff else 0 for sim in nn_new_found_matches_s2v_match_perc_pmass_sim['similarity']]
nn_new_found_matches_s2v_match_perc_pmass_sim['sim_label'] = simlabs
nn_new_found_matches_s2v_match_perc_pmass_sim.drop(['similarity'], axis = 1, inplace = True)
nn_new_found_matches_s2v_match_perc_pmass_sim.sort_values(by='s2v_score', ascending = False)<jupyter_output>C:\Users\joris\Documents\eScience_data\data\new_found_matches_s2v_match_perc_pmass_sim.pickle<jupyter_text>NN for learning similarity<jupyter_code>#from tensorflow import keras
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense

model_name = os.path.join(path_data, 'first_deep_nn_similarity')

# split into input (X) and output (y) variables
X = nn_found_matches_s2v_match_perc_pmass_sim.drop('sim_label', axis = 1)
y = nn_found_matches_s2v_match_perc_pmass_sim['sim_label']
if os.path.exists(model_name):
    model = load_model(model_name)
else:
    # define the keras model
    model = Sequential()
    model.add(Dense(12, input_dim=X.shape[1], activation='relu'))
    model.add(Dense(12, activation='relu'))
    model.add(Dense(16, activation='relu'))
    model.add(Dense(12, activation='relu'))
    model.add(Dense(8, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))
    # compile the keras model
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit the keras model on the dataset
    model.fit(X, y, epochs=30, batch_size = 16)
    
    model.save(model_name)
X_test = nn_new_found_matches_s2v_match_perc_pmass_sim.drop(['sim_label'], axis = 1)
y_test = nn_new_found_matches_s2v_match_perc_pmass_sim['sim_label']
loss, accuracy = model.evaluate(X_test, y_test)
print('Accuracy: %.2f' % (accuracy*100))
# make class predictions with the model
predictions_test = model.predict_classes(X_test)
# # summarize the first 5 cases
# for i in range(5):
#     print('%s => %d (expected %d)' % (X_test.iloc[i].tolist(), predictions_test[i], y_test.iloc[i]))
nn_new_testset_tested = []
for ID in range(len(nn_prep_new_found_matches_s2v_match_perc_pmass_sim)):
    current_match = nn_prep_new_found_matches_s2v_match_perc_pmass_sim[ID].copy()
    simlabs = [1 if sim >= sim_cutoff else 0 for sim in current_match['similarity']]
    predict_on = current_match.drop(['label', 'similarity'], axis = 1)
    predictions = model.predict_classes(predict_on)
    current_match['predictions'] = predictions
    correct_guess = [pred == lab for pred, lab in zip(predictions, simlabs)]
    current_match['correct_guess'] = correct_guess
    nn_new_testset_tested.append(current_match)
nn_new_testset_tested[0].sort_values(by=['s2v_score'], ascending=False).iloc[:20]
#if top hit is not true match, what is the average similarity of top hit above a threshold?
topn = 20
threshold = 0.4
new_false_m_simils_thresh = []
new_ids_top_false_hits_thresh = []
for ID in range(len(nn_new_testset_tested)):
    df_select = nn_new_testset_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    df_select = df_select[(df_select['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        top_hit = df_select.iloc[0]
        if top_hit['label'] == 0:
            new_false_m_simil = top_hit['similarity']
            print(ID, new_false_m_simil)
            new_ids_top_false_hits_thresh.append(ID)
            new_false_m_simils_thresh.append(new_false_m_simil)
print(len(new_false_m_simils_thresh))
plt.figure(figsize=(2,2))
plt.boxplot(new_false_m_simils_thresh)

plt.show()
#testing if top hit is above 0.6 similarity after filtering with nn
#look at top20
test_sim_mcos_top20 = []
test_sim_s2v_top20 = []
test_sim_filt_nn = []
ideal_case_top20 = []

mass_sim_thresh = 0.95
threshold = 0.4
topn = 20 #only look at top 20
sim_threshold = 0.6

for ID in range(len(nn_new_testset_tested)):
    # mass + mod cosine
    current_match = nn_new_testset_tested[ID].sort_values(by=['mod_cosine_score'], ascending=False).iloc[:topn]

    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['mod_cosine_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['mod_cosine_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_mcos_top20.append(best_bet)

    # Scenario 1: mass + Spec2Vec + threshold
    current_match = nn_new_testset_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    #inchikey_query = documents_query_s2v[ID]._obj.get("inchikey")[:14]
    #select top20
    #current_match = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending=False).iloc[:topn]

    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_s2v_top20.append(best_bet)
    
    # Scenario 2: Spec2Vec + threshold after filtering for non-similar matches with nn
    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['predictions'] == 1)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn.append(best_bet)
    
    # ideal case: how many queries have an actual match above s2v threshold and >0.6 tanimoto
    df_select = current_match[(current_match['s2v_score'] > threshold)]
    best_bet = -1
    if df_select.shape[0] > 0:
        if any(df_select['similarity'] > 0.6):
            best_bet = 1
    ideal_case_top20.append(best_bet)


#make arrays from lists
test_sim_mcos_arr_top20 = np.array(test_sim_mcos_top20)
test_sim_s2v_arr_top20 = np.array(test_sim_s2v_top20)
test_sim_filt_nn_arr = np.array(test_sim_filt_nn)
ideal_case_arr_top20 = np.array(ideal_case_top20)
### plt.style.use('ggplot')

ylabels = ['mass + mod cosine',
           'mass + Spec2Vec',
           'mass + Spec2Vec + filtering NN',
           'ideal case Spec2Vec>0.4']
xlabels = ['true matches', 'false matches', 'no matches found']
          #'mass + mod.cosine \n + as backup: Spec2Vec + mod.cosine']
data = np.array([[np.sum(test_sim_mcos_arr_top20 == 1), np.sum(test_sim_mcos_arr_top20 == 0), np.sum(test_sim_mcos_arr_top20 == -1)],
                 [np.sum(test_sim_s2v_arr_top20 == 1), np.sum(test_sim_s2v_arr_top20 == 0), np.sum(test_sim_s2v_arr_top20 == -1)],
                 [np.sum(test_sim_filt_nn_arr == 1), np.sum(test_sim_filt_nn_arr == 0), np.sum(test_sim_filt_nn_arr == -1)],
                 [np.sum(ideal_case_arr_top20 == 1), np.sum(ideal_case_arr_top20 == 0), np.sum(ideal_case_arr_top20 == -1)]])

dataframe = pd.DataFrame(data, columns=xlabels, index=ylabels)

#fig, ax = plt.subplots(figsize=(5, 5))
ax = dataframe.sort_values(by=['false matches'], ascending=True).plot.barh(stacked=True, edgecolor='none', figsize=(5, 3))

horiz_offset = 1.03
vert_offset = 1.
ax.legend(bbox_to_anchor=(horiz_offset, vert_offset))
plt.xlabel('Number of spectra')
plt.title('Compare if top hit is above 0.6 tanimoto similarity\n(>5 matching inchikey in library)')
#plt.figure(figsize=(5,5))
plt.show()<jupyter_output><empty_output><jupyter_text>NN for learning true hits<jupyter_code>#testing if top hit is a true hit after filtering for similarity with nn
#look at top20
test_m_new_s2v_top20 = []
test_m_new_filt_nn_top20 = []

mass_sim_thresh = 1
threshold = 0.4
topn = 20 #only look at top 20
#sim_threshold = 0.6

for ID in range(len(nn_new_testset_tested)):
    current_match = nn_new_testset_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    inchikey_query = new_documents_query_s2v[ID]._obj.get("inchikey")[:14]
    #select top20
    #current_match = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending=False).iloc[:topn]

    # Scenario 1: mass + Spec2Vec + threshold
    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        inchikey_match = new_documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        #sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (inchikey_query == inchikey_match)
    else:
        best_bet = -1 # meaning: not found
    test_m_new_s2v_top20.append(best_bet)
    
    # Scenario 2: Spec2Vec + threshold after filtering for non-similar matches with nn
    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['predictions'] == 1)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        #sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (inchikey_query == inchikey_match)
    else:
        best_bet = -1 # meaning: not found
    test_m_new_filt_nn_top20.append(best_bet)


#make arrays from lists
test_m_new_s2v_arr_top20 = np.array(test_m_new_s2v_top20)
test_m_new_filt_nn_arr_top20 = np.array(test_m_new_filt_nn_top20)
### plt.style.use('ggplot')

ylabels = ['mass + Spec2Vec',
           'mass + Spec2Vec + filtering NN']
xlabels = ['true matches', 'false matches', 'no matches found']
          #'mass + mod.cosine \n + as backup: Spec2Vec + mod.cosine']
data = np.array([[np.sum(test_m_new_s2v_arr_top20 == 1), np.sum(test_m_new_s2v_arr_top20 == 0), np.sum(test_m_new_s2v_arr_top20 == -1)],
                 [np.sum(test_m_new_filt_nn_arr_top20 == 1), np.sum(test_m_new_filt_nn_arr_top20 == 0), np.sum(test_m_new_filt_nn_arr_top20 == -1)]])

dataframe = pd.DataFrame(data, columns=xlabels, index=ylabels)

#fig, ax = plt.subplots(figsize=(5, 5))
ax = dataframe.sort_values(by=['false matches'], ascending=True).plot.barh(stacked=True, edgecolor='none', figsize=(5, 3))

horiz_offset = 1.03
vert_offset = 1.
ax.legend(bbox_to_anchor=(horiz_offset, vert_offset))
plt.xlabel('Number of spectra')
plt.title('Selecting top hit\n(>5 matching inchikey in library)')
#plt.figure(figsize=(5,5))
plt.show()<jupyter_output><empty_output><jupyter_text>NN with equal true/pos labels by oversampling or taking top 20<jupyter_code>#oversample true hits to get equal ratio
def oversample(matches, label_column):
    '''Oversamples the 1 label so that 1 and 0 label are equal in the df

    matches: pandas DataFrame, concatenation of all queries matching to a library
    label_column: str, the column that contains the label
    '''
    all_true_p = matches.copy()[matches[label_column]==1]
    len_all_true_p = all_true_p.shape[0]
    all_false_p = matches.copy()[matches[label_column]==0]
    len_all_false_p = all_false_p.shape[0]
    ratio = len_all_false_p/len_all_true_p
    decimals = ratio % 1
    np.random.seed(42)
    chosen_indices = np.random.choice(range(len_all_true_p), round(decimals*len_all_true_p))
    balanced_true_p = all_true_p.iloc[chosen_indices]
    for i in range(1,int(ratio)+1):
        balanced_true_p = balanced_true_p.append(all_true_p)
    matches_oversampled = balanced_true_p.append(all_false_p)
    np.random.seed(42)
    matches_oversampled = matches_oversampled.sample(frac=1)
    print('New shape:', matches_oversampled.shape)
    return matches_oversampled
nn_found_matches_s2v_match_perc_pmass_sim_oversampled = oversample(nn_found_matches_s2v_match_perc_pmass_sim, 'sim_label')
#check:
sim_labels_oversampled = list(nn_found_matches_s2v_match_perc_pmass_sim_oversampled['sim_label'])
sim_labels_oversampled.count(1), sim_labels_oversampled.count(0)
model_name_oversampled = os.path.join(path_data, 'first_deep_nn_similarity_oversampled')

# split into input (X) and output (y) variables
X = nn_found_matches_s2v_match_perc_pmass_sim_oversampled.drop('sim_label', axis = 1)
y = nn_found_matches_s2v_match_perc_pmass_sim_oversampled['sim_label']
if os.path.exists(model_name_oversampled):
    model_oversampled = load_model(model_name_oversampled)
else:
    # define the keras model
    model_oversampled = Sequential()
    model_oversampled.add(Dense(12, input_dim=X.shape[1], activation='relu'))
    model_oversampled.add(Dense(12, activation='relu'))
    model_oversampled.add(Dense(16, activation='relu'))
    model_oversampled.add(Dense(12, activation='relu'))
    model_oversampled.add(Dense(8, activation='relu'))
    model_oversampled.add(Dense(1, activation='sigmoid'))
    # compile the keras model
    model_oversampled.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit the keras model on the dataset
    model_oversampled.fit(X, y, epochs=30, batch_size = 16)
    
    model_oversampled.save(model_name_oversampled)
X_test = nn_new_found_matches_s2v_match_perc_pmass_sim.drop(['sim_label'], axis = 1)
y_test = nn_new_found_matches_s2v_match_perc_pmass_sim['sim_label']
loss, accuracy = model_oversampled.evaluate(X_test, y_test)
print('Accuracy: %.2f' % (accuracy*100))
predictions_test_oversampled = model_oversampled.predict_classes(X_test)
nn_new_testset_oversampled_tested = []
for ID in range(len(nn_prep_new_found_matches_s2v_match_perc_pmass_sim)):
    current_match = nn_prep_new_found_matches_s2v_match_perc_pmass_sim[ID].copy()
    simlabs = [1 if sim >= sim_cutoff else 0 for sim in current_match['similarity']]
    predict_on = current_match.drop(['label', 'similarity'], axis = 1)
    predictions = model_oversampled.predict_classes(predict_on)
    current_match['predictions'] = predictions
    correct_guess = [pred == lab for pred, lab in zip(predictions, simlabs)]
    current_match['correct_guess'] = correct_guess
    nn_new_testset_oversampled_tested.append(current_match)
nn_new_testset_oversampled_tested[0].sort_values(by=['s2v_score'], ascending=False).iloc[:20]
#testing if top hit is above 0.6 similarity after filtering with nn on oversampled data
#look at top20
test_sim_filt_nn_oversampled_top20 = []

threshold = 0.4
topn = 20 #only look at top 20
sim_threshold = 0.6

for ID in range(len(nn_new_testset_oversampled_tested)):
    current_match = nn_new_testset_oversampled_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    # Scenario 2: Spec2Vec + threshold after filtering for non-similar matches with nn
    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['predictions'] == 1)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn_oversampled_top20.append(best_bet)
    
#     # ideal case: how many queries have an actual match above s2v threshold and >0.6 tanimoto
#     df_select = current_match[(current_match['s2v_score'] > threshold)]
#     best_bet = -1
#     if df_select.shape[0] > 0:
#         if any(df_select['similarity'] > 0.6):
#             best_bet = 1
#     ideal_case_top20.append(best_bet)


#make arrays from lists
test_sim_filt_nn_oversampled_arr = np.array(test_sim_filt_nn_oversampled_top20)
### plt.style.use('ggplot')

ylabels = ['mass + mod cosine',
           'mass + Spec2Vec',
           'mass + Spec2Vec + filtering NN',
           'mass + Spec2Vec + filtering NN oversampled',
           'ideal case Spec2Vec>0.4']
xlabels = ['true matches', 'false matches', 'no matches found']
          #'mass + mod.cosine \n + as backup: Spec2Vec + mod.cosine']
data = np.array([[np.sum(test_sim_mcos_arr_top20 == 1), np.sum(test_sim_mcos_arr_top20 == 0), np.sum(test_sim_mcos_arr_top20 == -1)],
                 [np.sum(test_sim_s2v_arr_top20 == 1), np.sum(test_sim_s2v_arr_top20 == 0), np.sum(test_sim_s2v_arr_top20 == -1)],
                 [np.sum(test_sim_filt_nn_arr == 1), np.sum(test_sim_filt_nn_arr == 0), np.sum(test_sim_filt_nn_arr == -1)],
                 [np.sum(test_sim_filt_nn_oversampled_arr == 1), np.sum(test_sim_filt_nn_oversampled_arr == 0), np.sum(test_sim_filt_nn_oversampled_arr == -1)],
                 [np.sum(ideal_case_arr_top20 == 1), np.sum(ideal_case_arr_top20 == 0), np.sum(ideal_case_arr_top20 == -1)]])

dataframe = pd.DataFrame(data, columns=xlabels, index=ylabels)

#fig, ax = plt.subplots(figsize=(5, 5))
ax = dataframe.sort_values(by=['false matches'], ascending=True).plot.barh(stacked=True, edgecolor='none', figsize=(5, 3))

horiz_offset = 1.03
vert_offset = 1.
ax.legend(bbox_to_anchor=(horiz_offset, vert_offset))
plt.xlabel('Number of spectra')
plt.title('Compare if top hit is above 0.6 tanimoto similarity\n(>5 matching inchikey in library)')
#plt.figure(figsize=(5,5))
plt.show()
#top20

def make_s2v_selection(matches, topn):
    '''Return one df with topn matches (based on s2v_score) for each query
    
    matches: list of pandas DataFrames, library matching result with matching labels
    topn: int, number of top hits to select, sorted on s2v_score
    all_selected_matches: pandas DataFrames, concatonated selection of matches for each query
    '''
    selected_matches = []
    for df in matches:
        selected_matches.append(df.sort_values(by=['s2v_score'], ascending=False).iloc[:topn])
    all_selected_matches = selected_matches[0].append(selected_matches[1:])
    return all_selected_matches

nn_found_matches_s2v_match_perc_pmass_sim_top20 = make_s2v_selection(nn_prep_found_matches_s2v_match_perc_pmass_sim, 20)
#add simlabel
simlabs = [1 if sim >= sim_cutoff else 0 for sim in nn_found_matches_s2v_match_perc_pmass_sim_top20['similarity']]
nn_found_matches_s2v_match_perc_pmass_sim_top20['sim_label'] = simlabs
nn_found_matches_s2v_match_perc_pmass_sim_top20.drop(['similarity', 'label'], axis = 1, inplace = True)
#check label dist
simlabs.count(0), simlabs.count(1)
model_name_top20 = os.path.join(path_data, 'first_deep_nn_similarity_top20')

# split into input (X) and output (y) variables
X = nn_found_matches_s2v_match_perc_pmass_sim_top20.drop('sim_label', axis = 1)
y = nn_found_matches_s2v_match_perc_pmass_sim_top20['sim_label']
if os.path.exists(model_name_top20):
    model_top20 = load_model(model_name_top20)
else:
    # define the keras model
    model_top20 = Sequential()
    model_top20.add(Dense(12, input_dim=X.shape[1], activation='relu'))
    model_top20.add(Dense(12, activation='relu'))
    model_top20.add(Dense(16, activation='relu'))
    model_top20.add(Dense(12, activation='relu'))
    model_top20.add(Dense(8, activation='relu'))
    model_top20.add(Dense(1, activation='sigmoid'))
    # compile the keras model
    model_top20.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit the keras model on the dataset
    model_top20.fit(X, y, epochs=30, batch_size = 16) #add validation set?
    
    model_top20.save(model_name_top20)
#make test set top20 to compare performance
nn_new_found_matches_s2v_match_perc_pmass_sim_top20 = make_s2v_selection(nn_prep_new_found_matches_s2v_match_perc_pmass_sim, 20)
#add simlabel
simlabs = [1 if sim >= sim_cutoff else 0 for sim in nn_new_found_matches_s2v_match_perc_pmass_sim_top20['similarity']]
nn_new_found_matches_s2v_match_perc_pmass_sim_top20['sim_label'] = simlabs
nn_new_found_matches_s2v_match_perc_pmass_sim_top20.drop(['similarity', 'label'], axis = 1, inplace = True)
#check label dist
simlabs.count(0), simlabs.count(1)
#with complete test set
X_test = nn_new_found_matches_s2v_match_perc_pmass_sim.drop(['sim_label'], axis = 1)
y_test = nn_new_found_matches_s2v_match_perc_pmass_sim['sim_label']
loss, accuracy = model_top20.evaluate(X_test, y_test)
print('Accuracy: %.2f' % (accuracy*100))
#accuracy of models with top20 testset

X_test = nn_new_found_matches_s2v_match_perc_pmass_sim_top20.drop(['sim_label'], axis = 1)
y_test = nn_new_found_matches_s2v_match_perc_pmass_sim_top20['sim_label']

loss, accuracy = model.evaluate(X_test, y_test)
print('Accuracy first model: %.2f' % (accuracy*100))
loss, accuracy = model_oversampled.evaluate(X_test, y_test)
print('Accuracy oversampled model: %.2f' % (accuracy*100))
loss, accuracy = model_top20.evaluate(X_test, y_test)
print('Accuracy top20 model: %.2f' % (accuracy*100))
nn_new_testset_top20_tested = []
for ID in range(len(nn_prep_new_found_matches_s2v_match_perc_pmass_sim)):
    current_match = nn_prep_new_found_matches_s2v_match_perc_pmass_sim[ID].copy()
    simlabs = [1 if sim >= sim_cutoff else 0 for sim in current_match['similarity']]
    predict_on = current_match.drop(['label', 'similarity'], axis = 1)
    predictions = model_top20.predict_classes(predict_on)
    current_match['predictions'] = predictions
    correct_guess = [pred == lab for pred, lab in zip(predictions, simlabs)]
    current_match['correct_guess'] = correct_guess
    nn_new_testset_top20_tested.append(current_match)
# investigating bad matches
false_matches_oversampled = [i for i, score in enumerate(test_sim_filt_nn_oversampled_top20) if score == 0]
x = 0
print(false_matches_oversampled[x])
nn_new_testset_oversampled_tested[false_matches_oversampled[x]].sort_values(by=['s2v_score'], ascending=False).iloc[:30]<jupyter_output>11<jupyter_text>Trying to vary number of layers for top20 dataset<jupyter_code>#nn function
def train_nn(X_train, y_train, X_test, y_test, layers = [12, 12, 12, 12, 12, 1],
             model_loss = 'binary_crossentropy', activations = 'relu',
             last_activation = 'sigmoid', model_epochs = 20, model_batch_size = 16,
             save_name = False):
    '''Train a keras deep NN and test on test data, returns (model, accuracy)
    
    X_train: matrix like object like pd.DataFrame, training set
    y_train: list like object like np.array, training labels
    X_test: matrix like object like pd.DataFrame, test set
    y_test: list like object like np.array, test labels
    layers: list of ints, the number of layers is the len of this list while the elements
        are the amount of neurons per layer, default: [12, 12, 12, 12, 12, 1]
    model_loss: str, loss function, default: binary_crossentropy
    activations: str, the activation of the layers except the last one, default: relu
    last_activation: str, activation of last layer, default: sigmoid
    model_epochs: int, number of epochs, default: 20
    model_batch_size: int, batch size for updating the model, default: 16
    save_name: str, location for saving model, optional, default: False
    
    If save_name is not False and save_name exists this function will load existing model
    '''
    if os.path.exists(save_name) and save_name:
        print('Loading existing model')
        nn_model = load_model(save_name)
    else:
        # define the keras model
        nn_model = Sequential()
        #add first layer
        nn_model.add(Dense(layers[0], input_dim = X_train.shape[1], activation = activations))
        #add other layers
        for i in range(1,len(layers)-1): #skip first and last one
            nn_model.add(Dense(layers[i], activation = activations))
        #add last layer
        nn_model.add(Dense(layers[-1], activation = last_activation))
        # compile the keras model
        nn_model.compile(loss = model_loss, optimizer='adam', metrics=['accuracy'])
        # fit the keras model on the dataset
        nn_model.fit(X_train, y_train, epochs = model_epochs, batch_size = model_batch_size)
        
    #test set    
    loss, accuracy = nn_model.evaluate(X_test, y_test)
    if save_name:
        print(save_name)
    print('Accuracy: %.2f' % (accuracy*100))
    
    if save_name and not os.path.exists(save_name):
#         #adding accuracy to the model name
#         save_model = '{}_{:.2f}'.format(save_model, accuracy*100)
        nn_model.save(save_name)
    
    return (nn_model, accuracy)
layers_to_test = [[12]*10+[1],[16]*10+[1], [20]*20+[1], [20]*40+[1], [20]*30+[1], [12]*5+[24]*10+[12]*5+[1], [30]*20+[1]]
name = 'nn_top20_more_layers_100epochs'
epcs = 100 #epochs
models_acc = []
X_top20 = nn_found_matches_s2v_match_perc_pmass_sim_top20.drop('sim_label', axis = 1)
y_top20 = nn_found_matches_s2v_match_perc_pmass_sim_top20['sim_label']
X_test_top20 = nn_new_found_matches_s2v_match_perc_pmass_sim_top20.drop(['sim_label'], axis = 1)
y_test_top20 = nn_new_found_matches_s2v_match_perc_pmass_sim_top20['sim_label']
for i in range(len(layers_to_test)):
    new_name = name + '_' + str(i)
    save_new_name = os.path.join(path_data, new_name)
    print(save_new_name)
    model_acc = train_nn(X_top20, y_top20, X_test_top20, y_test_top20, layers = layers_to_test[i], model_epochs = epcs, save_name = save_new_name)
    models_acc.append(model_acc)
for i in range(len(models_acc)):
    print(i, models_acc[i][1], layers_to_test[i])
#test highest model, number 4
nn_new_testset_top20_tested_more_layers_4 = []
for ID in range(len(nn_prep_new_found_matches_s2v_match_perc_pmass_sim)):
    current_match = nn_prep_new_found_matches_s2v_match_perc_pmass_sim[ID].copy()
    simlabs = [1 if sim >= sim_cutoff else 0 for sim in current_match['similarity']]
    predict_on = current_match.drop(['label', 'similarity'], axis = 1)
    predictions = models_acc[4][0].predict_classes(predict_on)
    current_match['predictions'] = predictions
    correct_guess = [pred == lab for pred, lab in zip(predictions, simlabs)]
    current_match['correct_guess'] = correct_guess
    nn_new_testset_top20_tested_more_layers_4.append(current_match)<jupyter_output><empty_output><jupyter_text>Try vary layers for oversampled model<jupyter_code>layers_to_test = [[12]*10+[1],[16]*10+[1], [20]*20+[1], [20]*40+[1], [20]*30+[1], [12]*5+[24]*10+[12]*5+[1], [30]*20+[1]]
name = 'nn_oversampled_more_layers_30epochs'
epcs = 30 #epochs
models_acc_overs = []
X_overs = nn_found_matches_s2v_match_perc_pmass_sim_oversampled.drop('sim_label', axis = 1)
y_overs = nn_found_matches_s2v_match_perc_pmass_sim_oversampled['sim_label']
X_test_top20 = nn_new_found_matches_s2v_match_perc_pmass_sim_top20.drop(['sim_label'], axis = 1)
y_test_top20 = nn_new_found_matches_s2v_match_perc_pmass_sim_top20['sim_label']
for i in range(len(layers_to_test)):
    new_name = name + '_' + str(i)
    save_new_name = os.path.join(path_data, new_name)
    print(save_new_name)
    model_acc = train_nn(X_overs, y_overs, X_test_top20, y_test_top20, layers = layers_to_test[i], model_epochs = epcs, save_name = save_new_name)
    models_acc_overs.append(model_acc)
for i in range(len(models_acc_overs)):
    print(i, models_acc_overs[i][1], layers_to_test[i])<jupyter_output>0 0.59785 [12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 1]
1 0.62875 [16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 1]
2 0.6317 [20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 1]<jupyter_text>Plot results<jupyter_code>#testing if top hit is above 0.6 similarity after filtering with nn on oversampled data
#look at top20
test_sim_filt_nn_oversampled_top20 = []
test_sim_filt_nn_top20_top20 = []
test_sim_mcos_top20 = []
test_sim_s2v_top20 = []
test_sim_filt_nn = []
ideal_case_top20 = []
test_sim_filt_nn_top20_more_layers_4_top20 = []

mass_sim_thresh = 0.95
restr_nn = False # do mass+spec2vec restricitons on nn models
threshold = 0.4
topn = 20 #only look at top 20
sim_threshold = 0.6

for ID in range(len(nn_new_testset_top20_tested)):
    # mass + mod cosine
    current_match = nn_new_testset_tested[ID].sort_values(by=['mod_cosine_score'], ascending=False).iloc[:topn]

    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['mod_cosine_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['mod_cosine_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_mcos_top20.append(best_bet)

    # Scenario 1: mass + Spec2Vec + threshold
    current_match = nn_new_testset_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    #inchikey_query = documents_query_s2v[ID]._obj.get("inchikey")[:14]
    #select top20
    #current_match = found_matches_s2v[ID].sort_values(by=['s2v_score'], ascending=False).iloc[:topn]

    df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                  & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_s2v_top20.append(best_bet)
    
    # Scenario 3: (Spec2Vec + threshold) after filtering for non-similar matches with nn
    if not restr_nn:
        df_select = current_match[(current_match['predictions'] == 1)]
    else:
        df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                      & (current_match['predictions'] == 1)
                                      & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn.append(best_bet)
    
    # ideal case: how many queries have an actual match above s2v threshold and >0.6 tanimoto
    df_select = current_match[(current_match['s2v_score'] > threshold)]
    best_bet = -1
    if df_select.shape[0] > 0:
        if any(df_select['similarity'] > 0.6):
            best_bet = 1
    ideal_case_top20.append(best_bet)
    
    current_match = nn_new_testset_top20_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    # Scenario 4: (Spec2Vec + threshold) after filtering for non-similar matches with nn trained on top20
    if not restr_nn:
        df_select = current_match[(current_match['predictions'] == 1)]
    else:
        df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                      & (current_match['predictions'] == 1)
                                      & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn_top20_top20.append(best_bet)
    
    current_match = nn_new_testset_oversampled_tested[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    
    # Scenario 5: (Spec2Vec + threshold) after filtering for non-similar matches with nn trained on oversampled
    if not restr_nn:
        df_select = current_match[(current_match['predictions'] == 1)]
    else:
        df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                      & (current_match['predictions'] == 1)
                                      & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn_oversampled_top20.append(best_bet)

    # Scenario 6: (Spec2Vec + threshold) after filtering for non-similar matches with nn trained on top20 #4
    current_match = nn_new_testset_top20_tested_more_layers_4[ID].sort_values(by=['s2v_score'], ascending = False).iloc[:topn]
    if not restr_nn:
        df_select = current_match[(current_match['predictions'] == 1)]
    else:
        df_select = current_match[(current_match['mass_sim'] >= mass_sim_thresh)
                                      & (current_match['predictions'] == 1)
                                      & (current_match['s2v_score'] > threshold)]
    if df_select.shape[0] > 0:
        #best_match_ID = df_select.sort_values(by=['s2v_score'], ascending=False).index[0]
        #inchikey_match = documents_library_s2v[best_match_ID]._obj.get("inchikey")[:14]
        sim = df_select.iloc[0]['similarity'] #look at similarity of top hit
        best_bet = 1 * (sim >= sim_threshold)
    else:
        best_bet = -1 # meaning: not found
    test_sim_filt_nn_top20_more_layers_4_top20.append(best_bet)
    nn_new_testset_top20_tested_more_layers_4

#make arrays from lists
test_sim_filt_nn_oversampled_arr = np.array(test_sim_filt_nn_oversampled_top20)
test_sim_filt_nn_top20_arr = np.array(test_sim_filt_nn_top20_top20)
test_sim_mcos_arr_top20 = np.array(test_sim_mcos_top20)
test_sim_s2v_arr_top20 = np.array(test_sim_s2v_top20)
test_sim_filt_nn_arr = np.array(test_sim_filt_nn)
ideal_case_arr_top20 = np.array(ideal_case_top20)
test_sim_filt_nn_top20_more_layers_4_top20_arr = np.array(test_sim_filt_nn_top20_more_layers_4_top20)
### plt.style.use('ggplot')
if restr_nn:
    ylabels = ['mass + mod cosine',
               'mass + Spec2Vec',
               'mass + Spec2Vec + NN',
               'mass + Spec2Vec + NN oversampled',
               'mass + Spec2Vec + NN top20',
               'ideal case Spec2Vec>0.4',
               'mass + Spec2Vec + NN top20 #4']
else:
    ylabels = ['mass + mod cosine',
               'mass + Spec2Vec',
               'NN',
               'NN oversampled',
               'NN top20',
               'ideal case Spec2Vec>0.4',
               'NN top20 #4']
xlabels = ['true matches', 'false matches', 'no matches found']
          #'mass + mod.cosine \n + as backup: Spec2Vec + mod.cosine']
data = np.array([[np.sum(test_sim_mcos_arr_top20 == 1), np.sum(test_sim_mcos_arr_top20 == 0), np.sum(test_sim_mcos_arr_top20 == -1)],
                 [np.sum(test_sim_s2v_arr_top20 == 1), np.sum(test_sim_s2v_arr_top20 == 0), np.sum(test_sim_s2v_arr_top20 == -1)],
                 [np.sum(test_sim_filt_nn_arr == 1), np.sum(test_sim_filt_nn_arr == 0), np.sum(test_sim_filt_nn_arr == -1)],
                 [np.sum(test_sim_filt_nn_oversampled_arr == 1), np.sum(test_sim_filt_nn_oversampled_arr == 0), np.sum(test_sim_filt_nn_oversampled_arr == -1)],
                 [np.sum(test_sim_filt_nn_top20_arr == 1), np.sum(test_sim_filt_nn_top20_arr == 0), np.sum(test_sim_filt_nn_top20_arr == -1)],
                 [np.sum(ideal_case_arr_top20 == 1), np.sum(ideal_case_arr_top20 == 0), np.sum(ideal_case_arr_top20 == -1)],
                 [np.sum(test_sim_filt_nn_top20_more_layers_4_top20_arr == 1), np.sum(test_sim_filt_nn_top20_more_layers_4_top20_arr == 0),
                  np.sum(test_sim_filt_nn_top20_more_layers_4_top20_arr == -1)]])

dataframe = pd.DataFrame(data, columns=xlabels, index=ylabels)

#fig, ax = plt.subplots(figsize=(5, 5))
ax = dataframe.sort_values(by=['false matches'], ascending=True).plot.barh(stacked=True, edgecolor='none', figsize=(5, 3))

horiz_offset = 1.03
vert_offset = 1.
ax.legend(bbox_to_anchor=(horiz_offset, vert_offset))
plt.xlabel('Number of spectra')
plt.title('Compare if top hit is above 0.6 tanimoto similarity\n(>5 matching inchikey in library)')
#plt.figure(figsize=(5,5))
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11482673867137466, 'fraction_numerical': 0.028559559663516462, 'mean_word_length': 4.229926676925862, 'pattern_counts': {'":': 4, '<': 37, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '126877', 'license': 'Apache-2.0', 'n_tokens_mistral': 22200, 'n_tokens_neox': 20881, 'n_words': 4002, 'path': '.ipynb_checkpoints/2-library-matching-improvement-checkpoint.ipynb', 'repo_name': 'louwenjjr/improve_library_matching', 'size': '57774'}