<jupyter_start><jupyter_text>Import pre-processed dataset "AllPositive" and post process<jupyter_code>#loading data as pickled object goes a lot quicker
import time
import pickle
outfile = os.path.join(path_data, 'gnps_positive_ionmode_cleaned_by_matchms_and_lookups.pickle')
print(outfile)
start = time.time()
with open(outfile, 'rb') as inf:
        spectrums = pickle.load(inf)
end = time.time()
print('loading took {:.2f} s'.format(end-start))
print("number of spectra:", len(spectrums))
from matchms.filtering import normalize_intensities
from matchms.filtering import require_minimum_number_of_peaks
from matchms.filtering import select_by_mz
from matchms.filtering import select_by_relative_intensity
from matchms.filtering import reduce_to_number_of_peaks
from matchms.filtering import add_losses

def post_process_s2v(s):
    s = normalize_intensities(s)
    s = select_by_mz(s, mz_from=0, mz_to=1000)
    s = require_minimum_number_of_peaks(s, n_required=10)
    s = reduce_to_number_of_peaks(s, n_required=10, ratio_desired=0.5)
    if s is None:
        return None
    s_remove_low_peaks = select_by_relative_intensity(s, intensity_from=0.001)
    if len(s_remove_low_peaks.peaks) >= 10:
        s = s_remove_low_peaks
        
    s = add_losses(s, loss_mz_from=5.0, loss_mz_to=200.0)
    return s

# apply post processing steps to the data
spectrums_s2v = [post_process_s2v(s) for s in spectrums]

# omit spectrums that didn't qualify for analysis
spectrums_s2v = [s for s in spectrums_s2v if s is not None]

print("{} remaining spectra.".format(len(spectrums_s2v)))<jupyter_output>95320 remaining spectra.<jupyter_text>Inspect annotations<jupyter_code>Inchikeys = []
for spec in spectrums_s2v:
    Inchikeys.append(spec.get("inchikey"))
inchikeys_pd = pd.Series([x for x in Inchikeys if x])
inchikeys_pd.str[:14].value_counts()[:30]
suitable_inchikeys = pd.DataFrame(inchikeys_pd.str[:14].value_counts()[inchikeys_pd.str[:14].value_counts().values >= 5])
suitable_inchikeys.reset_index(level=suitable_inchikeys.index.names, inplace=True)
suitable_inchikeys.columns = (['inchikey14', 'occurences'])

# Important: sort values to make it reproducible (same occurences have random order otherwise!)
suitable_inchikeys = suitable_inchikeys.sort_values(['occurences', 'inchikey14'], ascending=False)
suitable_inchikeys.head(15)
print("Number of spectra with inchikeys that exist >= 5 times:", suitable_inchikeys.shape[0])<jupyter_output>Number of spectra with inchikeys that exist >= 5 times: 4079<jupyter_text>Randomly select 1000 inchikeys that exist >=5  times in the dataset<jupyter_code>num_spectra = 1000

np.random.seed(42) # to make it reproducible
selection = np.random.choice(suitable_inchikeys.shape[0], num_spectra, replace=False)
print(selection[:25])
selected_inchikeys = suitable_inchikeys['inchikey14'].values[selection]
selected_inchikeys[:25]
#Randomly select another 1000 inchikeys that exist >=5 times in the dataset and don't occur in the first list selected_inchikeys
new_suitable_inchikeys = suitable_inchikeys.copy()
drop_selection = suitable_inchikeys.index[selection] #get index names of first selection
new_suitable_inchikeys.drop(drop_selection, inplace = True) #remove first selection
print(new_suitable_inchikeys.shape[0])
print('Number of old selected inchikeys in new list:',len(set(new_suitable_inchikeys['inchikey14']) & set(selected_inchikeys)))

np.random.seed(42) # to make it reproducible
new_selection = np.random.choice(new_suitable_inchikeys.shape[0], num_spectra, replace=False)
print(new_selection[:25], len(new_selection))
new_selected_inchikeys = new_suitable_inchikeys['inchikey14'].values[new_selection]
print('Overlap between new and old list:', len(set(new_selected_inchikeys) & set(selected_inchikeys)))<jupyter_output>3079
Number of old selected inchikeys in new list: 0
[  63 2775 2016  331 2739  594 2244  346 2312 1472 2785  194 2370 2032
 2538  246  256  486  978 1961  144  801  744  457  416] 1000
Overlap between new and old list: 0<jupyter_text>Randomly pick one spectra for each of the chosen inchikeys<jupyter_code>selected_spectra = []
inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!

np.random.seed(42) # to make it reproducible
for inchikey in selected_inchikeys:
    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values
    selected_spectra.append(int(np.random.choice(matches,1)[0]))
#Pick one spectrum for each of the new chosen inchikeys
new_selected_spectra = []

np.random.seed(42) # to make it reproducible
for inchikey in new_selected_inchikeys:
    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values
    new_selected_spectra.append(int(np.random.choice(matches,1)[0]))<jupyter_output><empty_output><jupyter_text>Randomly pick 1000 inchikeys (spectra) that are unqiue in the dataset<jupyter_code>unique_inchikeys = pd.DataFrame(inchikeys_pd.str[:14].value_counts()[inchikeys_pd.str[:14].value_counts().values == 1])

unique_inchikeys.reset_index(level=unique_inchikeys.index.names, inplace=True)
unique_inchikeys.columns = (['inchikey14', 'occurences'])

# Important: sort values to make it reproducible (same occurences have random order otherwise!)
unique_inchikeys = unique_inchikeys.sort_values(['occurences', 'inchikey14'], ascending=False)
unique_inchikeys.head(15)
print(len(unique_inchikeys))
num_spectra = 1000

np.random.seed(42) # to make it reproducible
unique_selection = np.random.choice(unique_inchikeys.shape[0], num_spectra, replace=False)
print(unique_selection[:25])
selected_unique_inchikeys = unique_inchikeys['inchikey14'].values[unique_selection]
print(selected_unique_inchikeys[:25])

#get the spectrum that belongs with the unique chosen inchikeys
selected_unique_spectra = []
inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!

np.random.seed(42) # to make it reproducible
for inchikey in selected_unique_inchikeys:
    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values
    selected_unique_spectra.append(int(matches[0]))
print(len(selected_unique_spectra))
#Randomly select another 1000 inchikeys that are unique in the dataset and don't occur in the first list selected_unique_inchikeys
new_unique_inchikeys = unique_inchikeys.copy()
drop_selection = unique_inchikeys.index[unique_selection] #get index names of first selection
new_unique_inchikeys.drop(drop_selection, inplace = True) #remove first selection
print(new_unique_inchikeys.shape[0])
print('Number of old selected inchikeys in new list:',len(set(new_unique_inchikeys['inchikey14']) & set(selected_unique_inchikeys)))

np.random.seed(42) # to make it reproducible
new_unique_selection = np.random.choice(new_unique_inchikeys.shape[0], num_spectra, replace=False)
print(new_unique_selection[:25], len(new_unique_selection))
new_selected_unique_inchikeys = new_unique_inchikeys['inchikey14'].values[new_unique_selection]
print('Overlap between new and old list:', len(set(new_selected_unique_inchikeys) & set(selected_unique_inchikeys)))

#get the spectrum that belongs with the unique chosen inchikeys
new_selected_unique_spectra = []
inchikeys_pd = pd.Series([x for x in Inchikeys]) #include all even empty ones to get the IDs right!

np.random.seed(42) # to make it reproducible
for inchikey in new_selected_unique_inchikeys:
    matches = inchikeys_pd[inchikeys_pd.str[:14] == inchikey].index.values
    new_selected_unique_spectra.append(int(matches[0]))
print(len(new_selected_unique_spectra))<jupyter_output>4352
Number of old selected inchikeys in new list: 0
[ 505 1611 3107  511 1467  410 3678 2677 3370 4235 3853 3217  149 4332
 1295 2457 2589 1261  287 3038   17  755  305 3276 4084] 1000
Overlap between new and old list: 0
1000<jupyter_text>Add the two datasets together<jupyter_code>#add unique dataset together with old test set of 1000 spectra >5 occurrence
old_and_unique_test_set = selected_spectra + selected_unique_spectra
len(old_and_unique_test_set)
#add second unique test set together with new test set of 1000 spectra >5 occurrence
new_and_unique2_test_set = new_selected_spectra + new_selected_unique_spectra
len(new_and_unique2_test_set)<jupyter_output><empty_output><jupyter_text>Split spectra into library and query set<jupyter_code>old_and_unique_documents_library_s2v = [SpectrumDocument(s, n_decimals=2) for ind, s in enumerate(spectrums_s2v) if ind not in old_and_unique_test_set]
new_and_unique2_documents_library_s2v = [SpectrumDocument(s, n_decimals=2) for ind, s in enumerate(spectrums_s2v) if ind not in new_and_unique2_test_set]
old_and_unique_documents_query_s2v = [SpectrumDocument(spectrums_s2v[i], n_decimals=2) for i in old_and_unique_test_set]
new_and_unique2_documents_query_s2v = [SpectrumDocument(spectrums_s2v[i], n_decimals=2) for i in new_and_unique2_test_set]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09096032461677186, 'fraction_numerical': 0.04316952209197475, 'mean_word_length': 7.292523364485981, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '416759', 'license': 'Apache-2.0', 'n_tokens_mistral': 3315, 'n_tokens_neox': 2997, 'n_words': 718, 'path': '.ipynb_checkpoints/Generate-2000-queries-datasets-checkpoint.ipynb', 'repo_name': 'louwenjjr/improve_library_matching', 'size': '8872'}