<jupyter_start><jupyter_text>Iomega workflow Split cleaned data into subsetsHere we split the previously cleaned dataset (>150,000 spectra) into various subsets for further analysis.<jupyter_code>import os
import sys

ROOT = os.path.dirname(os.getcwd())
path_data = os.path.join(ROOT, 'data')<jupyter_output><empty_output><jupyter_text>Import data from fully pre-processed data<jupyter_code># Load fully processed dataset
from matchms.importing import load_from_json

filename = os.path.join(path_data,'gnps_all_cleand_by_matchms_and_pubchem_lookups.json')
reference_spectrums = load_from_json(filename)<jupyter_output><empty_output><jupyter_text>Select positive mode spectra only<jupyter_code>reference_spectrums_positive = [s.clone() for s in reference_spectrums if s.get("ionmode") == "positive"]
def count_annotations(spectra):
    inchi_lst = []
    smiles_lst = []
    inchikey_lst = []
    for i, spec in enumerate(spectra):
        inchi_lst.append(spec.get("inchi"))
        smiles_lst.append(spec.get("smiles"))
        inchikey = spec.get("inchikey")
        if inchikey is None:
            inchikey = spec.get("inchikey_inchi")
        inchikey_lst.append(inchikey)

    inchi_count = sum([1 for x in inchi_lst if x])
    smiles_count = sum([1 for x in smiles_lst if x])
    inchikey_count = sum([1 for x in inchikey_lst if x])
    print("Inchis:", inchi_count, "--", len(set(inchi_lst)), "unique")
    print("Smiles:", smiles_count, "--", len(set(smiles_lst)), "unique")
    print("Inchikeys:", inchikey_count, "--", 
          len(set([x[:14] for x in inchikey_lst if x])), "unique (first 14 characters)")
    
print("Number of spectra:", len(reference_spectrums_positive))
count_annotations(reference_spectrums_positive)
from matchms.exporting import save_as_json

filename = os.path.join(path_data,'gnps_positive_ionmode_cleaned_by_matchms_and_lookups.json')
save_as_json(reference_spectrums_positive, filename)
# or load results
"""from matchms.importing import load_from_json

filename = os.path.join(path_data,'gnps_positive_ionmode_cleaned_by_matchms_and_lookups.json')
reference_spectrums_positive = load_from_json(filename)"""<jupyter_output>RDKit WARNING: [20:50:33] Enabling RDKit 2019.09.3 jupyter extensions<jupyter_text>Select subset of unique InchiKeysCreate strongly reduced dataset to limit computation times for benchmarking.For every unique InchiKey (only considering first 14 characters), only one spectrum will be picked. In cases where many spectra exit for the same InchiKey, the one with most peaks above an relative intensity threshold is selected.<jupyter_code>import numpy

def count_higher_peaks(spectrum, threshold = 0.1):
    return numpy.sum(spectrum.peaks.intensities/spectrum.peaks.intensities.max() >= threshold)
# Get collection/dictionary of inchikeys
inchikey_collection = {}
for i, spec in enumerate(reference_spectrums_positive):
    inchikey = spec.get("inchikey")
    if inchikey:
        if inchikey[:14] in inchikey_collection:
            inchikey_collection[inchikey[:14]] += [i]
        else:
            inchikey_collection[inchikey[:14]] = [i]
len(inchikey_collection)<jupyter_output><empty_output><jupyter_text>Actual picking of the unique InchiKey spectra<jupyter_code>import numpy as np

intensity_thres = 0.01
n_peaks_required = 10
ID_picks = []

inchikey14_unique = [x for x in inchikey_collection.keys()]

# Loop through all unique inchiques (considering 14 first characters)
for inchikey14 in inchikey14_unique:
    specIDs = np.array(inchikey_collection[inchikey14])
    if specIDs.size == 1:
        ID_picks.append(specIDs[0])
    else:
        # Step 1 - select spectrum with sufficient peaks (e.g. 10 with intensity 0.01)
        num_peaks = np.array([count_higher_peaks(reference_spectrums_positive[specID], intensity_thres) for specID in specIDs])
        sufficient_peaks = np.where(num_peaks >= n_peaks_required)[0]
        if sufficient_peaks.size == 0:
            sufficient_peaks = np.where(num_peaks == max(num_peaks))[0]
        step1IDs = specIDs[sufficient_peaks]

        # Step 2 - select best spectrum qualities (according to gnps measure). 1 > 2 > 3
        qualities = np.array([int(reference_spectrums_positive[specID].get("library_class")) for specID in step1IDs])
        step2IDs = step1IDs[np.where(qualities == min(qualities))[0]]

        # Step 3 Select the ones with most peaks > threshold
        num_peaks = np.array([count_higher_peaks(reference_spectrums_positive[specID], intensity_thres) for specID in step2IDs])
        pick = np.argmax(num_peaks)
        ID_picks.append(step2IDs[pick])
#Check if indeed correct number of unique inchikeys:
test_inchikeys14 = []
for ID in ID_picks:
    test_inchikeys14.append(reference_spectrums_positive[ID].get("inchikey")[:14])
    
len(set(test_inchikeys14))
import json
filename = os.path.join(path_data,'unique_inchikeys_positive_ionmode_IDs200519.json')
with open(filename, 'w') as f:
    json.dump([int(x) for x in ID_picks], f)
spectrums_unique_inchikeys_positive = [reference_spectrums_positive[ID].clone() for ID in ID_picks]
len(spectrums_unique_inchikeys_positive)
count_annotations(spectrums_unique_inchikeys_positive)
from matchms.exporting import save_as_json

filename = os.path.join(path_data,'gnps_positive_ionmode_unique_inchikey_cleaned_by_matchms_and_lookups.json')
save_as_json(spectrums_unique_inchikeys_positive, filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09891123823583688, 'fraction_numerical': 0.016239158516331426, 'mean_word_length': 5.36150234741784, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '423956', 'license': 'Apache-2.0', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1757, 'n_words': 393, 'path': 'notebooks/iomega-2-split_data_into_subsets.ipynb', 'repo_name': 'iomega/spec2vec_gnps_data_analysis', 'size': '5419'}