<jupyter_start><jupyter_text>HDF5 for MGF filesTo avoid multiple MGF file accesses, we create a huge HDF5 file containing all MGF files. For testing, we created a small subset of all datasets. formatdata: [filename],[scanid], [precursor mass],[mz,mz,...],[intensity,intensity,...]labels: peptide sequences (one string) Notes!!For python 2.7, we use protocol=0<jupyter_code>from read_mgf import read_mgf
dataset = read_mgf('/Volumes/MSSHARE/Joonyong/KAIKO_Data/Original_mgf/Alverdy_Efae_1A_lys_13Jul13_Pippin_12-12-39.mgf')
len(dataset)
import sys
sys.getsizeof(dataset)
import pandas as pd

list_data = []
list_labels = []
for d in dataset:
    temp = []
    temp.append(str(d[0]))
    temp.append(str(d[1]))
    temp.append(','.join([str(s) for s in d[2]]) + ';' + ','.join([str(s) for s in d[3]]))
    list_data.append(temp)
    list_labels.append(','.join(d[4]))
df_data = pd.DataFrame(list_data)
df_labels = pd.DataFrame(list_labels)
print(sys.getsizeof(df_data))
print(df.memory_usage())
df.shape
import deepdish as dd
import numpy as np

dd.io.save('test.h5', {'data': list_data, 'labels': list_labels}, compression=True)
tt = dd.io.load('test.h5')
print(len(tt['data']))
print(len(tt['labels']))
print(tt['data'][0])
print(tt['labels'][0])<jupyter_output>6659
6659
['2031', '1909.0504280520513', '119.02627563476562,129.1024169921875,130.1060028076172,136.06167602539062,145.09689331054688,156.7694854736328,164.85572814941406,170.10325622558594,173.0915069580078,175.1190643310547,186.12355041503906,192.22216796875,283.1388244628906,354.1733093261719,397.3297119140625,437.5796203613281,437.91693115234375,438.2517395019531,448.7451171875,455.2225646972656,458.7539367675781,474.01434326171875,474.27203369140625,495.8350830078125,531.6235961914062,548.3114624023438,556.8167114257812,557.3126220703125,568.8131103515625,569.3073120117188,577.3114013671875,577.81494140625,578.3125,584.3173828125,595.3419189453125,598.8284301757812,599.3284301757812,600.344970703125,604.349853515625,607.34326171875,607.8345947265625,612.3402099609375,624.8550415039062,625.8565063476562,626.3489990234375,626.8512573242188,633.8519287109375,634.3541870117188,646.8584594726562,647.357177734375,647.8567504882812,655.868408203125,656.3654174804688,656.8717041015625,73[...]
{'fraction_non_alphanumeric': 0.14733127481252756, 'fraction_numerical': 0.3987648875165417, 'mean_word_length': 8.333333333333334, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '643624', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1495, 'n_tokens_neox': 962, 'n_words': 108, 'path': 'tool/mgf2binary/mgf2hdf5.ipynb', 'repo_name': 'anubhav0fnu/Kaiko', 'size': '2267'}