<jupyter_start><jupyter_text>Paper Figure 2: IORaw file accession times and pyRawFileReader interfaceThis notebook converts raw files and shows accesion times.<jupyter_code>import os
from test_files import get_files, prepare_test

multiple_file_test = ['PXD006109_HeLa12_1.raw','PXD006109_HeLa12_2.raw','PXD006109_HeLa12_3.raw','PXD006109_HeLa2_1.raw','PXD006109_HeLa2_2.raw','PXD006109_HeLa2_3.raw']
tmp_folder = os.path.join(os.getcwd(),'temp/')
test_folder = 'fig2'
import platform

print(platform.platform())
print(platform.processor())

get_files(tmp_folder, multiple_file_test)
prepare_test(multiple_file_test, tmp_folder, test_folder)
raw_path = os.path.join(os.path.join(tmp_folder, test_folder, multiple_file_test[0]))
print(f"File size of '{raw_path}' = {os.path.getsize(raw_path)/1024/1024/1024:.3f} GB")<jupyter_output>File size of 'F:\alphapept\sandbox\temp/fig2\PXD006109_HeLa12_1.raw' = 1.567 GB<jupyter_text>Interfacing files with pyRawFileReader<jupyter_code>from alphapept.pyrawfilereader import RawFileReader
rawFile = RawFileReader(raw_path)<jupyter_output><empty_output><jupyter_text>Metadata<jupyter_code>print(f"First scan: {rawFile.FirstSpectrumNumber}")
print(f"Last scan: {rawFile.LastSpectrumNumber}")
print(f"Total spectra: {rawFile.NumSpectra}")
print(f"Start time: {rawFile.StartTime}")
print(f"End time: {rawFile.EndTime}")

print("GetFileName: ", rawFile.GetFileName())
print("GetCreatorID: ", rawFile.GetCreatorID())

try:
    print(rawFile.source.GetInstrumentMethod(1)) # Error on MacOS
except:
    print("Count not access instrument method on macOS")<jupyter_output>First scan: 1
Last scan: 92650
Total spectra: 92650
Start time: 0.0032707386
End time: 100.00126
GetFileName:  F:\alphapept\sandbox\temp\fig2\PXD006109_HeLa12_1.raw
GetCreatorID:  thermo
                            Method of Q Exactive HF

OVERALL METHOD SETTINGS

Global Settings
Use lock masses                                                          off 
Lock mass injection                                                        ― 
Chrom. peak width (FWHM)                                                  15 s
Time
Method duration                                                       100.00 min
Customized Tolerances (+/-)
Lock Masses                                                                ― 
Inclusion                                                                  ― 
Exclusion                                                                  ― 
Neutral Loss                                                               ― 
Mass Tags                                  [...]<jupyter_text>Accessing a single scan<jupyter_code>scanNumber = 10000
print("GetMSOrderForScanNum: ", rawFile.GetMSOrderForScanNum(scanNumber))
print("GetScanEventForScanNum: ", rawFile.GetScanEventStringForScanNum(scanNumber))
print("GetNumberOfSourceFragmentsFromScanNum: ", rawFile.GetNumberOfSourceFragmentsFromScanNum(scanNumber))
if rawFile.GetNumberOfSourceFragmentsFromScanNum(scanNumber) > 0:
    print("GetSourceFragmentValueFromScanNum: ", rawFile.GetSourceFragmentValueFromScanNum(scanNumber, 0))
if rawFile.GetMSOrderForScanNum(scanNumber) > 1:
    print("GetIsolationWidthForScanNum: ", rawFile.GetIsolationWidthForScanNum(scanNumber))
    print("GetCollisionEnergyForScanNum: ", rawFile.GetCollisionEnergyForScanNum(scanNumber))
    print("GetActivationTypeForScanNum: ", rawFile.GetActivationTypeForScanNum(scanNumber))
    print("GetPrecursorMassForScanNum: ", rawFile.GetPrecursorMassForScanNum(scanNumber))
    print("GetPrecursorRangeForScanNum: ", rawFile.GetPrecursorRangeForScanNum(scanNumber))
print("GetMassAnalyzerTypeForScanNum: ", rawFile.GetMassAnalyzerTypeForScanNum(scanNumber))
print("GetNumberOfMassCalibratorsFromScanNum: ", rawFile.GetNumberOfMassCalibratorsFromScanNum(scanNumber))
print("GetMassCalibrationValueFromScanNum: ", rawFile.GetMassCalibrationValueFromScanNum(scanNumber, 0))
print("GetFirstSpectrumNumber: ", rawFile.GetFirstSpectrumNumber())
print("GetLastSpectrumNumber: ", rawFile.GetLastSpectrumNumber())
print("IsProfileScanForScanNum: ", rawFile.IsProfileScanForScanNum(scanNumber))
print("IsCentroidScanForScanNum: ", rawFile.IsCentroidScanForScanNum(scanNumber))

print("RTFromScanNum: ", rawFile.RTFromScanNum(scanNumber))
print("ScanNumFromRT: ", rawFile.ScanNumFromRT(30))
print("RTInSecondsFromScanNum: ", rawFile.RTInSecondsFromScanNum(scanNumber))
print("ScanNumFromRTInSeconds: ", rawFile.ScanNumFromRTInSeconds(30))

print("GetNumberOfMSOrdersFromScanNum: ", rawFile.GetNumberOfMSOrdersFromScanNum(scanNumber))
print("GetBasePeakForScanNum: ", rawFile.GetBasePeakForScanNum(scanNumber))
print("GetTrailerExtraForScanNum: ", rawFile.GetTrailerExtraForScanNum(scanNumber))
import matplotlib.pyplot as plt

masses, intens = rawFile.GetCentroidMassListFromScanNum(scanNumber)

print(f'masses {masses}')
print(f'intensities {intens}')

plt.figure(figsize=(10,5))
plt.vlines(masses, intens, 0)
plt.title(f'scanNumber {scanNumber}')
plt.show()<jupyter_output>masses [ 101.07131958  106.05017853  106.28302765  110.07151031  129.10220337
  136.07565308  147.11235046  156.07362366  169.09700012  187.10772705
  193.97605896  204.1350708   207.01774597  215.10255432  219.13264465
  230.07609558  266.12301636  272.17062378  273.1736145   323.42163086
  329.14291382  343.15542603  345.10177612  365.19311523  390.13040161
  392.23382568  395.16445923  444.16860962  474.1545105   484.24813843
  509.20657349  583.75164795  613.29223633  622.28826904  632.30969238
  638.24108887  649.81646729  650.34399414  651.30206299  727.33856201
  753.27416992  838.36077881  854.28295898  856.37255859  866.37121582
  867.32226562  953.36047363  954.36053467  971.39764404  982.33898926
 1068.40710449 1085.43310547 1086.44177246 1095.43969727 1183.45373535
 1200.46228027 1201.4708252 ]
intensities [  9817.5234375   20783.8203125    5325.89306641  26432.38085938
  12011.63378906   6990.6484375   21516.33398438   4777.22949219
  30521.54882812  53889.47265625   4654.[...]<jupyter_text>Loading file to RAM<jupyter_code>import time

from alphapept.io import load_thermo_raw
from alphapept.interface import import_raw_data

time_list = []


def ram_test():
    start_time = time.perf_counter()
    _ = load_thermo_raw(raw_path, n_most_abundant = 1000)
    end_time = time.perf_counter()
    _size = os.path.getsize(raw_path)/1024/1024/1024
    _time = end_time-start_time
    time_list.append([f'RAW to RAM (N=1, {_size:.1f}GB)', _time])
    print(f"Converting all raw files into HDF takes {end_time-start_time:.3f} seconds ({(end_time-start_time)/60:.3f} minutes)")<jupyter_output><empty_output><jupyter_text>Loading using AlphaPept<jupyter_code>def print_data_info(settings):
    total_size = 0
    print(f"{len(settings['experiment']['file_paths'])} raw files")
    for raw_path in settings['experiment']['file_paths']:
        _size = os.path.getsize(raw_path)/1024/1024/1024
        total_size += _size
        print(f"File size of '{raw_path}' = {_size:.3f} GB")
    print(f'Total size = {total_size:.3f} GB')
    return total_size

def load_alphapept():
    settings = {}
    settings['experiment'] = {}
    settings['experiment']['file_paths'] = []
    settings['general'] = {}
    settings['general']['n_processes'] = 1
    settings['raw'] = {}
    settings['raw']['n_most_abundant'] = 1000

    settings['experiment']['file_paths'] = []
    settings['experiment']['file_paths'].append(raw_path)

    try:
        os.remove(raw_path[:-3]+'ms_data.hdf')
    except:
        pass

    start_time = time.perf_counter()
    import_raw_data(settings, settings_parsed=True)
    end_time = time.perf_counter()
    _size = print_data_info(settings)
    _time = end_time-start_time
    time_list.append([f'RAW to HDF (N=1, {_size:.1f}GB)', _time])
    print(f"Converting all raw files into HDF takes {end_time-start_time:.3f} seconds ({(end_time-start_time)/60:.3f} minutes)")
import alphapept.io

def load_hdf(hdf_file):
    start_time = time.perf_counter()
    _reader = alphapept.io.MS_Data_File(hdf_file)
    query_data = _reader.read_DDA_query_data()
    end_time = time.perf_counter()
    _size = os.path.getsize(hdf_file[:-len('ms_data.hdf')]+'raw')/1024/1024/1024
    _time = end_time-start_time
    time_list.append([f'HDF to RAM (N=1, {_size:.1f}GB)', _time])
    print(f"Loading {hdf_file} takes {end_time-start_time:.3f} seconds ({(end_time-start_time)/60:.3f} minutes)")
    return query_data


def test_load_hdf():
    _ = load_hdf(raw_path[:-3]+'ms_data.hdf')<jupyter_output><empty_output><jupyter_text>Parallelized version  Single Core<jupyter_code>def test_single_core():
    settings = {}
    settings['experiment'] = {}
    settings['experiment']['file_paths'] = []
    settings['general'] = {}
    settings['general']['n_processes'] = 1
    settings['raw'] = {}
    settings['raw']['n_most_abundant'] = 1000

    settings['experiment']['file_paths'] = []
    for raw in multiple_file_test:
        settings['experiment']['file_paths'].append(os.path.join(os.path.join(tmp_folder, test_folder, raw)))

    cpu = 1
    settings['general']['n_processes'] = cpu
    for raw in settings['experiment']['file_paths']:
        try:
            os.remove(raw[:-4]+'.ms_data.hdf')
        except:
            pass


    start_time = time.perf_counter()
    import_raw_data(settings, settings_parsed=True)
    end_time = time.perf_counter()
    _size = print_data_info(settings)
    _time = end_time-start_time
    N = len(settings['experiment']['file_paths'])
    time_list.append([f'RAW to HDF (N={N}, {_size:.1f}GB, CPU={cpu})', _time])
    print(f"Converting all raw files into HDF takes {end_time-start_time:.3f} seconds ({(end_time-start_time)/60:.3f} minutes)")<jupyter_output><empty_output><jupyter_text>Multicore<jupyter_code>def test_multicore():
    settings = {}
    settings['experiment'] = {}
    settings['experiment']['file_paths'] = []
    settings['general'] = {}
    settings['general']['n_processes'] = 1
    settings['raw'] = {}
    settings['raw']['n_most_abundant'] = 1000

    settings['experiment']['file_paths'] = []
    for raw in multiple_file_test:
        settings['experiment']['file_paths'].append(os.path.join(os.path.join(tmp_folder, test_folder, raw)))

    
    cpu = 6
    settings['general']['n_processes'] = cpu
    for raw in settings['experiment']['file_paths']:
        try:
            os.remove(raw[:-4]+'.ms_data.hdf')
        except:
            pass

    start_time = time.perf_counter()
    import_raw_data(settings, settings_parsed=True)
    end_time = time.perf_counter()
    _size = print_data_info(settings)
    _time = end_time-start_time
    N = len(settings['experiment']['file_paths'])
    time_list.append([f'RAW to HDF (N={N}, {_size:.1f}GB, CPU={cpu})', _time])
    print(f"Converting all raw files into HDF takes {end_time-start_time:.3f} seconds ({(end_time-start_time)/60:.3f} minutes)")
## Test multiple times
time_list = []

for i in range(3):
    ram_test()
    load_alphapept()
    test_load_hdf()
    test_single_core()
    test_multicore()
    
    
    
time_list
time_list = [['RAW to RAM (N=1, 1.6GB)', 71.7460462],
 ['RAW to HDF (N=1, 1.6GB)', 69.3700454],
 ['HDF to RAM (N=1, 1.6GB)', 0.18922449999999458],
 ['RAW to HDF (N=6, 9.2GB, CPU=1)', 424.1473389],
 ['RAW to HDF (N=6, 9.2GB, CPU=6)', 82.68116509999993],
 ['RAW to RAM (N=1, 1.6GB)', 66.8809268],
 ['RAW to HDF (N=1, 1.6GB)', 71.11713600000007],
 ['HDF to RAM (N=1, 1.6GB)', 0.19676760000004379],
 ['RAW to HDF (N=6, 9.2GB, CPU=1)', 423.57570050000004],
 ['RAW to HDF (N=6, 9.2GB, CPU=6)', 81.60013860000004],
 ['RAW to RAM (N=1, 1.6GB)', 68.55210439999996],
 ['RAW to HDF (N=1, 1.6GB)', 69.33096609999984],
 ['HDF to RAM (N=1, 1.6GB)', 0.19891740000002756],
 ['RAW to HDF (N=6, 9.2GB, CPU=1)', 414.59822899999995],
 ['RAW to HDF (N=6, 9.2GB, CPU=6)', 132.97670340000013]]
import pandas as pd
df = pd.DataFrame(time_list)
mean_ = df.groupby(0).mean()
std_ = df.groupby(0).std()

mean_ = mean_.loc[[_[0] for _ in time_list[:5]]]
std_ = std_.loc[[_[0] for _ in time_list[:5]]]
import matplotlib.pyplot as plt
import numpy as np

dark_blue = '#17212b'
light_blue = '#3dc5ef'
teal= '#42dee1'
green = '#6eecb9'
yellow = '#eef5b3'


time_used = mean_.values.flatten()
time_std = std_.values.flatten()

labels = mean_.index.tolist()

hfont = {'fontname':'Arial', 'size':10}
plt.figure(figsize=(3.94,2.363)) #100x60
height=0.5
ind = list(range(len(labels)))[::-1]

text = ['Raw to memory', 'Raw to hdf', 'Hdf to memory',  'Raw to hdf (6 files, 1 core)',  'Raw to hdf (6 fles, 6 cores)']

for i, t in zip(ind, time_used):
    plt.text(t*1.5, i, f'{t:.1f}', **hfont, va='center')

plt.barh(ind[:3], time_used[:3], height, color =light_blue, xerr = time_std[:3], linewidth=1, edgecolor='black')
plt.barh(ind[3], time_used[3], height, color =teal, xerr = time_std[3], linewidth=1, edgecolor='black')
plt.barh(ind[4], time_used[4], height, color =green, xerr = time_std[4], linewidth=1, edgecolor='black')

plt.yticks(ind, text, **hfont)
plt.xticks(**hfont)

plt.xscale('log')
plt.xlim([0, 0.5*1e4])
plt.xticks([0.1, 1e0, 1e1, 1e2 , 1e3])
plt.xlabel('Time (seconds)', **hfont)
plt.tight_layout()
plt.savefig('./figures/02_io.pdf')  
plt.show()
display(std_)
display(mean_)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13995332379733494, 'fraction_numerical': 0.10103139351050215, 'mean_word_length': 4.2693375644585485, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '165611', 'license': 'Apache-2.0', 'n_tokens_mistral': 5663, 'n_tokens_neox': 4779, 'n_words': 817, 'path': 'sandbox/Figure_Notebooks/P_02_IO.ipynb', 'repo_name': 'PatrickvanZalm/alphapept', 'size': '13283'}