<commit_before>from __future__ import print_function

import argparse
import os
import pwd
import datetime
import sys

import requests
import tables
import pymzml

DEBUG = False


class Spectrum(tables.IsDescription):
    mz = tables.Float32Col(pos=0)
    rt = tables.Float32Col(pos=1)
    i = tables.Float32Col(pos=2)
    polarity = tables.UInt8Col(pos=3)
    ms_level = tables.UInt8Col(pos=4)
    precursor_MZ = tables.Float32Col(pos=5)
    precursor_intensity = tables.Float32Col(pos=6)
    collision_energy = tables.Float32Col(pos=7)


def mzml_to_hdf(in_file_name, out_file_name=None, debug=False):
    """Converts in_file (mzml) to binary and stores it in out_file
    """
    if not out_file_name:
        out_file_name = in_file_name.replace('.mzML', '.h5')

    FILTERS = tables.Filters(complib='lzo', complevel=1)
    out_file = tables.open_file(out_file_name, "w", filters=FILTERS)

    table = out_file.create_table('/', 'spectra', description=Spectrum)

    debug = debug or DEBUG
    if debug:
        print("STATUS: Converting %s to %s (mzML to HDF)" %
              (in_file_name, out_file_name), end='')

    # Extra accessions for pymzml to read
    extraAccessions = [
        ('MS:1000016', ['value', 'unitName']),  # scan start time
        ('MS:1000129', ['name']),  # negative scan
        ('MS:1000130', ['name']),  # positive scan
        ('MS:1000744', ['name', 'value']),  # selected ion m/z
        ('MS:1000042', ['name', 'value']),  # peak intensity
        ('MS:1000045', ['name', 'value']),  # collision energy
    ]

    mzml_reader = pymzml.run.Reader(in_file_name,
                                    extraAccessions=extraAccessions)

    for (ind, spectrum) in enumerate(mzml_reader):
        try:
            polarity = 1 if 'positive scan' in spectrum.keys() else 0
            ms_level = int(spectrum['ms level'])
            # check if scan start time exists. some thermo spectra
            #  are missing this value
            if 'scan start time' in spectrum.keys():
                rt = float(spectrum['scan start time'][0])
            else:
                rt = float(spectrum['MS:1000016'][0])
        except (KeyError, TypeError):
            continue

        precursor_MZ = 0.0
        precursor_intensity = 0.0
        collision_energy = 0.0

        if ms_level == 2:
            collision_energy = spectrum['collision energy'][1]
            if 'peak intensity' in spectrum.keys():
                precursor_intensity = spectrum['peak intensity'][1]
            precursor_MZ = spectrum['selected ion m/z'][0]

        mylist = []
        for mz, i in spectrum.peaks:
            mylist.append((mz, rt, i, polarity, ms_level,
                           precursor_MZ, precursor_intensity,
                           collision_energy))
        table.append(mylist)
        table.flush()

        if debug and not (ind % 100):
            sys.stdout.write('.')
            sys.stdout.flush()

    out_file.set_node_attr('/', "upload_date", datetime.datetime.utcnow())
    out_file.set_node_attr('/', "uploaded_by",
                           pwd.getpwuid(os.getuid())[0])

    if debug:
        print('\nSaving file')
    out_file.close()
    if debug:<commit_msg>Make the loader more robust<commit_after>from __future__ import print_function

import argparse
import os
import pwd
import datetime
import sys

import requests
import tables
import pymzml

DEBUG = False


class Spectrum(tables.IsDescription):
    mz = tables.Float32Col(pos=0)
    rt = tables.Float32Col(pos=1)
    i = tables.Float32Col(pos=2)
    polarity = tables.UInt8Col(pos=3)
    ms_level = tables.UInt8Col(pos=4)
    precursor_MZ = tables.Float32Col(pos=5)
    precursor_intensity = tables.Float32Col(pos=6)
    collision_energy = tables.Float32Col(pos=7)


def mzml_to_hdf(in_file_name, out_file_name=None, debug=False):
    """Converts in_file (mzml) to binary and stores it in out_file
    """
    if not out_file_name:
        out_file_name = in_file_name.replace('.mzML', '.h5')

    FILTERS = tables.Filters(complib='lzo', complevel=1)
    out_file = tables.open_file(out_file_name, "w", filters=FILTERS)

    table = out_file.create_table('/', 'spectra', description=Spectrum)

    debug = debug or DEBUG
    if debug:
        print("STATUS: Converting %s to %s (mzML to HDF)" %
              (in_file_name, out_file_name), end='')

    # Extra accessions for pymzml to read
    extraAccessions = [
        ('MS:1000016', ['value', 'unitName']),  # scan start time
        ('MS:1000129', ['name']),  # negative scan
        ('MS:1000130', ['name']),  # positive scan
        ('MS:1000744', ['name', 'value']),  # selected ion m/z
        ('MS:1000042', ['name', 'value']),  # peak intensity
        ('MS:1000045', ['name', 'value']),  # collision energy
    ]

    mzml_reader = pymzml.run.Reader(in_file_name,
                                    extraAccessions=extraAccessions)

    for (ind, spectrum) in enumerate(mzml_reader):
        try:
            polarity = 'positive scan' in spectrum or 'MS:1000130' in spectrum
            ms_level = spectrum['ms level']
            rt = spectrum.get('scan start time', spectrum['MS:1000016'])[0]
        except (KeyError, TypeError):
            continue

        precursor_MZ = 0.0
        precursor_intensity = 0.0
        collision_energy = 0.0

        if ms_level == 2:
            try:
                collision_energy = spectrum.get('collision energy',
                                                spectrum['MS:1000045'])[1]
                precursor_intensity = spectrum.get('peak intensity',
                                                   spectrum['MS:1000042'])[1]
                precursor_MZ = spectrum.get('selected ion m/z',
                                            spectrum['MS:1000744'])[0]
            except (KeyError, TypeError):
                continue

        mylist = []
        for mz, i in spectrum.peaks:
            mylist.append((mz, rt, i, polarity, ms_level,
                           precursor_MZ, precursor_intensity,
                           collision_energy))
        table.append(mylist)
        table.flush()

        if debug and not (ind % 100):
            sys.stdout.write('.')
            sys.stdout.flush()

    out_file.set_node_attr('/', "upload_date", datetime.datetime.utcnow())
    out_file.set_node_attr('/', "uploaded_by",
                           pwd.getpwuid(os.getuid())[0])

    if debug:
        print('\nSaving file')
    out_file.close()
    if debug:
{'fraction_non_alphanumeric': 0.12076923076923077, 'fraction_numerical': 0.031846153846153843, 'mean_word_length': 2.2602808425275827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7185344', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2002, 'n_words': 459}