<commit_before>""" mzml to h5 file conversion """

import argparse
import functools
import logging
import os
import re
import shutil
import subprocess
import sys
import time
import traceback

from datetime import datetime
from subprocess import check_output

from metatlas.datastructures.metatlas_objects import LcmsRun, store, retrieve
from metatlas.io.mzml_loader import mzml_to_hdf
from metatlas.io.mzml_loader import VERSION_TIMESTAMP
from metatlas.io.system_utils import send_mail

logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)

readonly_files = {}  # username (or uid) | a set of files associated with them
other_errors = {}  # info with user | list of error messages
patt = re.compile(r".+\/raw_data\/(?P<username>[^/]+)\/(?P<experiment>[^/]+)\/(?P<path>.+)")

MEMBERS_CMD = "getent group metatlas | cut -d: -f4"
ALL_USERS = set(sorted(subprocess.check_output(MEMBERS_CMD, shell=True, text=True).strip().split(',')))
REMOVE_USERS = {'msdata', 'jaws', 'jgi_dna', 'vrsingan', 'wjholtz',
                'mjblow', 'greensi', 'annau', 'jfroula', 'pasteur'}
USERS = tuple(ALL_USERS - REMOVE_USERS)
DEFAULT_USERNAME = 'smkosina'
EXPLICIT_USERNAMES = {"ag": "agolini", "ao": "arosborn"}


@functools.lru_cache
def _initials_to_username(initials):
    if initials == '':
        return None
    if initials in EXPLICIT_USERNAMES:
        return EXPLICIT_USERNAMES[initials]
    for user in USERS:
        if user.startswith(initials):
            return user
    for user in USERS:
        pat = re.compile(f"^{initials[0]}[a-z]{initials[1]}")
        if pat.match(user):
            return user
    return None


def move_file(src, dest):
    """ move file and create directories if needed """
    assert os.path.isfile(src)
    dest_dir = dest if os.path.isdir(dest) else os.path.dirname(dest)
    os.makedirs(dest_dir, exist_ok=True)
    shutil.move(src, dest)


def _file_name_to_username(file_name, default):
    """ extract initials from filename and convert to nersc username """
    initials_field = os.path.basename(file_name).split("_")[1].lower()
    for initials in initials_field.split("-"):  # from left to right
        username = _initials_to_username(initials.replace('_', ''))
        if username is not None:
            return username
    return default


def get_acqtime_from_mzml(mzml_file):
    start_time = None
    with open(mzml_file, "r", encoding="utf-8") as mzml:
        for line in mzml:
            if 'start_time' in line:
                start_time = line.split('start_time="')[1].split('"')[0].replace('T', ' ').rstrip('Z')
                break
    if start_time is not None and '-infinity' not in start_time:
        date_object = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
        utc_timestamp = int(time.mktime(date_object.timetuple()))
    else:
        utc_timestamp = int(0)
    return utc_timestamp


def convert(ind, fname):
    """Helper function, converts a single file"""
    logger.info("Converting file number %d: %s", ind + 1, fname)

    # Get relevant information about the file.
    username = _file_name_to_username(fname, DEFAULT_USERNAME)
    info = patt.match(os.path.abspath(fname))
    if info:
        info = info.groupdict()
    else:
        logger.error("Invalid path name: %s", fname)
        return
    dirname = os.path.dirname(fname)

    # Convert to HDF and store the entry in the database.
    try:
        hdf5_file = fname.replace('mzML', 'h5')
        logger.info("Generating h5 file: %s", hdf5_file)
        mzml_to_hdf(fname, hdf5_file, True)
        os.chmod(hdf5_file, 0o640)
        # Add this to the database unless it is already there
        try:
            runs = retrieve('lcmsrun', username='*', mzml_file=fname)<commit_msg>file convert create mode 660 files (#498)<commit_after>""" mzml to h5 file conversion """

import argparse
import functools
import logging
import os
import re
import shutil
import subprocess
import sys
import time
import traceback

from datetime import datetime
from subprocess import check_output

from metatlas.datastructures.metatlas_objects import LcmsRun, store, retrieve
from metatlas.io.mzml_loader import mzml_to_hdf
from metatlas.io.mzml_loader import VERSION_TIMESTAMP
from metatlas.io.system_utils import send_mail

logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)

readonly_files = {}  # username (or uid) | a set of files associated with them
other_errors = {}  # info with user | list of error messages
patt = re.compile(r".+\/raw_data\/(?P<username>[^/]+)\/(?P<experiment>[^/]+)\/(?P<path>.+)")

MEMBERS_CMD = "getent group metatlas | cut -d: -f4"
ALL_USERS = set(sorted(subprocess.check_output(MEMBERS_CMD, shell=True, text=True).strip().split(',')))
REMOVE_USERS = {'msdata', 'jaws', 'jgi_dna', 'vrsingan', 'wjholtz',
                'mjblow', 'greensi', 'annau', 'jfroula', 'pasteur'}
USERS = tuple(ALL_USERS - REMOVE_USERS)
DEFAULT_USERNAME = 'smkosina'
EXPLICIT_USERNAMES = {"ag": "agolini", "ao": "arosborn"}


@functools.lru_cache
def _initials_to_username(initials):
    if initials == '':
        return None
    if initials in EXPLICIT_USERNAMES:
        return EXPLICIT_USERNAMES[initials]
    for user in USERS:
        if user.startswith(initials):
            return user
    for user in USERS:
        pat = re.compile(f"^{initials[0]}[a-z]{initials[1]}")
        if pat.match(user):
            return user
    return None


def move_file(src, dest):
    """ move file and create directories if needed """
    assert os.path.isfile(src)
    dest_dir = dest if os.path.isdir(dest) else os.path.dirname(dest)
    os.makedirs(dest_dir, exist_ok=True)
    shutil.move(src, dest)


def _file_name_to_username(file_name, default):
    """ extract initials from filename and convert to nersc username """
    initials_field = os.path.basename(file_name).split("_")[1].lower()
    for initials in initials_field.split("-"):  # from left to right
        username = _initials_to_username(initials.replace('_', ''))
        if username is not None:
            return username
    return default


def get_acqtime_from_mzml(mzml_file):
    start_time = None
    with open(mzml_file, "r", encoding="utf-8") as mzml:
        for line in mzml:
            if 'start_time' in line:
                start_time = line.split('start_time="')[1].split('"')[0].replace('T', ' ').rstrip('Z')
                break
    if start_time is not None and '-infinity' not in start_time:
        date_object = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
        utc_timestamp = int(time.mktime(date_object.timetuple()))
    else:
        utc_timestamp = int(0)
    return utc_timestamp


def convert(ind, fname):
    """Helper function, converts a single file"""
    logger.info("Converting file number %d: %s", ind + 1, fname)

    # Get relevant information about the file.
    username = _file_name_to_username(fname, DEFAULT_USERNAME)
    info = patt.match(os.path.abspath(fname))
    if info:
        info = info.groupdict()
    else:
        logger.error("Invalid path name: %s", fname)
        return
    dirname = os.path.dirname(fname)

    # Convert to HDF and store the entry in the database.
    try:
        hdf5_file = fname.replace('mzML', 'h5')
        logger.info("Generating h5 file: %s", hdf5_file)
        mzml_to_hdf(fname, hdf5_file, True)
        os.chmod(hdf5_file, 0o660)  # this can be changed to 0o440 once everyone is on the current code
        # Add this to the database unless it is already there
        try:
            runs = retrieve('lcmsrun', username='*', mzml_file=fname)
{'fraction_non_alphanumeric': 0.12469151837901025, 'fraction_numerical': 0.006494349915573451, 'mean_word_length': 3.4663573085846866, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3283582', 'n_tokens_mistral': 2679, 'n_tokens_neox': 2611, 'n_words': 662}