<commit_before>import logging
import sys
import traceback
import dateutil
import pandas as pd
import pymzml
from celery import shared_task
from celery.schedules import crontab
from celery.task import periodic_task

from standards_review.models import Adduct, FragmentationSpectrum, Xic, LcInfo, MsInfo, InstrumentInfo
from standards_review.models import Molecule, Standard, ProcessingError
from django.conf import settings
from standards_review.tools import DatabaseLogHandler
import os
import datetime
import numpy as np

@shared_task
def add_batch_standard(metadata, csv_file):
    """
    handle a csv fil of standards
    header line should be "mcfid","name","formula", "inchi", "solubility", "vendor","vendor_id", "hmdb_id" , "chebi_id", "lipidmaps_id", "cas_id", "pubchem_id". "date","location","lot_num"

    To Be Set:
    ### Standard
    # mandatory
    molecule = models.ForeignKey(Molecule, default=Molecule.objects.all().filter(name='DUMMY'))
    MCFID = models.IntegerField(null=True, blank=True)# if blank MCFID == Standard.pk
    # optional
    vendor = models.TextField(null=True, blank=True)
    vendor_cat = models.TextField(null=True, blank=True)
    lot_num = models.TextField(null=True, blank=True)
    location = models.TextField(null=True, blank=True)
    purchase_date = models.DateField(null=True, blank=True)

    If Not Existing:
    ### Molecule
    # mandatory
    name = models.TextField(default = "")
    sum_formula = models.TextField(null=True)
    pubchem_id = models.TextField(null=True, blank=True)
    # Optional
    inchi_code = models.TextField(default="")
    exact_mass = models.FloatField(default=0.0)
    solubility   = models.TextField(null=True, blank=True)
    # External reference numbers
    hmdb_id = models.TextField(null=True, blank=True)
    chebi_id = models.TextField(null=True, blank=True)
    lipidmaps_id = models.TextField(null=True, blank=True)
    cas_id = models.TextField(null=True, blank=True)
    :param csv_file:
    :return:
    """
    error_list = []
    df = pd.read_csv(csv_file, sep="\t")
    logging.info('I read the file')
    df.columns = [x.replace(" ", "_").lower() for x in df.columns]
    logging.info("I replaced columns")
    # Fill missing values
    df['id'].fillna(-1, inplace=True)
    df.fillna("", inplace=True)
    df['id'] = df['id'].astype(str)
    logging.info("Shape: {}".format(df.shape))
    for row in df.iterrows():
        logging.info("row: {}".format(row))
        try:
            # clean up input
            entry = row[1]
            if entry['formula'] == '':
                raise ValueError('sum formula cannot be blank')
            # for tag in entry.keys():
            #    if entry[tag] != "":
            #        entry[tag] = entry[tag].encode("utf8") # make strings safe


            if entry['pubchem_id'] != "":
                molecule = Molecule.objects.all().filter(pubchem_id=entry['pubchem_id'])
            else:
                molecule = Molecule.objects.all().filter(name__iexact=entry['name'])  # filter lowercase

            if molecule.exists():
                molecule = molecule[0]
            else:
                molecule = Molecule(
                    name=entry["name"],
                    sum_formula=entry["formula"],
                    inchi_code=entry["inchi"],
                    solubility=entry["solubility"],
                    hmdb_id=entry["hmdb_id"],
                    chebi_id=entry["chebi_id"],
                    lipidmaps_id=entry["lipidmaps_id"],
                    cas_id=entry["cas_id"],
                    pubchem_id=entry["pubchem_id"],
                )
                logging.info("about to save " + molecule.name)
                logging.info(molecule)
                molecule.save()
                logging.info("Successfully saved " + molecule.name)

            s = Standard.objects.all().filter(inventory_id=entry['id'])
            if s.exists():  # standard already added, overwrite
                s = s[0]
                s.molecule=molecule
            else:
                s = Standard(molecule=molecule)
            s.vendor = entry["vendor"]
            s.vendor_cat = entry["vendor_id"]
            s.save()
        except:
            error_list.append([entry['name'], sys.exc_info()[1]])
            logging.warning("Failed for: {} with {}".format(entry['name'], sys.exc_info()[1]))

    return error_list


@shared_task
def handle_uploaded_files(metadata, mzml_filepath, d):
    print(__file__ + str(d.id))
    logger = logging.getLogger(__file__ + str(d.id))
    logger.addHandler(DatabaseLogHandler(d, level=logging.DEBUG))
    logger.debug("adding dataset {}".format(mzml_filepath))
    try:
        msrun = pymzml.run.Reader(mzml_filepath)
        ppm = float(metadata['mass_accuracy_ppm'])
        mz_tol_quad = float(metadata['quad_window_mz'])
        ionization_method = metadata['ionization_method']
        ion_analyzer = metadata['ion_analyzer']
        scan_time = []<commit_msg>fixed bug for batching import with no values for id, or pure numeric values.<commit_after>import logging
import sys
import traceback
import dateutil
import pandas as pd
import pymzml
from celery import shared_task
from celery.schedules import crontab
from celery.task import periodic_task

from standards_review.models import Adduct, FragmentationSpectrum, Xic, LcInfo, MsInfo, InstrumentInfo
from standards_review.models import Molecule, Standard, ProcessingError
from django.conf import settings
from standards_review.tools import DatabaseLogHandler
import os
import datetime
import numpy as np

@shared_task
def add_batch_standard(metadata, csv_file):
    """
    handle a csv fil of standards
    header line should be "mcfid","name","formula", "inchi", "solubility", "vendor","vendor_id", "hmdb_id" , "chebi_id", "lipidmaps_id", "cas_id", "pubchem_id". "date","location","lot_num"

    To Be Set:
    ### Standard
    # mandatory
    molecule = models.ForeignKey(Molecule, default=Molecule.objects.all().filter(name='DUMMY'))
    MCFID = models.IntegerField(null=True, blank=True)# if blank MCFID == Standard.pk
    # optional
    vendor = models.TextField(null=True, blank=True)
    vendor_cat = models.TextField(null=True, blank=True)
    lot_num = models.TextField(null=True, blank=True)
    location = models.TextField(null=True, blank=True)
    purchase_date = models.DateField(null=True, blank=True)

    If Not Existing:
    ### Molecule
    # mandatory
    name = models.TextField(default = "")
    sum_formula = models.TextField(null=True)
    pubchem_id = models.TextField(null=True, blank=True)
    # Optional
    inchi_code = models.TextField(default="")
    exact_mass = models.FloatField(default=0.0)
    solubility   = models.TextField(null=True, blank=True)
    # External reference numbers
    hmdb_id = models.TextField(null=True, blank=True)
    chebi_id = models.TextField(null=True, blank=True)
    lipidmaps_id = models.TextField(null=True, blank=True)
    cas_id = models.TextField(null=True, blank=True)
    :param csv_file:
    :return:
    """
    error_list = []
    df = pd.read_csv(csv_file, sep="\t")
    logging.info('I read the file')
    df.columns = [x.replace(" ", "_").lower() for x in df.columns]
    logging.info("I replaced columns")
    # Fill missing values
    df['id'] = df['id'].astype(str)
    df.fillna("", inplace=True)
    logging.info("Shape: {}".format(df.shape))
    for row in df.iterrows():
        logging.info("row: {}".format(row))
        try:
            # clean up input
            entry = row[1]
            if entry['formula'] == '':
                raise ValueError('sum formula cannot be blank')
            # for tag in entry.keys():
            #    if entry[tag] != "":
            #        entry[tag] = entry[tag].encode("utf8") # make strings safe


            if entry['pubchem_id'] != "":
                molecule = Molecule.objects.all().filter(pubchem_id=entry['pubchem_id'])
            else:
                molecule = Molecule.objects.all().filter(name__iexact=entry['name'])  # filter lowercase

            if molecule.exists():
                molecule = molecule[0]
            else:
                molecule = Molecule(
                    name=entry["name"],
                    sum_formula=entry["formula"],
                    inchi_code=entry["inchi"],
                    solubility=entry["solubility"],
                    hmdb_id=entry["hmdb_id"],
                    chebi_id=entry["chebi_id"],
                    lipidmaps_id=entry["lipidmaps_id"],
                    cas_id=entry["cas_id"],
                    pubchem_id=entry["pubchem_id"],
                )
                logging.info("about to save " + molecule.name)
                logging.info(molecule)
                molecule.save()
                logging.info("Successfully saved " + molecule.name)
            if entry['id'] == 'nan':
                s = Standard(molecule=molecule)
            else:
                s = Standard.objects.all().filter(inventory_id=entry['id'])
                if s.exists():  # standard already added, overwrite
                    s = s[0]
                    s.molecule=molecule
                else:
                    s = Standard(molecule=molecule)
            s.vendor = entry["vendor"]
            s.vendor_cat = entry["vendor_id"]
            s.save()
        except:
            error_list.append([entry['name'], sys.exc_info()[1]])
            logging.warning("Failed for: {} with {}".format(entry['name'], sys.exc_info()[1]))

    return error_list


@shared_task
def handle_uploaded_files(metadata, mzml_filepath, d):
    print(__file__ + str(d.id))
    logger = logging.getLogger(__file__ + str(d.id))
    logger.addHandler(DatabaseLogHandler(d, level=logging.DEBUG))
    logger.debug("adding dataset {}".format(mzml_filepath))
    try:
        msrun = pymzml.run.Reader(mzml_filepath)
        ppm = float(metadata['mass_accuracy_ppm'])
        mz_tol_quad = float(metadata['quad_window_mz'])
        ionization_method = metadata['ionization_method']
        ion_analyzer = metadata['ion_analyzer']
        scan_time = []
{'fraction_non_alphanumeric': 0.1253831701769999, 'fraction_numerical': 0.0016810046474834371, 'mean_word_length': 2.622492836676218, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5003631', 'n_tokens_mistral': 3095, 'n_tokens_neox': 2942, 'n_words': 619}