<commit_before>import smtplib
import mimetypes
import itertools
import time
import sys
import os
import glob
import pandas as pd
import metatlas.metatlas_objects as metob
import multiprocessing as mp
import numpy as np

from datetime import datetime, time as dtime
from collections import defaultdict
from metatlas.mzml_loader import VERSION_TIMESTAMP
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email import encoders
from subprocess import check_output
from metatlas.helpers import metatlas_get_data_helper_fun as ma_data
from metatlas.metatlas_objects import find_invalid_runs
from metatlas.system_utils import send_mail


# TO-DO: have these vars be defined from external YML (probably nersc.yml?)

# the lower the tolerance, the more relevant data you might see
# at a cost of being less flexible with defined peaks
# default is 15
tolerance = 15
# the lower the threshold, the more "peaks" you'll see
threshold = 1e5
# change standard if needed. This is ABMBA
std = 229.981116596
std_neg = 227.966564596

run_times = {}
save_path = '/project/projectdirs/metatlas/projects/istd_logs/'


def check_metatlas():
    """
    Checks for invalid runs.
    """
    invalid_runs = find_invalid_runs(_override=True)

    if invalid_runs:
        grouped = defaultdict(list)
        for run in invalid_runs:
            grouped[run.username].append(run.mzml_file)
        for (username, filenames) in grouped.items():<commit_msg>added comments to check_metatlas<commit_after>import smtplib
import mimetypes
import itertools
import time
import sys
import os
import glob
import pandas as pd
import metatlas.metatlas_objects as metob
import multiprocessing as mp
import numpy as np

from datetime import datetime, time as dtime
from collections import defaultdict
from metatlas.mzml_loader import VERSION_TIMESTAMP
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email import encoders
from subprocess import check_output
from metatlas.helpers import metatlas_get_data_helper_fun as ma_data
from metatlas.metatlas_objects import find_invalid_runs
from metatlas.system_utils import send_mail


# TO-DO: have these vars be defined from external YML (probably nersc.yml?)

# the lower the tolerance, the more relevant data you might see
# at a cost of being less flexible with defined peaks
# default is 15
tolerance = 15
# the lower the threshold, the more "peaks" you'll see
threshold = 1e5
# change standard if needed. This is ABMBA
std = 229.981116596
std_neg = 227.966564596

run_times = {}
save_path = '/project/projectdirs/metatlas/projects/istd_logs/'


def check_metatlas():
    """
    Checks for invalid runs.
    """
    invalid_runs = find_invalid_runs(_override=True)
    # Currently there exists a bug (?) where it shows empty files.
    # It could be a result of the database remembering invalid files
    # that were removed. Since there is no file associated with these runs,
    # there will not be any filename.
    # This can technically be fixed by running:
    # grouped = grouped.filter(False, str_list)
    # followed by a run to remove all usernames without any files,
    # but does not directly solve the problem.
    # Leaving this to someone in the future to resolve.
    if invalid_runs:
        grouped = defaultdict(list)
        for run in invalid_runs:
            grouped[run.username].append(run.mzml_file)
        for (username, filenames) in grouped.items():
{'fraction_non_alphanumeric': 0.05445823927765237, 'fraction_numerical': 0.016930022573363433, 'mean_word_length': 4.275297619047619, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4125586', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1048, 'n_words': 404}