<commit_before>"""
Implements the EN increasing depth check.
"""

import EN_spike_and_step_check
import numpy as np
from collections import Counter
import util.main as main

def test(p, parameters):
    """
    Runs the quality control check on profile p and returns a numpy array
    of quality control decisions with False where the data value has
    passed the check and True where it failed.
    """

    # Check if the QC of this profile was already done and if not
    # run the QC.
    query = 'SELECT en_increasing_depth_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
    qc_log = main.dbinteract(query)
    qc_log = main.unpack_row(qc_log[0])
    if qc_log[0] is not None:
        return qc_log[0]

    return run_qc(p, parameters)

def mask_index(mat, index):
    """
    update comparison matrix by setting (index,j) and (i,index) to 0 for all i,j
    corresponds to recomputing the matrix after qc[index] is set True.
    """

    n = len(mat)

    for i in range(n):
        mat[index, i] = 0
        mat[i, index] = 0

def run_qc(p, parameters):

    # Get z values from the profile.
    d    = p.z()
    mask = d.mask
    n    = p.n_levels()

    # Initialize qc array.
    qc = np.zeros(n, dtype=bool)

    # don't flag single-level profiles
    if n == 1:
        return qc

    # if all the depths are the same, flag all levels and finish immediately
    most_common_depth = Counter(d.data).most_common(1)
    if most_common_depth[0][1] == len(d.data):
        qc = np.ones(n, dtype=bool)
        uid = p.uid()
        return qc

    # Basic check on each level.
    qc[d < 0]     = True
    qc[d > 11000] = True

    # Now check for inconsistencies in the depth levels.
    comp       = np.ndarray((n, n), dtype=int)
    currentMax = 1

    # initialize matrix
    # Comp gets set to 1 if there is not an increase in depth.
    comp[:, :] = 0
    for i in range(n):
        if qc[i] or mask[i]: continue
        for j in range(n):<commit_msg>comments from Simon<commit_after>"""
Implements the EN increasing depth check.
"""

import EN_spike_and_step_check
import numpy as np
from collections import Counter
import util.main as main

def test(p, parameters):
    """
    Runs the quality control check on profile p and returns a numpy array
    of quality control decisions with False where the data value has
    passed the check and True where it failed.
    """

    # Check if the QC of this profile was already done and if not
    # run the QC.
    query = 'SELECT en_increasing_depth_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
    qc_log = main.dbinteract(query)
    qc_log = main.unpack_row(qc_log[0])
    if qc_log[0] is not None:
        return qc_log[0]

    return run_qc(p, parameters)

def mask_index(mat, index):
    """
    update comparison matrix by setting (index,j) and (i,index) to 0 for all i,j
    corresponds to recomputing the matrix after qc[index] is set True.
    """

    n = len(mat)

    for i in range(n):
        mat[index, i] = 0
        mat[i, index] = 0

def run_qc(p, parameters):

    # Get z values from the profile.
    d    = p.z()
    mask = d.mask
    n    = p.n_levels()

    # Initialize qc array.
    qc = np.zeros(n, dtype=bool)

    # Basic check on each level.
    qc[d < 0]     = True
    qc[d > 11000] = True

    # don't perform more sophisticated tests for single-level profiles
    if n == 1:
        return qc

    # if all the depths are the same, flag all levels and finish immediately
    most_common_depth = Counter(d.data).most_common(1)
    if most_common_depth[0][1] == len(d.data):
        qc = np.ones(n, dtype=bool)
        uid = p.uid()
        return qc

    # Now check for inconsistencies in the depth levels.
    comp       = np.ndarray((n, n), dtype=int)
    currentMax = 1

    # initialize matrix
    # Comp gets set to 1 if there is not an increase in depth.
    comp[:, :] = 0
    for i in range(n):
        if qc[i] or mask[i]: continue
        for j in range(n):
{'fraction_non_alphanumeric': 0.099601593625498, 'fraction_numerical': 0.009462151394422311, 'mean_word_length': 2.496083550913838, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1671025', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1286, 'n_words': 489}