<commit_before>#!/usr/bin/env python

from __future__ import print_function
from builtins import zip
from builtins import str
from builtins import range
import numpy as np
import os


## LOW-LEVEL VALIDATION FUNCTIONS

def requiredUnless(col_name, arg, dm, df, *args):
    """
    Arg is a string in the format "str1, str2, ..."
    Each string will be a column name.
    Col_name is required in df unless each column from arg is present.
    """
    # if column name is present, no need to check if it is required
    if col_name in df.columns:
        return None
    arg_list = arg.split(",")
    arg_list = [argument.strip('"') for argument in arg_list]
    msg = ""
    for a in arg_list:
        # ignore validations that reference a different table
        if "." in a:
            continue
        if a not in df.columns:
            msg += "{} column is required unless {} is present.  ".format(col_name, a)
    if msg:
        return msg
    else:
        return None
    return None


def requiredUnlessTable(col_name, arg, dm, df, con=None):
    """
    Col_name must be present in df unless
    arg (table_name) is present in contribution
    """
    table_name = arg
    if col_name in df.columns:
        return None
    elif not con:
        return None
    elif table_name in con.tables:
        return None
    else:
        return "{} column is required unless table {} is present".format(col_name, table_name)


def requiredIfGroup(col_name, arg, dm, df, *args):
    """
    Col_name is required if other columns of
    the group arg are present.
    """
    group_name = arg
    groups = set()
    columns = df.columns
    for col in columns:
        if col not in dm.index:
            continue
        group = dm.loc[col]['group']
        groups.add(group)
    if group_name in groups:
        if col_name in columns:
            return None
        else:
            return "{} column is required if column group {} is used".format(col_name, group_name)
    return None


def required(col_name, arg, dm, df, *args):
    """
    Col_name is required in df.columns.
    Return error message if not.
    """
    if col_name in df.columns:
        return None
    else:
        return '"{}" column is required'.format(col_name)

def isIn(row, col_name, arg, dm, df, con=None):
    """
    row[col_name] must contain a value from another column.
    If not, return error message.
    """
    #grade = df.apply(func, args=(validation_name, arg, dm), axis=1)
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif not con:
        return None
    # if it's in another table
    cell_values = [v.strip(" ") for v in cell_value.split(":")]
    if "." in arg:
        table_name, table_col_name = arg.split(".")
        if table_name not in con.tables:
            return "Must contain a value from {} table. Missing {} table.".format(table_name, table_name)
        if table_col_name not in con.tables[table_name].df.columns:
            return '{} table is missing "{}" column, which is required for validating "{}" column'.format(table_name, table_col_name, col_name)
        possible_values = con.tables[table_name].df[table_col_name].unique()
        for value in cell_values:
            if value not in possible_values:
                trunc_possible_values = [val.replace(' ', '') for val in possible_values if val]
                trunc_cell_value = cell_value.replace(' ', '')
                if trunc_cell_value not in trunc_possible_values:
                    if trunc_cell_value != value:
                        return 'This value (long): "{}" is not found in: {} column in {} table.  Also (short): {} is not in {}'.format(value, table_col_name, table_name, trunc_cell_value, arg)
                    else:
                        return 'This value: "{}" is not found in: {} column in {} table'.format(value, table_col_name, table_name)
                    break
    # if it's in the present table:
    else:
        possible_values = df[arg].unique()
        for value in cell_values:
            if value not in possible_values:
                return 'This value: "{}" is not found in: {} column'.format(value, arg)
                break
    return None

def checkMax(row, col_name, arg, *args):
    """
    row[col_name] must be less than or equal to arg.
    else, return error message.
    """
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif isinstance(cell_value, float):
        if np.isnan(cell_value):
            return None
    try:
        arg_val = float(arg)
    except ValueError:
        arg_val = row[arg]
    #arg = float(arg)
    try:
        if float(cell_value) <= float(arg_val):
            return None
        else:
            return "{} ({}) must be <= {} ({})".format(str(cell_value), col_name, str(arg_val), str(arg))
    # this happens when the value isn't a float (an error which will be caught elsewhere)
    except ValueError:
        return None

def checkMin(row, col_name, arg, *args):
    """
    row[col_name] must be greater than or equal to arg.
    else, return error message.
    """
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif isinstance(cell_value, float):
        if np.isnan(cell_value):
            return None
    try:
        arg_val = float(arg)
    except ValueError:
        if arg in row.index:
            arg_val = row[arg]
        else:
            return None
    try:
        if float(cell_value) >= float(arg_val):
            return None
        else:
            return "{} ({}) must be >= {} ({})".format(str(cell_value), col_name, arg_val, str(arg))
    # this happens when the value isn't a float (an error which will be caught elsewhere)
    except ValueError:
        return None

def cv(row, col_name, arg, current_data_model, df, con):
    """
    row[col_name] must contain only values from the appropriate controlled vocabulary<commit_msg>prevent validation error if trying to check for max/min but those values are missing<commit_after>#!/usr/bin/env python

from __future__ import print_function
from builtins import zip
from builtins import str
from builtins import range
import os
import numpy as np
from pmagpy import new_builder as nb


## LOW-LEVEL VALIDATION FUNCTIONS

def requiredUnless(col_name, arg, dm, df, *args):
    """
    Arg is a string in the format "str1, str2, ..."
    Each string will be a column name.
    Col_name is required in df unless each column from arg is present.
    """
    # if column name is present, no need to check if it is required
    if col_name in df.columns:
        return None
    arg_list = arg.split(",")
    arg_list = [argument.strip('"') for argument in arg_list]
    msg = ""
    for a in arg_list:
        # ignore validations that reference a different table
        if "." in a:
            continue
        if a not in df.columns:
            msg += "{} column is required unless {} is present.  ".format(col_name, a)
    if msg:
        return msg
    else:
        return None
    return None


def requiredUnlessTable(col_name, arg, dm, df, con=None):
    """
    Col_name must be present in df unless
    arg (table_name) is present in contribution
    """
    table_name = arg
    if col_name in df.columns:
        return None
    elif not con:
        return None
    elif table_name in con.tables:
        return None
    else:
        return "{} column is required unless table {} is present".format(col_name, table_name)


def requiredIfGroup(col_name, arg, dm, df, *args):
    """
    Col_name is required if other columns of
    the group arg are present.
    """
    group_name = arg
    groups = set()
    columns = df.columns
    for col in columns:
        if col not in dm.index:
            continue
        group = dm.loc[col]['group']
        groups.add(group)
    if group_name in groups:
        if col_name in columns:
            return None
        else:
            return "{} column is required if column group {} is used".format(col_name, group_name)
    return None


def required(col_name, arg, dm, df, *args):
    """
    Col_name is required in df.columns.
    Return error message if not.
    """
    if col_name in df.columns:
        return None
    else:
        return '"{}" column is required'.format(col_name)

def isIn(row, col_name, arg, dm, df, con=None):
    """
    row[col_name] must contain a value from another column.
    If not, return error message.
    """
    #grade = df.apply(func, args=(validation_name, arg, dm), axis=1)
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif not con:
        return None
    # if it's in another table
    cell_values = [v.strip(" ") for v in cell_value.split(":")]
    if "." in arg:
        table_name, table_col_name = arg.split(".")
        if table_name not in con.tables:
            return "Must contain a value from {} table. Missing {} table.".format(table_name, table_name)
        if table_col_name not in con.tables[table_name].df.columns:
            return '{} table is missing "{}" column, which is required for validating "{}" column'.format(table_name, table_col_name, col_name)
        possible_values = con.tables[table_name].df[table_col_name].unique()
        for value in cell_values:
            if value not in possible_values:
                trunc_possible_values = [val.replace(' ', '') for val in possible_values if val]
                trunc_cell_value = cell_value.replace(' ', '')
                if trunc_cell_value not in trunc_possible_values:
                    if trunc_cell_value != value:
                        return 'This value (long): "{}" is not found in: {} column in {} table.  Also (short): {} is not in {}'.format(value, table_col_name, table_name, trunc_cell_value, arg)
                    else:
                        return 'This value: "{}" is not found in: {} column in {} table'.format(value, table_col_name, table_name)
                    break
    # if it's in the present table:
    else:
        possible_values = df[arg].unique()
        for value in cell_values:
            if value not in possible_values:
                return 'This value: "{}" is not found in: {} column'.format(value, arg)
                break
    return None

def checkMax(row, col_name, arg, *args):
    """
    row[col_name] must be less than or equal to arg.
    else, return error message.
    """
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif isinstance(cell_value, float):
        if np.isnan(cell_value):
            return None
    try:
        arg_val = float(arg)
    except ValueError:
        if arg in row.index:
            arg_val = row[arg]
        else:
            return None
    if nb.is_null(arg_val):
        return None
    #arg = float(arg)
    try:
        if float(cell_value) <= float(arg_val):
            return None
        else:
            return "{} ({}) must be <= {} ({})".format(str(cell_value), col_name, str(arg_val), str(arg))
    # this happens when the value isn't a float (an error which will be caught elsewhere)
    except ValueError:
        return None

def checkMin(row, col_name, arg, *args):
    """
    row[col_name] must be greater than or equal to arg.
    else, return error message.
    """
    cell_value = row[col_name]
    if not cell_value:
        return None
    elif isinstance(cell_value, float):
        if np.isnan(cell_value):
            return None
    try:
        arg_val = float(arg)
    except ValueError:
        if arg in row.index:
            arg_val = row[arg]
        else:
            return None
    if nb.is_null(arg_val):
        return None
    try:
        if float(cell_value) >= float(arg_val):
            return None
        else:
            return "{} ({}) must be >= {} ({})".format(str(cell_value), col_name, arg_val, str(arg))
    # this happens when the value isn't a float (an error which will be caught elsewhere)
    except ValueError:
        return None

def cv(row, col_name, arg, current_data_model, df, con):
    """
    row[col_name] must contain only values from the appropriate controlled vocabulary
{'fraction_non_alphanumeric': 0.09656987743686765, 'fraction_numerical': 0.0004935428148391873, 'mean_word_length': 2.2534118276692534, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3485498', 'n_tokens_mistral': 3581, 'n_tokens_neox': 3510, 'n_words': 1261}