<commit_before>import pandas as pd
import genie
import pytest

DATABASE_DF = pd.DataFrame({
    'UNIQUE_KEY': ['test1', 'test2', 'test3'],
    "test": ['test1', 'test2', 'test3'],
    "foo": [1, 2, 3],
    "baz": [float('nan'), float('nan'), float('nan')]})
DATABASE_DF.index = ['1_3', '2_3', '3_5']


def test_valid__check_valid_df():
    genie.process_functions._check_valid_df(DATABASE_DF, "test")


def test_invalid__check_valid_df():
    with pytest.raises(ValueError, match="Must pass in pandas dataframe"):
        genie.process_functions._check_valid_df("foo", "test")
    with pytest.raises(
            ValueError,
            match="'error' column must exist in dataframe"):
        genie.process_functions._check_valid_df(DATABASE_DF, "error")


def test__get_left_diff_df():
    new_datadf = pd.DataFrame({
        'UNIQUE_KEY': ['test1', 'test2', 'test3', 'test4'],
        "test": ['test1', 'test2', 'test3', 'test4'],
        "foo": [1, 2, 3, 4],
        "baz": [float('nan'), float('nan'), float('nan'), 3.2]})
    get_diff = genie.process_functions._get_left_diff_df(
        new_datadf, DATABASE_DF, 'UNIQUE_KEY')
    expecteddf = new_datadf.loc[[3]]
    assert get_diff.equals(expecteddf[get_diff.columns])


def test_norows_get_left_diff_df():
    append_rows = genie.process_functions._get_left_diff_df(
        DATABASE_DF, DATABASE_DF, 'UNIQUE_KEY')
    assert append_rows.empty


def test_validation_get_left_diff_df():
    '''
    This checks to make sure that validation is called
    - In a situation where someone comments out the validation
    line, this will cause an error
    '''
    with pytest.raises(
            ValueError,
            match="'FOO' column must exist in dataframe"):
        genie.process_functions._get_left_diff_df(
            DATABASE_DF, DATABASE_DF, 'FOO')

<commit_msg>get left union calls checkdf twice, so must validate twice<commit_after>import pandas as pd
import genie
import pytest

DATABASE_DF = pd.DataFrame({
    'UNIQUE_KEY': ['test1', 'test2', 'test3'],
    "test": ['test1', 'test2', 'test3'],
    "foo": [1, 2, 3],
    "baz": [float('nan'), float('nan'), float('nan')]})
DATABASE_DF.index = ['1_3', '2_3', '3_5']


def test_valid__check_valid_df():
    genie.process_functions._check_valid_df(DATABASE_DF, "test")


def test_invalid__check_valid_df():
    with pytest.raises(ValueError, match="Must pass in pandas dataframe"):
        genie.process_functions._check_valid_df("foo", "test")
    with pytest.raises(
            ValueError,
            match="'error' column must exist in dataframe"):
        genie.process_functions._check_valid_df(DATABASE_DF, "error")


def test__get_left_diff_df():
    new_datadf = pd.DataFrame({
        'UNIQUE_KEY': ['test1', 'test2', 'test3', 'test4'],
        "test": ['test1', 'test2', 'test3', 'test4'],
        "foo": [1, 2, 3, 4],
        "baz": [float('nan'), float('nan'), float('nan'), 3.2]})
    get_diff = genie.process_functions._get_left_diff_df(
        new_datadf, DATABASE_DF, 'UNIQUE_KEY')
    expecteddf = new_datadf.loc[[3]]
    assert get_diff.equals(expecteddf[get_diff.columns])


def test_norows_get_left_diff_df():
    append_rows = genie.process_functions._get_left_diff_df(
        DATABASE_DF, DATABASE_DF, 'UNIQUE_KEY')
    assert append_rows.empty


def test_validation_get_left_diff_df():
    '''
    This checks to make sure that validation is called
    - In a situation where someone comments out the validation
    line, this will cause an error
    '''
    with pytest.raises(
            ValueError,
            match="'FOO' column must exist in dataframe"):
        genie.process_functions._get_left_diff_df(
            DATABASE_DF, DATABASE_DF, 'FOO')


def test_first_validation_get_left_union_df():
    '''
    This checks to make sure that validation is called
    - In a situation where someone comments out the 1st validation
    line, this will cause an error
    '''
    with pytest.raises(
            ValueError,
            match="'FOO' column must exist in dataframe"):
        genie.process_functions._get_left_union_df(
            DATABASE_DF, DATABASE_DF, 'FOO')


def test_second_validation_get_left_union_df():
    '''
    This checks to make sure that validation is called
    - In a situation where someone comments out the 2nd validation
    line, this will cause an error
    '''
    testing = DATABASE_DF.copy()
    testing['FOO'] = float('nan')
    with pytest.raises(
            ValueError,
            match="'FOO' column must exist in dataframe"):
        genie.process_functions._get_left_union_df(
            DATABASE_DF, DATABASE_DF, 'FOO')


{'fraction_non_alphanumeric': 0.12756756756756757, 'fraction_numerical': 0.013405405405405406, 'mean_word_length': 3.4523580365736284, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '349960', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1566, 'n_words': 367}