<commit_before>"""Validate post-ETL EIA 860 data and the associated derived outputs."""
import logging

import pytest

from pudl import validate as pv

logger = logging.getLogger(__name__)


@pytest.mark.parametrize(
    "df_name,cols", [
        ("bf_eia923", "all"),
        ("bga_eia860", "all"),
        ("frc_eia923", "all"),
        ("gen_eia923", "all"),
        ("gens_eia860", "all"),
        ("gf_eia923", "all"),
        ("gf_nonuclear_eia923", "all"),
        ("gf_nuclear_eia923", "all"),
        ("own_eia860", "all"),
        ("plants_eia860", "all"),
        ("pu_eia860", "all"),
        ("utils_eia860", "all"),
    ])
def test_no_null_cols_eia(pudl_out_eia, live_dbs, cols, df_name):
    """Verify that output DataFrames have no entirely NULL columns."""
    if not live_dbs:
        pytest.skip("Data validation only works with a live PUDL DB.")
    pv.no_null_cols(
        pudl_out_eia.__getattribute__(df_name)(),
        cols=cols, df_name=df_name)


@pytest.mark.parametrize(
    "df_name,raw_rows,monthly_rows,annual_rows", [
        ('bf_eia923', 1_309_942, 1_309_942, 109_807),
        ('bga_eia860', 117_930, 117_930, 117_930),
        ('frc_eia923', 560_377, 230_063, 22_686),
        ('gen_eia923', 559_570, 559_570, 46_718),
        ('gens_eia860', 490_824, 490_824, 490_824),
        ('gf_eia923', 2_507_870, 2_507_870, 214_709),
        ('gf_nonuclear_eia923', 2_492_481, 2_492_481, 213_422),
        ('gf_nuclear_eia923', 23_498, 23_498, 1_964),
        ('own_eia860', 79_311, 79_311, 79_311),
        ('plants_eia860', 171_148, 171_148, 171_148),
        ('pu_eia860', 170_370, 170_370, 170_370),
        ('utils_eia860', 113_357, 113_357, 113_357),
    ]
)
def test_minmax_rows(
    pudl_out_eia,
    live_dbs,
    raw_rows,
    annual_rows,
    monthly_rows,
    df_name
):
    """Verify that output DataFrames don't have too many or too few rows.<commit_msg>Adjust expected row counts for eia860m-2012-12<commit_after>"""Validate post-ETL EIA 860 data and the associated derived outputs."""
import logging

import pytest

from pudl import validate as pv

logger = logging.getLogger(__name__)


@pytest.mark.parametrize(
    "df_name,cols", [
        ("bf_eia923", "all"),
        ("bga_eia860", "all"),
        ("frc_eia923", "all"),
        ("gen_eia923", "all"),
        ("gens_eia860", "all"),
        ("gf_eia923", "all"),
        ("gf_nonuclear_eia923", "all"),
        ("gf_nuclear_eia923", "all"),
        ("own_eia860", "all"),
        ("plants_eia860", "all"),
        ("pu_eia860", "all"),
        ("utils_eia860", "all"),
    ])
def test_no_null_cols_eia(pudl_out_eia, live_dbs, cols, df_name):
    """Verify that output DataFrames have no entirely NULL columns."""
    if not live_dbs:
        pytest.skip("Data validation only works with a live PUDL DB.")
    pv.no_null_cols(
        pudl_out_eia.__getattribute__(df_name)(),
        cols=cols, df_name=df_name)


@pytest.mark.parametrize(
    "df_name,raw_rows,monthly_rows,annual_rows", [
        ('bf_eia923', 1_309_942, 1_309_942, 109_807),
        ('bga_eia860', 117_930, 117_930, 117_930),
        ('frc_eia923', 560_377, 230_063, 22_686),
        ('gen_eia923', 559_570, 559_570, 46_718),
        ('gens_eia860', 491_469, 491_469, 491_469),
        ('gf_eia923', 2_507_870, 2_507_870, 214_709),
        ('gf_nonuclear_eia923', 2_492_481, 2_492_481, 213_422),
        ('gf_nuclear_eia923', 23_498, 23_498, 1_964),
        ('own_eia860', 79_311, 79_311, 79_311),
        ('plants_eia860', 171_570, 171_570, 171_570),
        ('pu_eia860', 170_792, 170_792, 170_792),
        ('utils_eia860', 113_464, 113_464, 113_464),
    ]
)
def test_minmax_rows(
    pudl_out_eia,
    live_dbs,
    raw_rows,
    annual_rows,
    monthly_rows,
    df_name
):
    """Verify that output DataFrames don't have too many or too few rows.
{'fraction_non_alphanumeric': 0.1447609093284557, 'fraction_numerical': 0.1528612490201202, 'mean_word_length': 3.2533333333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7288529', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1522, 'n_words': 300}