<commit_before>"""Module to perform data cleaning functions on EPA IPM data tables."""

import logging
import pandas as pd
import numpy as np
import pudl
import pudl.constants as pc
from pudl.helpers import simplify_columns

logger = logging.getLogger(__name__)


def load_curves(epaipm_dfs, epaipm_transformed_dfs):
    """
    Pull and transform the load curve table from wide to tidy format.

    Args:
        epaipm_dfs (dictionary of pandas.DataFrame): Each entry in this
            dictionary of DataFrame objects corresponds to a table from
            EPA's IPM, as reported in the Excel spreadsheets they distribute.
        epa_ipm_transformed_dfs (dictionary of DataFrames)

    Returns: transformed dataframe.

    """
    lc = epaipm_dfs['load_curves_ipm'].copy()
    lc = simplify_columns(lc)
    # Melt the load curves
    melt_lc = lc.melt(
        id_vars=['region', 'month', 'day'],
        var_name='hour',
        value_name='load_mw'
    )

    melt_lc['hour'] = (
        melt_lc['hour'].str.replace('hour_', '').astype(int)
    )
    # IPM hour designations are 1-24. Convert to 0-23 to match datetime.
    melt_lc['hour'] -= 1

    # Group to easily create 8760 time_index
    grouped = melt_lc.groupby('region')

    df_list = []
    for _, df in grouped:
        df = df.sort_values(['month', 'day', 'hour'])
        df = df.reset_index(drop=True)
        df['time_index'] = df.index + 1
        df_list.append(df)

    tidy_load_curves = pd.concat(df_list)

    epaipm_transformed_dfs['load_curves_ipm'] = tidy_load_curves

    return epaipm_transformed_dfs


def transmission_single(epaipm_dfs, epaipm_transformed_dfs):
    """
    Pull and transform the transmission constraints between individual regions
    table, renaming columns.

    Args:
        epaipm_dfs (dictionary of pandas.DataFrame): Each entry in this
            dictionary of DataFrame objects corresponds to a table from
            EPA's IPM, as reported in the Excel spreadsheets they distribute.
        epa_ipm_transformed_dfs (dictionary of DataFrames)
<commit_msg>Add line to rename load curve column<commit_after>"""Module to perform data cleaning functions on EPA IPM data tables."""

import logging
import pandas as pd
import numpy as np
import pudl
import pudl.constants as pc
from pudl.helpers import simplify_columns

logger = logging.getLogger(__name__)


def load_curves(epaipm_dfs, epaipm_transformed_dfs):
    """
    Pull and transform the load curve table from wide to tidy format.

    Args:
        epaipm_dfs (dictionary of pandas.DataFrame): Each entry in this
            dictionary of DataFrame objects corresponds to a table from
            EPA's IPM, as reported in the Excel spreadsheets they distribute.
        epa_ipm_transformed_dfs (dictionary of DataFrames)

    Returns: transformed dataframe.

    """
    lc = epaipm_dfs['load_curves_ipm'].copy()
    lc = simplify_columns(lc)
    # Melt the load curves
    melt_lc = lc.melt(
        id_vars=['region', 'month', 'day'],
        var_name='hour',
        value_name='load_mw'
    )

    melt_lc['hour'] = (
        melt_lc['hour'].str.replace('hour_', '').astype(int)
    )
    # IPM hour designations are 1-24. Convert to 0-23 to match datetime.
    melt_lc['hour'] -= 1

    # Group to easily create 8760 time_index
    grouped = melt_lc.groupby('region')

    df_list = []
    for _, df in grouped:
        df = df.sort_values(['month', 'day', 'hour'])
        df = df.reset_index(drop=True)
        df['time_index'] = df.index + 1
        df_list.append(df)

    tidy_load_curves = pd.concat(df_list)
    tidy_load_curves = tidy_load_curves.rename(
        columns=pc.epaipm_rename_dict['load_curves_ipm']
    )

    epaipm_transformed_dfs['load_curves_ipm'] = tidy_load_curves

    return epaipm_transformed_dfs


def transmission_single(epaipm_dfs, epaipm_transformed_dfs):
    """
    Pull and transform the transmission constraints between individual regions
    table, renaming columns.

    Args:
        epaipm_dfs (dictionary of pandas.DataFrame): Each entry in this
            dictionary of DataFrame objects corresponds to a table from
            EPA's IPM, as reported in the Excel spreadsheets they distribute.
        epa_ipm_transformed_dfs (dictionary of DataFrames)

{'fraction_non_alphanumeric': 0.08516612072999533, 'fraction_numerical': 0.005615348619560131, 'mean_word_length': 3.134429400386847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3794565', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1323, 'n_words': 389}