<commit_before># -*- coding: utf-8 -*-
"""
Utilities for assisting with the benchmarking analysis process.
"""
from collections import namedtuple
import pandas as pd
import numpy as np

def calendar_to_fiscal(cal_year, cal_mo):
    """Converts a calendar year and month into a fiscal year and month.
    Returns (fiscal_year, fical_month) tuple.
    """
    if cal_mo <= 6:
        fiscal_year = cal_year
        fiscal_month = cal_mo + 6
    else:
        fiscal_year = cal_year + 1
        fiscal_month = cal_mo - 6

    return fiscal_year, fiscal_month

def fiscal_to_calendar(fiscal_year, fiscal_mo):
    """Converts a fiscal year and month into a calendar year and month for graphing purposes.
    Returns (calendar_year, calendar_month) tuple."""
    
    if fiscal_mo > 6:
        calendar_month = fiscal_mo - 6
        calendar_year = fiscal_year
    else:
        calendar_month = fiscal_mo + 6
        calendar_year = fiscal_year - 1
        
    return (calendar_year, calendar_month)

# Fiscal Month labels, both as a list and as a dictionary
mo_list = [
    'Jul',
    'Aug',
    'Sep',
    'Oct',
    'Nov',
    'Dec',
    'Jan',
    'Feb',
    'Mar',
    'Apr',
    'May',
    'Jun'
]

mo_dict = dict(zip(range(1,13), mo_list))

# A dictionary used to map column names to more standardized variable names.
name_changes = {
    'Electricity': 'electricity',
    'Natural Gas': 'natural_gas',
    'Oil #1': 'fuel_oil',
    'Refuse': 'refuse',
    'Sewer': 'sewer',
    'Steam': 'district_heat',
    'Water': 'water',
    'Total': 'total'
}

# If a function is needed to change one name, here is one:
def change_name(old_name):
    """This returns a new name if the old name is in the "name_changes"
    ditionary.  Otherwise, it just returns the old name.
    """
    return name_changes.get(old_name, old_name)

PeriodSplit = namedtuple('PeriodSplit', 'cal_year cal_mo bill_frac days_served')
def split_period(start_date, end_date):
    """Splits a range of service dates from a utility bill into pieces that
    fit within calendar months. For each piece, the number of days in that piece 
    and the fraction of the original date range are returned in a namedtuple.
    For the first and last day in the date range, it is assumed that only half
    the day is served (this is typically the meter reading day).
    """
    # make a daily series.  The value is the fraction of the day served,
    # 1.0 for days except the first and last.
    ser = pd.Series(data=1.0, index=pd.date_range(start_date, end_date))
    
    # the half days at the beginning and end
    ser.iloc[0] = 0.5
    ser.iloc[-1] = 0.5
    
    tot_days = ser.sum()    # total days served in the bill
    
    # break into monthly pieces and add up the days served
    pieces = ser.resample('M').sum()
    
    result = []
    for dt, days in pieces.items():
        result.append(
            PeriodSplit(cal_year=dt.year, cal_mo=dt.month, bill_frac=days/tot_days, days_served=days)
        )
    return result

def months_present(df, yr_col='fiscal_year', mo_col='fiscal_mo'):
    """Returns a list of the year/months present in a DataFrame.  Each item
    of the list is a two-tuple: (year, mo), and list is sorted from earliest
    to latest month.
    yr_col: the name of the column in the DataFrame containing the year.
    mo_col: the name of the column in the DataFrame containing the month.
    """
    yr_mo = set(zip(df[yr_col], df[mo_col]))
    yr_mo = list(yr_mo)
    yr_mo.sort()
    return yr_mo

def month_count(mo_present):
    """Returns a Pandas series that gives the number of months present for each
    year in the 'mo_present' list.  The 'mo_present' list is a list of two
    tuples:  (year, mo).  The returned series is indexed on year.
    """
    return pd.DataFrame(data=mo_present, columns=['year', 'month']).groupby('year').count()['month']

def add_month_count_column(df_target, df_detail, yr_col='fiscal_year', mo_col='fiscal_mo'):
    """Adds a 'month_count' column to the 'df_target' DataFrame, whose index
    must be a year value.  The value in the 'month_count' column is the number
    of months that are present in the detailed DataFrame 'df_detail' for each 
    year.  'yr_col' and 'mo_col' give the names of the columns in 'df_detail'
    that are used to determine the month counts by year.
    """
    mo_present = months_present(df_detail)
    mo_count = month_count(mo_present)
    df_target['month_count'] = mo_count

# List of all possible services
# I'm leaving out Oil #2 here.
all_services = [
        'Oil #1',
        'Natural Gas',
        'Electricity',
        'Steam',
        'Water',
        'Sewer',
        'Refuse'
]

all_energy_services = [
    'Oil #1',
    'Natural Gas',
    'Electricity',
    'Steam'
]

all_heat_services = [
    'Oil #1',
    'Natural Gas',
    'Steam'
]

def missing_services(services_present):
    """Returns a list of the Service Types that are *not* present in the 
    'services_present' input list.
    """
    return list(set(all_services) - set(services_present))

def missing_energy_services(services_present):
    """Returns a list of the Energy-related Service Types that are *not* present
    in the 'services_present' input list.
    """
    return list(set(all_energy_services) - set(services_present))

def add_columns(df, col_names):
    """Adds a new column to the DataFrame df for each column name in the list
    'col_names'.  Sets all the values to 0.0 for that column.
    """
    for col in col_names:
        df[col] = 0.0

def add_missing_columns(df_in, required_columns, fill_val=0.0):
    """Adds columns to the DataFrame 'df' if it does not contain all of the
    columns in the list 'required_columns'.  'fill_val' is the value that is used
    to fill the new columns.
    """
    missing_cols = set(required_columns) - set(df_in.columns)
    for col in missing_cols:
        df_in[col] = fill_val

    return df_in

def df_to_dictionaries(df, change_names={}, include_index=True):
    """Returns a list of dictionaries, one dictionary for each row in the 
    DataFrame 'df'.  The keys of the dictionary match the DataFrame column names,
    except for any substitututions provided in the 'change_names' dictionary;
    {'old_name': 'new_name', etc}.  If 'include_index' is True, the index 
    values are included in the dictionary keyed on the index name (unless changed
    in the 'change_names' dictionary)
    """
    # make a list of the final names to use
    names = list(df.columns.values)
    if include_index:
        names = [df.index.name] + names
        
    # apply name substitutions
    for i in range(len(names)):
        names[i] = change_names.get(names[i], names[i])
    
    result = []
    for ix, row in df.iterrows():
        vals = list(row.values)<commit_msg>Changed function documentation.<commit_after># -*- coding: utf-8 -*-
"""
Utilities for assisting with the benchmarking analysis process.
"""
from collections import namedtuple
import pandas as pd
import numpy as np

def calendar_to_fiscal(cal_year, cal_mo):
    """Converts a calendar year and month into a fiscal year and month.
    Returns (fiscal_year, fical_month) tuple.
    """
    if cal_mo <= 6:
        fiscal_year = cal_year
        fiscal_month = cal_mo + 6
    else:
        fiscal_year = cal_year + 1
        fiscal_month = cal_mo - 6

    return fiscal_year, fiscal_month

def fiscal_to_calendar(fiscal_year, fiscal_mo):
    """Converts a fiscal year and month into a calendar year and month for graphing purposes.
    Returns (calendar_year, calendar_month) tuple."""
    
    if fiscal_mo > 6:
        calendar_month = fiscal_mo - 6
        calendar_year = fiscal_year
    else:
        calendar_month = fiscal_mo + 6
        calendar_year = fiscal_year - 1
        
    return (calendar_year, calendar_month)

# Fiscal Month labels, both as a list and as a dictionary
mo_list = [
    'Jul',
    'Aug',
    'Sep',
    'Oct',
    'Nov',
    'Dec',
    'Jan',
    'Feb',
    'Mar',
    'Apr',
    'May',
    'Jun'
]

mo_dict = dict(zip(range(1,13), mo_list))

# A dictionary used to map column names to more standardized variable names.
name_changes = {
    'Electricity': 'electricity',
    'Natural Gas': 'natural_gas',
    'Oil #1': 'fuel_oil',
    'Refuse': 'refuse',
    'Sewer': 'sewer',
    'Steam': 'district_heat',
    'Water': 'water',
    'Total': 'total'
}

# If a function is needed to change one name, here is one:
def change_name(old_name):
    """This returns a new name if the old name is in the "name_changes"
    ditionary.  Otherwise, it just returns the old name.
    """
    return name_changes.get(old_name, old_name)

PeriodSplit = namedtuple('PeriodSplit', 'cal_year cal_mo bill_frac days_served')
def split_period(start_date, end_date):
    """Splits a range of service dates from a utility bill into pieces that
    fit within calendar months. For each piece, the number of days in that piece 
    and the fraction of the original date range are returned in a namedtuple.
    For the first and last day in the date range, it is assumed that only half
    the day is served (this is typically the meter reading day).
    """
    # make a daily series.  The value is the fraction of the day served,
    # 1.0 for days except the first and last.
    ser = pd.Series(data=1.0, index=pd.date_range(start_date, end_date))
    
    # the half days at the beginning and end
    ser.iloc[0] = 0.5
    ser.iloc[-1] = 0.5
    
    tot_days = ser.sum()    # total days served in the bill
    
    # break into monthly pieces and add up the days served
    pieces = ser.resample('M').sum()
    
    result = []
    for dt, days in pieces.items():
        result.append(
            PeriodSplit(cal_year=dt.year, cal_mo=dt.month, bill_frac=days/tot_days, days_served=days)
        )
    return result

def months_present(df, yr_col='fiscal_year', mo_col='fiscal_mo'):
    """Returns a list of the year/months present in a DataFrame.  Each item
    of the list is a two-tuple: (year, mo), and list is sorted from earliest
    to latest month.
    yr_col: the name of the column in the DataFrame containing the year.
    mo_col: the name of the column in the DataFrame containing the month.
    """
    yr_mo = set(zip(df[yr_col], df[mo_col]))
    yr_mo = list(yr_mo)
    yr_mo.sort()
    return yr_mo

def month_count(mo_present):
    """Returns a Pandas series that gives the number of months present for each
    year in the 'mo_present' list.  The 'mo_present' list is a list of two
    tuples:  (year, mo).  The returned series is indexed on year.
    """
    return pd.DataFrame(data=mo_present, columns=['year', 'month']).groupby('year').count()['month']

def add_month_count_column(df_target, df_detail, yr_col='fiscal_year', mo_col='fiscal_mo'):
    """Adds a 'month_count' column to the 'df_target' DataFrame, whose index
    must be a year value.  The value in the 'month_count' column is the number
    of months that are present in the detailed DataFrame 'df_detail' for each 
    year.  'yr_col' and 'mo_col' give the names of the columns in 'df_detail'
    that are used to determine the month counts by year.
    """
    mo_present = months_present(df_detail)
    mo_count = month_count(mo_present)
    df_target['month_count'] = mo_count

# List of all possible services
# I'm leaving out Oil #2 here.
all_services = [
        'Oil #1',
        'Natural Gas',
        'Electricity',
        'Steam',
        'Water',
        'Sewer',
        'Refuse'
]

all_energy_services = [
    'Oil #1',
    'Natural Gas',
    'Electricity',
    'Steam'
]

all_heat_services = [
    'Oil #1',
    'Natural Gas',
    'Steam'
]

def missing_services(services_present):
    """Returns a list of the Service Types that are *not* present in the 
    'services_present' input list.
    """
    return list(set(all_services) - set(services_present))

def missing_energy_services(services_present):
    """Returns a list of the Energy-related Service Types that are *not* present
    in the 'services_present' input list.
    """
    return list(set(all_energy_services) - set(services_present))

def add_columns(df, col_names):
    """Adds a new column to the DataFrame df for each column name in the list
    'col_names'.  Sets all the values to 0.0 for that column.
    """
    for col in col_names:
        df[col] = 0.0

def add_missing_columns(df_in, required_columns, fill_val=0.0):
    """Adds columns to the DataFrame 'df_in' if it does not contain all of the
    columns in the list 'required_columns'.  'fill_val' is the value that is used
    to fill the new columns.
    """
    missing_cols = set(required_columns) - set(df_in.columns)
    for col in missing_cols:
        df_in[col] = fill_val

    return df_in

def df_to_dictionaries(df, change_names={}, include_index=True):
    """Returns a list of dictionaries, one dictionary for each row in the 
    DataFrame 'df'.  The keys of the dictionary match the DataFrame column names,
    except for any substitututions provided in the 'change_names' dictionary;
    {'old_name': 'new_name', etc}.  If 'include_index' is True, the index 
    values are included in the dictionary keyed on the index name (unless changed
    in the 'change_names' dictionary)
    """
    # make a list of the final names to use
    names = list(df.columns.values)
    if include_index:
        names = [df.index.name] + names
        
    # apply name substitutions
    for i in range(len(names)):
        names[i] = change_names.get(names[i], names[i])
    
    result = []
    for ix, row in df.iterrows():
        vals = list(row.values)
{'fraction_non_alphanumeric': 0.09715378392292418, 'fraction_numerical': 0.004854011914392881, 'mean_word_length': 3.0422116527942924, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6834310', 'n_tokens_mistral': 4373, 'n_tokens_neox': 4259, 'n_words': 1545}