<jupyter_start><jupyter_text>--- IESO (Power) Data Analysis and Preparation---  Setup<jupyter_code># Libraries:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime

# set paths to data files
IESO_path = 'https://raw.githubusercontent.com/VernonNaidoo-Toronto/3253_ML_Group_7_Electricity_Project/master/RAW_DATA/IESO_Data/'
weather_path = 'https://raw.githubusercontent.com/VernonNaidoo-Toronto/3253_ML_Group_7_Electricity_Project/master/DATA/Weather_Data/'
# Set default plot styles
plt.style.use('seaborn')
plt.rcParams['figure.figsize'] = (16, 6)
plt.rcParams['axes.titlesize'] = 20
plt.rcParams['xtick.labelsize'] = 14
plt.rcParams['ytick.labelsize'] = 14
plt.rcParams['axes.labelsize'] = 14<jupyter_output><empty_output><jupyter_text>Load Ontario Power DEMAND DATA  Load files, concatenate, add date index to create hourly DataFrame: **power_supply_hourly**<jupyter_code>#2017 Zonal Demand File: PUB_DemandZonal_2017.csv
demand_2017 = pd.read_csv(IESO_path+'PUB_DemandZonal_2017.csv', skiprows=3, parse_dates=['Date'])
demand_2018 = pd.read_csv(IESO_path+'PUB_DemandZonal_2018.csv', skiprows=3, parse_dates=['Date'])
demand_2019 = pd.read_csv(IESO_path+'PUB_DemandZonal_2019.csv', skiprows=3, parse_dates=['Date'])

# Print summary of records loaded:
for year, df in [[2017,demand_2017], [2018,demand_2018], [2019,demand_2019]]:
  print(f'Loaded {df.shape[0]} records and {df.shape[1]} columns from the {year} file.')
# Concatenate DataFrames
power_demand_hourly = pd.concat([demand_2017, demand_2018, demand_2019], ignore_index=True) 

# Change column names to Python standard lowercase:
cols = [column_name.lower() for column_name in power_demand_hourly.columns] 
power_demand_hourly.columns = cols
#Move date column to index
power_demand_hourly.set_index('date', inplace=True)

# Show sample records
print('Shape:',power_demand_hourly.shape)
display(power_demand_hourly.sample(5))<jupyter_output>Shape: (26280, 14)<jupyter_text>Data Preparation  Check for missing values; check datatypes; add day column<jupyter_code>power_demand_hourly.info()
# Check for missing values: NONE FOUND
col_count = 0 # count columns with nulls
for col in power_demand_hourly.columns:
  null_count = power_demand_hourly[col].isna().sum()
  if null_count >0:
    print(f'The {col} column has {null_count} rows without values.')
    col_count+=1
print(f'{col_count} columns have some missing values.')
# Add weekday column
power_demand_hourly['day'] = power_demand_hourly.index.strftime('%A')
power_demand_hourly.head(3)<jupyter_output><empty_output><jupyter_text>Export CSV Files for Machine Learning Output Files:- **power_demand_hourly**- **power_demand_daily**  Create and export power_demand_hourly.csv<jupyter_code>#Save as csv:
filename_with_path = 'power_demand_hourly.csv'
power_demand_hourly.to_csv(filename_with_path)
#Test:
filename_with_path = 'power_demand_hourly.csv'
test = pd.read_csv(filename_with_path)
test.set_index(['date']).head(3)<jupyter_output><empty_output><jupyter_text>Create and export power_demand_daily.csv<jupyter_code># Resample to daily totals
power_demand_daily = power_demand_hourly.resample('D').sum()

# Drop unwanted columns:
power_demand_daily.drop(columns='hour', inplace=True)

# Add weekday column
power_demand_daily['day'] = power_demand_daily.index.strftime('%A')
power_demand_daily.head(3)
#Save as csv:
filename_with_path = 'power_demand_daily.csv'
power_demand_daily.to_csv(filename_with_path)
#Test:
filename_with_path = 'power_demand_daily.csv'
test = pd.read_csv(filename_with_path)
test.set_index(['date']).head(3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11746987951807229, 'fraction_numerical': 0.029846659364731652, 'mean_word_length': 8.109725685785536, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '253011', 'license': 'MIT', 'n_tokens_mistral': 1431, 'n_tokens_neox': 1285, 'n_words': 251, 'path': 'code_samples/VN_load_demand_data.ipynb', 'repo_name': 'CommonSenseMachineLearning/OntarioPower', 'size': '3652'}