<jupyter_start><jupyter_text>Script to load electricity data into pandas DFThis script loads data into dataframes to be used by other notebooks.<jupyter_code>from datetime import timedelta, date
import pandas as pd
from os import listdir
from os.path import isfile, join

# Put this to True to print intermediate results and debuging information
dbg = False
# Put this to True to call the data scraping script to refresh the load data
update_data = False
# This is the path where the data are located
power_data = './power_data/'
temp_data = './temp_data/'
# Load load profile files from the path. 
# There are multiple excel files downloaded from the TSOC website for the relevant periods.
if update_data:
    %run ./power_data_scrap.ipynb

datafiles = [f for f in listdir(power_data) if isfile(join(power_data, f))]
# We only need certain columns and read the files and concatenate to a single DF
# If there is an overlap it might create issues.
cols = 'A,B,C,D,F'
mynames = ["DATE", "CAP", "WIND", "PV", "CONV"]
dftmp = pd.read_excel(power_data+datafiles[0], usecols = cols, skiprows = 2,  index_col = 0, header = None, names = mynames, verbose=dbg, parse_dates=True)
for f in datafiles[1:]:
    dftmp = pd.concat([dftmp, pd.read_excel(power_data+f, usecols = cols, skiprows = 2,  index_col = 0, header = None, names = mynames, verbose=dbg, parse_dates=True)])
if dbg:
    print(dftmp)
# Since we might have multi-year data, extract just the two years we like to compare.
df19 = dftmp[dftmp.index.year == 2019].copy()
# In the 2020 one, there are nan values for the days after today, so we filter them out.
df20 = dftmp[(dftmp.index.year == 2020) & (dftmp.index.date < date.today())].copy()
if dbg:
    print(df19,df20)
# Create the columns with the total load and penetration
# Load=Conventional + PV + WIND
df19["LOAD19"] = df19["CONV"] + df19["WIND"] + df19["PV"]
df19["PEN19"] = 100.0*(df19["WIND"] + df19["PV"]) / df19["LOAD19"]
df20["LOAD20"] = df20["CONV"] + df20["WIND"] + df20["PV"]
df20["PEN20"] = 100.0*(df20["WIND"] + df20["PV"]) / df20["LOAD20"]
if dbg:
    print(df19, df20)
# Read the weekly heating degree days. Downloaded from https://www.degreedays.net/
# Careful that these are given with the week comensing the date in column A (Monday)
csv_file = temp_data + 'LCRA_HDD_15.5C.csv'
cols = [0, 1]
dfHDD = pd.read_csv(csv_file, usecols = cols,  index_col = 0, skiprows = 6, header = 0, parse_dates=True)  
dfHDD["Datetime"] = pd.to_datetime(dfHDD.index, dayfirst=True, errors='coerce')
dfHDD.set_index('Datetime', inplace=True)
dfHDD19 = dfHDD[dfHDD.index.year == 2019].copy()
dfHDD20 = dfHDD[dfHDD.index.year == 2020].copy()
if dbg:
    print(dfHDD19,dfHDD20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12804428044280441, 'fraction_numerical': 0.03911439114391144, 'mean_word_length': 4.8806941431670285, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '626724', 'license': 'MIT', 'n_tokens_mistral': 1039, 'n_tokens_neox': 945, 'n_words': 313, 'path': 'import_data.ipynb', 'repo_name': 'SPS-L/Cyprus-load-data-analysis-2020-blog', 'size': '2710'}