<jupyter_start><jupyter_text>Naive/persistence modelAssumes each day's day prices predict the next in the same hour<jupyter_code>import json
import csv
import pandas as pd
pd.set_option('display.max_columns', 500)
pd.options.mode.chained_assignment = None  # default='warn'
import numpy as np
import geopandas as gpd
import shapely
from shapely.geometry import Point, MultiPoint, Polygon, MultiPolygon
from shapely.affinity import scale
import matplotlib.pyplot as plt
import glob
import os
import datetime
import pytz
from pytz import timezone
#Loading AS price data; downselecting to reg down prices
prices = pd.read_csv("../data/AS_price_vol.csv")
price = prices[['date','hr_beg','price_DAH_REGDN']]
price['dt'] = pd.to_datetime(price['date']+ " " + price['hr_beg'].astype(str)+ ":00")

#converting back to central time
utc = timezone('UTC')
tz = timezone('America/Chicago')
price['UTC'] = price['dt'].apply(lambda x: utc.localize(x))
price['Central'] = price['UTC'].apply(lambda x: x.astimezone(tz))
price['hr_beg'] = price['Central'].dt.hour
price['date'] = price['Central'].dt.date
price['year'] = price['Central'].dt.year
price.drop(columns=['dt','UTC','Central'], inplace=True)
#Lagging by 48!! hours
price_24 = price['price_DAH_REGDN'].shift(48)
df = pd.concat([price,price_24], axis=1)
df.columns = ['date','hr_beg','price','year','price_24']
df.dropna(inplace=True)
df.reset_index(drop=True,inplace=True)
#Downselecting to 2018
df_2018 = df[df['year']==2018]
#Calculating number of spikes captured in 2018
spike = np.percentile(df['price'][(df['year']<2018) & (df['year']>=2014)], 90)
df_2018['Actual spike'] = df_2018['price'] >= spike
df_2018['Predicted spike'] = df_2018['price_24'] >= spike
df_2018['Both spike'] = (df_2018['price'] >= spike) & (df_2018['price_24'] >= spike)
df_2018['Neither spike'] = (df_2018['price'] < spike) & (df_2018['price_24'] < spike)
spikes_correct = sum(df_2018['Both spike']) / sum(df_2018['Actual spike'])
overall_correct = (sum(df_2018['Both spike']) + sum(df_2018['Neither spike'])) / len(df_2018)
print("Spikes correctly predicted, 2018: " + str(spikes_correct))
print("Overall hours correctly predicted, 2018: " + str(overall_correct))
#Calculating number of spikes captured, 2008-2017
df = df[(df['year']>=2014) & (df['year']<=2017)]
df['Actual spike'] = df['price'] >= spike
df['Predicted spike'] = df['price_24'] >= spike
df['Both spike'] = (df['price'] >= spike) & (df['price_24'] >= spike)
df['Neither spike'] = (df['price'] < spike) & (df['price_24'] < spike)
spikes_correct = sum(df['Both spike']) / sum(df['Actual spike'])
overall_correct = (sum(df['Both spike']) + sum(df['Neither spike'])) / len(df)
print("Spikes correctly predicted, 2014-2017: " + str(spikes_correct))
print("Overall hours correctly predicted, 2014-2017: " + str(overall_correct))<jupyter_output>Spikes correctly predicted, 2014-2017: 0.4586894586894587
Overall hours correctly predicted, 2014-2017: 0.8916847144648353<jupyter_text>Outputting for optimization team<jupyter_code>results = df_2018[['date','hr_beg','price','price_24']]
results.columns = ['date','hour','Actual_price','Predicted_price_48hr']
results.drop_duplicates(subset=['date','hour','Actual_price'], inplace=True)
results.reset_index(drop=True, inplace=True)
results.shape
fig = plt.figure(figsize=(9,3))
plt.plot(results['Actual_price'], label="Actual", color='teal')
plt.plot(results['Predicted_price_48hr'], label="Predicted", color='orange')
plt.legend()
plt.title("Predicted DAH Regulation Down Prices, 2018 (persistence model)")
plt.xlabel('Hour of year')
plt.ylabel("$/MWh")
plt.savefig("persistence_results_48.png")

results.to_csv("persistence_results_48.csv", index=False)<jupyter_output><empty_output><jupyter_text>Autocorrelation<jupyter_code>from statsmodels.graphics.tsaplots import plot_acf, plot_pacf

fig, ax = plt.subplots(2, figsize=(12,6))
ax[0] = plot_acf(df['price'].dropna(), ax=ax[0], lags=120)
ax[1] = plot_pacf(df['price'].dropna(), ax=ax[1], lags=120)
plt.suptitle("Autocorrelation, regulation down price, 2014-2017")
plt.savefig("autocorr.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17902439024390243, 'fraction_numerical': 0.06, 'mean_word_length': 8.341685649202734, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '92184', 'license': 'MIT', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1420, 'n_words': 293, 'path': 'MM_Code/model_naive.ipynb', 'repo_name': 'nickolasclarke/anciML', 'size': '4100'}