<jupyter_start><jupyter_text>Load and preprocess data<jupyter_code>!ls
# DAM(Day ahead Market) data
df_dam = pd.read_csv ('20200101-20211217 ERCOT Day-Ahead Price.csv')

df_dam.head
df_dam.dtypes

df_dam.zone.apply(str)

df_dam['Date']= pd.to_datetime(df_dam['Date'])
df_dam.dtypes
df_dam.head
df_dam = df_dam [df_dam['zone']=='LZ_NORTH']
df_dam.head
# Real time data
df_realtime = pd.read_csv ('20210101-20211217 ERCOT Real-time Price.csv')

df_realtime['Date']= pd.to_datetime(df_realtime['Date'])
df_realtime.head(3)
df_realtime.Zone.apply(str)

df_realtime = df_realtime [df_realtime['Zone']=='LZ_NORTH']
df_realtime.head<jupyter_output><empty_output><jupyter_text>Analysis ( ignore Feb for now)<jupyter_code>df_dam['month'] = df_dam['Date'].dt.month
df_dam['year'] = df_dam['Date'].dt.year
df_dam['day'] = df_dam['Date'].dt.date
df_dam = df_dam[df_dam['year'] == 2021]
df_realtime['month'] = df_realtime['Date'].dt.month
df_realtime['year'] = df_realtime['Date'].dt.year
df_realtime['day'] = df_realtime['Date'].dt.date
df_realtime = df_realtime[df_realtime['year'] == 2021]
df_dam.head
#ignore Feb
df_dam = df_dam [df_dam['month'] != 2]
df_realtime = df_realtime [df_realtime['month'] != 2]
df_realtime.describe()
df_dam.describe()
df_realtime.groupby('month').mean()
df_dam.groupby('month').mean()
df_realtime.drop('year', axis=1, inplace=True)

df_realtime.groupby('month').mean().plot(legend=True)
df_dam.drop('year', axis=1, inplace=True)

df_dam.groupby('month').mean().plot(legend=True)
df_price_ng = df_dam.groupby('month').mean().reset_index()
df_price_ng.head
#https://www.eia.gov/dnav/ng/hist/rngwhhdm.htm
df_price_ng['natural_gas_price'] = [2.71, 2.62,2.66, 2.91, 3.26, 3.84, 4.07, 5.16, 5.51, 5.05, 3.6]
df_price_ng.columns
df_price_ng.plot(x ='month', y='natural_gas_price', kind = 'line', title="2021 Monthly NG price", 
                      grid=True, legend=False)<jupyter_output><empty_output><jupyter_text>Curtail for REAL TIME<jupyter_code>df_realtime = df_realtime.rename(columns={'Price': 'price'})
df_realtime['SMA'] = df_realtime['price'].rolling(2).mean()

#df_realtime['SMA3'] = df_realtime['price'].rolling(3).mean()


                                           
curtail_analysis = pd.DataFrame(columns = ['curtail', 'average', 'uptime'])

for curtail in range(50,105, 5):
    print("*"*100)
    print("Curtail at "+ str(curtail))
    df_curtail = df_realtime [df_realtime['SMA'] <= curtail]
    print(df_curtail.groupby('month').mean())
    df_curtail.groupby('month').mean().plot(legend=True)
    print("Overall Average ", df_curtail['price'].mean())
    print("Uptime percentage ", df_curtail['Date'].count()/df_realtime['Date'].count())
    curtail_analysis = curtail_analysis.append({'curtail' : curtail, 'average' :df_curtail['price'].mean(), 'uptime' : df_curtail['Date'].count()/df_realtime['Date'].count()}, 
                ignore_index = True)
curtail_analysis
curtail_analysis.plot(x ='curtail', y='average', kind = 'line', title="Average after curtail", 
                      xticks=range(50,105,5),
                      grid=True, legend=False)

curtail_analysis.plot(x ='curtail', y='uptime', kind = 'line', title="Uptime after curtail", 
                      xticks=range(50,105,5),
                      #yticks=range(0.84,1.0,0.02),
                      grid=True, legend=False)<jupyter_output><empty_output><jupyter_text>Curtail for DAM<jupyter_code>curtail_analysis.drop(curtail_analysis.index, inplace=True)
curtail_analysis.head

#pd.DataFrame(columns = ['curtail', 'average', 'uptime'])

for curtail in range(50,105, 5):
    print("*"*100)
    print("Curtail at "+ str(curtail))
    df_curtail = df_dam [df_dam['price'] <= curtail]
    print(df_curtail.groupby('month').mean())
    df_curtail.groupby('month').mean().plot(legend=True)
    print("Overall Average ", df_curtail['price'].mean())
    print("Uptime percentage ", df_curtail['Date'].count()/df_dam['Date'].count())
    curtail_analysis = curtail_analysis.append({'curtail' : curtail, 'average' :df_curtail['price'].mean(), 'uptime' : df_curtail['Date'].count()/df_dam['Date'].count()}, 
                ignore_index = True)
                                           
curtail_analysis
curtail_analysis.plot(x ='curtail', y='average', kind = 'line', title="Average after curtail", 
                      xticks=range(50,105,5),
                      grid=True, legend=False)

curtail_analysis.plot(x ='curtail', y='uptime', kind = 'line', title="Uptime after curtail", 
                      xticks=range(50,105,5),
                      #yticks=range(0.84,1.0,0.02),
                      grid=True, legend=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17602915951972556, 'fraction_numerical': 0.030445969125214408, 'mean_word_length': 4.393063583815029, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '423612', 'license': 'MIT', 'n_tokens_mistral': 1810, 'n_tokens_neox': 1680, 'n_words': 243, 'path': 'notebook/Realtime_DAM_analysis.ipynb', 'repo_name': 'mingchen62/ercot-dam-notif', 'size': '4664'}