<jupyter_start><jupyter_text>Residential demand estimation<jupyter_code>%matplotlib inline
# import packages

import pandas as pd
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt
df = pd.read_csv("../data/raw/Residential/AUS_data.csv",
                index_col=['Economy','Year'])
df.head()
df.tail()
df.info()
# split in to training and test sets

feature_list = ["GDP","CSGDP","POP","POPU","GDPC"]

train = (df.loc['01_AUS',:]
           .loc[1980:2000])
train.head()
test = (df.loc['01_AUS',:]
          .loc[2001:2016])
test.tail()
# training
y = train.loc[:,"RETT"]
X_train = train.loc[:,feature_list]

# test
X_test = test.loc[:,feature_list]
X_train.head()
X_test.tail()
model = sm.OLS(y,X_train)
fit_model = model.fit()
fit_model.summary()
# test model

results = fit_model.predict(X_test)
results
fig, ax = plt.subplots()
ax.plot(train.loc[:,'RETT'], 'o', label="Train")
ax.plot(results, 'o', label="Test")
ax.plot(df.loc['01_AUS',"RETT"], '-g', label="Actual")
ax.legend(loc="best")<jupyter_output><empty_output><jupyter_text>next steps- add lags?  ARIMA<jupyter_code>from statsmodels.tsa.arima_model import ARIMA
df = pd.read_csv("../data/raw/Residential/AUS_simple.csv",
                index_col=['Economy','Date'])
df.head()
df.info()
feature_list = ["GDP","CSGDP","POP","POPU","GDPC"]

train = (df.loc['01_AUS',:]
           .iloc[1:21]
           .drop(columns="Year"))
train.tail()
y = train[['RETT']]
X_train = train.drop(columns=y)
X_train.tail()
model = ARIMA(y, (0,1,0),X_train, freq='A')
model_fit = model.fit(disp=0)
print(model_fit.summary())
test = (df.loc['01_AUS',:]
           .iloc[21:37]
           .drop(columns="Year"))
test.tail()
X_test = test.drop(columns="RETT")
X_test.head()
forecast = model_fit.forecast(steps=16,exog=X_test)
forecast = model_fit.forecast?
forecast = model_fit.forecast
forecast = model_fit.predict
ARIMA?<jupyter_output><empty_output><jupyter_text>Prophet<jupyter_code>from fbprophet import Prophet
df = pd.read_csv("../data/raw/Residential/AUS_data.csv")
df.head()
df['ds'] = pd.to_datetime(df.Year,format='%Y')
df.head()
df.tail()
simple_df = df[['RETT','GDP','POP','ds']]
simple_df.head()
simple_df = simple_df.rename(columns={"RETT":"y"})
simple_df.head()
m = Prophet(daily_seasonality=False,
            weekly_seasonality=False,
            yearly_seasonality=False,
            seasonality_mode='additive',
            growth='linear')

m.add_regressor('GDP')
m.add_regressor('POP')
m.fit(simple_df)
# add future GDP

df_future_macro = pd.read_csv('../data/raw/Industry/MacroAssumptions_datetime.csv',
                             index_col=['Economy'])

df_future_macro = df_future_macro.loc['AUS',:]
df_future_macro = df_future_macro.reset_index(drop=True)

df_future_macro.head()
df_future_macro = df_future_macro.rename(columns={"Population":"POP"})
df_future_macro.head()
future = m.make_future_dataframe(periods=34,
                                 freq = 'Y',
                                 include_history=True)
future.head()
future.tail()
df1 = simple_df.drop(columns=['y','ds'])
df1.head()
df2 = df_future_macro.drop(columns=['Year'])
df2.head()
df_future = pd.concat([df1,df2],
                      ignore_index=True)

df_future.head()
df_future.tail()
future = future.join(df_future)
future.head()
future.tail()
forecast = m.predict(future)
forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']]
fig1 = m.plot(forecast)
future.iloc[20:40,:]<jupyter_output><empty_output><jupyter_text>SciKit-Learn<jupyter_code>from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(simple_df[["GDP","POP"]],simple_df[['ds']])
model.fit(X_train,y)
predictions = model.predict(X_test)
predictions
model2 = LinearRegression()
model2.fit(simple_df[["GDP","POP"]],simple_df[['y']])
predictions2 = model2.predict(future[["GDP","POP"]])
predictions2
fig, ax = plt.subplots()
ax.plot(future[['ds']],predictions2, 'o', label="Predictions")
ax.plot(simple_df[['ds']],simple_df[["y"]], 'o', label="Actual")
ax.legend(loc="best")<jupyter_output><empty_output><jupyter_text>Using own functions<jupyter_code>def run_regression(models, economies, df, x, y):
    """
    Perform linear regression for one or multiple economies.
    economy = list of economies
    models = {economy: LinearRegression() for economy in economies}
    The function returns a dictionary of economy-model pairs. That is,
    each economy will have its own set of coefficients.
    """
    for economy, model in models.items():
            (model.fit(df.loc[economy, x],
                df.loc[economy, y]))
    return models  
def run_prediction(models, economies, df, ResultsColumn):
    """
    Use coefficients from run_regression to generate predictions.
    Pass a dataframe df with the X and Y data. 
    ResultsColumn = name of prediction results
    """
    df_list =[]
    # run predictions
    for economy, model in models.items():
            #years = df['Year']
            #years.reset_index(drop=True, inplace=True)
            prediction = model.predict(df.loc[economy,:])
            df_name = pd.DataFrame(prediction, columns=ResultsColumn)
            #df_name.insert(loc=0,column='Year',value=years)
            df_name.insert(loc=0,column='Economy',value=economy)
            #df_list.append(df_name)
            
            # adding to the input df
            df2 = df.loc[economy,:]
            df2.insert(loc=0,column="Results",value=prediction)
            df2.insert(loc=0,column='Economy',value=economy)
            df2 = df2.reset_index()
            df2 = df2.set_index(['Economy','Year'])
            df_list.append(df2)

    # combine individual economy dataframes to one dataframe
    dfResults = pd.concat(df_list, sort=True)
    return dfResults
#    return df2
# create plotting function

def plot2(economies, df, figurename, Plotylabel, share_x, share_y):
    """
    Line plot for 21 economies. 
    Economies = economies to plot
    df = dataframe of data to plot. Note: each line must be a column.
    Plotylabel = y label for graph
    share_x = share the x axis (true or False)
    share_y = share the y axis (True or False)
    """
    print('Preparing plots...')
    # Create the 'figure'
    plt.style.use('tableau-colorblind10')
    
    # multiple line plot
    fig, axes = plt.subplots(nrows=3, ncols=7, sharex=share_x, sharey=share_y, figsize=(16,12))
    for ax, economy,num in zip(axes.flatten(), economies, range(1,22)):
        print('Creating plot for %s...' %economy)
        df11=df[df['Economy']==economy]
    
        for column in df11.drop(['Economy','Year'], axis=1):
            ax.plot(df11['Year'], df11[column], marker='', linewidth=1.5, label=economy)
            ax.set_title(economy)
            ax.set_ylabel(Plotylabel)
        ax.label_outer()
    
    #plt.tight_layout()
    fig.legend( list(df.drop(['Economy','Year'], axis=1)), bbox_to_anchor=(0,0,1,0.25), loc='lower center', ncol=9)
    fig.savefig(figurename,dpi=200)
    print('Figure saved as %s' % figurename)
    print('Preparing to show the figure...')
    plt.show()
df = pd.read_csv("../data/raw/Residential/AUS_data.csv",
                index_col=['Economy','Year'])
df.head()
economies = df.index.unique(level=0)
economies
# get list of economies and create economy-model pairs

models = {economy: LinearRegression() for economy in economies}
x = ['GDP','POP']
y = ['RETT']
df.head()
run_regression(models,economies,df,x,y)
# make predictions

df_future = pd.read_csv("../data/raw/Residential/MacroAssumptions_datetime-AUS.csv",
                        index_col=['Economy'])
df_future.head()
df2 = df_future.drop(columns=["Year"],axis=1)
df2 = df2.rename(columns={"Year2":"Year","Population":"POP"})
df2 = df2.reset_index(drop=False).set_index(['Economy','Year'])

#df2 = df2.drop(columns=["Year"],axis=1)
ResultsColumn = ["y_prediction"]
df2.head()
results = run_prediction(models, economies, df2, ResultsColumn)
results = results.rename(columns={"Results":"Predicted RETT"})
results.tail()
x = results.loc['01_AUS']
x.head()
fig, ax = plt.subplots()
ax.plot(x.index,x[['Predicted RETT']], 'o', label="Predictions")
#ax.plot(simple_df[['ds']],simple_df[["y"]], 'o', label="Actual")
#ax.legend(loc="best")<jupyter_output><empty_output><jupyter_text>Cleaned up using own functions<jupyter_code>%matplotlib inline
# import packages

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
def run_regression(models, economies, df, x, y):
    """
    Perform linear regression for one or multiple economies.
    economy = list of economies
    models = {economy: LinearRegression() for economy in economies}
    The function returns a dictionary of economy-model pairs. That is,
    each economy will have its own set of coefficients.
    """
    for economy, model in models.items():
            (model.fit(df.loc[economy, x],
                df.loc[economy, y]))
    return models  
def run_prediction(models, economies, df, ResultsColumn):
    """
    Use coefficients from run_regression to generate predictions.
    Pass a dataframe df with the X and Y data. 
    ResultsColumn = name of prediction results
    """
    df_list =[]
    # run predictions
    for economy, model in models.items():
            # make prediction
            prediction = model.predict(df.loc[economy,:])
            
            # adding to the input df
            df2 = df.loc[economy,:]
            df2.insert(loc=0,column="Results",value=prediction)
            df2.insert(loc=0,column='Economy',value=economy)
            df2 = df2.reset_index()
            df2 = df2.set_index(['Economy','Year'])
            df_list.append(df2)

    # combine individual economy dataframes to one dataframe
    dfResults = pd.concat(df_list, sort=True)
    return dfResults
df = pd.read_csv("../data/raw/Residential/AUS_Data.csv",
                index_col=['Economy','Year'])
economies = df.index.unique(level=0)
economies
# get list of economies and create economy-model pairs

models = {economy: LinearRegression() for economy in economies}
x = ['GDP','POP']
y = ['RETT']
df.tail()
run_regression(models,economies,df,x,y)
# make predictions

df_future = pd.read_csv("../data/raw/Residential/MacroAssumptions_datetime.csv",
                        index_col=['Economy'])
df_future.tail()
df2 = df_future.drop(columns=["Year"],axis=1)
df2 = df2.rename(columns={"Year2":"Year","Population":"POP"})
df2 = df2.reset_index(drop=False).set_index(['Economy','Year'])

#df2 = df2.drop(columns=["Year"],axis=1)
ResultsColumn = ["y_prediction"]
df2.tail()
results = run_prediction(models, economies, df2, ResultsColumn)
results = results.rename(columns={"Results":"Predicted RETT"})
results.tail()
x = results.loc['01_AUS']
x.head()
fig, ax = plt.subplots()
ax.plot(x.index,x[['Predicted RETT']], 'o', label="Predictions")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15524988537368178, 'fraction_numerical': 0.013480055020632738, 'mean_word_length': 3.943789664551224, 'pattern_counts': {'":': 8, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '186625', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3881, 'n_tokens_neox': 3761, 'n_words': 742, 'path': 'notebooks/Residential_draft.ipynb', 'repo_name': 'wogandavid/Platts-Cleaner', 'size': '10905'}