<jupyter_start><jupyter_text><NAME>  2018801021  IE582-Homework3<jupyter_code>pip install pandas

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import datetime as dt
import cv2
import itertools
import warnings
warnings.filterwarnings('ignore')
import matplotlib.pylab as plb
from sklearn.linear_model import LinearRegression
from sklearn import linear_model
data=pd.read_csv("C:/Users/negin/Desktop/GercekZamanliTuketim.csv", sep=';', encoding='cp1252', 
                thousands='.',decimal=',')


data.columns=["Date","Hour","Consumption(MWh)"]
data["DateTime"] = data["Date"] + " " +data["Hour"]


data
data.set_index('Date',inplace=True)
#data["Consumption(MWh)"]=data["Consumption(MWh)"].astype(float)
data["Consumption(MWh)"]=data["Consumption(MWh)"].astype(float)
data=data.reset_index()
data["Lag_48"]=data["Consumption(MWh)"].shift(48)
data.dropna(how='any',inplace=True)
data["Lag_168"]=data["Consumption(MWh)"].shift(168)
data.dropna(how='any',inplace=True)
data.columns= ['Date','Hour','Consumption','DateTime','Lag_48','Lag_168']

data.head()

#test_data=data[33578:]
#train_data=data[data['Date']< '2019-11-01 00:00:00']
test_data=data[data['Date']>= '2019-11-01']
train_data=data[data['Date']< '2019-11-01']
#Part a MAPE values for the test period

def MAPE(actual, prediction):
    actual=np.array(actual)
    prediction=np.array(prediction)
    return np.mean(np.abs((actual-prediction)/actual))*100

MAPE_Lag_48=MAPE(test_data['Consumption'],test_data['Lag_48'])
print("MAPE value for Lag_48 is:" + "%" +str(MAPE_Lag_48))


MAPE_Lag_168=MAPE(test_data['Consumption'],test_data['Lag_168'])
print("MAPE value for Lag_168 is:" + "%" +str(MAPE_Lag_168))<jupyter_output>MAPE value for Lag_168 is:%2.63293095245023<jupyter_text>Part a) As we can see the MAPE value for Lag_168 is lower than the MAPE value for Lag_48 because we use more data (from past values) and our prediction is more accurate.<jupyter_code>#creating Long format for part b
long_format=data.copy()
long_format["Lag_48"]=data["Consumption"].shift(48)
long_format["Lag_168"]=data["Consumption"].shift(168)
long_format.dropna(how='any',inplace=True)
long_format
#Part b Linear Regression

test_long=long_format[long_format['Date']>= '2019-11-01']
train_long=long_format[long_format['Date']< '2019-11-01']
X=train_long[['Lag_48','Lag_168']]
Y=train_long['Consumption']
regression=LinearRegression()
regression.fit(X,Y)
y_true=test_long['Consumption']
x_predict=test_long[['Lag_48','Lag_168']]
y_predict=regression.predict(x_predict)
linear_mape=MAPE(y_true,y_predict)
print('MAPE for linear regression is  : '  + str(linear_mape))
print('Coefficient for lag48:' + str(regression.coef_[0])  + ' and coefficient for lag168 :'  + str (regression.coef_[1]))
print('Intercept value is' + str(regression.intercept_))<jupyter_output>MAPE for linear regression is  : 2.920088150346613
Coefficient for lag48:0.325443700079999 and coefficient for lag168 :0.6252477343538021
Intercept value is1607.7293629522683<jupyter_text>Part b) The MAPE value for linear regression model is 2.92 percent which is a value between using the Lag48 and Lag168.Is is something reasonable and the coefficients for lag48 and lag168 are as 0.325 and 0.625 respectively. Also the intercept value is 1607.72.<jupyter_code># Part C 
hours= ['00:00','01:00','02:00','03:00','04:00','05:00','06:00','07:00','08:00','09:00','10:00','11:00','12:00',
       '13:00','14:00','15:00','16:00','17:00','18:00','19:00','20:00','21:00','22:00','23:00']
MAPE_results=[]
regression=LinearRegression()
for i in range (0,24):
    train_prime=train_long[train_long.Hour==hours[i]]
    test_prime=test_long[test_long.Hour==hours[i]]
    X=train_prime[['Lag_48','Lag_168']]
    Y=train_prime['Consumption']
    regression=LinearRegression()
    regression.fit(X,Y)
    y_true=test_prime['Consumption']
    x_predict=test_prime[['Lag_48','Lag_168']]
    y_predict=regression.predict(x_predict)
    linear_mape=MAPE(y_true,y_predict)
    print('MAPE value for linear regression for ' + hours[i]+ 'is : ' + str(linear_mape))
    print('')
    MAPE_results.append(linear_mape)<jupyter_output>MAPE value for linear regression for 00:00is : 3.263860468820378

MAPE value for linear regression for 01:00is : 3.0582014362530265

MAPE value for linear regression for 02:00is : 3.557967961952439

MAPE value for linear regression for 03:00is : 3.308944242651312

MAPE value for linear regression for 04:00is : 2.73843043482726

MAPE value for linear regression for 05:00is : 3.1497461978707832

MAPE value for linear regression for 06:00is : 2.8006660148879003

MAPE value for linear regression for 07:00is : 1.953952596400262

MAPE value for linear regression for 08:00is : 2.287296229147997

MAPE value for linear regression for 09:00is : 2.7174221192955375

MAPE value for linear regression for 10:00is : 4.45818922580818

MAPE value for linear regression for 11:00is : 4.751831391341398

MAPE value for linear regression for 12:00is : 5.141847995101261

MAPE value for linear regression for 13:00is : 5.494837448568943

MAPE value for linear regression for 14:00is : 6.680840801733301

MAPE val[...]<jupyter_text>Part c) The highest value for MAPE is 6.68 which belongs to 14:00 and the lowest value is 0.13 and it belongs to hour 21:00.Each hour has differenet MAPE values.<jupyter_code>#Part d
data1=pd.read_csv("C:/Users/negin/Desktop/data1.csv", encoding='unicode_escape', sep=';')
consumption = data1['Tüketim Miktarý (MWh)'].tolist()
dates = data1['Tarih'].tolist()
hours = data1['Saat'].tolist()
datetimes =data1['Tarih'].tolist()


for i in range (0,data1.shape[0]):
    datetimes[i] = datetimes[i].replace('.','/') + " " + hours[i]
    dates[i] = dates[i].replace('.','/')
    consumption[i] = float(consumption[i].replace('.','').replace(',','.'))
    
datetimes_list = [dt.datetime.strptime(date, '%d/%m/%Y %H:%M') for date in datetimes]
dates_list = [dt.datetime.strptime(date, '%d/%m/%Y') for date in dates]

data1['Datetime'] = datetimes_list
data1['Tarih'] = dates_list
data1['Tüketim Miktarý (MWh)'] = consumption

ne = []
ka = []
lag_data1 = []
lag_data2 = []
ne = []
ka = []

for i in range (0,data1.shape[0]):
    ne = (data1[data1['Tarih'] == data1.iloc[i]['Tarih'] \
    - dt.timedelta(days=7)]['Tüketim Miktarý (MWh)']).values
    ka = (data1[data1['Tarih'] == data1.iloc[i]['Tarih'] \
    - dt.timedelta(days=2)]['Tüketim Miktarý (MWh)']).values
    lag_data1.append(ne)
    lag_data2.append(ka)
    ne = []
    ka = []
column_1 = []
column_2 = []
for i in range(0,24):
    column_1.append('Lag_day7_hour_' + str(i))
    column_2.append('Lag_day2_hour_' + str(i))
lags1 = pd.DataFrame(lag_data1,columns=column_1)
lags2 = pd.DataFrame(lag_data2,columns=column_2)
lags1['Datetime'] = data1['Datetime']
lags2['Datetime'] = data1['Datetime']
lags = pd.merge(lags1, lags2, how='left', on=['Datetime'])
data1_wide = pd.merge(data1, lags, how='left', on=['Datetime'])
data1_wide = (data1_wide.dropna()).reset_index(drop=True)
data1_wide.head()
train_wide=data1_wide[data1_wide['Tarih']  < '2019-11-01']
test_wide=data1_wide[data1_wide['Tarih']  >= '2019-11-01']
condition_for_train=column_data1+column_data2
Lasso_Results=[]
for i in range (0,24):
    train_data_wide=train_wide[train_wide.Saat==hours[i]]
    test_data_wide=test_wide[test_wide.Saat==hours[i]]
    regression=linear_model.LassoCV(cv=10, normalize=True, tol=0.001, selection='random', random_state=43,alphas=np.logspace(-4,2,100))
    X=train_data_wide[condition_for_train]
    Y=train_data_wide['Tüketim Miktarý (MWh)']
    regression.fit(X,Y)
    y_true=test_data_wide['<NAME> (MWh)']
    x_predict=test_data_wide[condition_for_train]
    y_predict=regression.predict(x_predict)
    mape_for_lasso=MAPE(y_true,y_predict)
    print('MAPE value for Lasso regression for ('  +  hours[i] + ')is :' +  str(mape_for_lasso))
    print('')
    mape_for_lasso=float(mape_for_lasso)
    Lasso_Results.append(mape_for_lasso)<jupyter_output>MAPE value for Lasso regression for (00:00)is :1.8748210921098103

MAPE value for Lasso regression for (01:00)is :1.6570314651123332

MAPE value for Lasso regression for (02:00)is :1.6681734802711696

MAPE value for Lasso regression for (03:00)is :1.6877996121610221

MAPE value for Lasso regression for (04:00)is :2.1103369327132193

MAPE value for Lasso regression for (05:00)is :1.5374506464111268

MAPE value for Lasso regression for (06:00)is :1.6734708439977222

MAPE value for Lasso regression for (07:00)is :2.1114648037352515

MAPE value for Lasso regression for (08:00)is :1.7801274825256261

MAPE value for Lasso regression for (09:00)is :0.7785404748368039

MAPE value for Lasso regression for (10:00)is :1.3331985975447824

MAPE value for Lasso regression for (11:00)is :2.157684654900181

MAPE value for Lasso regression for (12:00)is :2.093839100023719

MAPE value for Lasso regression for (13:00)is :3.553826257056949

MAPE value for Lasso regression for (14:00)is :3.9462375536417245[...]<jupyter_text>Part d) as we can see from the results the MAPE values are lower for Lasso regression and it predicts better. Maximum MAPE score is 3.94 which belongs to hour 14:00 and the minimum value is 0.77 and it belongs to 09:00.<jupyter_code>#Part F
plt.figure(figsize=(10,8))
a=24*[MAPE_Lag_48]
b=24*[MAPE_Lag_168]
c=24*[linear_mape]
d=MAPE_results
e=Lasso_Results
cases=[a,b,c,d,e]
plt.boxplot(cases)
plt.xticks([1, 2, 3, 4, 5], ['MAPE_lag_48', 'MAPE_lag_168', 'linear_mape', 'mape_hours', 'mape_lasso'])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475650713685978, 'fraction_numerical': 0.11775818639798488, 'mean_word_length': 6.415564202334631, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '523610', 'license': 'MIT', 'n_tokens_mistral': 4358, 'n_tokens_neox': 3639, 'n_words': 750, 'path': 'IE582_Homework3.ipynb', 'repo_name': 'BU-IE-582/fall19-neginkazemi-2', 'size': '9556'}