<jupyter_start><jupyter_text>**Importing Packages**<jupyter_code>import pandas as pd
import numpy as np

from keras.models import load_model
from keras.models import Sequential
from keras.layers import Dense, Conv1D, GRU, Bidirectional, Dropout
from keras.utils import plot_model

import matplotlib.pyplot as plt
import time
import matplotlib.pyplot as plt
%matplotlib inline
from IPython.display import display
import datetime
import math
import warnings
warnings.filterwarnings("ignore")
import glob
from sklearn.metrics import mean_squared_error, mean_absolute_error<jupyter_output><empty_output><jupyter_text>*Initializing Window size*<jupyter_code>window_size = 10<jupyter_output><empty_output><jupyter_text>**Functions**<jupyter_code>def read_label():
    label = {}
    for i in range(1,7):
        hi = '/content/gdrive/MyDrive/energy_disaggregation/windowGRU/data/house_{}/labels.dat'.format(i)
        label[i] = {}
        with open(hi) as f:
            for line in f:
                splitted_line = line.split(' ')
                label[i][int(splitted_line[0])] = splitted_line[1].strip() + '_' + splitted_line[0]
    return label


#-------------------------------------------------------------------------------------------------------------------

def read_merge_data(house):
    path = '/content/gdrive/MyDrive/energy_disaggregation/windowGRU/data/house_{}/'.format(house)
    
    
    file = path + 'channel_1.dat'

    df = pd.read_table(file, sep = ' ', names = ['unix_time', labels[house][1]], 
                                       dtype = {'unix_time': 'int64', labels[house][1]:'float64'}) 

    num_apps = len(glob.glob(path + 'channel*'))

    
    for i in range(2, num_apps + 1):
        file = path + 'channel_{}.dat'.format(i)
        data = pd.read_table(file, sep = ' ', names = ['unix_time', labels[house][i]], 
                                       dtype = {'unix_time': 'int64', labels[house][i]:'float64'})
        df = pd.merge(df, data, how = 'inner', on = 'unix_time')
    df['timestamp'] = df['unix_time'].astype("datetime64[s]")
    df = df.set_index(df['timestamp'].values)
    df.drop(['unix_time','timestamp'], axis=1, inplace=True)
    return df

#-------------------------------------------------------------------------------------------------------------------

def uniform_label(df):
  if 'kitchen_outlets_7' in df.columns:   
    df = df.rename(columns={'kitchen_outlets_7': 'kitchen'})

  if 'kitchen_outlets_5' in df.columns:
    df = df.rename(columns={'kitchen_outlets_5': 'kitchen'})

  if 'kitchen_outlets_3' in df.columns:
    df = df.rename(columns={'kitchen_outlets_3': 'kitchen'})

  if 'kitchen_outlets_24' in df.columns:
    df = df.rename(columns={'kitchen_outlets_24': 'kitchen'})

    

  return df

#----------------------------------------------------------------------------------------------------------------------

def add_houses(df_list, appliance):
  merged_df = pd.DataFrame()
  for df in df_list:
    df = uniform_label(df)

    df["aggregate_mains"] = df['mains_1'] + df['mains_2']
    df = df[['aggregate_mains',appliance]]
    
    merged_df = pd.concat([merged_df,df])
  return merged_df

#-----------------------------------------------------------------------------------------------------------------------

def max_value(df):
  return max(df['aggregate_mains'])


#-----------------------------------------------------------------------------------------------------------------------


def preprocess_data(df, appliance, nmax):

  df['aggregate_mains'] = df['aggregate_mains']/ nmax

  df[appliance] = df[appliance]/ nmax
  
  return df

#----------------------------------------------------------------------------------------------------------------------
def create_window_chunk(df, sequence_length):
  result = []
  for index in range(len(df) - sequence_length):
      result.append(df[index: index + sequence_length])

  return result

#----------------------------------------------------------------------------------------------------------------------

def _create_model(window_size):
    '''Creates the GRU architecture described in the paper
    '''
    model = Sequential()

    # 1D Conv
    model.add(Conv1D(16, 4, activation='relu', input_shape=(window_size,1), padding="same", strides=1))

    #Bi-directional GRUs
    model.add(Bidirectional(GRU(64, activation='relu', return_sequences=True), merge_mode='concat'))
    model.add(Dropout(0.5))
    model.add(Bidirectional(GRU(128, activation='relu', return_sequences=False), merge_mode='concat'))
    model.add(Dropout(0.5))

    # Fully Connected Layers
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(1, activation='linear'))

    model.compile(loss='mse', optimizer='adam')
    print(model.summary())

    return model

#---------------------------------------------------------------------------------------------------------------------<jupyter_output><empty_output><jupyter_text>**Data Preprocessing**<jupyter_code>df_dic = {}
dates = {}<jupyter_output><empty_output><jupyter_text>*Reading and storing labels of all the houses in a dictionary*<jupyter_code>labels = read_label()
labels<jupyter_output><empty_output><jupyter_text>*Converting raw dataset of a house into Pandas Dataframe and storing all the dataframes in a dictionary*<jupyter_code>for i in range(1,7):
    df_dic[i] = read_merge_data(i)
    
print(df_dic)<jupyter_output>{1:                      mains_1  mains_2  ...  washer_dryer_19  washer_dryer_20
2011-04-18 13:22:13   222.20   118.83  ...              0.0              0.0
2011-04-18 13:22:16   223.17   119.19  ...              0.0              0.0
2011-04-18 13:22:20   223.60   118.92  ...              0.0              0.0
2011-04-18 13:22:23   222.91   119.16  ...              0.0              0.0
2011-04-18 13:22:26   222.94   118.83  ...              0.0              0.0
...                      ...      ...  ...              ...              ...
2011-05-24 19:56:20   235.73    38.65  ...              0.0              0.0
2011-05-24 19:56:23   235.03    38.66  ...              0.0              0.0
2011-05-24 19:56:27   235.46    38.61  ...              0.0              0.0
2011-05-24 19:56:30   235.98    38.77  ...              0.0              0.0
2011-05-24 19:56:34   235.29    38.83  ...              0.0              0.0

[406748 rows x 20 columns], 2:                      mains_1  mains_2  .[...]<jupyter_text>*Checking the Number of days of available data of different houses*<jupyter_code>for i in range(1,7):
    dates[i] = [str(time)[:10] for time in df_dic[i].index.values]
    dates[i] = sorted(list(set(dates[i])))
    print('House {0} data contain {1} days from {2} to {3}.'.format(i,len(dates[i]),dates[i][0], dates[i][-1]))
    print(dates[i], '\n')
for i in range(1,7):
    print('House {} data has shape: '.format(i), df_dic[i].shape)
    display(df_dic[i].tail(3))<jupyter_output>House 1 data has shape:  (406748, 20)<jupyter_text>*Preparing training data - house 1 & 4, Appliance- Kitchen_outlet*<jupyter_code>df_list = [df_dic[1], df_dic[4]]
dfs = add_houses(df_list, 'kitchen')
dfs<jupyter_output><empty_output><jupyter_text>*Creating window chunk and normalizing dataframe*<jupyter_code>nmax = max_value(dfs)
df = preprocess_data(dfs, 'kitchen', nmax)

df
sequence_length = window_size


x = create_window_chunk(df['aggregate_mains'], sequence_length)

y = df['kitchen'][sequence_length:]
x = np.array(x)
y = np.array(y)<jupyter_output><empty_output><jupyter_text>*reshaping the data*<jupyter_code>x_train = np.reshape(x, (x.shape[0], x.shape[1], 1))<jupyter_output><empty_output><jupyter_text>**Creating Model**<jupyter_code>model = _create_model(window_size)<jupyter_output>WARNING:tensorflow:Layer gru will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU
WARNING:tensorflow:Layer gru_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running[...]<jupyter_text>*training model*<jupyter_code>start = time.time()
model.fit(x_train, y, epochs=2, batch_size=128, shuffle=True, validation_split=0.33)
print('Finish trainning. Time: ', time.time() - start)<jupyter_output>Epoch 1/2
4370/4370 [==============================] - 392s 82ms/step - loss: 9.9237e-07 - val_loss: 4.6604e-06
Epoch 2/2
4370/4370 [==============================] - 360s 82ms/step - loss: 6.3970e-07 - val_loss: 5.2425e-06
Finish trainning. Time:  751.3553023338318<jupyter_text>*Saving the trained model*<jupyter_code>model.save("gdrive/MyDrive/saved_model/kitchen_outlet_train_1_4_test_5_6_epochs_2_batch_128_window_10_validation_split_0.33_model.h5")<jupyter_output><empty_output><jupyter_text>*Preparing and preprocessing test data - house 5 & 6*<jupyter_code>df_list = [df_dic[5],df_dic[6]]
dfs = add_houses(df_list, 'kitchen')
df_test = preprocess_data(dfs, "kitchen", nmax)

print(df_test.head())
sequence_length = window_size

x_test = create_window_chunk(df_test['aggregate_mains'], sequence_length)

y_test = df_test['kitchen'][sequence_length:]

x_test = np.array(x_test)
y_test = np.array(y_test)
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))<jupyter_output><empty_output><jupyter_text>*Predicting on the test*<jupyter_code>pred_test = model.predict(x_test, batch_size=128)

pred_test = np.reshape(pred_test, (len(pred_test)))<jupyter_output><empty_output><jupyter_text>*Denormalized the test and predicted data*<jupyter_code>y_test_d = y_test * nmax
y_pred_d = pred_test* nmax<jupyter_output><empty_output><jupyter_text>**Data Visualization**<jupyter_code>plt.figure(figsize = (30, 8))
plt.plot(y_test, color='green', label="ground kitchen outlet")
plt.plot(pred_test, color='blue', label="predicted kitchen outlet")

plt.legend
plt.show<jupyter_output><empty_output><jupyter_text>*More Zooming into the graph*<jupyter_code>plt.figure(figsize = (30, 8))
plt.plot(pred_test[50000:100000], color='green', label="predicted kitchen outlet")
plt.plot(y_test[50000:100000], color='blue', label="ground kitchen outlet")

plt.legend
plt.show<jupyter_output><empty_output><jupyter_text>**Evaluation Metrics**<jupyter_code># Calculating MAE on test data
mean_absolute_error(y_test_d, y_pred_d)
# Calculating RMSE on test data
math.sqrt(mean_squared_error(y_test_d, y_pred_d))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21028921524461663, 'fraction_numerical': 0.046670871249662134, 'mean_word_length': 2.714859437751004, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '13198', 'license': 'MIT', 'n_tokens_mistral': 4008, 'n_tokens_neox': 3644, 'n_words': 767, 'path': 'Training_Notebooks/WindowGRU_Kitchen_Outlet.ipynb', 'repo_name': 'pferido/Energy-Disaggregation', 'size': '11099'}