<jupyter_start><jupyter_text>Recurrent Neural Network  Part 1 - Data Preprocessing  Importing the libraries<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import datetime as dt<jupyter_output><empty_output><jupyter_text>Importing the training set<jupyter_code>dataset_train = pd.read_csv("no2.csv")<jupyter_output><empty_output><jupyter_text>Cleaning the data<jupyter_code>col_to_remove_no2 =('wkt_geom', 'id', 'timestam_1', 'unit_NO2', 'unit_PM10', 'unit_RH', 'unit_P', 'value_PM10', 'value_RH', 'value_P')
for name in col_to_remove_no2:
  if name in dataset_train.columns:
    dataset_train = dataset_train.drop(name,1)


dates=[]
for i in range(len(dataset_train)):
  dates.append(dataset_train['timestamp_'][i])

cleaned_dates_no2=[]
for date in dates:
  value = dt.datetime.strptime(date, '%Y-%m-%d')
  cleaned_dates_no2.append(value)
dates=[] #emptying list

dataset_train['dates'] = cleaned_dates_no2
##creating list of dataframes for each no2 sensor
no2_sensors = dataset_train.label.unique()

dataset_per_no2_sensor =[] #list of dataframes for each NO2 sensor

for sensor in no2_sensors:
  dataset_per_no2_sensor.append(dataset_train.loc[dataset_train['label'] == sensor])

print(dataset_per_no2_sensor[0])
#CHECKING AND MITIGATING FOR NAN VALUES
window_size = 20 #select a window size to calculate average values for cells with no data
count =0
for dataset in dataset_per_no2_sensor: 
  for r in range(len(dataset)):
    if np.isnan(dataset.iloc[r,2]):
      count+=1
     # print(np.isnan(dataset.iloc[r,2]))
      window = range(window_size, -window_size)
      mean_values=[]
      for w in window:
        if dataset.iloc[r+w,2] >=0:
          mean_values.append(dataset.iloc[r+w,2])
        if len(mean_values)!=0:
          mean = (sum(mean_values))/len(mean_values)
          dataset.iloc[r,2]=mean
    if np.isnan(dataset.iloc[r,2]) :
      mean = dataset['value_NO2'].mean()
      dataset.iloc[r,2] = mean
#break into test and train dataset

no2_train=[]
no2_test=[]
train_split_ratio = 0.8
for dataset in dataset_per_no2_sensor:
  print(dataset.isnull().any())
  train_len = int(train_split_ratio*len(dataset))
  test_len = len(dataset) - train_len

  no2_train.append(dataset[:train_len])
  no2_test.append(dataset[train_len:-1])<jupyter_output><empty_output><jupyter_text>The model<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout, Embedding
from keras.regularizers import l2
from tensorflow.keras import activations

for i in range(len(no2_train)):
  training_set = no2_train[i]
  training_set= training_set.iloc[:,2:3].values


  test_set = no2_test[i]
  test_set= test_set.iloc[:,2:3].values

  sc = MinMaxScaler(feature_range=(0,1))
  training_set_sc = sc.fit_transform(training_set)  

  print(training_set_sc)
  print(len(training_set_sc))

  x_train=[]
  y_train=[]
  for x in range(144, len(training_set)): 
    x_train.append(training_set_sc[x-144:x,0])#should make 1198 records:60 columns
    y_train.append(training_set_sc[x,0])
  x_train, y_train = np.array(x_train), np.array(y_train)

  x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1],1))# shape - batch_size, timestep, number of indicators

  RNNregressor = Sequential()

  RNNregressor.add(LSTM(units=50, return_sequences=True, input_shape=(x_train.shape[1],1)))
  RNNregressor.add(Dropout(0.2))##20% dropout

  RNNregressor.add(LSTM(units=50, return_sequences=True))
  RNNregressor.add(Dropout(0.2))##20% dropout

  RNNregressor.add(LSTM(units=50, return_sequences=True))
  RNNregressor.add(Dropout(0.2))##20% dropout

  RNNregressor.add(LSTM(units=50))
  RNNregressor.add(Dropout(0.2))##20% dropout

  RNNregressor.add(Dense(units=1))

  RNNregressor.compile(optimizer = 'Adam', loss = 'mean_squared_error')#experiment with different optimizeers

  RNNregressor.fit(x=x_train, y = y_train, epochs = 50, batch_size = 32 )

  dataset_concat=dataset_per_no2_sensor[i]
  dataset_concat = dataset_concat.iloc[:,2:3]


  inputs = dataset_concat[len(dataset_concat)-len(test_set)-144 : ].values

  inputs = inputs.reshape(-1,1)

  inputs = sc.transform(inputs)

  x_test=[]
  for y in range(144, 144+120): #create upper bound for 20 financial days of jab 2k17, using timestep = 60
    x_test.append(inputs[y-144:y,0])#should make 1198 records:60 columns
  
  x_test = np.array(x_test)
  x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1],1))# shape - batch_size, timestep, number of indicators

  predicted_stock_price = RNNregressor.predict(x_test)
  predicted_stock_price = sc.inverse_transform(predicted_stock_price)

  plt.plot(test_set[0:120], color ='red', label = "Real NO2")
  plt.plot(predicted_stock_price, color ='blue', label = "predicted No2")

  plt.title('No2 prediction prediction')
  plt.xlabel('records')
  plt.ylabel('No2 levels')
  plt.legend()
  plt.show()


  

# Use range for columns to ensure the right shape of the data for the np array --- all rows:one column
#we use the open column to decide the trends of the data<jupyter_output>[[0.61415525]
 [0.42170293]
 [0.42170293]
 ...
 [0.40357239]
 [0.39833468]
 [0.35616438]]
2894
Epoch 1/50
86/86 [==============================] - 17s 193ms/step - loss: 0.0145
Epoch 2/50
86/86 [==============================] - 17s 197ms/step - loss: 0.0113
Epoch 3/50
86/86 [==============================] - 18s 207ms/step - loss: 0.0111
Epoch 4/50
86/86 [==============================] - 17s 203ms/step - loss: 0.0115
Epoch 5/50
86/86 [==============================] - 18s 206ms/step - loss: 0.0106
Epoch 6/50
86/86 [==============================] - 18s 206ms/step - loss: 0.0102
Epoch 7/50
86/86 [==============================] - 18s 206ms/step - loss: 0.0101
Epoch 8/50
86/86 [==============================] - 18s 206ms/step - loss: 0.0096
Epoch 9/50
86/86 [==============================] - 18s 208ms/step - loss: 0.0095
Epoch 10/50
86/86 [==============================] - 18s 209ms/step - loss: 0.0092
Epoch 11/50
86/86 [==============================] - 18s 207ms/step - loss: 0.0092
E[...]
{'fraction_non_alphanumeric': 0.17886311662855275, 'fraction_numerical': 0.06452139823587064, 'mean_word_length': 5.159959758551308, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '465152', 'license': 'CC0-1.0', 'n_tokens_mistral': 2526, 'n_tokens_neox': 2228, 'n_words': 463, 'path': 'Deep learning model/Senshagen_project.ipynb', 'repo_name': 'yttehs123/Coppernicus_project_zwolle', 'size': '6122'}