<jupyter_start><jupyter_text>Read data<jupyter_code>import pandas as pd
import numpy as np

data = pd.read_csv("AvgAlKras.csv")
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 43819 entries, 0 to 43818
Data columns (total 2 columns):
Unnamed: 0    43819 non-null object
AlKras        43819 non-null float64
dtypes: float64(1), object(1)
memory usage: 684.8+ KB<jupyter_text>Data split<jupyter_code>import math

def split_data(data, frame_size, n_frames):
    X, y = [], []
    for i in range(len(data)):
        # find the end of this pattern
        end_ix = i + frame_size
        # check if we are beyond the sequence
        if end_ix > len(data)-1:
            break
        # gather input and output parts of the pattern
        seq_x, seq_y = data[i:end_ix], data[end_ix]
        X.append(seq_x)
        y.append(seq_y)
    return np.array(X), np.array(y)
# 
frame_size = 24
n_frames = math.floor(len(data) / frame_size)
n_features = 1

X, y = split_data(data.iloc[:,1].values, frame_size, n_frames)
X = X.reshape(-1,frame_size,n_features)
print('Data type: ', type(X))
print('X shape:', X.shape, 'y shape:', y.shape)<jupyter_output>Data type:  <class 'numpy.ndarray'>
X shape: (43795, 24, 1) y shape: (43795,)<jupyter_text>Data split<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=1)

print(np.isnan(X_train).any())
print(np.isnan(y_train).any())
print(np.isnan(X_val).any())
print(np.isnan(y_val).any())
print(np.isnan(X_test).any())
print(np.isnan(y_test).any())<jupyter_output>False
False
False
False
False
False<jupyter_text>LSTM<jupyter_code>from tensorflow import keras
from keras import optimizers
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout, Flatten<jupyter_output>Using TensorFlow backend.<jupyter_text>Construct model<jupyter_code>model = Sequential()

model.add(LSTM(20,activation='relu',return_sequences=True, input_shape=(frame_size, n_features)))
model.add(LSTM(20,activation='relu',return_sequences=False))

model.add(Dense(1))

adam = optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-8, decay=0.0, amsgrad=False)

model.compile(optimizer=adam, loss="mse", metrics=None)<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_3 (LSTM)                (None, 24, 20)            1760      
_________________________________________________________________
lstm_4 (LSTM)                (None, 20)                3280      
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 21        
=================================================================
Total params: 5,061
Trainable params: 5,061
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Train model<jupyter_code>history = model.fit(x=X_train,y=y_train, batch_size=None, epochs=5, validation_data=(X_val,y_val))
import matplotlib.pyplot as plt
%matplotlib inline

fig = plt.figure(figsize=(10,6))

plt.plot(history.history["loss"], label='train loss', color='darkmagenta')
plt.plot(history.history["val_loss"], label='val loss', color='red')
plt.legend(fontsize='x-large')
plt.xlabel('epoch', fontsize=14)
plt.ylabel('loss', fontsize=14)<jupyter_output><empty_output><jupyter_text>Test model<jupyter_code>predictions = model.predict(X_test)
limit = 168
x = list(range(len(y_test[:limit])))
plt.figure(figsize=(20,3))
plt.scatter(x, y_test[:limit], label='test', color='darkmagenta', alpha=0.5)
plt.scatter(x, predictions[:limit], label='prediction', color='red', alpha=0.5)
plt.ylabel('value', fontsize=14)
plt.xlabel('sample', fontsize=14)
plt.legend(fontsize='x-large')
from sklearn.metrics import r2_score
r2_score(y_test[:limit], predictions[:limit])
from keras.models import load_model

model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'
del model  # deletes the existing model

#model = load_model('my_model.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1581146970048758, 'fraction_numerical': 0.03134432319479916, 'mean_word_length': 5.517397881996974, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '211908', 'license': 'MIT', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1477, 'n_words': 294, 'path': 'hourly_pred/Prediction.ipynb', 'repo_name': 'tynski/AirPolutionPrediction', 'size': '4307'}