<jupyter_start><jupyter_text>**Data Preprocess**<jupyter_code>import pandas as pd
df = pd.read_csv('/content/MCIRD_aaic2021_train.csv', header=0)
df.head()

df1 = df[['subscriber_ecid', 'data_usage_volume']]
df1.head()<jupyter_output><empty_output><jupyter_text>خیلی از مقادیر دیتاست -0.0139766307813943 این هستن و میشه به راحتی اینارو جایگزین کرد<jupyter_code>unique_sub_id = set(df1['subscriber_ecid'].values)
len(unique_sub_id)<jupyter_output><empty_output><jupyter_text>use just sample with 69 element<jupyter_code>import numpy as np
data_list = []
for sub_id in unique_sub_id:
    # print(sub_id)
    temp = df1[df1['subscriber_ecid']==sub_id].values
    if temp.shape[0] < 69:
        print(temp.shape)
        print(sub_id)
    else:
        data_list.append(temp)<jupyter_output>(60, 2)
-QXHomYaJxYXi
(61, 2)
0o-xDa8uTNBGu
(1, 2)
37v4v4PPObMC_
(18, 2)
1EN04BS-9nKgc
(1, 2)
32ez6CX89v6KZ
(68, 2)
-DgEYYT0gqMqr
(67, 2)
-XU6p4P-782mp
(2, 2)
28gWxNYMU_2dg
(53, 2)
0T7ixhiDdZ8TL
(22, 2)
-gjfIaG2oxwzj<jupyter_text>data plot<jupyter_code>import matplotlib.pyplot as plt
for j in range(len(data_list)) :
  one_sample = data_list[j]
  y = one_sample[:,1]
  x = []

  for i in range(len(one_sample[:,1])):
    x.append(i)
  plt.plot(x,y)

plt.show()<jupyter_output><empty_output><jupyter_text>**LSTM Model**<jupyter_code>X = []
y = []
mem_step = 14

for item in data_list:
  for i in range(mem_step, 69):
      X.append(item[i-mem_step:i, 1:2])
      # print(item[i-mem_step:i, 1:2])
      y.append(item[i, 1:2])

X, y = np.array(X), np.array(y)
X, y = X.astype('float32'), y.astype('float32')
 from sklearn.model_selection import train_test_split
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
import tensorflow as tf
import pandas as pd
import numpy as np
from tensorflow import keras
import matplotlib.pyplot as plt

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout

# check GPU availability
print("GPU is available :)" if tf.config.list_physical_devices("GPU") else "Not available :(")

import warnings
warnings.filterwarnings("ignore")
from tensorflow.keras.backend import clear_session 
clear_session()
keras_reg = Sequential()

keras_reg.add(LSTM(units = 64, return_sequences = True, input_shape = (X.shape[1], 1)))
keras_reg.add(Dropout(0.4))

keras_reg.add(LSTM(units = 100, return_sequences = True))
keras_reg.add(Dropout(0.4))

keras_reg.add(LSTM(units = 100, return_sequences = True))
keras_reg.add(Dropout(0.2))

keras_reg.add(LSTM(units = 16))
keras_reg.add(Dropout(0.2))

# keras_reg.add(Dense(units=32, activation='relu' ))
keras_reg.add(Dense(units=16, activation='relu' ))
keras_reg.add(Dense(units = 1))
keras_reg.compile(optimizer = 'adam',
                  loss = 'mean_squared_error',
                  metrics='mse')
# keras_reg.compile(optimizer = 'rmsprop', loss = 'mean_squared_error')
history = keras_reg.fit(X_train, y_train,validation_split=0.2,shuffle=False,epochs = 30,verbose = 1)
from math import sqrt

def plot_loss(history):
  fig = plt.figure(figsize=(7,7))
  plt.plot(history.history['mse'], label='mse')
  plt.plot(history.history['val_mse'], label='val_mse')
  plt.xlabel('Epoch')
  plt.ylabel('Error')
  plt.legend()
  plt.grid(True)

plot_loss(history)

predicted_stock_price = keras_reg.predict(X_test)
from sklearn.metrics import mean_squared_error
print(sqrt(mean_squared_error(y_test, predicted_stock_price)))<jupyter_output>2.784862872023127
{'fraction_non_alphanumeric': 0.1392369020501139, 'fraction_numerical': 0.04441913439635535, 'mean_word_length': 5.640831758034026, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '667000', 'license': 'Apache-2.0', 'n_tokens_mistral': 1585, 'n_tokens_neox': 1469, 'n_words': 262, 'path': 'LSTM.ipynb', 'repo_name': 'zahraDehghanian97/SCINet', 'size': '3512'}