<jupyter_start><jupyter_text>Primary Stock Forecast Algorithm  Tensorflow Version Check<jupyter_code># Install TensorFlow
# !pip install -q tensorflow-gpu==2.0.0-beta1

try:
  %tensorflow_version 2.x  # Colab only.
except Exception:
  pass

import tensorflow as tf
print(tf.__version__)<jupyter_output>`%tensorflow_version` only switches the major version: 1.x or 2.x.
You set: `2.x  # Colab only.`. This will be interpreted as: `2.x`.


TensorFlow 2.x selected.
2.2.0<jupyter_text>Imports<jupyter_code># More imports
from tensorflow.keras.layers import Input, LSTM, GRU, SimpleRNN, Dense, GlobalMaxPool1D
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import SGD, Adam

import numpy as np
import plotly.express as px
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
plt.style.use('seaborn-poster')<jupyter_output><empty_output><jupyter_text>Reading the Data<jupyter_code># yes, you can read dataframes from URLs!
df = pd.read_csv('ITC.csv')
df.head()
print(df.columns)<jupyter_output>Index(['Unnamed: 0', 'Date ', 'series ', 'OPEN ', 'HIGH ', 'LOW ',
       'PREV. CLOSE ', 'ltp ', 'close ', 'vwap ', '52W H ', '52W L ',
       'VOLUME ', 'VALUE ', 'No of trades '],
      dtype='object')<jupyter_text>Cleaning the Data - Removing Duplicates of Dates- Sorting by Date<jupyter_code>df['Date '] = pd.to_datetime(df['Date '])
df = df.sort_values(by='Date ')
len(df['Date '].unique())
df = df.drop_duplicates(subset='Date ', keep='first')<jupyter_output><empty_output><jupyter_text>Simple Time Series Plot to get a general idea of the graph and stock price<jupyter_code>time_series = px.line(df ,x='Date ', y='close ')
time_series.show()<jupyter_output><empty_output><jupyter_text>Approach 1: (The Wrong Approach)<jupyter_code># Start by doing the WRONG thing - trying to predict the price itself
series = df['close '].values.reshape(-1, 1)
# Normalize the data
# Note: I didn't think about where the true boundary is, this is just approx.
scaler = StandardScaler()
scaler.fit(series[:len(series) // 2])
series = scaler.transform(series).flatten()
### build the dataset
# let's see if we can use T past values to predict the next value
T = 2000
D = 1
X = []
Y = []
for t in range(len(series) - T):
  x = series[t:t+T]
  X.append(x)
  y = series[t+T]
  Y.append(y)

X = np.array(X).reshape(-1, T, 1) # Now the data should be N x T x D
Y = np.array(Y)
N = len(X)
print("X.shape", X.shape, "Y.shape", Y.shape)
### try autoregressive RNN model
i = Input(shape=(T, 1))
x = LSTM(5)(i)
x = Dense(1)(x)
model = Model(i, x)
model.compile(
  loss='mse',
  optimizer=Adam(lr=0.1),
)

# train the RNN
r = model.fit(
  X[:-N//2], Y[:-N//2],
  epochs=80,
  validation_data=(X[-N//2:], Y[-N//2:]),
)
fig = px.line()
fig.add_scatter(y=r.history['loss'], mode='lines', name='loss')
fig.add_scatter(y=r.history['val_loss'], mode='lines', name='val_loss')
fig.show()
# One-step forecast using true targets
outputs = model.predict(X)
print(outputs.shape)
predictions = outputs[:,0]



fig = px.line()
fig.add_scatter(y=Y, name='actual')
fig.add_scatter(y=predictions, name='predictions')
fig.show()
# plt.plot(Y, label='targets')
# plt.plot(predictions, 'ro', label='predictions')
# plt.legend()
# plt.show()
# Multi-step forecast
validation_target = Y[-N//2:]
validation_predictions = []

# first validation input
last_x = X[-N//2] # 1-D array of length T

while len(validation_predictions) < len(validation_target):
  p = model.predict(last_x.reshape(1, T, 1))[0,0] # 1x1 array -> scalar
  
  # update the predictions list
  validation_predictions.append(p)
  
  # make the new input
  last_x = np.roll(last_x, -1)
  last_x[-1] = p

fig = px.line()
fig.add_scatter(y=validation_target, name='forecast target')
fig.add_scatter(y=validation_predictions, name='forecast predictions')
fig.show()
# plt.plot(validation_target, label='forecast target')
# plt.plot(validation_predictions, label='forecast prediction')
# plt.legend()<jupyter_output><empty_output><jupyter_text>**THIS HORRIBLY FAILS ON THE VALIDATION SET**  Approach 2: LSTM Returns Approach<jupyter_code># calculate returns by first shifting the data
df['PREV. CLOSE'] = df['close '].shift(1) # move everything up 1

# so now it's like
# close / prev close
# x[2] x[1]
# x[3] x[2]
# x[4] x[3]
# ...
# x[t] x[t-1]
df.head()
# then the return is
# (x[t] - x[t-1]) / x[t-1]
df['Return'] = (df['close '] - df['PREV. CLOSE']) / df['PREV. CLOSE']
df.head()
# Now let's try an LSTM to predict returns
fig = px.histogram(x=df['Return'])
fig.show()
series = df['Return'].values[1:].reshape(-1, 1)

# Normalize the data
# Note: I didn't think about where the true boundary is, this is just approx.
scaler = StandardScaler()
scaler.fit(series[:len(series) // 2])
series = scaler.transform(series).flatten()
### build the dataset
# let's see if we can use T past values to predict the next value
T = 10
D = 1
X = []
Y = []
for t in range(len(series) - T):
  x = series[t:t+T]
  X.append(x)
  y = series[t+T]
  Y.append(y)

X = np.array(X).reshape(-1, T, 1) # Now the data should be N x T x D
Y = np.array(Y)
N = len(X)
print("X.shape", X.shape, "Y.shape", Y.shape)
### try autoregressive RNN model
i = Input(shape=(T, 1))
x = LSTM(5)(i)
x = Dense(1)(x)
model = Model(i, x)
model.compile(
  loss='mse',
  optimizer=Adam(lr=0.01),
)

# train the RNN
r = model.fit(
  X[:-N//2], Y[:-N//2],
  epochs=80,
  validation_data=(X[-N//2:], Y[-N//2:]),
)
# # Plot loss per iteration
# import matplotlib.pyplot as plt
# plt.plot(r.history['loss'], label='loss')
# plt.plot(r.history['val_loss'], label='val_loss')
# plt.legend()

fig = px.line()
fig.add_scatter(y=r.history['loss'], mode='lines', name='loss')
fig.add_scatter(y=r.history['val_loss'], mode='lines', name='val_loss')
fig.show()
# One-step forecast using true targets
outputs = model.predict(X)
print(outputs.shape)
predictions = outputs[:,0]

# plt.plot(Y, label='targets')
# plt.plot(predictions, label='predictions')
# plt.legend()
# plt.show()

fig = px.line()
fig.add_scatter(y=Y, name='actual')
fig.add_scatter(y=predictions, name='predictions')
fig.show()

# Multi-step forecast
validation_target = Y[-N//2:]
validation_predictions = []

# first validation input
last_x = X[-N//2] # 1-D array of length T

while len(validation_predictions) < len(validation_target):
  p = model.predict(last_x.reshape(1, T, 1))[0,0] # 1x1 array -> scalar
  
  # update the predictions list
  validation_predictions.append(p)
  
  # make the new input
  last_x = np.roll(last_x, -1)
  last_x[-1] = p

# plt.plot(validation_target, label='forecast target')
# plt.plot(validation_predictions, label='forecast prediction')
# plt.legend()


fig = px.line()
fig.add_scatter(y=validation_target, name='forecast target')
fig.add_scatter(y=validation_predictions, name='forecast predictions')
fig.show()
# Now turn the full data into numpy arrays

# Not yet in the final "X" format!
input_data = df[['OPEN ', 'HIGH ', 'LOW ', 'close ', 'VOLUME ']].values
targets = df['Return'].values
# Now make the actual data which will go into the neural network
T = 10 # the number of time steps to look at to make a prediction for the next day
D = input_data.shape[1]
N = len(input_data) - T # (e.g. if T=10 and you have 11 data points then you'd only have 1 sample)
# normalize the inputs
Ntrain = len(input_data) * 2 // 3
scaler = StandardScaler()
scaler.fit(input_data[:Ntrain + T - 1])
input_data = scaler.transform(input_data)
# Setup X_train and Y_train
X_train = np.zeros((Ntrain, T, D))
Y_train = np.zeros(Ntrain)

for t in range(Ntrain):
  X_train[t, :, :] = input_data[t:t+T]
  Y_train[t] = (targets[t+T] > 0)
# Setup X_test and Y_test
X_test = np.zeros((N - Ntrain, T, D))
Y_test = np.zeros(N - Ntrain)

for u in range(N - Ntrain):
  # u counts from 0...(N - Ntrain)
  # t counts from Ntrain...N
  t = u + Ntrain
  X_test[u, :, :] = input_data[t:t+T]
  Y_test[u] = (targets[t+T] > 0)
# make the RNN
i = Input(shape=(T, D))
x = LSTM(50)(i)
x = Dense(1, activation='sigmoid')(x)
model = Model(i, x)
model.compile(
  loss='binary_crossentropy',
  optimizer=Adam(lr=0.001),
  metrics=['accuracy'],
)
# train the RNN
r = model.fit(
  X_train, Y_train,
  batch_size=32,
  epochs=1000, ##CHANGE BACK TO 300 OR LESS
  validation_data=(X_test, Y_test),
)
# plot the loss
# plt.plot(r.history['loss'], label='loss')
# plt.plot(r.history['val_loss'], label='val_loss')
# plt.legend()
# plt.show()

fig = px.line()
fig.add_scatter(y=r.history['loss'], mode='lines', name='loss')
fig.add_scatter(y=r.history['val_loss'], mode='lines', name='val_loss')
fig.show()
# Plot accuracy per iteration
# plt.plot(r.history['accuracy'], label='accuracy')
# plt.plot(r.history['val_accuracy'], label='val_accuracy')
# plt.legend()
# plt.show()

fig = px.line()
fig.add_scatter(y=r.history['accuracy'], mode='lines', name='accuracy')
fig.add_scatter(y=r.history['val_accuracy'], mode='lines', name='val_accuracy')
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17002336708579058, 'fraction_numerical': 0.014242795148547902, 'mean_word_length': 4.847755367599219, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '539064', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3510, 'n_tokens_neox': 3450, 'n_words': 866, 'path': 'src/old_src/ai-backend/Completed LSTM.ipynb', 'repo_name': 'sumukshashidhar/toreda', 'size': '8987'}