<jupyter_start><jupyter_text>Stock Market Prediction using Numerical and Textual Analysis  <NAME> <EMAIL> https://www.linkedin.com/in/mallikarjun-jamadarkhani-950929101/  1 Import libraries<jupyter_code># download nltk if not installed previously
# nltk.download('vader_lexicon')
# download yfinance
pip install yfinance --upgrade --no-cache-dir
# importing libraries
import warnings
warnings.filterwarnings('ignore')

import math
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout, Dense, Activation

import nltk
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
nltk.download('vader_lexicon')

from sklearn import preprocessing, metrics
from sklearn.preprocessing import MinMaxScaler

import yfinance as yf<jupyter_output>[nltk_data] Downloading package vader_lexicon to /root/nltk_data...
[nltk_data]   Package vader_lexicon is already up-to-date!<jupyter_text>2 Downloading Datasets | Stock Price Data | Stock News Headlines Data<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# reading the datasets into pandas
stock_price = yf.download('^BSESN', start='2015-01-01', end='2020-12-31')
stock_headlines = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/ML_Datasets/india-news-headlines.csv')
# displaying stock price dataset
stock_price.head()
# displaying stock news headlines dataset
stock_headlines.head()<jupyter_output><empty_output><jupyter_text>3 Feature Engineering<jupyter_code># displaying number of records in both stock_price and stock_headlines datasets
len(stock_price), len(stock_headlines)
# checking for null values in both the datasets
stock_price.isna().any(), stock_headlines.isna().any()<jupyter_output><empty_output><jupyter_text>3.1 Stock Price Data<jupyter_code># dropping duplicates
stock_price = stock_price.drop_duplicates()

# filtering the important columns required
stock_price = stock_price.filter(['Close', 'Open', 'High', 'Low', 'Volume'])

# sorting the data according to the index i.e 'Date'
stock_price = stock_price.sort_index(ascending=True, axis=0)
stock_price<jupyter_output><empty_output><jupyter_text>3.2 Stock News Headlines Data<jupyter_code># dropping duplicates
stock_headlines = stock_headlines.drop_duplicates()

# coverting the datatype of column 'Date' from type string to type 'datetime'
stock_headlines['publish_date'] = stock_headlines['publish_date'].astype(str)
stock_headlines['publish_date'] = stock_headlines['publish_date'].apply(lambda x: x[0:4]+'-'+x[4:6]+'-'+x[6:8])
stock_headlines['publish_date'] = pd.to_datetime(stock_headlines['publish_date']).dt.normalize()

# filtering the important columns required
stock_headlines = stock_headlines.filter(['publish_date', 'headline_text'])

# grouping the news headlines according to 'Date'
stock_headlines = stock_headlines.groupby(['publish_date'])['headline_text'].apply(lambda x: ','.join(x)).reset_index()

# setting column 'Date' as the index column
stock_headlines.set_index('publish_date', inplace= True)

# sorting the data according to the index i.e 'Date'
stock_headlines = stock_headlines.sort_index(ascending=True, axis=0)
stock_headlines<jupyter_output><empty_output><jupyter_text>3.3 Combined Stock Data<jupyter_code># concatenating the datasets stock_price and stock_headlines
stock_data = pd.concat([stock_price, stock_headlines], axis=1)

# dropping the null values if any
stock_data.dropna(axis=0, inplace=True)

# displaying the combined stock_data
stock_data<jupyter_output><empty_output><jupyter_text>3.4 Calculating Sentiment Scores<jupyter_code># adding empty sentiment columns to stock_data for later calculation
stock_data['compound'] = ''
stock_data['negative'] = ''
stock_data['neutral'] = ''
stock_data['positive'] = ''
stock_data.head()

  
# importing requires libraries to analyze the sentiments
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import unicodedata

# instantiating the Sentiment Analyzer
sid = SentimentIntensityAnalyzer()

# calculating sentiment scores
stock_data['compound'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['compound'])
stock_data['negative'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['neg'])
stock_data['neutral'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['neu'])
stock_data['positive'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['pos'])

# displaying the stock data
stock_data.head()<jupyter_output><empty_output><jupyter_text>4 Finalising Stock Data and writing to Disk<jupyter_code># dropping the 'headline_text' which is unwanted now
stock_data.drop(['headline_text'], inplace=True, axis=1)

# rearranging the columns of the whole stock_data
stock_data = stock_data[['Close', 'compound', 'negative', 'neutral', 'positive', 'Open', 'High', 'Low', 'Volume']]

# displaying the final stock_data
stock_data.head()
# writing the prepared stock_data to disk
stock_data.to_csv('stock_data.csv')<jupyter_output><empty_output><jupyter_text>5 Reading Stock Data<jupyter_code># re-reading the stock_data into pandas dataframe
stock_data = pd.read_csv('stock_data.csv', index_col = False)

# renaming the column
stock_data.rename(columns={'Unnamed: 0':'Date'}, inplace = True)

# setting the column 'Date' as the index column
stock_data.set_index('Date', inplace=True)

# displaying the stock_data
stock_data.head()<jupyter_output><empty_output><jupyter_text>6 Feature Engineering of Stock Data<jupyter_code># displaying the shape i.e. number of rows and columns of stock_data
stock_data.shape
# checking for null values
stock_data.isna().any()
# displaying stock_data statistics
stock_data.describe(include='all')
# displaying stock_data information
stock_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 1473 entries, 2015-01-02 to 2020-12-30
Data columns (total 9 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   Close     1473 non-null   float64
 1   compound  1473 non-null   float64
 2   negative  1473 non-null   float64
 3   neutral   1473 non-null   float64
 4   positive  1473 non-null   float64
 5   Open      1473 non-null   float64
 6   High      1473 non-null   float64
 7   Low       1473 non-null   float64
 8   Volume    1473 non-null   float64
dtypes: float64(9)
memory usage: 115.1+ KB<jupyter_text>7 EDA of Stock Data<jupyter_code># setting figure size
plt.figure(figsize=(16,10))

# plotting close price
stock_data['Close'].plot()

# setting plot title, x and y labels
plt.title("Close Price")
plt.xlabel('Date')
plt.ylabel('Close Price ($)')
# calculating 7 day rolling mean
stock_data.rolling(7).mean().head(20)
# setting figure size
plt.figure(figsize=(16,10))

# plotting the close price and a 30-day rolling mean of close price
stock_data['Close'].plot()
stock_data.rolling(window=30).mean()['Close'].plot()
# displaying stock_data
stock_data<jupyter_output><empty_output><jupyter_text>8 Data Preparation for Modelling<jupyter_code># calculating data_to_use
percentage_of_data = 1.0
data_to_use = int(percentage_of_data*(len(stock_data)-1))

# using 80% of data for training
train_end = int(data_to_use*0.8)
total_data = len(stock_data)
start = total_data - data_to_use

# printing number of records in the training and test datasets
print("Number of records in Training Data:", train_end)
print("Number of records in Test Data:", total_data - train_end)
# predicting one step ahead
steps_to_predict = 1

# capturing data to be used for each column
close_price = stock_data.iloc[start:total_data,0] #close
compound = stock_data.iloc[start:total_data,1] #compound
negative = stock_data.iloc[start:total_data,2] #neg
neutral = stock_data.iloc[start:total_data,3] #neu
positive = stock_data.iloc[start:total_data,4] #pos
open_price = stock_data.iloc[start:total_data,5] #open
high = stock_data.iloc[start:total_data,6] #high
low = stock_data.iloc[start:total_data,7] #low
volume = stock_data.iloc[start:total_data,8] #volume

# printing close price
print("Close Price:")
close_price
# shifting next day close
close_price_shifted = close_price.shift(-1) 

# shifting next day compound
compound_shifted = compound.shift(-1) 

# concatenating the captured training data into a dataframe
data = pd.concat([close_price, close_price_shifted, compound, compound_shifted, volume, open_price, high, low], axis=1)

# setting column names of the revised stock data
data.columns = ['close_price', 'close_price_shifted', 'compound', 'compound_shifted','volume', 'open_price', 'high', 'low']

# dropping nulls
data = data.dropna()    
data.head(10)<jupyter_output><empty_output><jupyter_text>9 Setting the Target Variable and the Feature Dataset<jupyter_code># setting the target variable as the shifted close_price
y = data['close_price_shifted']
y
# setting the features dataset for prediction  
cols = ['close_price', 'compound', 'compound_shifted', 'volume', 'open_price', 'high', 'low']
x = data[cols]
x<jupyter_output><empty_output><jupyter_text>9.1 Scaling the Target Variable and the Feature Dataset<jupyter_code># scaling the feature dataset
scaler_x = preprocessing.MinMaxScaler (feature_range=(-1, 1))
x = np.array(x).reshape((len(x) ,len(cols)))
x = scaler_x.fit_transform(x)

# scaling the target variable
scaler_y = preprocessing.MinMaxScaler (feature_range=(-1, 1))
y = np.array (y).reshape ((len( y), 1))
y = scaler_y.fit_transform (y)

# displaying the scaled feature dataset and the target variable
x, y<jupyter_output><empty_output><jupyter_text>9.2 Dividing the dataset into Training and Test<jupyter_code># preparing training and test dataset
X_train = x[0 : train_end,]
X_test = x[train_end+1 : len(x),]    
y_train = y[0 : train_end] 
y_test = y[train_end+1 : len(y)]  

# printing the shape of the training and the test datasets
print('Number of rows and columns in the Training set X:', X_train.shape, 'and y:', y_train.shape)
print('Number of rows and columns in the Test set X:', X_test.shape, 'and y:', y_test.shape)<jupyter_output>Number of rows and columns in the Training set X: (1177, 7) and y: (1177, 1)
Number of rows and columns in the Test set X: (293, 7) and y: (293, 1)<jupyter_text>9.3 Reshaping the Feature Dataset<jupyter_code># reshaping the feature dataset for feeding into the model
X_train = X_train.reshape (X_train.shape + (1,)) 
X_test = X_test.reshape(X_test.shape + (1,))

# printing the re-shaped feature dataset
print('Shape of Training set X:', X_train.shape)
print('Shape of Test set X:', X_test.shape)<jupyter_output>Shape of Training set X: (1177, 7, 1)
Shape of Test set X: (293, 7, 1)<jupyter_text>10 Stock Data Modelling<jupyter_code># setting the seed to achieve consistent and less random predictions at each execution
np.random.seed(2016)

# setting the model architecture
model=Sequential()
model.add(LSTM(100,return_sequences=True,activation='tanh',input_shape=(len(cols),1)))
model.add(Dropout(0.1))
model.add(LSTM(100,return_sequences=True,activation='tanh'))
model.add(Dropout(0.1))
model.add(LSTM(100,activation='tanh'))
model.add(Dropout(0.1))
model.add(Dense(1))

# printing the model summary
model.summary()
# compiling the model
model.compile(loss='mse' , optimizer='adam')

# fitting the model using the training dataset
model.fit(X_train, y_train, validation_split=0.2, epochs=10, batch_size=8, verbose=1)<jupyter_output>Epoch 1/10
118/118 [==============================] - 10s 31ms/step - loss: 0.0563 - val_loss: 0.0610
Epoch 2/10
118/118 [==============================] - 2s 21ms/step - loss: 0.0261 - val_loss: 0.0548
Epoch 3/10
118/118 [==============================] - 2s 21ms/step - loss: 0.0119 - val_loss: 0.0069
Epoch 4/10
118/118 [==============================] - 2s 21ms/step - loss: 0.0028 - val_loss: 0.0049
Epoch 5/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0025 - val_loss: 0.0027
Epoch 6/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0022 - val_loss: 0.0025
Epoch 7/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0020 - val_loss: 0.0023
Epoch 8/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0022 - val_loss: 0.0023
Epoch 9/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0018 - val_loss: 0.0024
Epoch 10/10
118/118 [==============================] - 2s 20ms/step - loss: 0.0017 - val_l[...]<jupyter_text>10.1 Saving the Model to disk<jupyter_code># saving the model as a json file
model_json = model.to_json()
with open('model.json', 'w') as json_file:
    json_file.write(model_json)
    
# serialize weights to HDF5
model.save_weights('model.h5')
print('Model is saved to the disk')<jupyter_output>Model is saved to the disk<jupyter_text>11 Predictions<jupyter_code># performing predictions
predictions = model.predict(X_test) 

# unscaling the predictions
predictions = scaler_y.inverse_transform(np.array(predictions).reshape((len(predictions), 1)))

# printing the predictions
print('Predictions:')
predictions[0:5]<jupyter_output>Predictions:<jupyter_text>12 Model Evaluation<jupyter_code># calculating the training mean-squared-error
train_loss = model.evaluate(X_train, y_train, batch_size = 1)

# calculating the test mean-squared-error
test_loss = model.evaluate(X_test, y_test, batch_size = 1)

# printing the training and the test mean-squared-errors
print('Train Loss =', round(train_loss,4))
print('Test Loss =', round(test_loss,4))
# calculating root mean squared error
root_mean_square_error = np.sqrt(np.mean(np.power((y_test - predictions),2)))
print('Root Mean Square Error =', round(root_mean_square_error,4))
# calculating root mean squared error using sklearn.metrics package
rmse = metrics.mean_squared_error(y_test, predictions)
print('Root Mean Square Error (sklearn.metrics) =', round(np.sqrt(rmse),4))<jupyter_output>Root Mean Square Error (sklearn.metrics) = 38672.2948<jupyter_text>13 Plotting the Predictions against unseen data<jupyter_code># unscaling the test feature dataset, x_test
X_test = scaler_x.inverse_transform(np.array(X_test).reshape((len(X_test), len(cols))))

# unscaling the test y dataset, y_test
y_train = scaler_y.inverse_transform(np.array(y_train).reshape((len(y_train), 1)))
y_test = scaler_y.inverse_transform(np.array(y_test).reshape((len(y_test), 1)))
# plotting
plt.figure(figsize=(16,10))

# plt.plot([row[0] for row in y_train], label="Training Close Price")
plt.plot(predictions, label="Predicted Close Price")
plt.plot([row[0] for row in y_test], label="Testing Close Price")
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=2)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13706954760297096, 'fraction_numerical': 0.0350438892640108, 'mean_word_length': 6.361332007952286, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 80, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '90734', 'license': 'Apache-2.0', 'n_tokens_mistral': 5362, 'n_tokens_neox': 4900, 'n_words': 1273, 'path': 'stock_market_prediction.ipynb', 'repo_name': 'Mallikarjun29/Stock_price_prediction', 'size': '14849'}