<jupyter_start><jupyter_text>GRIP The Sparks Foundation(TSF)Task - 7 Stock Market Prediction using Numerical and Textual AnalysisIn this task,- Create a hybrid model for stock price/performance prediction using numerical analysis of historical stock prices, and sentimental analysis of news headlines - Stock to analyze and predict - SENSEX (S&P BSE SENSEX)Steps - **Stock Price Data | Stock News Headlines Data** - Step 1 - Importing the Library - Step 2 - Importing the Dataset - Step 3 - Feature Engineering - Step 4 - Finalising Stock Data andwriting to Disk - Step 5 - Reading Stock Data - Step 6 - Feature Engineering of Stock Data - Step 7 - EDA off Stock Data - Step 8 - Data preparation for Modelling - Step 9 - Setting the Target Variable and the Feature Dataset - Step 10 - Stock Data Modelling Saving the Model to disk - Step 11 - Predictions - Step 12 - Model Evaluation - Step 13 - Plotting the Predictions against unseen dataCreated by - <NAME><jupyter_code># download nltk if not installed previously
nltk.download('vader_lexicon')
!pip install yfinance --upgrade --no-cache-dir<jupyter_output>Collecting yfinance
  Downloading https://files.pythonhosted.org/packages/7a/e8/b9d7104d3a4bf39924799067592d9e59119fcfc900a425a12e80a3123ec8/yfinance-0.1.55.tar.gz
Requirement already satisfied, skipping upgrade: pandas>=0.24 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.1.5)
Requirement already satisfied, skipping upgrade: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.19.5)
Requirement already satisfied, skipping upgrade: requests>=2.20 in /usr/local/lib/python3.7/dist-packages (from yfinance) (2.23.0)
Requirement already satisfied, skipping upgrade: multitasking>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from yfinance) (0.0.9)
Collecting lxml>=4.5.1
[?25l  Downloading https://files.pythonhosted.org/packages/d2/88/b25778f17e5320c1c58f8c5060fb5b037288e162bd7554c30799e9ea90db/lxml-4.6.2-cp37-cp37m-manylinux1_x86_64.whl (5.5MB)
[K     |████████████████████████████████| 5.5MB 4.9MB/s 
[?25hRequirement already satisfied, skipping upgrade:[...]<jupyter_text>STEP 1 -Importing the Library<jupyter_code># importing libraries
import warnings
warnings.filterwarnings('ignore')

import math
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout, Dense, Activation

import nltk
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *

from sklearn import preprocessing, metrics
from sklearn.preprocessing import MinMaxScaler<jupyter_output><empty_output><jupyter_text>STEP 2 - Importing the Dataset<jupyter_code># reading the datasets into pandas
import yfinance as yf
stock_price = yf.download('^BSESN', start='2015-01-01', end='2021-01-01')
stock_headlines = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/india-news-headlines.csv')
# displaying stock price dataset
stock_price.head()
# displaying stock news headlines dataset
stock_headlines.head()<jupyter_output><empty_output><jupyter_text>STEP 3 - Feature Engineering<jupyter_code># displaying number of records in both stock_price and stock_headlines datasets
len(stock_price), len(stock_headlines)
# checking for null values in both the datasets
stock_price.isna().any(), stock_headlines.isna().any()<jupyter_output><empty_output><jupyter_text>**Stock Price Data**<jupyter_code># dropping duplicates
stock_price = stock_price.drop_duplicates()

# coverting the datatype of column 'Date' from type object to type 'datetime'
stock_price.reset_index(inplace=True)
stock_price['Date'] = pd.to_datetime(stock_price['Date'],format='%Y-%m-%d')

# filtering the important columns required
stock_price = stock_price.filter(['Date', 'Close', 'Open', 'High', 'Low', 'Volume'])

# setting column 'Date' as the index column
stock_price.set_index('Date', inplace= True)

# sorting the data according to the index i.e 'Date'
stock_price = stock_price.sort_index(ascending=True, axis=0)
stock_price<jupyter_output><empty_output><jupyter_text>**Stock News Headlines Data**<jupyter_code># dropping duplicates
stock_headlines = stock_headlines.drop_duplicates()

# coverting the datatype of column 'Date' from type string to type 'datetime'
stock_headlines['publish_date'] = stock_headlines['publish_date'].astype(str)
stock_headlines['publish_date'] = stock_headlines['publish_date'].apply(lambda x: x[0:4]+'-'+x[4:6]+'-'+x[6:8])
stock_headlines['publish_date'] = pd.to_datetime(stock_headlines['publish_date']).dt.normalize()

# filtering the important columns required
stock_headlines = stock_headlines.filter(['publish_date', 'headline_text'])

# grouping the news headlines according to 'Date'
stock_headlines = stock_headlines.groupby(['publish_date'])['headline_text'].apply(lambda x: ','.join(x)).reset_index()

# setting column 'Date' as the index column
stock_headlines.set_index('publish_date', inplace= True)

# sorting the data according to the index i.e 'Date'
stock_headlines = stock_headlines.sort_index(ascending=True, axis=0)
stock_headlines<jupyter_output><empty_output><jupyter_text>**Combined Stock Data**<jupyter_code># concatenating the datasets stock_price and stock_headlines
stock_data = pd.concat([stock_price, stock_headlines], axis=1)

# dropping the null values if any
stock_data.dropna(axis=0, inplace=True)

# displaying the combined stock_data
stock_data<jupyter_output><empty_output><jupyter_text>**Calculating Sentiment Scores**<jupyter_code># adding empty sentiment columns to stock_data for later calculation
stock_data['compound'] = ''
stock_data['negative'] = ''
stock_data['neutral'] = ''
stock_data['positive'] = ''
stock_data.head()
# importing requires libraries to analyze the sentiments
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import unicodedata

# instantiating the Sentiment Analyzer
sid = SentimentIntensityAnalyzer()

# calculating sentiment scores
stock_data['compound'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['compound'])
stock_data['negative'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['neg'])
stock_data['neutral'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['neu'])
stock_data['positive'] = stock_data['headline_text'].apply(lambda x: sid.polarity_scores(x)['pos'])

# displaying the stock data
stock_data.head()<jupyter_output><empty_output><jupyter_text>STEP 4 - Finalising Stock Data and writing to Disk<jupyter_code># dropping the 'headline_text' which is unwanted now
stock_data.drop(['headline_text'], inplace=True, axis=1)

# rearranging the columns of the whole stock_data
stock_data = stock_data[['Close', 'compound', 'negative', 'neutral', 'positive', 'Open', 'High', 'Low', 'Volume']]

# displaying the final stock_data
stock_data.head()
# writing the prepared stock_data to disk
stock_data.to_csv('stock_data.csv')<jupyter_output><empty_output><jupyter_text>STEP 5 - Reading Stock Data<jupyter_code># re-reading the stock_data into pandas dataframe
stock_data = pd.read_csv('stock_data.csv', index_col = False)

# renaming the column
stock_data.rename(columns={'Unnamed: 0':'Date'}, inplace = True)

# setting the column 'Date' as the index column
stock_data.set_index('Date', inplace=True)

# displaying the stock_data
stock_data.head()<jupyter_output><empty_output><jupyter_text>STEP 6 - Feature Engineering of Stock Data<jupyter_code># displaying the shape i.e. number of rows and columns of stock_data
stock_data.shape
# checking for null values
stock_data.isna().any()
# displaying stock_data statistics
stock_data.describe(include='all')
# displaying stock_data information
stock_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 4877 entries, 2001-01-02 to 2020-06-30
Data columns (total 9 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   Close     4877 non-null   float64
 1   compound  4877 non-null   float64
 2   negative  4877 non-null   float64
 3   neutral   4877 non-null   float64
 4   positive  4877 non-null   float64
 5   Open      4877 non-null   float64
 6   High      4877 non-null   float64
 7   Low       4877 non-null   float64
 8   Volume    4877 non-null   float64
dtypes: float64(9)
memory usage: 381.0+ KB<jupyter_text>STEP 7 - EDA of Stock Data<jupyter_code># setting figure size
plt.figure(figsize=(16,10))

# plotting close price
stock_data['Close'].plot()

# setting plot title, x and y labels
plt.title("Close Price")
plt.xlabel('Date')
plt.ylabel('Close Price ($)')
# calculating 7 day rolling mean
stock_data.rolling(7).mean().head(20)
# setting figure size
plt.figure(figsize=(16,10))

# plotting the close price and a 30-day rolling mean of close price
stock_data['Close'].plot()
stock_data.rolling(window=30).mean()['Close'].plot()
# displaying stock_data
stock_data<jupyter_output><empty_output><jupyter_text>STEP 8 - Data Preparation for Modelling<jupyter_code># calculating data_to_use
percentage_of_data = 1.0
data_to_use = int(percentage_of_data*(len(stock_data)-1))

# using 80% of data for training
train_end = int(data_to_use*0.8)
total_data = len(stock_data)
start = total_data - data_to_use

# printing number of records in the training and test datasets
print("Number of records in Training Data:", train_end)
print("Number of records in Test Data:", total_data - train_end)
# predicting one step ahead
steps_to_predict = 1

# capturing data to be used for each column
close_price = stock_data.iloc[start:total_data,0] #close
compound = stock_data.iloc[start:total_data,1] #compound
negative = stock_data.iloc[start:total_data,2] #neg
neutral = stock_data.iloc[start:total_data,3] #neu
positive = stock_data.iloc[start:total_data,4] #pos
open_price = stock_data.iloc[start:total_data,5] #open
high = stock_data.iloc[start:total_data,6] #high
low = stock_data.iloc[start:total_data,7] #low
volume = stock_data.iloc[start:total_data,8] #volume

# printing close price
print("Close Price:")
close_price
# shifting next day close
close_price_shifted = close_price.shift(-1) 

# shifting next day compound
compound_shifted = compound.shift(-1) 

# concatenating the captured training data into a dataframe
data = pd.concat([close_price, close_price_shifted, compound, compound_shifted, volume, open_price, high, low], axis=1)

# setting column names of the revised stock data
data.columns = ['close_price', 'close_price_shifted', 'compound', 'compound_shifted','volume', 'open_price', 'high', 'low']

# dropping nulls
data = data.dropna()    
data.head(10)<jupyter_output><empty_output><jupyter_text>STEP 9 - Setting the Target Variable and the Feature Dataset<jupyter_code># setting the target variable as the shifted close_price
y = data['close_price_shifted']
y
# setting the features dataset for prediction  
cols = ['close_price', 'compound', 'compound_shifted', 'volume', 'open_price', 'high', 'low']
x = data[cols]
x<jupyter_output><empty_output><jupyter_text>**Scaling the Target Variable and the Feature Dataset**Since we are using LSTM to predict stock prices, which is a time series data, it is important to understand that LSTM can be very sensitive to the scale of the data. Right now, if the data is observed, it is present in different scales. Therefore, it is important to re-scale the data so that the range of the dataset is same, for almost all records. Here a feature range of (-1,1) is used.<jupyter_code># scaling the feature dataset
scaler_x = preprocessing.MinMaxScaler (feature_range=(-1, 1))
x = np.array(x).reshape((len(x) ,len(cols)))
x = scaler_x.fit_transform(x)

# scaling the target variable
scaler_y = preprocessing.MinMaxScaler (feature_range=(-1, 1))
y = np.array (y).reshape ((len( y), 1))
y = scaler_y.fit_transform (y)

# displaying the scaled feature dataset and the target variable
x, y<jupyter_output><empty_output><jupyter_text>**Dividing the dataset into Training and Test**Normally for any other dataset train_test_split from sklearn package is used, but for time series data like stock prices which is dependent on date, the dataset is divided into train and test dataset in a different way as shown below. In timeseries data, an observation for a particular date is always dependent on the previous date records.<jupyter_code># preparing training and test dataset
X_train = x[0 : train_end,]
X_test = x[train_end+1 : len(x),]    
y_train = y[0 : train_end] 
y_test = y[train_end+1 : len(y)]  

# printing the shape of the training and the test datasets
print('Number of rows and columns in the Training set X:', X_train.shape, 'and y:', y_train.shape)
print('Number of rows and columns in the Test set X:', X_test.shape, 'and y:', y_test.shape)<jupyter_output>Number of rows and columns in the Training set X: (3900, 7) and y: (3900, 1)
Number of rows and columns in the Test set X: (974, 7) and y: (974, 1)<jupyter_text>**Reshaping the Feature Dataset**<jupyter_code># reshaping the feature dataset for feeding into the model
X_train = X_train.reshape (X_train.shape + (1,)) 
X_test = X_test.reshape(X_test.shape + (1,))

# printing the re-shaped feature dataset
print('Shape of Training set X:', X_train.shape)
print('Shape of Test set X:', X_test.shape)<jupyter_output>Shape of Training set X: (3900, 7, 1)
Shape of Test set X: (974, 7, 1)<jupyter_text>STEP 10 - Stock Data Modelling<jupyter_code># setting the seed to achieve consistent and less random predictions at each execution
np.random.seed(2016)

# setting the model architecture
model=Sequential()
model.add(LSTM(100,return_sequences=True,activation='tanh',input_shape=(len(cols),1)))
model.add(Dropout(0.1))
model.add(LSTM(100,return_sequences=True,activation='tanh'))
model.add(Dropout(0.1))
model.add(LSTM(100,activation='tanh'))
model.add(Dropout(0.1))
model.add(Dense(1))

# printing the model summary
model.summary()
# compiling the model
model.compile(loss='mse' , optimizer='adam')

# fitting the model using the training dataset
model.fit(X_train, y_train, validation_split=0.2, epochs=10, batch_size=8, verbose=1)<jupyter_output>WARNING:tensorflow:From C:\Users\Monica\anaconda3\lib\site-packages\tensorflow\python\ops\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
Train on 3120 samples, validate on 780 samples
Epoch 1/10
3120/3120 [==============================] - 14s 4ms/step - loss: 0.0335 - val_loss: 0.1489
Epoch 2/10
3120/3120 [==============================] - 13s 4ms/step - loss: 0.0090 - val_loss: 0.0014
Epoch 3/10
3120/3120 [==============================] - 12s 4ms/step - loss: 0.0025 - val_loss: 3.4762e-04
Epoch 4/10
3120/3120 [==============================] - 13s 4ms/step - loss: 0.0021 - val_loss: 8.7089e-04
Epoch 5/10
3120/3120 [==============================] - 14s 4ms/step - loss: 0.0020 - val_loss: 8.4524e-04
Epoch 6/10
3120/3120 [==============================] - 14s 4ms/step - loss: 0.0015 - val_loss: 0.0013
Epoch 7/10
3120/3120 [==============================] - 14s 5ms/s[...]<jupyter_text>**Saving the Model to disk**<jupyter_code># saving the model as a json file
model_json = model.to_json()
with open('model.json', 'w') as json_file:
    json_file.write(model_json)
    
# serialize weights to HDF5
model.save_weights('model.h5')
print('Model is saved to the disk')<jupyter_output>Model is saved to the disk<jupyter_text>STEP 11 - Predictions<jupyter_code># performing predictions
predictions = model.predict(X_test) 

# unscaling the predictions
predictions = scaler_y.inverse_transform(np.array(predictions).reshape((len(predictions), 1)))

# printing the predictions
print('Predictions:')
predictions[0:5]<jupyter_output>Predictions:<jupyter_text>STEP 12 - Model Evaluation<jupyter_code># calculating the training mean-squared-error
train_loss = model.evaluate(X_train, y_train, batch_size = 1)

# calculating the test mean-squared-error
test_loss = model.evaluate(X_test, y_test, batch_size = 1)

# printing the training and the test mean-squared-errors
print('Train Loss =', round(train_loss,4))
print('Test Loss =', round(test_loss,4))
# calculating root mean squared error
root_mean_square_error = np.sqrt(np.mean(np.power((y_test - predictions),2)))
print('Root Mean Square Error =', round(root_mean_square_error,4))
# calculating root mean squared error using sklearn.metrics package
rmse = metrics.mean_squared_error(y_test, predictions)
print('Root Mean Square Error (sklearn.metrics) =', round(np.sqrt(rmse),4))<jupyter_output>Root Mean Square Error (sklearn.metrics) = 50.3044<jupyter_text>STEP 13 - Plotting the Predictions against unseen data<jupyter_code># unscaling the test feature dataset, x_test
X_test = scaler_x.inverse_transform(np.array(X_test).reshape((len(X_test), len(cols))))

# unscaling the test y dataset, y_test
y_train = scaler_y.inverse_transform(np.array(y_train).reshape((len(y_train), 1)))
y_test = scaler_y.inverse_transform(np.array(y_test).reshape((len(y_test), 1)))
# plotting
plt.figure(figsize=(16,10))

# plt.plot([row[0] for row in y_train], label="Training Close Price")
plt.plot(predictions, label="Predicted Close Price")
plt.plot([row[0] for row in y_test], label="Testing Close Price")
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=2)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1262091465857707, 'fraction_numerical': 0.03560185450174575, 'mean_word_length': 6.093788063337393, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 88, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '243082', 'license': 'MIT', 'n_tokens_mistral': 6208, 'n_tokens_neox': 5691, 'n_words': 1593, 'path': 'TSF_Task_7.ipynb', 'repo_name': 'abhijeetraj22/TSF-GRIP_Tasks', 'size': '17486'}