<jupyter_start><jupyter_text>GRIP - Task 5 - Stock Sentiment Analysis By <NAME>  Importing libraries<jupyter_code>import numpy as np
import pandas as pd
import re
from textblob import TextBlob
import nltk
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import matplotlib.pyplot as plt
from IPython.display import clear_output
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Importing news datasets<jupyter_code>headlines = pd.read_csv('india-news-headlines.csv')
headlines.head()<jupyter_output><empty_output><jupyter_text>Create date time column<jupyter_code>def change_date(row):
    row = str(row)
    new_row = row[:4] + '/' + row[4:6] + '/' + row[6:]
    return new_row
headlines['publish_date'] = headlines['publish_date'].apply(change_date)
headlines['publish_date'] = pd.to_datetime(headlines['publish_date'])
headlines.head()<jupyter_output><empty_output><jupyter_text>Data cleaning<jupyter_code>def clean_text(row):
    row = str(row).lower()
    new_row = re.sub("[^a-zA-Z]", " ", row) + ' '
    return new_row
headlines['headline_text'] = headlines['headline_text'].apply(clean_text)
headlines.drop('headline_category', axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Headlines data merging<jupyter_code>news = headlines.groupby('publish_date').sum()
news.head()
news.iloc[2]['headline_text']
news.rename(columns={'headline_text': 'Headlines'}, inplace=True)
news.index.names = ['Date']<jupyter_output><empty_output><jupyter_text>Calculating Polaity and Subjectiity of Headlines data<jupyter_code>polarity = lambda x: TextBlob(x).sentiment.polarity
subjectivity = lambda x: TextBlob(x).sentiment.subjectivity
news['Polarity'] = news['Headlines'].apply(polarity)
news['Subjectivity'] = news['Headlines'].apply(subjectivity)
news.head()
news['Sentiment'] = news['Polarity'].apply(lambda x: 'Positive' if x>=0.2 else ("Negative" if x<=-0.2 else "Neutral"))
news['Opinion'] = news['Subjectivity'].apply(lambda x: 'Public' if x>=0.40 else 'Factual')
news.head()<jupyter_output><empty_output><jupyter_text>Graph shows different sentiment measures from Headlines data<jupyter_code>positive = news[news['Sentiment']=='Positive']
negative = news[news['Sentiment']=='Negative']
neutral = news[news['Sentiment']=='Neutral']
plt.rcParams['figure.figsize'] = [10, 8]
plt.scatter(x=positive.index, y=positive['Polarity'], c='green', label='Positive')
plt.scatter(x=negative.index, y=negative['Polarity'], c='red', label='Negative')
plt.scatter(x=neutral.index, y=neutral['Polarity'], c='b', label='Neutral')
plt.xlabel('Dates')
plt.ylabel('Polarity')
plt.legend()
public = news[news['Opinion']=='Public']
fact = news[news['Opinion']=='Factual']
plt.scatter(x=public.index, y=public.Subjectivity, c='orange', label='Public')
plt.scatter(x=fact.index, y=fact.Subjectivity, c='blue', label='Fact')
plt.xlabel('Dates')
plt.ylabel('Opinion')
plt.legend()<jupyter_output><empty_output><jupyter_text>Importing stock dataset<jupyter_code>stocks = pd.read_csv('^BSESN.csv')
stocks['Date'] = pd.to_datetime(stocks['Date'])
stocks.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 4869 entries, 0 to 4868
Data columns (total 7 columns):
 #   Column     Non-Null Count  Dtype         
---  ------     --------------  -----         
 0   Date       4869 non-null   datetime64[ns]
 1   Open       4795 non-null   float64       
 2   High       4795 non-null   float64       
 3   Low        4795 non-null   float64       
 4   Close      4795 non-null   float64       
 5   Adj Close  4795 non-null   float64       
 6   Volume     4795 non-null   float64       
dtypes: datetime64[ns](1), float64(6)
memory usage: 266.4 KB<jupyter_text>Merging news and stocks dataset together<jupyter_code>stocks_final = news.merge(stocks, how='inner', on='Date', left_index=True)
stocks_final.dropna(inplace=True)
stocks_final.drop(['Sentiment', 'Opinion', 'Volume'], axis=1, inplace=True)
stocks_final.set_index('Date', inplace=True)<jupyter_output><empty_output><jupyter_text>Saving new News dataset with polarity and subjectivity values<jupyter_code>news.to_csv('news_analysis.csv')
stocks_final.head()<jupyter_output><empty_output><jupyter_text>Closing price trend<jupyter_code>plt.figure(figsize=(12,8))
stocks_final['Close'].plot()
plt.title("S&P BSE SENSEX")
stocks_final['Close'][-500:].plot(figsize=(10,6))
plt.title("S&P BSE SENSEX")<jupyter_output><empty_output><jupyter_text>Sentiment scores for each day<jupyter_code>def getSIA(text): 
    sia = SentimentIntensityAnalyzer()
    sentiment = sia.polarity_scores(text)
    return sentiment

compound = []
neg = []
neu = []
pos = []
SIA = 0

for i in range(0, len(stocks_final['Headlines'])):
    clear_output(wait=True)
    SIA = getSIA(stocks_final['Headlines'][i])
    print((i/len(stocks_final['Headlines']))*100, '%')
    compound.append(SIA['compound'])
    neg.append(SIA['neg'])
    neu.append(SIA['neu'])
    pos.append(SIA['pos'])
stocks_final['Compound'] =compound
stocks_final['Negative'] =neg
stocks_final['Neutral'] =neu
stocks_final['Positive'] = pos
stocks_final.to_csv('stocks_final.csv')
stocks_final['Change'] = stocks_final['Close'] - stocks_final['Open']
stocks_final['Change'] = stocks_final['Change'].apply(lambda x: 1 if x>=0 else 0)<jupyter_output><empty_output><jupyter_text>Final dataset to train and test<jupyter_code>stocks_final.head()
keep_columns = [ 'Open',  'High', 'Low', 'Subjectivity', 'Polarity', 'Compound', 'Negative', 'Neutral' ,'Positive',  'Change' ]
final_df = stocks_final[keep_columns]
X= np.array(final_df.drop('Change', axis=1))
y= np.array(final_df['Change'])<jupyter_output><empty_output><jupyter_text>Creating training and testing datasets<jupyter_code>from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 0)<jupyter_output><empty_output><jupyter_text>Model used is LDA<jupyter_code>from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
model = LinearDiscriminantAnalysis()
model.fit(x_train, y_train)
predictions = model.predict(x_test)<jupyter_output><empty_output><jupyter_text>Classification Report which suggests that our accuracy is 88%<jupyter_code>from sklearn.metrics import classification_report
print(classification_report(y_test, predictions))<jupyter_output>precision    recall  f1-score   support

           0       0.88      0.90      0.89       521
           1       0.88      0.85      0.86       433

    accuracy                           0.88       954
   macro avg       0.88      0.87      0.87       954
weighted avg       0.88      0.88      0.88       954
{'fraction_non_alphanumeric': 0.1474397590361446, 'fraction_numerical': 0.025150602409638553, 'mean_word_length': 5.6211365902293124, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '292398', 'license': 'MIT', 'n_tokens_mistral': 2454, 'n_tokens_neox': 2311, 'n_words': 411, 'path': 'Task 5 - GRIP.ipynb', 'repo_name': 'bhaskar-29/TSF-Grip', 'size': '6650'}