<jupyter_start><jupyter_text>Stock Sentiment Analysis using News Headlines<jupyter_code># importing the libraries
import pandas as pd
# loading the Dataset into pandas Dataframe
df = pd.read_csv('Data.csv', encoding='ISO-8859-1')
df.head()
df.shape
df.info()
# splitting Dataset into train and testing
train = df[df['Date'] < '20150101']
test = df[df['Date'] > '20141231']
# Removing Punctuations
data = train.iloc[:, 2:27]
data.replace("[^a-zA-Z]", ' ', regex=True, inplace=True)

# Renaming the column name for each of the column access
list1 = [i for i in range(25)]
new_index = [str(i) for i in list1]
data.columns = new_index
data.head()
# converting headings to lowercase
for index in new_index:
    data[index] = data[index].str.lower()
data.head(2)
# this way we can convert all the heading into single paragraph
" ".join(str(x)for x in data.iloc[1, 0:25])
# we can to convert all the heading in each row in one paragraph for applying bag of words

headline = []
for row in range(0, len(data.index)):
    headline.append(" ".join(str(x)for x in data.iloc[row, 0:25]))
headline[0]
# Converting the list into DataFrame
df1 = pd.DataFrame(headline, columns=['headlines'])
# removing the words which has lenghth of less than 2
df1['headlines'] = df1['headlines'].apply(lambda x: " ".join([w for w in x.split() if len(w)>2]))
# Removing the stopwords
from nltk.corpus import stopwords

def cleaning_stopwords(text):
    return " ".join([word for word in str(text).split() if word not in stopwords.words('english')])
df1['headlines'] = df1['headlines'].apply(lambda text: cleaning_stopwords(text))
df1.head()
# converting dataframe into list for converting words to vectors using bag of words method
head = []
for x in df1.headlines:
    head.append(x)
# importing libraries for model building
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
# Implementation of Bag of Words
countvector = CountVectorizer(ngram_range=(2, 2))
traindataset = countvector.fit_transform(head)
# implement RandomForest Classifier
randomclassifier=RandomForestClassifier(n_estimators=200,criterion='entropy')
randomclassifier.fit(traindataset,train['Label'])
## Import library to check accuracy
from sklearn.metrics import classification_report,confusion_matrix,accuracy_score<jupyter_output><empty_output><jupyter_text>perform data Cleaning for test Data<jupyter_code># Removing Punctuations for test data
data1 = test.iloc[:, 2:27]
data1.replace("[^a-zA-Z]", ' ', regex=True, inplace=True)

# Renaming the column name for each of the column access
list2 = [i for i in range(25)]
new_index1 = [str(i) for i in list2]
data1.columns = new_index1
# converting headings to lowercase
for index in new_index1:
    data1[index] = data1[index].str.lower()
headline1 = []
for row in range(0, len(data1.index)):
    headline1.append(" ".join(str(x)for x in data1.iloc[row, 0:25]))
df2 = pd.DataFrame(headline1, columns=['headlines1'])
# removing the words which has lenghth of less than 2
df2['headlines1'] = df2['headlines1'].apply(lambda x: " ".join([w for w in x.split() if len(w)>2]))
# Removing the stopwords
from nltk.corpus import stopwords

def cleaning_stopwords(text):
    return " ".join([word for word in str(text).split() if word not in stopwords.words('english')])
df2['headlines1'] = df2['headlines1'].apply(lambda text: cleaning_stopwords(text))
head1 = []
for x in df2.headlines1:
    head1.append(x)
## Predict for the Test Dataset

testdataset = countvector.transform(head1)
predictions1 = randomclassifier.predict(testdataset)
matrix1 = confusion_matrix(test['Label'],predictions1)
print(matrix1)
score1=accuracy_score(test['Label'],predictions1)
print(score1)
report1=classification_report(test['Label'],predictions1)
print(report1)<jupyter_output>[[133  53]
 [  0 192]]
0.8597883597883598
              precision    recall  f1-score   support

           0       1.00      0.72      0.83       186
           1       0.78      1.00      0.88       192

    accuracy                           0.86       378
   macro avg       0.89      0.86      0.86       378
weighted avg       0.89      0.86      0.86       378
{'fraction_non_alphanumeric': 0.11918465227817746, 'fraction_numerical': 0.04460431654676259, 'mean_word_length': 4.5099075297225895, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '231803', 'license': 'Apache-2.0', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1358, 'n_words': 402, 'path': 'Stock_Sentiment_Analysis.ipynb', 'repo_name': 'Balajilp/Stock-Sentiment_Analysis-NLP-', 'size': '4170'}