<jupyter_start><jupyter_text>Taking News headlines<jupyter_code>data1 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2015.csv")
data2 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2016.csv")
data3 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2017.csv")
data4 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2018.csv")
data5 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2019.csv")
data6 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2020.csv")
data7 = pd.read_csv("Datasets of Stocks Market and Stock News Articles\Datasets of Stocks Market and Stock News Articles\Stocks News Articles Datasets\Headlines_2021.csv")

df_news = pd.concat([data1, data2, data3, data4, data5, data6, data7])
print(len(data1), len(data2), len(data3),len(data4), len(data5), len(data6),len(data7))
print("Total Rows in dataframe:", len(df_news))
df_news.head()

df_news = df_news.drop_duplicates()

df_news['Date'] = pd.to_datetime(df_news['Date'])
print(df_news.describe())
print("="*50)
df_news.info()
print("="*50)
print(df_news.shape)
print("="*50)
print(df_news.isnull().sum())
print("="*50)<jupyter_output>Date                                          Headlines
count                  2342                                               2342
unique                 2335                                               2332
top     2019-11-20 00:00:00  MasterCard is expanding its mobile wallet in S...
freq                      2                                                  3
first   2015-01-01 00:00:00                                                NaN
last    2021-06-23 00:00:00                                                NaN
==================================================
<class 'pandas.core.frame.DataFrame'>
Int64Index: 2342 entries, 0 to 173
Data columns (total 2 columns):
 #   Column     Non-Null Count  Dtype         
---  ------     --------------  -----         
 0   Date       2342 non-null   datetime64[ns]
 1   Headlines  2342 non-null   object        
dtypes: datetime64[ns](1), object(1)
memory usage: 54.9+ KB
===========================================[...]<jupyter_text>Sentiment scores<jupyter_code>#Create a function to get subjectivity

def getSubjectivity(df_news):
    return TextBlob(df_news).sentiment.subjectivity

#create a function to get the polarity

def getPolarity(df_news):
    return TextBlob(df_news).sentiment.polarity

#create two new solumns called "Subjectivity" & "Polarity"

df_news['Subjectivity'] = df_news['Headlines'].apply(getSubjectivity)
df_news['Polarity'] = df_news['Headlines'].apply(getPolarity)

#show the data"
df_news.head()
#create a function o get the text sentiment

def getSentiment(score):
    if score < 0:
        return 'Negative'
    elif score == 0:
        return 'neutral'
    else:
        return 'Positve'
#create a column to store the text sentiments

df_news['Sentiment'] = df_news['Polarity'].apply(getSentiment)

#show the data
df_news.head()
df_news.Sentiment.value_counts()
df_news['Sentiment'].value_counts()

#plot & visualize the counts

plt.title('Sentiment Analysis')
plt.xlabel('Sentiment')
plt.ylabel('Counts')
df_news['Sentiment'].value_counts().plot(kind='bar')
enc = LabelEncoder()
df_news['Sentiment'] = enc.fit_transform(df_news['Sentiment'])
df_news.head()
#df.merge(df_news, left_index=True, right_index=True, how='inner')
df.set_index(pd.to_datetime(df['Date']), inplace=True)
df_news.set_index(pd.to_datetime(df_news['Date']), inplace=True)
df.head()
final_df=pd.merge(df_news,df, how='inner', left_index=True, right_index=True)

final_df.head()
final_df.shape
#final_df.drop_duplicates(subset ="Volume",
                     #keep = False, inplace = True)
print(final_df.describe())
print("="*50)
final_df.info()
print("="*50)
print(final_df.shape)
print("="*50)
print(final_df.isnull().sum())
print("="*50)
print(df.dtypes)
print("="*50)
corr=df.corr()
print(corr)
print("="*50)
print(df.info())
print("="*50)
#date is exclude intetionally
final_df=final_df[['Sentiment','Close/Last','Volume','Open','High','Low']]
final_df.head(5)
from sklearn.preprocessing import MinMaxScaler
scaler=MinMaxScaler(feature_range=(0,1))
final_df=scaler.fit_transform(np.array(final_df).reshape(-1,1))
print(final_df)

##splitting dataset into train and test split
training_size=int(len(final_df)*0.70)
test_size=len(final_df)-training_size
train_data,test_data=final_df[0:training_size,:],final_df[training_size:len(final_df),:1]
training_size,test_size

train_data

test_data
import numpy
# convert an array of values into a dataset matrix
def create_dataset(dataset, time_step=1):
    dataX, dataY = [], []
    for i in range(len(dataset)-time_step-1):
        a = dataset[i:(i+time_step), 0]   ###i=0, 0,1,2,3-----99   100 
        dataX.append(a)
        dataY.append(dataset[i + time_step, 0])
    return numpy.array(dataX), numpy.array(dataY)
# reshape into X=t,t+1,t+2,t+3 and Y=t+4
time_step = 100
X_train, y_train = create_dataset(train_data, time_step)
X_test, ytest = create_dataset(test_data, time_step)
print(X_train.shape), print(y_train.shape)

print(X_test.shape), print(ytest.shape)

# reshape input to be [samples, time steps, features] which is required for LSTM
X_train =X_train.reshape(X_train.shape[0],X_train.shape[1] , 1)
X_test = X_test.reshape(X_test.shape[0],X_test.shape[1] , 1)

### Create the Stacked LSTM model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
model=Sequential()
model.add(LSTM(50,return_sequences=True,input_shape=(100,1)))
model.add(LSTM(50,return_sequences=True))
model.add(LSTM(50))
model.add(Dense(1))
model.compile(loss='mean_squared_error',optimizer='adam')
model.summary()

model.fit(X_train,y_train,validation_data=(X_test,ytest),epochs=100,batch_size=64,verbose=1)
### Lets Do the prediction and check performance metrics
train_predict=model.predict(X_train)
test_predict=model.predict(X_test)
test_predict
#saving the model
model.save('my_model.h5')

model = load_model('my_model.h5')

##Transformback to original form
train_predict=scaler.inverse_transform(train_predict)
test_predict=scaler.inverse_transform(test_predict)
test_predict
### Calculate RMSE performance metrics
import math
from sklearn.metrics import mean_squared_error
math.sqrt(mean_squared_error(y_train,train_predict))

### Test Data RMSE
math.sqrt(mean_squared_error(ytest,test_predict))
### Plotting 
# shift train predictions for plotting
look_back=100
trainPredictPlot = numpy.empty_like(final_df)
trainPredictPlot[:, :] = np.nan
trainPredictPlot[look_back:len(train_predict)+look_back, :] = train_predict
# shift test predictions for plotting
testPredictPlot = numpy.empty_like(final_df)
testPredictPlot[:, :] = numpy.nan
testPredictPlot[len(train_predict)+(look_back*2)+1:len(final_df)-1, :] = test_predict
# plot baseline and predictions
plt.plot(scaler.inverse_transform(final_df))
plt.plot(trainPredictPlot)
plt.plot(testPredictPlot)
plt.show()
len(test_data)

x_input=test_data[341:].reshape(1,-1)
x_input.shape
temp_input=list(x_input)
temp_input=temp_input[0].tolist()
temp_input

# demonstrate prediction for next 10 days
from numpy import array

lst_output=[]
n_steps=100
i=0
while(i<30):
    
    if(len(temp_input)>100):
        #print(temp_input)
        x_input=np.array(temp_input[1:])
        print("{} day input {}".format(i,x_input))
        x_input=x_input.reshape(1,-1)
        x_input = x_input.reshape((1, n_steps, 1))
        #print(x_input)
        yhat = model.predict(x_input, verbose=0)
        print("{} day output {}".format(i,yhat))
        temp_input.extend(yhat[0].tolist())
        temp_input=temp_input[1:]
        #print(temp_input)
        lst_output.extend(yhat.tolist())
        i=i+1
    else:
        x_input = x_input.reshape((1, n_steps,1))
        yhat = model.predict(x_input, verbose=0)
        print(yhat[0])
        temp_input.extend(yhat[0].tolist())
        print(len(temp_input))
        lst_output.extend(yhat.tolist())
        i=i+1
    

print(lst_output)
day_new=np.arange(1,101)
day_pred=np.arange(101,131)
len(final_df)

plt.plot(day_new,scaler.inverse_transform(df[2416:]))
plt.plot(day_pred,scaler.inverse_transform(lst_output))

df3=df.tolist()
df3.extend(lst_output)
plt.plot(df3[1200:]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13864273696017948, 'fraction_numerical': 0.03118339876612451, 'mean_word_length': 4.527588344699318, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '429929', 'license': 'MIT', 'n_tokens_mistral': 3221, 'n_tokens_neox': 3021, 'n_words': 614, 'path': 'Machine Learning Phase/Th_Neural Network_concat.ipynb', 'repo_name': 'Artificial-Incoherence/-Artificial-Incoherence-Stock-Price-and-News-Analysis', 'size': '8915'}