<jupyter_start><jupyter_text>import pandas_datareader.data as webimport datetimeimport numpy as npimport matplotlib.pyplot as plt%matplotlib inlinefrom numpy import arrayimport sklearnfrom sklearn import preprocessingfrom sklearn.tree import DecisionTreeRegressorfrom sklearn.linear_model import LinearRegressionfrom sklearn.model_selection import train_test_splitimport tensorflow as tffrom keras.models import Sequentialfrom keras.layers import Densefrom keras.layers import LSTM<jupyter_code>start = datetime.datetime(2014, 1, 1)
end = datetime.datetime(2020, 6, 30)
axisgold = web.DataReader("AXISGOLD.NS", 'yahoo', start, end)
bslgoldetf = web.DataReader("BSLGOLDETF.NS", 'yahoo', start, end)
#cangold = web.DataReader("CANGOLD.BO", 'yahoo', start, end)
#hfcmgetf = web.DataReader("HFCMFGETF.BO", 'yahoo', start, end)
#ipgetf = web.DataReader("IPGETF.BO", 'yahoo', start, end)
idbigold = web.DataReader("IDBIGOLD.NS", 'yahoo', start, end)
kotakgold = web.DataReader("KOTAKGOLD.NS", 'yahoo', start, end)
qgoldhalf = web.DataReader("QGOLDHALF.NS", 'yahoo', start, end)
#relgold = web.DataReader("RELGOLD.BO", 'yahoo', start, end)
#religarego = web.DataReader("RELIGAREGO.BO", 'yahoo', start, end)
#setfgold = web.DataReader("SETFGOLD.NS", 'yahoo', start, end)
goldshare = web.DataReader("GOLDSHARE.NS", 'yahoo', start, end)
goldbees = web.DataReader("GOLDBEES.NS", 'yahoo', start, end)
axisgold.to_csv('AXISGOLD.csv')
bslgoldetf.to_csv('BSLGOLDETF.csv')
idbigold.to_csv('IDBIGOLD.csv')
kotakgold.to_csv('KOTAKGOLD.csv')
qgoldhalf.to_csv('QGOLDHALF.csv')
#setfgold.to_csv('SETFGOLD.csv')
goldshare.to_csv('GOLDSHARE.csv')
goldbees.to_csv('GOLDBEES.csv')
axisgold['Open'].plot(label = 'axisgold', figsize= (15,7))
bslgoldetf['Open'].plot(label = 'bslgoldetf')
idbigold['Open'].plot(label = 'idbigold')
kotakgold['Open'].plot(label = 'kotakgold')
qgoldhalf['Open'].plot(label = 'qgoldhalf')
goldshare['Open'].plot(label = 'goldshare')
goldbees['Open'].plot(label = 'goldbees')
plt.title('Gold ETF stock price')
plt.legend()
#We are going to predict the price of one Gold ETF below. Similarly all other predictions can also be made. Let us chose Axisgold ETF
import numpy as np
import pandas as pd
import json

data = pd.read_csv('AXISGOLD.csv')
print(data.head())

print("\n")
print("Open   --- mean :", np.mean(data['Open']),  "  \t Std: ", np.std(data['Open']),  "  \t Max: ", np.max(data['Open']),  "  \t Min: ", np.min(data['Open']))
print("High   --- mean :", np.mean(data['High']),  "  \t Std: ", np.std(data['High']),  "  \t Max: ", np.max(data['High']),  "  \t Min: ", np.min(data['High']))
print("Low    --- mean :", np.mean(data['Low']),   "  \t Std: ", np.std(data['Low']),   "  \t Max: ", np.max(data['Low']),   "  \t Min: ", np.min(data['Low']))
print("Close  --- mean :", np.mean(data['Close']), "  \t Std: ", np.std(data['Close']), "  \t Max: ", np.max(data['Close']), "  \t Min: ", np.min(data['Close']))
print("Volume --- mean :", np.mean(data['Volume']),"  \t Std: ", np.std(data['Volume']),"  \t Max: ", np.max(data['Volume']),"  \t Min: ", np.min(data['Volume']))
data.info()
print("Dataframe contains Data from from " + str(data['Date'].min()) + " to " + str(data['Date'].max()))
data.head()
df = data.dropna()
df.shape
# Select Adjusted close data
df = df[['Adj Close']]
print(df.shape)
df.head(10)
# display the data 
plt.figure(figsize=(12,6))
# all data points
df.plot(title='Adjusted close', color='blue') 
plt.show()
#Preprocessing the data by scaling
#minmaxscaler = preprocessing.MinMaxScaler(feature_range=(0,1))
#scaler = MinMaxScalar(feature_range=(0,1))
#df = minmaxscaler.fit_transform(df)
#print(df)
#Create a variable to predict x days into the future
future_days = 380
#Create a new columns (target) shifted 'x' days up
df['Prediction'] = df[['Adj Close']].shift(-future_days)
df.tail(4)
#Create feature dataset (x), convert to numpy array and remove the last 'x' rows/days
X = np.array(df.drop(['Prediction'], 1))[:-future_days]
print(X)
#Create a target data set (y) and convert to numpy array to get all target values except last x data points
y = np.array(df['Prediction'])[:-future_days]
print(y)
#Split data 80% training 20% testing
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
#Create models
#Create decision tree model
tree = DecisionTreeRegressor().fit(x_train, y_train)

#Create linear decision model
lr = LinearRegression().fit(x_train, y_train)
#Get last x fows of future
x_future = df.drop(['Prediction'], 1)[:-future_days]
x_future = x_future.tail(future_days)
x_future = np.array(x_future)
x_future
#Show tree prediction
tree_prediction = tree.predict(x_future)
print(tree_prediction)
print()

#Show model linear regression prediction
lr_prediction = lr.predict(x_future)
print(lr_prediction)
#Visualize data
predictions = lr_prediction

valid = df[X.shape[0]:]
valid['Predictions'] = predictions
plt.figure(figsize = (16,8))
plt.title('Model')
plt.xlabel('Days')
plt.ylabel('Adj Close price USD ($)')
plt.plot(df['Adj Close'])
plt.plot(valid[['Adj Close', 'Predictions']])
plt.legend(['Original','Validation', 'Prediction'])
plt.show()
#Visualize data
predictions = tree_prediction

valid = df[X.shape[0]:]
valid['Predictions'] = predictions
plt.figure(figsize = (16,8))
plt.title('Model')
plt.xlabel('Days')
plt.ylabel('Adj Close price USD ($)')
plt.plot(df['Adj Close'])
plt.plot(valid[['Adj Close', 'Predictions']])
plt.legend(['Original','Validation', 'Prediction'])
plt.show()
#Modeling uing LSTM model
import numpy

data_1 = pd.read_csv('AXISGOLD.csv')

# Select Adjusted close data
data = data_1[['Adj Close']]
print(data_1.shape)
data_1.head(4)
from sklearn.preprocessing import MinMaxScaler
scaler=MinMaxScaler(feature_range=(0,1))
data=scaler.fit_transform(np.array(data).reshape(-1,1))
print(data)
#Split data 80% training 20% testing

training_size=int(len(data)*0.8)
test_size=len(data)-training_size
train_data,test_data=data[0:training_size],data[training_size:len(data)]
training_size,test_size
train_data
def create_dataset(dataset, time_step=1):
    dataX, dataY = [], []
    for i in range(len(dataset)-time_step-1):
        a = dataset[i:(i+time_step), 0]   ###i=0, 0,1,2,3-----99   100 
        dataX.append(a)
        dataY.append(dataset[i + time_step, 0])
    return numpy.array(dataX), numpy.array(dataY)
#reshape into X=t,t+1,t+2,t+3 and Y=t+4
time_step = 1
x_train, y_train = create_dataset(train_data, time_step)
x_test, y_test = create_dataset(test_data, time_step)
#x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
print(x_train.shape), print(y_train.shape)
print(x_test.shape), print(y_test.shape)

# reshape input to be [samples, time steps, features] which is required for LSTM
x_train =x_train.reshape(x_train.shape[0],x_train.shape[1] , 1)
x_test = x_test.reshape(x_test.shape[0],x_test.shape[1] , 1)
x_train.shape, x_test.shape
x_train
### Create the Stacked LSTM model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
from keras import backend as K
K.set_image_data_format('channels_last')  
keras.backend.image_data_format()  
model=Sequential()
model.add(LSTM(50,return_sequences=True,input_shape=(100,1)))
model.add(LSTM(50,return_sequences=True))
model.add(LSTM(50))
model.add(Dense(1))
model.compile(loss='mse',optimizer='adam')
model.summary()
model.fit(x_train,y_train,validation_data=(x_test,y_test),epochs=100,batch_size=64,verbose=1)
import tensorflow as tf
### Lets Do the prediction and check performance metrics
train_predict=model.predict(x_train)
test_predict=model.predict(x_test)
##Transformback to original form
train_predict=scaler.inverse_transform(train_predict)
test_predict=scaler.inverse_transform(test_predict)
### Calculate RMSE performance metrics
import math
from sklearn.metrics import mean_squared_error
math.sqrt(mean_squared_error(y_train,train_predict))
### Test Data RMSE
math.sqrt(mean_squared_error(y_test,test_predict))
# shift train predictions for plotting
look_back=100
trainPredictPlot = numpy.empty_like(data)
trainPredictPlot[:, :] = np.nan
trainPredictPlot[look_back:len(train_predict)+look_back, :] = train_predict
# shift test predictions for plotting
testPredictPlot = numpy.empty_like(data)
testPredictPlot[:, :] = numpy.nan
testPredictPlot[len(train_predict)+(look_back*2)-196:len(data)+100, :] = test_predict
# plot baseline and predictions
plt.plot(scaler.inverse_transform(data))
plt.plot(trainPredictPlot)
plt.plot(testPredictPlot)
plt.legend(['Original','Training', 'Test'])
plt.show()

len(data)
#Predicting to next 30 days data using previous 100 days
x_input=data[1435:].reshape(1,-1)
x_input.shape
temp_input=list(x_input)
temp_input=temp_input[0].tolist()
len(temp_input)
temp_input
from numpy import array

lst_output=[]
n_steps=100
i=0
while(i<30):
    
    if(len(temp_input)>100):
        #print(temp_input)
        x_input=np.array(temp_input[1:])
        print("{} day input {}".format(i,x_input))
        x_input=x_input.reshape(1,-1)
        x_input = x_input.reshape((1, n_steps, 1))
        #print(x_input)
        yhat = model.predict(x_input, verbose=0)
        print("{} day output {}".format(i,yhat))
        temp_input.extend(yhat[0].tolist())
        temp_input=temp_input[1:]
        #print(temp_input)
        lst_output.extend(yhat.tolist())
        i=i+1
    else:
        x_input = x_input.reshape((1, n_steps,1))
        yhat = model.predict(x_input, verbose=0)
        print(yhat[0])
        temp_input.extend(yhat[0].tolist())
        print(len(temp_input))
        lst_output.extend(yhat.tolist())
        i=i+1
    

print(lst_output)
day_new=np.arange(1,101)
day_pred=np.arange(101,131)
import matplotlib.pyplot as plt
len(data)
plt.plot(day_new,scaler.inverse_transform(data[1435:]))
plt.plot(day_pred,scaler.inverse_transform(lst_output))
df3=data.tolist()
df3.extend(lst_output)
plt.plot(df3[1500:])
df3=scaler.inverse_transform(df3).tolist()
plt.plot(df3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16176764647070585, 'fraction_numerical': 0.01749650069986003, 'mean_word_length': 6.044366197183098, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '546775', 'license': 'MIT', 'n_tokens_mistral': 3835, 'n_tokens_neox': 3655, 'n_words': 743, 'path': 'GoldETF_pric_ predictor_bad_loss-Copy1.ipynb', 'repo_name': 'AishSAcharya/GoldETF_price_analysis', 'size': '10002'}