<jupyter_start><jupyter_text>Deep learning Prediction about Stock Price using convolutionThis is a standard Deep simple neural network using convolution to predict any stock price based on temporal data, analysis and some date knowledge. Parametter part based on a lineare regression work from ""<jupyter_code>%%capture
import sys 
import pandas as pd 
import numpy as np
# parrameters
import warnings
warnings.filterwarnings('ignore')
# Choose your ticker
tickerSymbol = ["BTC-CAD","ETH-CAD","BTC-USD","CAD=X","CADEUR=X","CADUSD=X","XAD5.MI","GOOG","ARGX","^DJI",
                "DOGE-USD","DOGE-EUR","EUR","USD","BTC","^CMC200",
                "ETH-USD","ATOM-USD","ADA-USD","^GDAXI","^GSPC","^FCHI","ALGO-USD",
                "ADA-EUR","ADA-CAD","SOL-CAD","^HSI",
                "SOL-EUR","ETH-EUR","BTC-EUR"]

# Choose date range - format should be 'YYYY-MM-DD' 
startDate = '2008-01-01' # as strings
endDate = '2021-12-02' # as strings

# Define key model parameters
# Set days out to predict 
shifts = [7]

# Set a training percentage
train_pct = .85

# Plotting dimensions
w = 16 # width
h = 4 # height<jupyter_output><empty_output><jupyter_text>Some stock you may use to train (the more data the better) ["BTC-USD","BTC-EUR","CAD=X","CADEUR=X","CADUSD=X","XAD5.MI","GOOG","ARGX","^DJI",                "DOGE-USD","DOGE-EUR","EUR","USD","BTC","^CMC200",                "ETH-USD","ATOM","ADA-USD",                "ADA-EUR","ADA-CAD","SOL-CAD",                "SOL-EUR","ETH-EUR","ETH-CAD","BTC-CAD"]  ["BTC-CAD","ETH-CAD","BTC-USD","CAD=X","CADEUR=X","CADUSD=X","XAD5.MI","GOOG","ARGX","^DJI",                "DOGE-USD","DOGE-EUR","EUR","USD","BTC","^CMC200",                "ETH-USD","ATOM-USD","ADA-USD","^GDAXI","^GSPC","^FCHI","ALGO-USD",                "ADA-EUR","ADA-CAD","SOL-CAD","^HSI",                "SOL-EUR","ETH-EUR","BTC-EUR"]                 All the depencies part ▶<jupyter_code>%%capture
# Check if local computer has the library yfinance. If not, install. Then Import it.
!python -m pip install yfinance # Check if the machine has yfinance, if not, download yfinance
import yfinance as yf # Import library to access Yahoo finance stock data
# Add financial information and indicators 
!python -m pip install ta # Download ta
from ta import add_all_ta_features # Library that does financial technical analysis 
# Install fastai to use the date function
!python -m pip install fastai # Download fastai 
import fastai.tabular # Library that does date factors
!python -m pip install plotly # Download plotly 
import plotly.graph_objs as go  # Import the graph ojbects 

# Import sklearn modules that will help with modeling building
!python -m pip install sklearn # Download sklearn 
from sklearn.metrics import mean_squared_error # Install error metrics 
from sklearn.linear_model import LinearRegression # Install linear regression model
from sklearn.preprocessing import StandardScaler # to scale for ann
# Go through each shift....
import pickle
from joblib import dump, load
from google.colab import files

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.preprocessing import MinMaxScaler<jupyter_output><empty_output><jupyter_text>Downloading the data from internetusing Yahoo API because it s free, we can get many data about the any stock we want so let s get at it.<jupyter_code># Create ticker yfinance object
data = []
i = 0
for t in tickerSymbol:
  print('-----------------------  iteration :' + str(i))
  tickerData = yf.Ticker(t)
  # Create historic data dataframe and fetch the data for the dates given. 
  df = tickerData.history(start = startDate, end = endDate)

  # to date format so we can get some juicy details
  date_change = '%Y-%m-%d'
  df['Date'] = df.index
  df['Date'] = pd.to_datetime(df['Date'], format = date_change)
  Dates = df['Date']

  # add financial details
  df = add_all_ta_features(df, "Open", "High", "Low", "Close", "Volume", fillna=True)

  # Define the date parts 
  fastai.tabular.add_datepart(df,'Date', drop = 'True')
  # Ensure the correct format
  df['Date'] = pd.to_datetime(df.index.values, format = date_change)
  # Add the date parts
  fastai.tabular.add_cyclic_datepart(df, 'Date', drop = 'True')

  data.append(df)
  i +=1

# Print statement showing the download is done

# Show what the first 5 rows of the data frame
# Note the dataframe has:
#   - Date (YYY-MM-DD) as an index
#   - Open (price the stock started as)
#   - High (highest price stock reached that day)
#   - Low (lowest price stock reached that day)
#   - Close (price the stock ended the day as)
#   - Volume (how many shares were traded that day)
#   - Dividends (any earnings shared to shareholders)
#   - Stock Splits (any stock price changes)
# Create ticker yfinance object

print('-----------------------')
print('Done!')
data[0].head()<jupyter_output>-----------------------  iteration :0
-----------------------  iteration :1
-----------------------  iteration :2
-----------------------  iteration :3
-----------------------  iteration :4
-----------------------  iteration :5
-----------------------  iteration :6
-----------------------  iteration :7
-----------------------  iteration :8
-----------------------  iteration :9
-----------------------  iteration :10
-----------------------  iteration :11
-----------------------  iteration :12
-----------------------  iteration :13
-----------------------  iteration :14
-----------------------  iteration :15
-----------------------  iteration :16
-----------------------  iteration :17
-----------------------  iteration :18
-----------------------  iteration :19
-----------------------  iteration :20
-----------------------  iteration :21
-----------------------  iteration :22
-----------------------  iteration :23
-----------------------  iteration :24
-----------------------  iteration [...]<jupyter_text>Some usefull function<jupyter_code># Ensure column types are correct

def CorrectColumnTypes(df):
  # Input: dataframe 
  # ouptut: dataframe (with column types changed)

  # Numbers
  for col in df.columns[1:80]:
      df[col] = df[col].astype('float')

  for col in df.columns[-10:]:
      df[col] = df[col].astype('float')

  # Categories 
  for col in df.columns[80:-10]:
      df[col] = df[col].astype('category').cat.codes
      # Corriger le type, 
      # Soit transformer les cat en one hot soit 
      # Soit cat code sur tout le dataset 

  return df 
# Create the lags 
def CreateLags(df,lag_size):
  # inputs: dataframe , size of the lag (int)
  # ouptut: dataframe ( with extra lag column), shift size (int)

  # add lag
  shiftdays = lag_size
  shift = -shiftdays
  df['Close_lag'] = df['Close'].shift(shift)
  return df, shift

# Split the testing and training data 
def SplitData(df, train_pct, shift):
  # inputs: dataframe , training_pct (float between 0 and 1), size of the lag (int)
  # ouptut: x train dataframe, y train data frame, x test dataframe, y test dataframe, train data frame, test dataframe

  train_pt = int(len(df)*train_pct)
  
  train = df.iloc[:train_pt,:]
  test = df.iloc[train_pt:,:]
  
  x_train = train.iloc[:shift,1:-1]
  y_train = train['Close_lag'][:shift]
  x_test = test.iloc[:shift,1:-1]
  y_test = test['Close'][:shift]

  return x_train, y_train, x_test, y_test, train, test
# Function to make the plots
def PlotModelResults_Plotly(train, test, pred, ticker, w, h, shift_days,name,i):
  # inputs: train dataframe, test dataframe, predicted value (list), ticker ('string'), width (int), height (int), shift size (int), name (string)
  # output: None

  # Create lines of the training actual, testing actual, prediction 
  D1 = go.Scatter(x=train.index,y=train['Close'],name = 'Train Actual') # Training actuals
  D2 = go.Scatter(x=test.index[:shift],y=test['Close'],name = 'Test Actual') # Testing actuals
  D3 = go.Scatter(x=test.index[:shift],y=pred,name = 'Our Prediction') # Testing predction

  # Combine in an object  
  line = {'data': [D1,D2,D3],
          'layout': {
              'xaxis' :{'title': 'Date'},
              'yaxis' :{'title': '$'},
              'title' : name + ' - ' + tickerSymbol[i] + ' - ' + str(shift_days)
          }}
  # Send object to a figure 
  fig = go.Figure(line)

  # Show figure
  fig.show()
# Regreesion Function
from sklearn import linear_model
from keras.callbacks import EarlyStopping,ModelCheckpoint,ReduceLROnPlateau
def CalcProfit(test_df,pred,j):
  pd.set_option('mode.chained_assignment', None)
  test_df['pred'] = np.nan
  test_df['pred'].iloc[:-j] = pred
  test_df['change'] = test_df['Close_lag'] - test_df['Close'] 
  test_df['change_pred'] = test_df['pred'] - test_df['Close'] 
  test_df['MadeMoney'] = np.where(test_df['change_pred']/test_df['change'] > 0, 1, -1) 
  test_df['profit'] = np.abs(test['change']) * test_df['MadeMoney']
  profit_dollars = test['profit'].sum()
  print('Would have made: $ ' + str(round(profit_dollars,1)))
  profit_days = len(test_df[test_df['MadeMoney'] == 1])
  print('Percentage of good trading days: ' + str( round(profit_days/(len(test_df)-j),2))     )

  return test_df, profit_dollars
df.head()
def pgcd(a,b):
    """pgcd(a,b): calcul du 'Plus Grand Commun Diviseur' entre les 2 nombres entiers a et b"""
    if b==0:
        return a
    else:
        r=a%b
        return pgcd(b,r)<jupyter_output><empty_output><jupyter_text>Defining the model : Convlutinal and linear typeYou may change some part of it, if you got more data maybe a bigger model will improve the prediction, if not maybe a smaller. The basic idea here is that the convolution learn to link the different variation style of the stock price during 20 day then it allow the linear part to learn from the past variation to improve it s computation, this allow us to be more consistant on mid term prediction.<jupyter_code>from keras.utils.vis_utils import plot_model
#################################################################
inputs = keras.Input(shape=(20,110 ), name="digits")
norm = layers.BatchNormalization()(inputs)
Conv1 = layers.Conv1D(filters=64, kernel_size=10, activation='relu')(norm)
Conv2 = layers.Conv1D(filters=64, kernel_size=5, activation='relu')(Conv1)
Conv3 = layers.Conv1D(filters=64, kernel_size=3, activation='relu')(Conv2)
norm2 = layers.BatchNormalization()(Conv3)
Max2 = layers.MaxPooling1D(2)(norm2)
Flaterning = layers.Flatten()(Max2)

Dense1 = layers.Dense(64, activation="linear")(Flaterning)
Dense2 = layers.Dense(64, activation="linear")(Dense1)
Dense3 = layers.Dense(64, activation="linear")(Dense2)
Dense4 = layers.Dense(64, activation="linear")(Dense3)

outputs = layers.Dense(1, activation="linear", name="predictions")(Dense4)
#################################################################
model = keras.Model(inputs=inputs, outputs=outputs)

model.compile(
      optimizer=keras.optimizers.Adam(
        learning_rate=0.01),  # Optimizer
      # Loss function to minimize
      loss=keras.losses.MeanSquaredError())

earlyStopping = EarlyStopping(monitor='val_loss', patience=15, verbose=0, mode='min')
checkpointer=ModelCheckpoint("weight", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='auto', period=1),
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, epsilon=1e-4, mode='min')

# summary du modèle
print(model.summary())

# graph
plot_model(model, to_file='multilayer_perceptron_graph.png')<jupyter_output>WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.
WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.
Model: "model_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 digits (InputLayer)         [(None, 20, 110)]         0         
                                                                 
 batch_normalization_4 (Batc  (None, 20, 110)          440       
 hNormalization)                                                 
                                                                 
 conv1d_6 (Conv1D)           (None, 11, 64)            70464     
                                                                 
 conv1d_7 (Conv1D)           (None, 7, 64)             20544     
                                [...]<jupyter_text>Trainning the model<jupyter_code>i = 0
ShiftModelList = []
for j in shifts: 
  regList = []
  i = 0
  print('#####################################################################')
  print(' days out: ' + str(j))
  print()
  
  print("prediction for values: " + tickerSymbol[i])
  print('---------------------------------')
  df = pd.concat(data)

  df_lag, shift = CreateLags(df,j)
  df_lag = CorrectColumnTypes(df_lag)
  x_train, y_train, x_test, y_test, train, test = SplitData(df, train_pct, shift)
  
  # Linear Regression
  print("Linear Regression")
    #################################################################
    # inputs: x train data, y train data, x test data, y test data (all dataframe's)
    # output: the predicted values for the test data (list)
    # utiliser le model keras, normaliser les données
    ################################################################# 

    #for ltsm and convolutionnal network
  Ymax = np.max(np.array(y_train))
  pgcd= 20

  if not x_train.shape[0]%pgcd == 0 :
    x_train = x_train[:- (x_train.shape[0]%pgcd)]
    y_train = y_train[:- (y_train.shape[0]%pgcd)]
  if not x_test.shape[0]%pgcd == 0 :
    x_test = x_test[:- (x_test.shape[0]%pgcd)]
    y_test = y_test[:- (y_test.shape[0]%pgcd)]

  x_train = np.array(x_train).reshape((int(x_train.shape[0]/pgcd), pgcd, x_train.shape[1])) 
  x_test = np.array(x_test).reshape((int(x_test.shape[0]/pgcd), pgcd, 110)) 

  ya = []
  for ii in range(len(y_train)):
    if ii%20 == 0:
      ya.append(y_train[ii])
  
  y_train = np.array(ya) /Ymax

  ya = []
  for ii in range(len(y_test)):
    if ii%20 == 0:
      ya.append(y_test[ii])
  y_test = np.array(ya) / Ymax
  #for ltsm and convolutionnal network


  #__________________________________________________________________#
  model.fit(x_train,y_train,batch_size=50,epochs=100, shuffle=True,validation_data=(x_test,y_test), callbacks=[earlyStopping, reduce_lr_loss,checkpointer] )
  lr_pred = model.predict( x_test).reshape(-1) * Ymax
  #PlotModelResults_Plotly(train, test, lr_pred, tickerSymbol, w, h, j, 'Deep Learning',i)

  regList.append(model)
  i+=1
  print('--------------------------------')
  ShiftModelList.append(regList)
  print('#####################################################################')
print(lr_pred/Ymax)
print(y_test)
D2 = go.Scatter(x=np.array(range(lr_pred.shape[0])),y=y_test*Ymax,name = 'Test Actual') # Testing actuals
D3 = go.Scatter(x=np.array(range(lr_pred.shape[0])),y=lr_pred,name = 'Our Prediction') # Testing predction

  # Combine in an object  
line = {'data': [D2,D3],
          'layout': {
              'xaxis' :{'title': 'Date'},
              'yaxis' :{'title': '$'},
              'title' : "Deep Learning" + ' - ' + "Test " + ' - Day shift = ' + str(shifts)
          }}
  # Send object to a figure 
fig = go.Figure(line)

  # Show figure
fig.show()<jupyter_output><empty_output><jupyter_text>testing it on an entire stock price  it is what it is don t forget this is style a self made model from a studdent, but still it is a solid algorithm that predict 14 day in the future.<jupyter_code>df = data[-1]
prediction = []
df_lag, shift = CreateLags(df,j)
df_lag = CorrectColumnTypes(df_lag)
for i in range(int(df.shape[0]-20)):
  df2 = np.array(df.drop(columns=["Close","Close_lag"]))[i:i+20,:].reshape((1, 20, 110)) 
  prediction.append(model.predict(df2).reshape(-1))
  
D2 = go.Scatter(x=np.array(range(np.array(prediction).shape[0])),y=np.array(prediction).reshape(-1)*Ymax,name = 'prediction ') # Testing actuals
D3 = go.Scatter(x=np.array(range(np.array(prediction).shape[0])),y=df["Close_lag"],name = 'truth') # Testing predction
#D4 = go.Scatter(x=np.array(range(np.array(prediction).shape[0])),y=df["Close"],name = 'Close actual') # Testing predction
  # Combine in an object  
line = {'data': [D2,D3],
          'layout': {
              'xaxis' :{'title': 'Date'},
              'yaxis' :{'title': '$'},
              'title' : "Deep Learning" + ' - ' + "Test " + ' - Day shift = ' + str(shifts)
          }}
  # Send object to a figure 
fig = go.Figure(line)

  # Show figure
fig.show()
df2 = np.array(df.drop(columns=["Close","Close_lag"]))
prediction = []
for i in range(20):
  df3 = df2[df2.shape[0]-20 -(i+1):-(i + 1),:].reshape((1, 20, 110)) 
  prediction.append(model.predict(df3).reshape(-1))
df4 = np.array(df["Close"])
D2 = go.Scatter(x=np.array(range(np.array(prediction).shape[0])),y=np.array(prediction)[::-1].reshape(-1)*Ymax,name = 'prediction Actual') # Testing actuals
D3 = go.Scatter(x=np.array(range(np.array(prediction).shape[0])),y=df4[df4.shape[0]-20 + shifts[0]:],name = 'what we know') # Testing actuals

# Combine in an object  
line = {'data': [D2,D3],
          'layout': {
              'xaxis' :{'title': 'Date'},
              'yaxis' :{'title': '$'},
              'title' : "Deep Learning" + ' - ' + "Test " + ' - Day shift = ' + str(shifts)
          }}
  # Send object to a figure 
fig = go.Figure(line)

  # Show figure
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1988548525622674, 'fraction_numerical': 0.017578013169195533, 'mean_word_length': 3.148693586698337, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '472223', 'license': 'MIT', 'n_tokens_mistral': 5864, 'n_tokens_neox': 5540, 'n_words': 1450, 'path': 'Stock_Market_DeepLearning_Convolutionnal.ipynb', 'repo_name': 'Axle-Bucamp/StockPricePrediction', 'size': '17465'}