<jupyter_start><jupyter_text>Get the data<jupyter_code>from datetime import datetime
import pandas as pd
import pandas_datareader.data as web

import re
import numpy as np
import matplotlib.pyplot as plt
# Load the data from web api or csv file (limit: 5 api call per minute or 500 per day)
from_api = False
from_csv = True

stocks = ['AC.TO', 'SHOP.TO', 'AAV.TO', 'ABX.TO', 'ABST.TO']

if from_api == True:
    # 500 days
    start = datetime(2019, 6, 22)
    end = datetime(2021, 6, 18)

    stocks_df = web.DataReader(stocks[0], "av-daily", start=start,
                                end=end, 
                                api_key='<KEY>')

    for stock in stocks:
        stock_df = web.DataReader(stock, "av-daily", start=start,
                            end=end, 
                            api_key='<KEY>')
        stocks_df = pd.concat([stocks_df, stock_df], axis=1)

        stocks_df.to_csv('./datasets/can_ohlc_data.csv', index=False)

if from_csv == True:
    stocks_df = pd.read_csv('./datasets/can_ohlc_data.csv')
stocks_df
# Select high, low and volume columns to drop
drop_columns_lists = [[5*k+1, 5*k+2, 5*k+4] for k in range(len(stocks))]
# merge list of lists into one list
drop_columns = [j for i in drop_columns_lists for j in i]
stocks_df.drop(stocks_df.iloc[:, drop_columns], axis=1, inplace=True)
stocks_df
# Convert to O and C format
df_adj_list_all = []
for k in range(len(stocks)):
    # make a dictionary of open and close values for all columns
    df_adj_list=[{'O'+str(i+1):stocks_df.iloc[i, 2*k], 'C'+str(i+1):stocks_df.iloc[i, 2*k+1]} for i in range(len(stocks_df))]
    # merge dictionaries into single one
    df_adj_dict = {k: v for d in df_adj_list for k, v in d.items()}
    # append the dictionary to the list of all stocks
    df_adj_list_all.append(df_adj_dict)
# df_adj_list_all
df_OC = pd.DataFrame(df_adj_list_all)
df_OC
# Is the NaN values in any rows?
df_OC.isnull().any(axis=1)<jupyter_output><empty_output><jupyter_text>Exploretory Data Analysis<jupyter_code># we use regular expression to select Close columns in data
data_cols = df_OC.columns
r = re.compile("^C")
close_cols = filter(r.match, data_cols)

df_OC[close_cols].T.plot(figsize=(10,6), legend=False)
memory_days = 10
num_cols = 2 # open and close columns
start_day = 1
last_day = 440 # from start_day to last_day, make 10 day windows. So 450 days for training and 55 days for backtesting
X_windows = [0] # X_windows are stored as days starts at day=1 (workaround)


# scale a dataframe so the first column (opening price) is 1.0. Divide from the last to the first column.
def scaler(df):
    for column in reversed(df.columns):
        df.loc[:, column] = df.loc[:, column] / df.iloc[:,0]
    return df

for day in range(start_day, last_day+1):
    # select the 10 day window
    rolling_window = df_OC.iloc[:, day*num_cols-num_cols:(day+memory_days)*num_cols-num_cols]
    # scale data to 'O1' column
    rolling_window_scaled = scaler(rolling_window.copy())
    X_windows.append(rolling_window_scaled)
    # set equal column names for all dataframe to concatenate
    X_windows[day].columns = X_windows[start_day].columns

# concatenate all 10 day windows in one dataframe
X_windows_list = [X_windows[day] for day in range(start_day, last_day+1)]    
X_windows_con = pd.concat(X_windows_list) 
# Split the data into train, test and backtest data. The train data is used in machine learning to fit the model and the test data is to ckeck the model on one signle 10 day window.
# The backtest data is used to check the strategy by using the prediction of the model.
n_stocks = 5
train_data = X_windows_con.iloc[:(last_day-1)*n_stocks, :]
test_data = X_windows_con.iloc[(last_day-1)*n_stocks:, :]
backtest_data = df_OC.iloc[:, num_cols*(last_day+memory_days):]
train_data
train_data.filter(regex='^O', axis=1).T.plot(figsize=(10,6), legend=False)
corr_matrix = train_data.corr()

import seaborn as sns
# Generate a mask for the upper triangle (taken from seaborn example gallery)
mask = np.zeros_like(corr_matrix, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
plt.subplots(figsize = (11,9))
sns.heatmap(corr_matrix,
            annot=True,
            mask = mask,
            cmap = 'RdBu_r', ## in order to reverse the bar replace "RdBu" with "RdBu_r"
            linewidths=.9, 
            linecolor='white',
            fmt='.2g',
            center = 0,
            square=True)            
test_data
backtest_data.head()
# remove 10th day close column of test data
X_test = test_data.drop(columns = ['C10'])

# set the upward change in price as 1 and the downward as 0
y_test = (test_data.loc[:,'C10'] > test_data.loc[:,'O10']).astype('uint8')
# We set 9 days data to be the predictor of the 10th day
X_train = train_data.drop(columns = ['C10'])

# set the upward change in price as 1 and the downward as 0
y_train = (train_data.loc[:,'C10'] > train_data.loc[:,'O10']).astype('uint8')
# find the correlation between the interday and intraday stock movement
x = train_data.loc[:,'O10']/train_data.loc[:,'C9']
y = train_data.loc[:,'C10']/train_data.loc[:,'O10']

m, b = np.polyfit(x, y, 1)
r = y.corr(x)
plt.scatter(x, y)
plt.plot(x, m*x+b, '-r')
plt.text(1,1.15, 'r={:.2f}'.format(r), fontsize=16)
plt.xlim((.3,2.5))
plt.ylim((.3,2.5))
plt.xlabel("O10 / C9")
plt.ylabel("C10 / O10")
plt.title("Correlation between interday and intraday stock movement")
plt.savefig('images/correlation_interday_intraday.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn import metrics,preprocessing
from sklearn.model_selection import cross_val_predict
forest_clf = RandomForestClassifier(random_state=42)
y_predict = cross_val_predict(forest_clf, X_train, y_train, cv=5)
params = forest_clf.get_params()
print(params)
from sklearn.metrics import confusion_matrix
conf_mx = confusion_matrix(y_train, y_predict)
# normalize confusion matrix
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
norm_conf_mx
plt.matshow(norm_conf_mx, cmap=plt.cm.gray)
plt.show()
from sklearn.metrics import f1_score

f1_score(y_train, y_predict)
forest_clf.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Predict on the test set<jupyter_code>y_pred = forest_clf.predict(X_test)
conf_mx_test = confusion_matrix(y_test, y_pred)
conf_mx_test
# normalize confusion matrix
row_sums = conf_mx_test.sum(axis=1, keepdims=True)
norm_conf_mx_test = conf_mx_test / row_sums
norm_conf_mx_test
plt.matshow(norm_conf_mx_test, cmap=plt.cm.gray)
plt.show()
f1_score(y_test, y_pred)
y_pred
StID = range(1,len(y_pred)+1)
predictions = pd.DataFrame(y_pred, columns=['movement'], index=StID)
predictions
predictions.to_csv('predictions/can_predictions.csv')<jupyter_output><empty_output><jupyter_text>Applying the strategy<jupyter_code>from trade_stat_logger.logger import SimpleLogger

logger = SimpleLogger()

backtest_days = np.arange(memory_days, len(backtest_data.columns)/2, 1, dtype=int)

for day in backtest_days:
    # get the close and open of 9 days and the 10th day open
    X_backtest = backtest_data.iloc[:, 2*(day-memory_days):2*day-1]
    
    # predict the 10th close
    y_predict_10 = forest_clf.predict(X_backtest)
    
    # apply the strategy to all tickers in dataset in that specific day
    for index, rows in X_backtest.iterrows():
        share_price_open = backtest_data.iloc[index-1, 2*day-2]
        share_price_close = backtest_data.iloc[index-1, 2*day-1]
        # if the 10th close is higher than open, then buy. if it's lower, do nothing (no short selling for now)
        if y_predict_10[index-1] == 1:
            logger.log(security = index, shares = 1, share_price = share_price_open)
            logger.log(security = index, shares = -1, share_price = share_price_close)
# Analyze the performance of the strategy
logger.graph_statistics(show_window=False)
plt.savefig('./images/can_graph_statistics.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11388299862791568, 'fraction_numerical': 0.02083073468878633, 'mean_word_length': 4.381208053691275, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '463346', 'license': 'MIT', 'n_tokens_mistral': 2995, 'n_tokens_neox': 2855, 'n_words': 748, 'path': 'data_reader.ipynb', 'repo_name': 'm-tari/stock_drirection_predict_ml', 'size': '8039'}