<jupyter_start><jupyter_text>Predictive Modelling: LightGBM  Imports<jupyter_code>%load_ext autoreload
%autoreload 2

# Pandas and numpy
import pandas as pd
import numpy as np

# 
from IPython.display import display, clear_output
import sys
import time

# Libraries for Visualization
import matplotlib.pyplot as plt
import seaborn as sns
from src.visualization.visualize import plot_corr_matrix, plot_multi, plot_norm_dist, plot_feature_importances

# Some custom tools
from src.data.tools import check_for_missing_vals

# 
from src.models.predict_model import avg_model, run_combinations
#from src.models.train_model import run_combinations 

# Alpaca API
import alpaca_trade_api as tradeapi

# Pickle
import pickle
import os
from pathlib import Path

# To load variables from .env file into system environment
from dotenv import find_dotenv, load_dotenv

from atomm.Indicators import MomentumIndicators
from atomm.DataManager.main import MSDataManager
from atomm.Tools import calc_open_position, calc_returns
from src.visualization.visualize import plot_confusion_matrix
from atomm.Methods import BlockingTimeSeriesSplit, PurgedKFold           


import time

# scikit-learn
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score, TimeSeriesSplit
from xgboost import XGBClassifier
from sklearn.metrics import classification_report, confusion_matrix, plot_confusion_matrix
from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score
from sklearn.model_selection import train_test_split, TimeSeriesSplit
from xgboost import XGBClassifier

#from sklearn.ensemble import BaggingClassifier
#from sklearn.multiclass import OneVsRestClassifier
#from sklearn.neighbors import KNeighborsClassifier
#from sklearn.ensemble import RandomForestClassifier
#from sklearn.linear_model import LogisticRegression
from lightgbm import LGBMClassifier

# For BayesianHyperparameter Optimization
from atomm.Models.Tuning import search_space#, BayesianSearch
from hyperopt import space_eval

# Visualization libraries
import seaborn as sns
import matplotlib.pyplot as plt
from pandas.plotting import scatter_matrix
import matplotlib.gridspec as gridspec
#import matplotlib.style as style
from scipy import stats
##### Load environment variables
load_dotenv(find_dotenv())
R_STATE = 16818<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>data_base_dir = os.environ.get('DATA_DIR_BASE_PATH')

fname = os.path.join(data_base_dir, 'processed', 'index.h5')
fname = Path(fname)
#fname = '../data/processed/index.h5'
# Load dataset from HDF storage
with pd.HDFStore(fname) as storage:
    djia = storage.get('nyse/cleaned/rand_symbols')
    y_2c = storage.get('nyse/engineered/target_two_class')
    y_3c = storage.get('nyse/engineered/target_three_class')
    df_moments = storage.get('nyse/engineered/features')
    ws = storage.get('nyse/engineered/weights')

    #print(storage.info())
    
# Create copies of the pristine data
X = df_moments.copy()
y = y_3c.copy()
y2 = y_2c.copy()
prices = djia.copy()
print(df_moments.shape, y_3c.shape, y_2c.shape, djia.shape, ws.shape)
forecast_horizon = [1, 3, 5, 7, 10, 15, 20, 25, 30]
input_window_size = [3, 5, 7, 10, 15, 20, 25, 30]
ti_list = ['macd', 'rsi', 'stoc', 'roc', 'bbu', 'bbl', 'ema', 'atr', 'adx', 'cci', 'williamsr', 'stocd']
symbol_list = df_moments.columns.get_level_values(0).unique()
df_moments.columns.get_level_values(1).unique()<jupyter_output><empty_output><jupyter_text>Imputing missing values<jupyter_code>X.shape
check_for_missing_vals(X)<jupyter_output>No missing values found in dataframe<jupyter_text>Prices values<jupyter_code>prices.shape
check_for_missing_vals(prices)
y_3c.shape
check_for_missing_vals(y_3c)
y2.shape
check_for_missing_vals(y2)<jupyter_output>No missing values found in dataframe<jupyter_text>No missing values, and sizes of ```y.shape[0]``` and```X.shape[0]``` match.<jupyter_code>check_for_missing_vals(ws)
ws.shape<jupyter_output><empty_output><jupyter_text>Scaling the features<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler
#scale = MinMaxScaler()
scale = StandardScaler()
scaled = scale.fit_transform(X)
scaled.shape
X_scaled = pd.DataFrame(data=scaled, columns=X.columns)
#X_scaled = scaled<jupyter_output><empty_output><jupyter_text>Train-Test Split<jupyter_code># Use 90/10 train/test splits
test_p = .2
# Scaled, three-class
test_size = int((1 - test_p) * X_scaled.shape[0])
X_train, X_test, y_train, y_test = X_scaled[:test_size], X_scaled[test_size:], y_3c[:test_size], y_3c[test_size:]
prices_train, prices_test = djia[:test_size], djia[test_size:]
# Unscaled, two-class
test_size = int((1 - test_p) * X.shape[0])
X_train, X_test, y_train, y_test = X[:test_size], X[test_size:], y2[:test_size], y2[test_size:]

# Scaled, two-class
test_size = int((1 - test_p) * X.shape[0])
X_train, X_test, y_train, y_test = X_scaled[:test_size], X_scaled[test_size:], y2[:test_size], y2[test_size:]
prices_train, prices_test = djia[:test_size], djia[test_size:]
#test_size = test_p
#X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_3c, test_size=test_size, random_state=101)
# Splitting sample weights
ws_train, ws_test = ws[:test_size], ws[test_size:]
prices_train, prices_test = prices[:test_size], prices[test_size:]<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>symbol_list
symbol = 'AOS'
n1 = 15
n2 = 15
n_estimators = 10
# set up cross validation splits
tscv = TimeSeriesSplit(n_splits=5)
btscv = BlockingTimeSeriesSplit(n_splits=5)
#ppcv = PurgedKFold(n_splits=5)
# Creates a list of features for a given lookback window (n1)
features = [f'{x}_{n1}' for x in ti_list]
# Creates a list of all features
all_features = [f'{x}_{n}' for x in ti_list for n in input_window_size]<jupyter_output><empty_output><jupyter_text>Define custom loss functions w/ penalty for FP / FN<jupyter_code># Define custom loss function which adds penalty for falsely classifying as positive

def custom_loss_function(y_true, y_pred):
    beta = 1E-1
    p    = 1. / (1 + np.exp(-y_pred))
    grad = p * ((beta -1) * y_true + 1) - beta * y_true
    hess = ((beta - 1) * y_true + 1) * p * (1.0 - p)
    return grad, hess
    
def logloss_penalize_fp(y_true, y_pred):
    beta = 1.3
    p    = 1. / (1 + np.exp(-y_pred))
    st = (y_true - beta * (y_true - 1))
    # Gradient: ((beta-(1-beta)*y_true) - y_true) * p
    grad = st * p - y_true
    # Hessian: ((beta-(1-beta)*y_true) - y_true) * p * (1 - p)
    hess = st * p * (1.0 - p) 
    return grad, hess<jupyter_output><empty_output><jupyter_text>Single lookback/lookahead w/o hyperparameter optimization<jupyter_code>clf_lgbm = LGBMClassifier(random_state=R_STATE)
clf_lgbm.set_params(
    **{'objective': logloss_penalize_fp},
    metrics = ['logloss'],
    #learning_rate=1E-1,
    #num_iterations=200,
)
X_tr=X_train[symbol][[f'{x}_{n1}' for x in ti_list]]
y_tr=y_train[symbol][f'signal_{n2}']

clf_lgbm.fit(
    X=X_tr,
    y=y_tr
)
y_pred_lgbm1 = clf_lgbm.predict(X_test[symbol][[f'{x}_{n1}' for x in ti_list]])
print(classification_report(y_pred_lgbm1, y_test[symbol][f'signal_{n2}']))

plot_confusion_matrix(
    clf_lgbm,
    X_test[symbol][[f'{x}_{n1}' for x in ti_list]],
    y_test[symbol][f'signal_{n2}'],
    #normalize='all'
)

calc_returns(y_pred_lgbm1, signals_baseline=1, prices=djia[symbol][test_size:])<jupyter_output><empty_output><jupyter_text>Single lookback/lookahead with hyperparameter optimization<jupyter_code>def returns_score(y_true, y_pred, data):
    return calc_returns(y_pred, signals_baseline = 1, prices = data)['Cum_Returns_Strat'][-1]

from atomm.Models.Tuning import search_space
from atomm.Models.Tuning import BayesianOptimizer
from hyperopt import space_eval
w_tr = ws_train[symbol][f'weight_{n2}']/ws_train[symbol][f'weight_{n2}'].max()
w_te = ws_test[symbol][f'weight_{n2}']/ws_test[symbol][f'weight_{n2}'].max()
features
feature_list = ['macd_10', 'stoc_10']
opt = BayesianOptimizer(
    param_space=search_space(LGBMClassifier),
    model=LGBMClassifier,
    X_train=X_train[symbol][feature_list],
    y_train=y_train[symbol][f'signal_{n2}'],
    #weight_train=w_tr,
    X_test=X_test[symbol][feature_list],
    y_test=y_test[symbol][f'signal_{n2}'],
    #weight_test=w_te,
    valid_data=prices_train[symbol]['Close'],
    valid_metric=returns_score,
    num_eval=10,
)
from hyperopt import hp
opt.set_params(objective=logloss_penalize_fp)
#opt.set_params(metrics='logloss')
opt.set_params(cv=hp.choice('cv', ['btscv', 'tscv']))
opt.set_params(random_state=R_STATE)
test_score, clf_best, best_param = opt.optimize()
y_pred_bsearch_lgbm = clf_best.predict(X_test[symbol][feature_list])
print('Recall Score: ', recall_score(y_pred_bsearch_lgbm, y_test[symbol][f'signal_{n2}'], average='weighted'))
print(classification_report(y_pred_bsearch_lgbm, y_test[symbol][f'signal_{n2}']))
plot_confusion_matrix(
    clf_best,
    X_test[symbol][feature_list], y_test[symbol][f'signal_{n2}'],
)


calc_returns(y_pred_bsearch_lgbm, signals_baseline=1, prices=prices_test[symbol])
returns = prices_test[symbol].pct_change().copy()
returns['pred'] = y_pred_bsearch_lgbm

fig, ax1 = plt.subplots(sharex=True)

color = 'tab:red'
ax1.set_xlabel('date')
ax1.set_ylabel('preds', color=color)
ax1.plot(returns.index, returns['pred'], color=color)
ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

color = 'tab:blue'
ax2.set_ylabel('returns', color=color)  # we already handled the x-label with ax1
ax2.plot(returns.index, prices_test[symbol]['Close'], color=color)
ax2.tick_params(axis='y', labelcolor=color)
#ax2.hlines(0, dt.datetime(2019, 1, 1), dt.datetime(2019, 5, 10))
#plt.xlim(dt.datetime(2019, 1, 1), dt.datetime(2019, 5, 10))
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1227433182047403, 'fraction_numerical': 0.0178517397881997, 'mean_word_length': 6.592649310872894, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '617562', 'license': 'MIT', 'n_tokens_mistral': 3865, 'n_tokens_neox': 3717, 'n_words': 750, 'path': 'notebooks/06e_Predictive_Modeling-lightgbm.ipynb', 'repo_name': 'robindoering86/capstone_nf', 'size': '9915'}