<jupyter_start><jupyter_text>Ch10 Model<jupyter_code>import pandas as pd
import numpy as np 
import matplotlib.pyplot as plt 


from adv_finance import bars, labeling, utils, features, stats
from sklearn.metrics import roc_curve, classification_report, confusion_matrix, accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.utils import resample
from sklearn.utils import shuffle

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

%load_ext autoreload
%autoreload 2
plt.style.use('seaborn-talk')
plt.style.use('bmh')
pd.set_option('display.max_rows', 100)

data = pd.read_parquet('/nfs/data/interim_2018/TRADE_A233740_DB.parq')
data = data.loc[~data.index.duplicated()]

data.head()<jupyter_output><empty_output><jupyter_text>Primary Model<jupyter_code># Compute sides 
# data['side'] = np.nan 
%%time 

# CUSUM Filter 
daily_vol = stats.get_daily_vol(data['close'], lookback=100)
threshold = daily_vol.mean() * 0.2
t_events = labeling.cusum_filter(data['close'], threshold)
v_barriers = labeling.add_vertical_barrier(t_events=t_events, close=data['close'], num_days=1)


# Side Decision by Cusum Filter & Triple Barrier 
pt_sl = [1, 1]
min_ret = 0.01 
t_side_events = labeling.get_events(close=data['close'],
                                            t_events=t_events,
                                            pt_sl=pt_sl,
                                            target=daily_vol,
                                            min_ret=min_ret,
                                            num_threads=8,
                                            vertical_barrier_times=v_barriers,
                                            side_prediction=None)


side_labels = labeling.get_bins(t_side_events, data['close'])<jupyter_output>/nfs/code/adv_finance/adv_finance/labeling/labeling.py:111: FutureWarning: 
Passing list-likes to .loc or [] with any missing label will raise
KeyError in the future, you can use .reindex() as an alternative.

See the documentation here:
https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike
  target = target.loc[t_events]
2019-06-15 17:32:23.052601 100.0 apply_pt_sl_on_t1 done after 0.07 minutes. Remaining 0.0 minutes.<jupyter_text>Metal Label<jupyter_code>%%time
pt_sl = [1, 2]
min_ret = 0.02
t_barrier_events = labeling.get_events(close=data['close'],
                                            t_events=t_events,
                                            pt_sl=pt_sl,
                                            target=daily_vol,
                                            min_ret=min_ret,
                                            num_threads=8,
                                            vertical_barrier_times=v_barriers,
                                           side_prediction=side_labels['bin'])


labels = labeling.get_bins(t_barrier_events, data['close'])<jupyter_output>/nfs/code/adv_finance/adv_finance/labeling/labeling.py:111: FutureWarning: 
Passing list-likes to .loc or [] with any missing label will raise
KeyError in the future, you can use .reindex() as an alternative.

See the documentation here:
https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike
  target = target.loc[t_events]
/nfs/code/adv_finance/adv_finance/labeling/labeling.py:123: FutureWarning: 
Passing list-likes to .loc or [] with any missing label will raise
KeyError in the future, you can use .reindex() as an alternative.

See the documentation here:
https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike
  side_ = side_prediction.loc[target.index]<jupyter_text>Features<jupyter_code>%%time

raw_data = data.copy()

# Log Returns
raw_data['log_ret'] = np.log(raw_data['close']).diff()

# Momentum
raw_data['mom1'] = raw_data['close'].pct_change(periods=1)
raw_data['mom3'] = raw_data['close'].pct_change(periods=3)
raw_data['mom5'] = raw_data['close'].pct_change(periods=5)

# Volatility
raw_data['volatility_15'] = raw_data['log_ret'].rolling(window=15, min_periods=15, center=False).std()
raw_data['volatility_50'] = raw_data['log_ret'].rolling(window=50, min_periods=50, center=False).std()



# Serial Correlation (Takes about 4 minutes)
window_autocorr = 30
raw_data['autocorr_1'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=1), raw=False)
raw_data['autocorr_3'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=3), raw=False)
raw_data['autocorr_5'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=5), raw=False)

# Get the various log -t returns
raw_data['log_t1'] = raw_data['log_ret'].shift(1)
raw_data['log_t3'] = raw_data['log_ret'].shift(3)
raw_data['log_t5'] = raw_data['log_ret'].shift(5)

# Remove look ahead bias
raw_data = raw_data.shift(1)
# Get features at event dates 
X = raw_data 

# Drop unwanted columns 
try: 
    X.drop(['open', 'high', 'low', 'close', 'vol'], axis=1, inplace=True)

except Exception as e: 
    print(e)

    
y = labels.loc[X.index, 'bin']
y = y.dropna()
X = X.dropna()
com_idx = y.index.join(X.index).join(labels.index)

X = X.loc[com_idx]
y = y.loc[com_idx]<jupyter_output><empty_output><jupyter_text>Balance Classes<jupyter_code># Split data into training, validation and test sets
X_training_validation = X['2018-01-01':'2018-10-31']
y_training_validation = y['2018-01-01':'2018-10-31']
X_train, X_validate, y_train, y_validate = train_test_split(X_training_validation, y_training_validation, test_size=0.2, shuffle=False)
train_df = pd.concat([y_train, X_train], axis=1, join='inner')
train_df['bin'].value_counts()
# Upsample the training data to have a 50 - 50 split
# https://elitedatascience.com/imbalanced-classes
majority = train_df[train_df['bin'] == 0]
minority = train_df[train_df['bin'] == 1]

new_minority = resample(minority, 
                   replace=True,     # sample with replacement
                   n_samples=majority.shape[0],    # to match majority class
                   random_state=42)

train_df = pd.concat([majority, new_minority])
train_df = shuffle(train_df, random_state=42)

train_df['bin'].value_counts()<jupyter_output><empty_output><jupyter_text>Fit a Model<jupyter_code>%%time 

depth = 2
n_estimator = 10


# Refit a new model with best params, so we can see feature importance
rf2 = RandomForestClassifier(max_depth=depth, n_estimators=n_estimator, criterion='entropy', random_state=42)
rf2.fit(X_train, y_train.values.ravel())<jupyter_output>CPU times: user 20.3 ms, sys: 4 µs, total: 20.3 ms
Wall time: 19.8 ms<jupyter_text>Bet Sizing<jupyter_code>from adv_finance.labeling import get_signal
y_prob = rf2.predict_proba(X_train)
y_df = pd.Series(y_prob[:, 1], index=X_train.index)
t_barrier_events.head()
bet_size = get_signal(y_df, t_barrier_events0)
df_betsize = pd.DataFrame(y_df, columns=['Prob'])
df_betsize['BetSize'] = bet_size<jupyter_output><empty_output><jupyter_text>APPENDIX<jupyter_code>import pickle

with open('t_barrier_events.pickle', 'wb') as f:
    pickle.dump(t_barrier_events.to_dict(), f)


with open('y_prob.pickle', 'wb') as f:
    pickle.dump(y_df.to_dict(), f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12057596554972413, 'fraction_numerical': 0.023415421881308032, 'mean_word_length': 3.931652289316523, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '606922', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2533, 'n_words': 472, 'path': 'examples/Ch10/Ch10 Model.ipynb', 'repo_name': 'cw-jang/adv_finance', 'size': '7431'}