<jupyter_start><jupyter_text><jupyter_code>%matplotlib inline
import sys

!pip install category_encoders==2.*

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from category_encoders import OneHotEncoder, OrdinalEncoder
from sklearn.ensemble import RandomForestRegressor
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.metrics import accuracy_score
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import SelectKBest
    !pip install eli5
    !pip install pandas-profiling==2.*
    !pip install pdpbox
    !pip install shap

import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='xgboost')
df = pd.read_csv('/content/weekly asset classes.csv', parse_dates=['date'], index_col='date')
df = df.iloc[14:]
df.info()
# df['gold'] = df['gold'].str.strip(',').astype(float)
df.shape
df.head(1)
def wrangle(df):

  df = df.copy()

  # Remove the most extreme 1 % 
  df = df[(df['gold'] >= np.percentile(df['gold'], 0.5)) &
          (df['gold'] <= np.percentile(df['gold'], 99.5)) &
          (df['rocgold'] >= np.percentile(df['rocgold'], 0.5)) &
          (df['rocgold'] <= np.percentile(df['rocgold'], 99.5)) &
          (df['goldbps'] >= np.percentile(df['goldbps'], 0.5)) &
          (df['goldbps'] <= np.percentile(df['goldbps'], 99.5)) &

          (df['copper'] >= np.percentile(df['copper'], 0.5)) &
          (df['copper'] <= np.percentile(df['copper'], 99.5)) &
          (df['roccop'] >= np.percentile(df['roccop'], 0.5)) &
          (df['roccop'] <= np.percentile(df['roccop'], 99.5)) &
          (df['copbps'] >= np.percentile(df['copbps'], 0.5)) &
          (df['copbps'] <= np.percentile(df['copbps'], 99.5)) &

          (df['wti'] >= np.percentile(df['wti'], 0.5)) &
          (df['wti'] <= np.percentile(df['wti'], 99.5)) &
          (df['rocwti'] >= np.percentile(df['rocwti'], 0.5)) &
          (df['rocwti'] <= np.percentile(df['rocwti'], 99.5)) &
          (df['wtibps'] >= np.percentile(df['wtibps'], 0.5)) &
          (df['wtibps'] <= np.percentile(df['wtibps'], 99.5)) &

          (df['dxy'] >= np.percentile(df['dxy'], 0.5)) &
          (df['dxy'] <= np.percentile(df['dxy'], 99.5)) &
          (df['rocdxy'] >= np.percentile(df['rocdxy'], 0.5)) &
          (df['rocdxy'] <= np.percentile(df['rocdxy'], 99.5)) &
          (df['dxybps'] >= np.percentile(df['dxybps'], 0.5)) &
          (df['dxybps'] <= np.percentile(df['dxybps'], 99.5)) &

          (df['eur'] >= np.percentile(df['eur'], 0.5)) &
          (df['eur'] <= np.percentile(df['eur'], 99.5)) &
          (df['roceur'] >= np.percentile(df['roceur'], 0.5)) &
          (df['roceur'] <= np.percentile(df['roceur'], 99.5)) &
          (df['eurbps'] >= np.percentile(df['eurbps'], 0.5)) &
          (df['eurbps'] <= np.percentile(df['eurbps'], 99.5)) &

          (df['10yr'] >= np.percentile(df['10yr'], 0.5)) &
          (df['10yr'] <= np.percentile(df['10yr'], 99.5)) &
          (df['roc10yr'] >= np.percentile(df['roc10yr'], 0.5)) &
          (df['roc10yr'] <= np.percentile(df['roc10yr'], 99.5)) &
          (df['10yrbps'] >= np.percentile(df['10yrbps'], 0.5)) &
          (df['10yrbps'] <= np.percentile(df['10yrbps'], 99.5)) &

          (df['10yr2yr Spread'] >= np.percentile(df['10yr2yr Spread'], 0.5)) &
          (df['10yr2yr Spread'] <= np.percentile(df['10yr2yr Spread'], 99.5)) &
          (df['Agri'] >= np.percentile(df['Agri'], 0.5)) &
          (df['Agri'] <= np.percentile(df['Agri'], 99.5)) &
          (df['CRB'] >= np.percentile(df['CRB'], 0.5)) &
          (df['CRB'] <= np.percentile(df['CRB'], 99.5)) &
          (df['VIXCLS'] >= np.percentile(df['VIXCLS'], 0.5)) &
          (df['VIXCLS'] <= np.percentile(df['VIXCLS'], 99.5))]


  # Rename columns
  df = df.rename(columns={'gold': 'Gold', 
                          'rocgold': 'Gold ROC',
                          'goldbps': 'Gold bps',
                          'copper': 'Copper', 
                          'roccop': 'Copper ROC',
                          'copbps': 'Copper bps',
                          'wti': 'WTI',
                          'rocwti': 'WTI ROC',
                          'wtibps': 'WTI bps',
                          'dxy': 'DXY',
                          'rocdxy': 'DXY ROC',
                          'dxybps': 'DXY bps',
                          'eur': 'EUR',
                          'roceur': 'EUR ROC',
                          'eurbps': 'EUR bps',
                          '10yr': '10yr Rate',
                          'roc10yr': '10yr ROC',
                          '10yrbps': '10yr bps',
                          'regime': 'Regime'})
  
  # Turn Gold col into float instead of a string
  # df['Gold'] = df['Gold'].str.strip(',').astype(float)
  # df['Gold'] = pd.to_numeric(df['Gold'], downcast='float')
  # Rename column `regime`
  df['Regime'] = df['Regime'].replace(1.0, 'Regime 1')
  df['Regime'] = df['Regime'].replace(2.0, 'Regime 2')
  df['Regime'] = df['Regime'].replace(3.0, 'Regime 3')
  df['Regime'] = df['Regime'].replace(4.0, 'Regime 4')

  # Create CPI + or - column
  result = [] 
  for value in df['Regime']: 
    if value == 'Regime 1': 
      result.append("-") 
    elif value == 'Regime 2': 
      result.append('+') 
    elif value == 'Regime 3': 
      result.append('+') 
    else:
      result.append('-')

  df['CPI'] = result

        
  
  # Create GDP + or - column
  result1 = []
  for value in df['Regime']:
    if value == 'Regime 1': 
      result1.append('+') 
    elif value == 'Regime 2': 
      result1.append('+') 
    elif value == 'Regime 3': 
      result1.append('-') 
    else:
      result1.append('-')

  df['GDP'] = result1  

  # cols to poptentially drop `droppers`
  # droppers = ['Gold bps', 'Copper bps', 'WTI bps', 'DXY bps', 'EUR bps', 'Agri', 'CRB', 'Regime', '10yr Rate']  #'10yr bps'
  # droppers_alt = ['Gold ROC', 'Copper ROC', 'WTI ROC', 'DXY ROC', 'EUR ROC', '10yr ROC', 'Agri', 'CRB', 'Regime']
  
  # Keep only some columns
  # keep_cols = ['Gold',            #
              # 'Copper', 
              # 'WTI', 
              # 'DXY', 
              # 'EUR',
              # '10yr Rate',
              # 'Regime',
              # 'Gold bps']


  
  # df = df[keep_cols]
  # df = df.drop(columns=droppers_alt)
  # df['GDP Positive'] = (df['GDP'] == '+')
  df = df.drop(['10yr bps','10yr ROC', '10yr2yr Spread', 'EUR bps', 'Copper bps', 'Regime', 'Gold bps', 
                'Copper ROC', 'WTI ROC', 'WTI ROC', 'WTI bps', 'EUR', 'CPI'], axis=1)  #, 'Agri', 'CRB', 'Regime'
  # df = df[droppers]
  return df

df = wrangle(df)
df.info()
# mask = df['regime'] == 'Regime 3'
# df[mask]
# target is the regime column
# categorical
target = '10yr Rate'

y = df[target]
X = df.drop(target , axis=1)
cutoff1 = '2016-01-01'
cutoff2 = '2018-07-01'
# cutoff3 = '2020-01-01'
mask1 = (df.index <= cutoff1)
train = df[mask1]
val = df[~mask1]

mask1 = df.index < cutoff1
mask2 = (df.index >= cutoff1) & (df.index < cutoff2)
mask3 = (df.index >= cutoff2) 

val = df[mask2]
test = df[mask3]
y_train = train[target]
X_train = train.drop([target], axis=1)

y_val = val[target]
X_val = val.drop(target, axis=1)

y_test = test[target]
X_test = test.drop(target, axis=1)
# assert df.shape[0] == train.shape[0] + val.shape[0]
assert df.shape[0] == train.shape[0] + val.shape[0] + test.shape[0]
# Regression --> mean absolute error
from sklearn.metrics import mean_absolute_error
 
print('Baseline MAE:', mean_absolute_error(y_train, [y_train.mean()]*len(y_train)))
# print('Baseline Accuracy:', y_train.value_counts(normalize=True).max())
# Model 1: Linear Regression

model = make_pipeline(
    OrdinalEncoder(),   #OneHotEncoder(use_cat_names=True),
    # SimpleImputer(strategy='mean'),
    StandardScaler(),
    LinearRegression()  #LinearRegression()  RandomForestRegressor(random_state=42)
)

model.fit(X_train, y_train);
print('Training MAE:', mean_absolute_error(y_train, model.predict(X_train)))
print('Validation MAE:', mean_absolute_error(y_val, model.predict(X_val)))
print('Test MAE:', mean_absolute_error(y_test, model.predict(X_test)))
import numpy as np
coefficients = np.round(model.named_steps['linearregression'].coef_, 2)
list(zip(X_train.columns, coefficients))
# Model 2: Random Forest
model_f = make_pipeline(
    OrdinalEncoder(),
    SimpleImputer(strategy='mean'),
    StandardScaler(),
    RandomForestRegressor(n_jobs=-1, n_estimators=10)
)

model_f.fit(X_train, y_train);
print('Training MAE:', mean_absolute_error(y_train, model_f.predict(X_train)))
print('Validation MAE:', mean_absolute_error(y_val, model_f.predict(X_val)))
print('Test MAE:', mean_absolute_error(y_test, model_f.predict(X_test)))
# from sklearn.metrics import classification_report, plot_confusion_matrix, plot_roc_curve

# # On validation data only
# print('Random Forest')
# print(classification_report(y_val, model.predict(X_val)))
from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
from xgboost import XGBClassifier, XGBRegressor
# Model 3: Gradient Boosting Regressor

model_grad = make_pipeline(
    OrdinalEncoder(),
    SimpleImputer(),
    GradientBoostingRegressor(n_estimators=22)
)

model_grad.fit(X_train, y_train);
print('Training MAE:', mean_absolute_error(y_train, model_grad.predict(X_train)))
print('Validation MAE:', mean_absolute_error(y_val, model_grad.predict(X_val)))
print('Test MAE:', mean_absolute_error(y_test, model_grad.predict(X_test)))
# Model 4: XGB Regressor

model_xgb = make_pipeline(
    OrdinalEncoder(),
    SimpleImputer(),
    XGBRegressor(n_jobs=-1, n_estimators=43)
)

model_xgb.fit(X_train, y_train);
print('Training MAE:', mean_absolute_error(y_train, model_xgb.predict(X_train)))
print('Validation MAE:', mean_absolute_error(y_val, model_xgb.predict(X_val)))
print('Test MAE:', mean_absolute_error(y_test, model_xgb.predict(X_test)))<jupyter_output>Training MAE: 0.14635500267914836
Validation MAE: 0.32577631413435737
Test MAE: 0.45798841214441993<jupyter_text>Feature Importances<jupyter_code>model_xgb.named_steps['xgbclassifier'].feature_importances_
feature = 'Monthly Debts'

isolated = pdp_isolate(
    model=model_gb,
    dataset=X_val,
    model_features=X_val.columns,
    feature=feature
)

pdp_plot(isolated, feature_name=feature);<jupyter_output><empty_output><jupyter_text>PDP Plots<jupyter_code># View Feature
import matplotlib.pyplot as plt
plt.rcParams['figure.dpi'] = 72
from pdpbox.pdp import pdp_isolate, pdp_interact
feature_imp = model.named_steps['xgbclassifier'].feature_importances_
features = X_train.columns

pd.Series(feature_imp, features).sort_values(ascending=False).head(7).plot(kind='barh');
features = ['Copper', 'DXY']

interaction = pdp_interact(
    model=model_xgb,
    dataset=X_val,
    model_features=X_val.columns,
    features=features
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17905918057663125, 'fraction_numerical': 0.029545657413192896, 'mean_word_length': 3.3697347893915754, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '80879', 'license': 'MIT', 'n_tokens_mistral': 4343, 'n_tokens_neox': 4046, 'n_words': 759, 'path': 'module4-model-interpretation/10yr_regression_weekly_assetclasses.ipynb', 'repo_name': 'drewamorbordelon/DS-Unit-2-Applied-Modeling', 'size': '11203'}