<jupyter_start><jupyter_text>Quantile Regression Toy Models<jupyter_code>%reload_ext autoreload
%autoreload 2
from pathlib import Path
import numpy as np
import pandas as pd
import sklearn
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
dir_datasets = Path('../../soil/datasets/')
pth_wise3_data = dir_datasets/'ISRIC_WISE/orgc_stock_ISRIC_WISE_clip.csv'
wise3 = pd.read_csv(pth_wise3_data)
wise3.drop('Unnamed: 0', axis=1, inplace=True)
vars_X = ['LATDD', 'LONDD']
var_Y = 'orgc_stock_mineral'
wise3[vars_X + [var_Y]].head()
X = wise3[vars_X].values
Y = wise3[var_Y].values
# Y_scaled = preprocessing.scale(Y)
X_scaled = pd.DataFrame(preprocessing.scale(X), columns=vars_X)
X_train, X_test, Y_train, Y_test = train_test_split(
    X_scaled, Y, test_size=0.2, random_state=5)<jupyter_output><empty_output><jupyter_text>Helpers<jupyter_code>def correctPcnt(df):
    correct = 0
    obs = df.shape[0]
    for i in range(obs):
        if df.loc[i,0.01] <= df.loc[i,'actual'] <= df.loc[i,0.99]:
            correct += 1
    print(correct/obs)
    
    

def showIntervals(ax, df):
    x = np.arange(len(df))
    y50 = df[0.5].values
    y95 = df[0.95].values
    y05 = df[0.05].values
    
    ax.scatter(x, y50, marker='.')
    ax.fill_between(x, y05, y95, color='red', alpha=0.1)
    return ax<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor(
    n_estimators=200, random_state=0, min_samples_split=20)
%%time

rf.fit(X_train, Y_train)
%%time 

# Get the predictions of all trees for all observations
# Each observation has N predictions from the N trees

pred_Q = pd.DataFrame()
for pred in rf.estimators_:
    temp = pd.Series(pred.predict(X_test))#.round(2))
    pred_Q = pd.concat([pred_Q,temp],axis=1)
    
pred_Q.columns = range(len(rf.estimators_))
pred_Q.columns.name = 'tree'
pred_Q.index.name = 'sample'
pred_Q.head()
quantiles = [0.01, 0.05, 0.50, 0.95 , 0.99]
RF_actual_pred = pd.DataFrame()

for q in quantiles:
    s = pred_Q.quantile(q=q, axis=1)
    RF_actual_pred = pd.concat([RF_actual_pred,s],axis=1,sort=False)

RF_actual_pred.columns = quantiles
RF_actual_pred['actual'] = Y_test
RF_actual_pred['interval'] = RF_actual_pred[np.max(quantiles)] - RF_actual_pred[np.min(quantiles)]
RF_actual_pred = RF_actual_pred.sort_values('interval')
RF_actual_pred = RF_actual_pred#.round(2)

# Get the R-squared
r2 = sklearn.metrics.r2_score(RF_actual_pred['actual'], RF_actual_pred[0.5])#.round(2)
print('R2 score is {}'.format(r2) )  # 0.81

# Get the correct percentage
correctPcnt(RF_actual_pred) # 0.9509

# Show the intervals
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 5))
ax = showIntervals(ax, RF_actual_pred) 
ax.set_ylabel(var_Y)
ax.set_xlabel('sample');
ax.set_title('Samples sorted by confidence interval');<jupyter_output>R2 score is 0.1532692863651801
0.7804054054054054
{'fraction_non_alphanumeric': 0.13530797711208348, 'fraction_numerical': 0.04207337596768765, 'mean_word_length': 5.270042194092827, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '543487', 'license': 'Apache-2.0', 'n_tokens_mistral': 1240, 'n_tokens_neox': 1175, 'n_words': 221, 'path': 'nbs/QRF_toy_models.ipynb', 'repo_name': 'qAp/earthshotsoil', 'size': '2971'}