<jupyter_start><jupyter_text>It seems like linear regression may do a very good job at explaining variability of our target (since ideally we'd want an R-squared score of at least 80% but our R-squared scores are 100% for both train and test sets).<jupyter_code>from sklearn.linear_model import Ridge
from sklearn.model_selection import cross_val_score

pipeline = make_pipeline(
    ce.OneHotEncoder(use_cat_names=True), 
    SimpleImputer(strategy='mean'), 
    StandardScaler(), 
    SelectKBest(k=18), 
    Ridge(alpha=0.1)
)

k = 3
scoresLM = cross_val_score(pipeline, X_train, y_train, cv=k, 
                         scoring='neg_mean_absolute_error')
print(f'MAE for {k} folds:', -scoresLM)
-scoresLM.mean()
# I had trouble loading xgboost for some reason- I used the following command in terminal
# conda install -c conda-forge xgboost

from xgboost import XGBClassifier

pipeline101 = make_pipeline(
    ce.OrdinalEncoder(),
    XGBClassifier(n_estimators=100,
                  random_state=42,
                  n_jobs=-1)
)
pipeline101.fit(X_train, y_train)
from sklearn.metrics import accuracy_score

y_pred = pipeline101.predict(X_test)
print('Training Accuracy:', pipeline101.score(X_train, y_train))
print('Validation Accuracy:', pipeline101.score(X_val, y_val))
print('Test Accuracy', accuracy_score(y_test, y_pred))
k = 2
scoresLMp101 = cross_val_score(pipeline101, X_train, y_train, cv=k, 
                         scoring='neg_mean_absolute_error')
print(f'MAE for {k} folds:', -scoresLMp101)
print(f'Mean of cross-validation scores for {k} folds:',-scoresLMp101.mean())
!pip install eli5

import eli5
from eli5.sklearn import PermutationImportance

# Ignore warnings

transformers = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median')
)

X_train_transformed = transformers.fit_transform(X_train)
X_val_transformed = transformers.transform(X_val)

pipeline103 = make_pipeline(
    ce.OrdinalEncoder(),
    XGBClassifier(booster= 'gbtree',n_estimators=30, max_depth=5,
                  random_state=30,
                  n_jobs=-1)
)
pipeline103.fit(X_train_transformed, y_train)


feature_names = X_val.columns.tolist()

permuter = PermutationImportance(
    pipeline103,
    scoring='accuracy',
    n_iter=5,
    random_state=42
)

permuter.fit(X_val_transformed, y_val)

eli5.show_weights(
    permuter,
    top=None,
    feature_names=feature_names)
column  = 'BC_Vert'

# Fit without column
pipeline103.fit(X_train.drop(columns=column), y_train)
score_without = pipeline103.score(X_val.drop(columns=column), y_val.drop(columns=column))
print(f'Validation Accuracy without {column}: {score_without}')

# Fit with column
pipeline103.fit(X_train, y_train)
score_with = pipeline103.score(X_val, y_val)
print(f'Validation Accuracy with {column}: {score_with}')

# Compare the error with & without column
print(f'Drop-Column Importance for {column}: {score_with - score_without}')
from sklearn.metrics import plot_confusion_matrix
confusion_matrix = plot_confusion_matrix(pipeline103, X_test, y_test, values_format='.0f', xticks_rotation='vertical')

plt.show()
import matplotlib.pyplot as plt

plt.hist([nn_features['BC_Days'],nn_features['AB_Days']],range=[0,20])
plt.title('Comparison of BC and AB Days')
plt.xlabel('Days between Event Points')
plt.ylabel('Days')
plt.ylim(0,20)
plt.legend(["BC_Days","AB_Days"])

plt.show()
pairplot_nn_features = nn_features[['bs_1_or_sb_0','AB_Days','BC_Days','CD_Days','DE_Days']]
pairplot_nn_features.dtypes
pairplot_nn_features['bs_1_or_sb_0'] = pairplot_nn_features['bs_1_or_sb_0'].astype(str).astype(float)
pairplot_nn_features.dtypes
pairplot_nn_features
type(pairplot_nn_features)
# sns_plot=sns.pairplot(pairplot_nn_features)
# sns_plot.fig.suptitle('Comparing NN features', y=1.02)
# sns_plot.savefig("prettypairplots.png")
# plt.show()
# this isn't working for now so I'm going to enter things manually
from io import StringIO
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd

TESTDATA = StringIO("""bs_1_or_sb_0;AB_Days;BC_Days;CD_Days;DE_Days;
1.0;1;1;1;2;
0.0;2;1;1;1;
0.0;3;1;1;7;
0.0;2;1;3;2;
0.0;5;4;5;1;
1.0;1;1;7;1;
1.0;1;2;2;2;
1.0;2;1;1;1;
0.0;3;2;1;2;
0.0;14;2;1;1;
0.0;2;2;1;1;
1.0;20;1;1;3;
1.0;2;1;1;1;
1.0;1;1;3;3;
0.0;2;2;1;3;
1.0;1;1;1;1;
0.0;1;1;1;1;
1.0;2;1;4;6;
1.0;3;1;5;1;
1.0;2;1;4;3;
0.0;1;3;1;2;
    """)

freq_data = pd.read_csv(TESTDATA, sep=";")

# first try there was a weird column called "Unnamed: 4" so we're dropping it

freq_data = freq_data.iloc[:, :-1]

sns_plot = sns.pairplot(freq_data, hue="bs_1_or_sb_0", height=2.5)

sns_plot.fig.suptitle('Comparing NN features', y=1.02)
sns_plot.savefig("prettypairplots.png")

plt.show()
type(TESTDATA)
freq_data.dtypes
freq_data
type(freq_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13327745180217937, 'fraction_numerical': 0.04945515507124895, 'mean_word_length': 5.556318681318682, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '107295', 'license': 'MIT', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1865, 'n_words': 351, 'path': '4_of_5_Results_and_Conclusions/Neural_Net.ipynb', 'repo_name': 'bitcoin-candlestick-ML/Double-Bottome-Baby-Final-Prod', 'size': '4772'}