<jupyter_start><jupyter_text>pkg imports & configuration<jupyter_code># basix
import numpy as np
import pandas as pd
import seaborn as sns
import math
import random
import time

# prep
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV

#modeling
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans

# other
from sklearn.metrics import classification_report
from sklearn import tree
from sklearn.cluster import MiniBatchKMeans

%matplotlib inline   
import matplotlib.pyplot as plt
pd.options.display.float_format = "{:,.4f}".format
RANDOM_SEED = 8675309
responseVar = "label"
# setup for consolidated table, later
table = {"rfc":{"r2-train":None},
         "rfc_with_pca":{"r2-train":None},
         "adj_rfc_with_pca":{"r2-train":None},
        }<jupyter_output><empty_output><jupyter_text>BRING IN THE DATA<jupyter_code>mnist_train = pd.read_csv('train.csv')
mnist_test = pd.read_csv('test.csv')
mnist = pd.concat([mnist_test, mnist_train.drop(columns = "label")]) # combined dataset<jupyter_output><empty_output><jupyter_text>eda<jupyter_code>sum(mnist_train.isna().sum()) # checks for missing data
pd.DataFrame(mnist_train["label"].describe(include='all')) # AKA mnist_train.label.describe()
print("most common label (mode) :", mnist_train["label"].mode()[0])<jupyter_output>most common label (mode) : 1<jupyter_text>splitting train data and folding, model set-up<jupyter_code>X = mnist_train.loc[:, mnist_train.columns != responseVar]
y = mnist_train[responseVar]
# splitting og data once for simplicity
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    train_size=0.7,
                                                    test_size=0.3,
                                                    shuffle=True,
                                                    random_state=RANDOM_SEED)

folds = KFold(n_splits = 3, shuffle = True, random_state = RANDOM_SEED)


param_grid = { 
    'n_estimators': [100, 200, 500], # nbr of trees in forest
    'max_depth' : [3, 4, 5],
}

model = RandomForestClassifier(bootstrap=True, max_features = 'sqrt', n_estimators = 10)<jupyter_output><empty_output><jupyter_text>model 1 : intial, raw random forest classifier<jupyter_code>print("approximate start time : ", time.asctime())
start1 = time.perf_counter() # CPU time or real time in seconds since process start or since prev call to perf_counter(). v precise.
rfc = GridSearchCV(estimator = model, 
                   param_grid = param_grid, 
                   scoring= 'r2', 
                   cv = folds, 
                   verbose = 1,
                   return_train_score=True) 

model_fit_rfc = rfc.fit(X_train, y_train.values.ravel())  
stop1 = time.perf_counter()
tm1 = round(stop1 - start1)
print("approximate completion time, initial rfc : ", time.asctime())
print("time of execution in seconds : ", tm1)
print("time of execution in minutes : ", round(tm1/60, 1))
table["rfc"]["r2-train"] = rfc.score(X,y)
print("model used : ", rfc.best_estimator_)
#model_fit_rfc.best_estimator_
mnist_test_pred = rfc.predict(mnist_test)
rfc_preds = pd.DataFrame({'ImageId':range(1, 28001), 'Label':mnist_test_pred})
rfc_preds.to_csv("rfc_preds.csv", index = False)
rfc_preds.head() # test predictions<jupyter_output><empty_output><jupyter_text>PCA on the combined train/test sets<jupyter_code>print("approximate start time : ", time.asctime())
start2 = time.perf_counter()
scaled_mnist = mnist.apply(lambda x: (x - min(x))/(max(x) - min(x)), axis=1)
scaled_mnist.describe()[["pixel774", "pixel775", "pixel4"]] # response and random explanatory summary stats
pca = PCA(n_components = 0.95) # goal : to explain 95% explanatory variable variance
X_train_pca = pca.fit(mnist)
X_test_pca = pca.transform(mnist)
pca
pca.score(X_train)
stop2 = time.perf_counter()
tm2 = round(stop2 - start2)
print("approximate completion time : ", time.asctime())
print("time of execution in seconds : ", tm2)
print("time of execution in minutes : ", round(tm2/60, 1))
print("optimized component count : ", pca.n_components_)<jupyter_output>optimized component count :  154<jupyter_text>model 2 : rf classifier using identified PCA's derived from entire dataset<jupyter_code>print("approximate start time : ", time.asctime())
start3 = time.perf_counter()
rfc_with_pca = GridSearchCV(estimator = model, 
                        param_grid = param_grid, 
                        scoring= 'r2', 
                        cv = folds, 
                        verbose = 1,
                        return_train_score=True) 

#model_fit_with_pca = rfc_with_pca.fit(X_train_pca, y_train.values.ravel())  
X_train_pca.
y_train
help(rfc_with_pca.fit)
stop3 = time.perf_counter()
tm3 = round(stop3 - start3)
print("approximate completion time : ", time.asctime())
print("time of execution in seconds : ", tm3)
print("time of execution in minutes : ", round(tm3/60, 1))
table["rfc_with_pca"]["r2-train"] = rfc_with_pca.score(X_pca, y)
print("optimized model used : ", rfc_with_pca.best_estimator_)
#print("optimized model used : ", model_fit_with_pca.best_estimator_)
mnist_test_pred_with_pca = rfc_with_pca.predict(mnist_test)
rfc_with_pca_preds = pd.DataFrame({'ImageId':range(1,28001),'Label': mnist_test_pred_with_pca})
rfc_with_pcs_preds.to_csv("rfc_preds.csv", index = False)
rfc_with_pca_preds.head() # test predictions<jupyter_output><empty_output><jupyter_text>PCA on train set only<jupyter_code>print("approximate start time : ", time.asctime())
start4 = time.perf_counter()
pca_train = PCA(n_components = 0.95) # goal : to explain 95% explanatory variable variance
X_train_adj = pca_train.fit(mnist_train)
X_test_adj = pca_train.transform(mnist_train)
stop4 = time.perf_counter()
tm4 = round(stop4 - start4)
print("approximate completion time : ", time.asctime())
print("time of execution in seconds : ", tm4)
print("time of execution in minutes : ", round(tm4/60, 1))
print("optimized component count : ", pca_train.n_components_)<jupyter_output><empty_output><jupyter_text>model 3 : rf classifier using adjusted PCA : without any response variable data<jupyter_code>print("approximate start time : ", time.asctime())
start5 = time.perf_counter()
adj_rfc_with_pca = GridSearchCV(estimator = model, 
                        param_grid = param_grid, 
                        scoring= 'r2', 
                        cv = folds, 
                        verbose = 1,
                        return_train_score=True) 

model_fit_adj = adj_rfc_with_pca.fit(X_train_adj, y_train.values.ravel())  
stop5 = time.perf_counter()
tm5 = round(stop5 - start5)
print("approximate completion time : ", time.asctime())
print("time of execution in seconds : ", tm5)
print("time of execution in minutes : ", round(tm5/60, 1))
table["adj_rfc_with_pca"]["r2-train"] = adj_rfc_with_pca.score(X_train_adj, y_train) #?
print("model used : ", adj_rfc_with_pca.best_estimator_)
mnist_test_pred_adj = adj_rfc_with_pca.predict(mnist_test)
adj_rfc_preds = pd.DataFrame({'ImageId':range(1, 28001), 'Label':mnist_test_pred_adj})
adj_rfc_preds.to_csv("rfc_preds.csv", index = False)
adj_rfc_preds.head() # test predictions
table<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11768726079825041, 'fraction_numerical': 0.018179332968835428, 'mean_word_length': 4.171024734982332, 'pattern_counts': {'":': 6, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '185555', 'license': 'MIT', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2483, 'n_words': 520, 'path': 'Fall2020/MSDS422/mnist_data/asn5.ipynb', 'repo_name': 'kbridge14/regression_things', 'size': '7316'}