<jupyter_start><jupyter_text>data import + clean<jupyter_code>import pandas as pd
import numpy as np
# import random as rn
import sklearn
# from scipy import stats
# import math
import re

from sklearn.preprocessing import  LabelEncoder #OneHotEncoder
from sklearn import ensemble

from sklearn.model_selection import GridSearchCV,cross_val_score
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error,accuracy_score

# from sklearn.ensemble import RandomForestClassifier
from sklearn import linear_model
totDF = pd.read_csv('../../data/processed/Cleaned_Data_Set.csv')
sample_per_year = 2000

dwnSmplDF = totDF.groupby('birth_year',group_keys = False).apply(lambda x: x.sample(sample_per_year))
r1 = re.compile('.*reporting')
r2 = re.compile('.*imputed')

cols_to_drop1 = list(filter((r1.match), totDF.columns))
cols_to_drop2 = list(filter((r2.match), totDF.columns))
cols_to_drop3 = ['admit_NICU']
cols_to_drop = cols_to_drop1 + cols_to_drop2 + cols_to_drop3

cols_to_keep = [col for col in totDF.columns if col not in cols_to_drop]

X = dwnSmplDF[cols_to_keep].copy()
target = dwnSmplDF[['admit_NICU']].copy()
# X = totDF[cols_to_keep].copy()
# target = totDF[['admit_NICU']].copy()

numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
catDF = X.select_dtypes(include=object).copy()
numDF = X.select_dtypes(include=numerics).copy() #only numeric columns

le = LabelEncoder()
catDF = catDF.apply(le.fit_transform)
encoded_target = target.apply(le.fit_transform)

cl_df = pd.concat([numDF,catDF],axis=1)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>randomForest = ensemble.RandomForestClassifier()
grid_para_forest = [{
	'n_estimators': np.linspace(50,int(np.sqrt(len(cl_df))),10,dtype=int),
	'min_samples_split' : [100,10,2],
	'min_samples_leaf' : [100,10,1]
}]
randomForest.set_params(random_state=108)
grid_search_forest = GridSearchCV(randomForest, grid_para_forest, scoring='accuracy', cv=5, n_jobs=-1)
%time grid_search_forest.fit(cl_df, encoded_target) #put in the df here
print(
    '''Random Forest
sample size: {0}
best param : {1}
best score : {2}
r2         : {3}'''\
      .format(len(encoded_target), \
              grid_search_forest.best_params_,\
              grid_search_forest.best_score_, \
              r2_score(encoded_target, grid_search_forest.predict(cl_df)))
     )<jupyter_output>Random Forest
sample size: 10000
best param : {'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100}
best score : 0.9491
r2         : 1.0<jupyter_text>GLM<jupyter_code>logit_1 = linear_model.LogisticRegression()
logit_1.set_params(C=1e4, n_jobs=-1)
%time logit_1.fit(cl_df, encoded_target)
print('sample size: {0} \nscore: {1}\nr2: {2}'.format(len(encoded_target), \
                                                      logit_1.score(cl_df, encoded_target), \
                                                      r2_score(encoded_target, logit_1.predict(cl_df))))
print('sample size: {0} \nscore: {1}\nr2: {2}'.format(len(encoded_target), \
                                                      logit_1.score(cl_df, encoded_target), \
                                                      r2_score(encoded_target, logit_1.predict(cl_df))))
print('sample size: {0} \nscore: {1}\nr2: {2}'.format(len(encoded_target), \
                                                      logit_1.score(cl_df, encoded_target), \
                                                      r2_score(encoded_target, logit_1.predict(cl_df))))
print('sample size: {0} \nscore: {1}\nr2: {2}'.format(len(encoded_target), \
                                                      logit_1.score(cl_df, encoded_target), \
                                                      r2_score(encoded_target, logit_1.predict(cl_df))))
print('sample size: {0} \nscore: {1}\nr2: {2}'.format(len(encoded_target), \
                                                      logit_1.score(cl_df, encoded_target), \
                                                      r2_score(encoded_target, logit_1.predict(cl_df))))
#11min 54sec<jupyter_output>sample size: 1000000 
score: 0.940666
r2: 0.27437829660503155
{'fraction_non_alphanumeric': 0.12620423892100194, 'fraction_numerical': 0.03636801541425819, 'mean_word_length': 3.059628543499511, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '121801', 'license': 'FTL', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1382, 'n_words': 242, 'path': 'notebooks/richardkim/RK_GLM.ipynb', 'repo_name': 'richardthkim/CDC_capstone', 'size': '4152'}