<jupyter_start><jupyter_text>Functions for cleaning and ML<jupyter_code>#Clean the df and encode numeric values
def cleanDF (df):
    r1 = re.compile('.*reporting')
    r2 = re.compile('.*imputed')

    cols_to_drop1 = list(filter((r1.match), df.columns))
    cols_to_drop2 = list(filter((r2.match), df.columns))
    cols_to_drop3 = ['admit_NICU']
    cols_to_drop = cols_to_drop1 + cols_to_drop2 + cols_to_drop3

    cols_to_keep = [col for col in df.columns if col not in cols_to_drop]

    X_and_target = df[cols_to_keep + ['admit_NICU']].copy()

    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    catDF = X_and_target.select_dtypes(include=object).copy()
    numDF = X_and_target.select_dtypes(include=numerics).copy() #only numeric columns

    le = LabelEncoder()
    catDF = catDF.apply(le.fit_transform)

    concat_df = pd.concat([numDF,catDF],axis=1)
    return concat_df

def GLM_CV_Lasso (cleanDF,sample_per_year):
    
    dwnSmplDF = cleanDF.groupby('birth_year',group_keys = False).apply(lambda x: x.sample(sample_per_year))
    encoded_target = dwnSmplDF['admit_NICU']

    glm_lassoCV = linear_model.LogisticRegressionCV(#Cs = int(1e4),
                                                cv = 5,
                                                penalty = 'l1',
                                                solver = 'saga',
                                                n_jobs = -1,
                                                random_state = 108
                                               ).fit(cleanDF, encoded_target)
    print('sample size : %d\n' % (sample_per_year*5))
    %time glm_lassoCV.fit(cleanDF, encoded_target)
    print('\nscore    : {0}'.format(glm_lassoCV.score(cleanDF, encoded_target)))
    print('-'*50)
    return glm_lassoCV

def Forest_CV(X,Y):
    randomForest = ensemble.RandomForestClassifier()
    grid_para_forest = [{
        'n_estimators': np.linspace(50,int(np.sqrt(len(cl_df))),10,dtype=int),
        'min_samples_leaf' : range(100,1000,100)
    }]
    randomForest.set_params(random_state=108)
    gs_forest = GridSearchCV(randomForest, grid_para_forest, scoring='accuracy', 
                                      cv=5, n_jobs=-1)
    %time gs_forest.fit(X, Y)
    return gs_forest

def RF (X,Y,gs):
    bestRF = ensemble.RandomForestClassifier()
    best_params = gs.best_params_
    bestRF.set_params(random_state=108,n_jobs= -1,oob_score = True,**best_params)
    bestRF.fit(X,Y)
    return bestRF

def pred_resultsRF(fullDF, gsf, rf):
    X_bal = fullDF.drop('admit_NICU',axis=1)
    y_pred = rf.predict(X_bal)

    #Use sklearn's confusion_matrix on real and predicted y
    from sklearn.metrics import confusion_matrix
    cf = confusion_matrix(fullDF.admit_NICU, y_pred)
    print(cf/len(fullDF)*100)

    print(
        '''Random Forest
    best param : {0}
    best score : {1}
    r2         : {2}'''\
          .format(gsf.best_params_,\
                  gsf.best_score_, \
                  r2_score(fullDF.admit_NICU, y_pred))
         )
    return

def pred_resultsGLM(glm):
    print('sample size: {0} \nscore: {1}\nr2: {2}'
          .format(len(encoded_target), \
            glm.score(cl_df, encoded_target), \
            r2_score(encoded_target, glm.predict(cl_df))))
    return

def feature_select(X,rf):
    sampRF_coefs = pd.DataFrame({'col' :list(X.columns), 
                           'features': rf.feature_importances_})
    return sampRF_coefs.nlargest(20,'features')<jupyter_output><empty_output><jupyter_text>Load data and clean<jupyter_code>totDF = pd.read_csv('../data/raw/Cleaned_data_set.csv')
cl_df = cleanDF(totDF)<jupyter_output><empty_output><jupyter_text>GLM<jupyter_code>logit_1 = GLM_CV_Lasso(cl_df,20000)
pred_resultsGLM(logit_1)<jupyter_output><empty_output><jupyter_text>Create a col of Y and N<jupyter_code>nicu_allY = cl_df.loc[cl_df['admit_NICU']==2]
nicu_allN = cl_df.loc[cl_df['admit_NICU']==0]
nicu_YN = pd.concat([nicu_allY,nicu_allN],axis=0)<jupyter_output><empty_output><jupyter_text>RF on Balanced Sample per year<jupyter_code>sample_per_year = 10000
bal_dwnSmplY = nicu_allY.groupby('birth_year',group_keys = False).apply(lambda x: x.sample(sample_per_year))
bal_dwnSmplN = nicu_allN.groupby('birth_year',group_keys = False).apply(lambda x: x.sample(sample_per_year))
bal_dwnSmpl = pd.concat([bal_dwnSmplY,bal_dwnSmplN],axis=0)

bal_target = bal_dwnSmpl.admit_NICU #target
bal_X = bal_dwnSmpl.drop('admit_NICU',axis=1) #X<jupyter_output><empty_output><jupyter_text>Train RF with best params by training a Grid search for hyp param selection<jupyter_code>bal_gs = Forest_CV(bal_X,bal_target)
bal_rf = RF(bal_X,bal_target,bal_gs)
pred_resultsRF(nicu_YN,bal_gs,bal_rf)
feature_select(bal_X,bal_rf)<jupyter_output>[[808182  93293]
 [ 16978  69463]]
Random Forest
    best param : {'min_samples_leaf': 100, 'n_estimators': 894}
    best score : 0.84804
    r2         : -0.3980022248736994<jupyter_text>Save model for later use if needed<jupyter_code>bal_100k = '../data/processed/bal100K_model.sav'
pickle.dump(bal_gs, open(bal_100k, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1234592056349051, 'fraction_numerical': 0.029544120524359226, 'mean_word_length': 3.3580562659846547, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '233985', 'license': 'FTL', 'n_tokens_mistral': 2012, 'n_tokens_neox': 1881, 'n_words': 288, 'path': 'notebooks/Machine_Learning.ipynb', 'repo_name': 'ConnorHaas03/CDC_capstone', 'size': '5111'}