<jupyter_start><jupyter_text>Setting up connection<jupyter_code>params = json.load(open('psql_psycopg2.password'))

try:
    conn = psycopg2.connect(**params)
    conn.autocommit = True
    cur = conn.cursor()

except:
    print('Unable to connect to database')
    
import sys
sys.path.insert(0, '/home/lnolan/git_dir/babies')
from babysaver import features
from babysaver import models
from babysaver import evaluation<jupyter_output><empty_output><jupyter_text>Pulling in a sample of the vital statistics<jupyter_code># listing only features that are more or less present in 707G
vital_features = ['rf_diab_rec_c', 'rf_gest_rec_c', 'rf_phyp_rec_c', 'rf_ghyp_rec_c', 
            'rf_eclam_rec_c', 'rf_ppoutc_rec_c', 'rf_ppterm_rec_c', 'dplural_c', 
            'sexdis_rec_f', 'pay_rec_c', 'ip_gono_rec_c', 'ip_syph_rec_c',
            'ip_chlam_rec_c', 'ip_hepatb_rec_c', 'ip_hepatc_rec_c',
            'meduc_rec_n', 'cig_0_rec_n', 'cig_1_rec_n', 'cig_2_rec_n',
            'cig_3_rec_n', 'pay_rec_rec_c', 'pwgt_r_rec_n']
vital_outcome = "advb_otc"
vital_table = "vitals_all"<jupyter_output><empty_output><jupyter_text>Applying pipeline to vital statistics<jupyter_code>df = features.general_data_getter(vital_features, vital_outcome, vital_table, n_rows=100000, conn = conn, create_ref=True)
df['dataframe'].head()<jupyter_output><empty_output><jupyter_text>Proportion of adverse births in the sample<jupyter_code>print df['dataframe']['advb_otc'].mean()
print df['features']
help(models.machine_learner)<jupyter_output>Help on function machine_learner in module babysaver.models:

machine_learner(data_dct, clf_library, cv, make_evals=False, pkl_folder='pickles', verbose=False, eval_folder='evals', n_folds=5, stratified=True, bagging=False, test_size=0.3, increment=6, k=0.1, start_date=None, end_date=None, **kwargs)
    This function is customized for temporal cross-validation and 
    precision-at-k evaluation.
    
    data_dct - dictionary containing data, features, outcome from 
               features.data_getter() 
    clf_library - list of classifier dictionaries in following format
            {
                'clf': Classifier(),
                'param_dict': {
                    'param1': [list of parameter values],
                    'param2': [list of parameter values]
                } 
            }
    cv - specify type of cross-validation as string, current options are:
            'temporal_cv', 'train_test_split', 'kfold_cv'
    pkl_folder - folder to save pickle files 
    n_folds[...]<jupyter_text>Specifying classifiers and running the pipeline<jupyter_code>classifiers = {
    'clf': LogisticRegression,
    'param_dict': {
        'C': [0.001, 0.01, 1, 10],
        'penalty': ["l1", "l2"]
        }
    }

# only return the first item 
models_dict, pkl_dct = models.machine_learner(df, classifiers, "kfold_cv", k=[0.3])<jupyter_output>You have not specified a folder for the pickle files. By default, they will be saved in ./pickles/ - do you want to continue? (y/n)y
machine_learner: finished running models
machine_learner: pickle files available in pickles/
machine_learner: total runtime was 0:00:29.226414<jupyter_text>Running evaluation<jupyter_code>evaluation_df = evaluation.dict_to_dataframe(models_dict, pkl_dct)
pd.DataFrame(evaluation_df.sort('precision at 0.3 mean', ascending=False)['pickle_file'])
pd.DataFrame(evaluation_df.sort('precision at 0.3 mean', ascending=False))
df['features']<jupyter_output><empty_output><jupyter_text>Looking at coefficient importances<jupyter_code>from sklearn.externals import joblib

lr = joblib.load('pickles/LogisticRegression1.pkl')
pd.DataFrame({'features': df['features'], 'coef': lr.coef_[0]})
pd.DataFrame({'features': df['features'], 'coef': lr.coef_[0]}).sort('coef')<jupyter_output><empty_output><jupyter_text>SWITCHING GEARS A BIT: Correlation matrix for vital statistics  Just the variables that roughly correspond to 707<jupyter_code>vars707 = ['rf_diab_c_rec', 'rf_gest_c_rec', 'rf_phyp_c_rec', 'rf_ghyp_c_rec', 
            'rf_eclam_c_rec', 'rf_ppoutc_c_rec', 'rf_ppterm_c_rec', 'dplural_c',
            'sexdis_rec_f', 'meduc_rec_n', 'cig_0_rec_n', 'cig_1_rec_n', 
            'cig_2_rec_n', 'cig_3_rec_n', 'pay_rec_rec_c', 'advb_otc']
            
# initializing pandas data frame with the new_cols_list title - basically building an empty correlation matrix
ORs= pd.DataFrame(data=None, index=vars707, columns=vars707)
ORs.head()<jupyter_output><empty_output><jupyter_text>Load from postgres<jupyter_code>for_heatmap = pd.read_sql('SELECT rf_diab_c_rec, rf_gest_c_rec, rf_phyp_c_rec, rf_ghyp_c_rec,' + 
            'rf_eclam_c_rec, rf_ppoutc_c_rec, rf_ppterm_c_rec, dplural_c,' 
            'sexdis_rec_f, meduc_rec_n, cig_0_rec_n, cig_1_rec_n,' +
            'cig_2_rec_n, cig_3_rec_n, pay_rec_rec_c, pwgt_r_rec_n, advb_otc FROM forheatmap ORDER BY RANDOM() LIMIT 100000', conn)
# Checking types - some things will need to be recast
[(for_heatmap[i].dtype,i) for i in for_heatmap.columns.values]
# making sure that everything is a float

for_heatmap[['dplural_c', 'pay_rec_rec_c']] = for_heatmap[['dplural_c', 'pay_rec_rec_c']].astype(float)
# looking at all variables to see distribution
print "rf_diab_c_rec"
print for_heatmap['rf_diab_c_rec'].value_counts(dropna = False)
print "rf_gest_c_rec"
print for_heatmap['rf_gest_c_rec'].value_counts(dropna = False)
print "rf_gest_c_rec"
print for_heatmap['rf_gest_c_rec'].value_counts(dropna = False)
print "rf_phyp_c_rec"
print for_heatmap['rf_phyp_c_rec'].value_counts(dropna = False)
print "rf_ghyp_c_rec"
print for_heatmap['rf_ghyp_c_rec'].value_counts(dropna = False)
print "rf_eclam_c_rec"
print for_heatmap['rf_eclam_c_rec'].value_counts(dropna = False)
print "rf_ppoutc_c_rec"
print for_heatmap['rf_ppoutc_c_rec'].value_counts(dropna = False)
print "rf_ppterm_c_rec"
print for_heatmap['rf_ppterm_c_rec'].value_counts(dropna = False)
print "dplural_c" # NEEDS TO BE RECODED TO >1
print for_heatmap['dplural_c'].value_counts(dropna = False)
print "sexdis_rec_f"
print for_heatmap['sexdis_rec_f'].value_counts(dropna = False)
print "meduc_rec_n" # NEEDS TO BE RECODED TO (probably) more than high school, etc.
print for_heatmap['meduc_rec_n'].value_counts(dropna = False)
print "cig_0_rec_n"
print for_heatmap['cig_0_rec_n'].value_counts(dropna = False)
# checking on how many missings there are for an often missing variables
for_heatmap['rf_diab_c_rec'].value_counts(dropna = False)
for_heatmap.head()
# dropping na's
for_heatmap_complete = for_heatmap.dropna()
for_heatmap_complete.head()
len(for_heatmap_complete) # quickly checking sample size<jupyter_output><empty_output><jupyter_text>Recoding whether there is more than 1 fetus<jupyter_code>def babynum(f):
    if f > 1:
        return 1
    else: 
        return 0

dplural_c_dum = [1 if int(x) > 1 else 0 for x in for_heatmap_complete['dplural_c']]
type(for_heatmap_complete['dplural_c'][0]) # checking type...<jupyter_output><empty_output><jupyter_text>Recoding mother's education to 0/1<jupyter_code>def meduc_rec(f):
    if f < 3:
        return 1
    else:
        return 0

meduc_rec_n_dum = [meduc_rec(x) for x in for_heatmap_complete['meduc_rec_n']]<jupyter_output><empty_output><jupyter_text>Recoding cigs<jupyter_code>def cigs(f):
    if f > 0:
        return 1
    else:
        return 0
    
cig_0_rec_n_dum = [cigs(x) for x in for_heatmap_complete['cig_0_rec_n']]
cig_1_rec_n_dum = [cigs(x) for x in for_heatmap_complete['cig_1_rec_n']]
cig_2_rec_n_dum = [cigs(x) for x in for_heatmap_complete['cig_2_rec_n']]
cig_3_rec_n_dum = [cigs(x) for x in for_heatmap_complete['cig_3_rec_n']]<jupyter_output><empty_output><jupyter_text>Recoding payment method - 1 is Medicaid<jupyter_code>def pay(f):
    if f == 1:
        return 1
    else: 
        return 0

pay_rec_rec_c_dum = [pay(x) for x in for_heatmap_complete['pay_rec_rec_c']]
type(cig_3_rec_n_dum) # the recoded vars are in list format
# Getting all vars, including recoded, into one data frame

test = pd.DataFrame({'cigs_before_preg': cig_0_rec_n_dum,
        'cigs_in_first_tri': cig_1_rec_n_dum,
        'cigs_in_second_tri': cig_2_rec_n_dum,
        'cigs_in_third_tri': cig_3_rec_n_dum,
        'medicaid_payer': pay_rec_rec_c_dum,
        'diabetes': for_heatmap_complete['rf_diab_c_rec'],
        'gestational_diab': for_heatmap_complete['rf_gest_c_rec'],
        'prepreg_hyp': for_heatmap_complete['rf_phyp_c_rec'],
        'gest_hyp': for_heatmap_complete['rf_ghyp_c_rec'],
        'ecclampsia': for_heatmap_complete['rf_eclam_c_rec'],
        'poor_prev_preg_out': for_heatmap_complete['rf_ppoutc_c_rec'],
        'previous_preterm': for_heatmap_complete['rf_ppterm_c_rec'],
        'multi_fetus': dplural_c_dum,
        'sexually_trans_dis': for_heatmap_complete['sexdis_rec_f'],
        'advb_otc': for_heatmap_complete['advb_otc']})

test.head()
# all pairs of columns
varsformap = ['cigs_before_preg', 'cigs_in_first_tri', 'cigs_in_second_tri', 'cigs_in_third_tri',
             'medicaid_payer', 'diabetes', 'gestational_diab', 'prepreg_hyp', 'gest_hyp', 'ecclampsia',
             'poor_prev_preg_out', 'previous_preterm', 'multi_fetus', 'sexually_trans_dis', 'advb_otc']
forints = list(itertools.combinations(varsformap, 2))
 
# creating the shell for the correlation matrix
phis = pd.DataFrame(data=None, index=varsformap, columns=varsformap)
phis.head()<jupyter_output><empty_output><jupyter_text>Defining a function to compute the phi correlation (a measure of association for two binary variables)<jupyter_code>import math
def phi(thingy):
    num = (thingy[1,1]*thingy[0,0]) - (thingy[1][0]*thingy[0][1])
    denom = math.sqrt(thingy[2,1]*thingy[2,0]*thingy[0,2]*thingy[1,2])
    return num/denom
# computing the correlation and appending it to the empty data frame

for comb in forints:
    mini = test[[comb[0], comb[1]]]
    thingy = pd.crosstab(test[comb[0]], test[comb[1]], margins = True)
    thingy = np.array(thingy)
    #print phi(thingy)
    phi_corr = phi(thingy)
    phis.loc[comb[1], comb[0]] = phi_corr

test = test.fillna(value=0)

print phis.head()
# another option is to compute the odds ratios from bivariate logististic regression models 
# for comb in forints:
    # print comb[0], comb[1]
    # logit = sm.Logit(test[comb[0]], test[comb[1]])
    # result = logit.fit()
    # OR = np.exp(result.params)[0] # extracting just odds ratio
    # ORs.loc[comb[1], comb[0]] = OR<jupyter_output><empty_output><jupyter_text>Plotting a heat map of the phi correlations<jupyter_code>sns.set(style="white")

# replace NAs with 0 - or else heat map will not run
phis = phis.fillna(value=0)

# Generate a mask for the upper triangle
mask = np.zeros_like(phis, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)

pic = sns.heatmap(phis, mask=mask, xticklabels=varsformap, yticklabels=varsformap)<jupyter_output><empty_output><jupyter_text>Rerunning pipeline just with women whose birth was paid for by Medicaid<jupyter_code># listing only features that are more or less present in 707G
vital_features = ['rf_diab_rec_c', 'rf_phyp_rec_c',
            'rf_ppoutc_rec_c', 'rf_ppterm_rec_c', 'dplural_c', 
            'sexdis_rec_f', 'ip_gono_rec_c', 'ip_syph_rec_c',
            'ip_chlam_rec_c', 'ip_hepatb_rec_c', 'ip_hepatc_rec_c',
            'meduc_rec_n', 'cig_0_rec_n', 'cig_1_rec_n', 'cig_2_rec_n',
            'cig_3_rec_n', 'pay_rec_rec_c', 'pwgt_r_rec_n']
vital_outcome = "advb_otc"
vital_table = "justMedicaid"
                  
                  # not fair to include things during preg: 'rf_gest_rec_c', 'rf_ghyp_rec_c', 'rf_eclam_rec_c'

df = features.general_data_getter(vital_features, vital_outcome, vital_table, n_rows=100000, conn = conn, create_ref=True)

classifiers = {
    'clf': LogisticRegression,
    'param_dict': {
        'C': [0.001, 0.01, 1, 10],
        'penalty': ["l1", "l2"]
        }
    }

# only return the first item 
models_dict, pkl_dct = models.machine_learner(df, classifiers, "kfold_cv", k=[0.1])


evaluation_df = evaluation.dict_to_dataframe(models_dict, pkl_dct)
pd.DataFrame(evaluation_df.sort('precision at 0.1 mean', ascending=False)['pickle_file'])
pd.DataFrame(evaluation_df.sort('precision at 0.1 mean', ascending=False))
from sklearn.externals import joblib

lr = joblib.load('pickles/LogisticRegression1.pkl')
pd.DataFrame({'features': df['features'], 'coef': lr.coef_[0]}).sort('coef')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11826407238369765, 'fraction_numerical': 0.015133317319240933, 'mean_word_length': 5.143630103295623, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '455160', 'license': 'MIT', 'n_tokens_mistral': 4978, 'n_tokens_neox': 4730, 'n_words': 926, 'path': 'notebooks/vital_stats_example.ipynb', 'repo_name': 'dssg/babies-public', 'size': '12489'}