<jupyter_start><jupyter_text>Genentech Cervical Cancer - Feature Selectionhttps://www.kaggle.com/c/cervical-cancer-screening/<jupyter_code># imports
import sys # for stderr
import numpy as np
import pandas as pd
import sklearn as skl
from sklearn import metrics
import matplotlib.pyplot as plt
%matplotlib inline
# settings 
%logstop
%logstart  -o 'cc_feature_selection.log' rotate
plt.style.use('ggplot')
# constants
# plt.rcParams['figure.figsize'] = (10.0, 10.0)
# pd.set_option('display.max_rows', 50)
# pd.set_option('display.max_columns', 50)
# versions 
import sys
print(pd.datetime.now())
print('Python: '+sys.version)
print('numpy: '+np.__version__)
print('pandas: '+pd.__version__)
print('sklearn: '+skl.__version__)
from sqlalchemy import create_engine
engine = create_engine('postgresql://paulperry:@localhost:5432/ccancer')
from pyace import ace<jupyter_output><empty_output><jupyter_text>Load<jupyter_code>fdir = './features/'
train_file = './input/patients_train.csv.gz'
train = pd.read_csv(train_file)
train.drop('patient_gender', axis=1, inplace=True)
train.set_index('patient_id', inplace=True)
train[:3]
files =[
'diag_code_0_1000.csv.gz',
'diag_code_1000_2000.csv.gz',
'diag_code_2000_3000.csv.gz',
'diag_code_3000_4000.csv.gz',
'diag_code_4000_5000.csv.gz',
'diag_code_5000_6000.csv.gz',
'diag_code_6000_7000.csv.gz',
'diag_code_7000_8000.csv.gz',
'diag_code_8000_9000.csv.gz',
'diag_code_9000_10000.csv.gz',
'diag_code_10000_11000.csv.gz',
'diag_code_11000_12000.csv.gz',
'diag_code_12000_13000.csv.gz',
'diag_code_13000_14000.csv.gz',
'diag_code_14000_15000.csv.gz',
'diag_code_15000_16000.csv.gz',
]
len(files)
# # 
# tab = pd.read_csv(fdir+files[0])
# tab.set_index('patient_id', inplace=True)

# for f in files[1:]:
#     tab2 = pd.read_csv(fdir+f)
#     tab2.set_index('patient_id', inplace=True)
#     tab = tab.merge(tab2, left_index=True, right_index=True, how='left')
#     gc.collect()
#     print(f)<jupyter_output><empty_output><jupyter_text>Run<jupyter_code>nnn = 4
import datetime
start = datetime.datetime.now()
print(start)
tab = pd.read_csv(fdir+files[nnn])
tab.set_index('patient_id', inplace=True)
tab.shape
dfall = pd.merge(train, tab, left_index=True, right_index=True, how='left')
cat_cols = ['patient_age_group','patient_state','ethinicity','household_income','education_level']
ranks = ace(dfall, 'is_screener', cat_cols=[])
df_ranks = pd.DataFrame(ranks, index=dfall.columns, columns=['ace','mean'])
df_ranks = df_ranks.sort_values(by='ace', ascending=False)
top_ranks = df_ranks[df_ranks.ace > 0]
top_ranks[:20]
top_ranks.to_csv('diagnosis_ranks_'+str(nnn)+'.csv')
import gc
gc.collect()
end   = datetime.datetime.now()
print('run time: '+str(end-start)+' at: '+str(end))
break
top_ranks = pd.read_csv('diagnosis_ranks_15.csv')
top_ranks.set_index('Unnamed: 0', inplace=True)
top_ranks[:10]
qlist = list(top_ranks[top_ranks.ace > 0.003891].index)
for c in cat_cols: 
    qlist.remove(c)
qlist_str = "('"+qlist[0]+"'"
for c in qlist[1:]: 
    qlist_str=qlist_str+",'"+c+"'"
qlist_str=qlist_str+')'
qlist_str
q = 'select * from diagnosis_code where diagnosis_code in '+qlist_str

diag_codes = pd.read_sql_query(q, engine)
diag_codes
diag_codes.to_csv('diagnosis_top_codes.csv', mode='a', header=False)
qlist
tab[qlist].to_csv('diagnosis_top_'+str(nnn)+'.csv')<jupyter_output><empty_output><jupyter_text>Merge feature values<jupyter_code>diag_top_features = [
    'diagnosis_top_0.csv',
    'diagnosis_top_3.csv',
    'diagnosis_top_4.csv',
    'diagnosis_top_6.csv',
    'diagnosis_top_10.csv',
    'diagnosis_top_12.csv',
    'diagnosis_top_15.csv'
    ]
dff = pd.read_csv(diag_top_features[0])
dff.set_index('patient_id', inplace=True)
print(dff.shape)

for f in diag_top_features[1:]:
    df2 = pd.read_csv(f)
    df2.set_index('patient_id', inplace=True)
    dff = dff.merge(df2, left_index=True, right_index=True, how='outer')
    gc.collect()
    print(f)
gc.collect()
dff.shape
dff[:5]
dff.columns
big_table = pd.read_csv(fdir+'train_big_table.csv.gz')
big_table.set_index('patient_id', inplace=True)
big_table.shape
big_table[:2]
dff[:2]
dff.columns
big_table.columns
bad_cols = ['CLINIC', 'INPATIENT', 'OTHER', 'OUTPATIENT', 'UNKNOWN', 
            '0001', '0002', '0003', '0004', '0005', '0006', 
            'HX01', 'HX02', 'HX03', 'HX04', 'HX05', 'HXPR',
            'pract_screen_pct', 'cbsa_pct', 'age_pct', 'state_pct',
            '632','650', u'57452', u'57454', u'57455', u'57456',
           u'81252', u'90696', u'G0143', u'S4020', u'S4023']
# take only a subset of the features, the rest I think is junk

cols = list(big_table.columns)
cols = [x for x in cols if x not in bad_cols]

test_cols = list(cols)
test_cols.remove('is_screener')
bigt = big_table[cols].merge(dff, left_index=True, right_index=True, how='left')
bigt.columns
bigt.shape
bigt.to_csv(fdir+'train_big_table.csv')
big_table_encoded = pd.read_csv(fdir+'train_big_table_encoded.csv.gz')
big_table_encoded.set_index('patient_id', inplace=True)
big_table_encoded.shape
big_table_encoded[:2]
dffd = dff.copy()
dffd[dffd > 0] = 1
dffd[:10]
bigte = big_table_encoded[cols].merge(dffd,left_index=True, right_index=True, how='left')
bigte.shape
bigte[:10]
bigte.to_csv(fdir+'train_big_table_encoded.csv')<jupyter_output><empty_output><jupyter_text>Procedures<jupyter_code>procs = pd.read_csv(fdir+'procedure/procedure_counts_selected.csv.gz')
procs.shape
procs.set_index('patient_id', inplace=True)
procs[:2]
print(bigt.shape)
bigtp = bigt.merge(procs, left_index=True, right_index=True, how='left')
bigtp.shape
bigtp.to_csv(fdir+'train_big_table.csv')
print(bigte.shape)
bigtep = bigte.merge(procs, left_index=True, right_index=True, how='left')
bigtep.shape
bigtep.to_csv(fdir+'train_big_table_encoded.csv')<jupyter_output><empty_output><jupyter_text>test_top<jupyter_code>test_diagnosis_top = pd.read_csv('test_diagnosis_top.csv')
test_diagnosis_top.shape
#test_diagnosis_top.set_index('patient_id', inplace=True)
test_diagnosis_top[:5]
test_pivot = test_diagnosis_top.pivot(index='patient_id', columns='diagnosis_code', values='diagnosis_code_count')
test_pivot[:5]
test_pivot.shape
test_big_table = pd.read_csv(fdir+'test_big_table.csv.gz')
test_big_table.set_index('patient_id', inplace=True)
test_big_table = test_big_table[test_cols]
test_big_table.shape
test_big_table[:4]
test_big_table.info()
test_bigt = test_big_table.merge(test_pivot, left_index=True, right_index=True, how='left')
test_bigt.shape
test_bigt[:5]
test_bigt.to_csv(fdir+'test_big_table.csv')
test_big_table_encoded = pd.read_csv(fdir+'test_big_table_encoded.csv.gz')
test_big_table_encoded.set_index('patient_id', inplace=True)
test_big_table_encoded = test_big_table_encoded[test_cols]
test_big_table_encoded.shape
test_big_table_encoded.info()
test_big_table_encoded[:2]
test_dffd = test_pivot.copy()
test_dffd[test_dffd > 0] = 1
test_dffd[:10]
test_bigte = test_big_table_encoded.merge(test_dffd,left_index=True, right_index=True, how='left')
test_bigte.shape
test_bigte[:10]
test_bigte.to_csv(fdir+'test_big_table_encoded.csv')
dff.shape
test_pivot.shape
sorted_cols = [u'401.9', u'462', u'496', u'585.3', u'616.0', u'616.10', u'620.2',
       u'622.10', u'622.11', u'623.5', u'625.3', u'625.9', u'626.0', u'626.2',
       u'626.4', u'626.8', u'646.83', u'648.93', u'650', u'795.00',
       u'V22.0', u'V22.1', u'V22.2', u'V24.2', u'V25.2', u'V27.0', u'V28.3',
       u'V70.0', u'V74.5']
dff[sorted_cols].to_csv('train_diagnosis_top.csv')
test_pivot.to_csv('test_diagnosis_top.csv')
test_big_table.shape, big_table.shape
test_bigt.columns
len(test_bigt.columns)
len(big_table.columns)
len(bigte.columns)
test_bigt.drop('632_y', axis=1, inplace=True)
test_bigt.rename(columns={'632_x':'632'}, inplace=True)
test_bigte.drop('632_y', axis=1, inplace=True)
test_bigte.rename(columns={'632_x':'632'}, inplace=True)<jupyter_output><empty_output><jupyter_text>Check results<jupyter_code>train_diagnosis_top = fdir+'train_diagnosis_top.csv.gz'
train_diagnosis_top = pd.read_csv(train_diagnosis_top)
train_diagnosis_top.set_index('patient_id', inplace=True)
train_diagnosis_top[:3]
train_diagnosis_top.shape
test_diagnosis_top = fdir+'test_diagnosis_top.csv.gz'
test_diagnosis_top = pd.read_csv(test_diagnosis_top)
test_diagnosis_top.set_index('patient_id', inplace=True)
test_diagnosis_top[:3]
test_diagnosis_top.shape
set(test_diagnosis_top.columns) - set(train_diagnosis_top.columns)
pd.read_csv(fdir+'diagnosis_top_codes.csv')
test_diagnosis_top.columns
train_632 = pd.read_csv(fdir+'train_big_table.csv.gz')
train_632.set_index('patient_id', inplace=True)
train_632['632'][:5]
train_diagnosis_top['632'] = train_632['632']
train_diagnosis_top.sort_index(axis=1, inplace=True)
train_diagnosis_top.columns
train_diagnosis_top.to_csv(fdir+'train_diagnosis_top.csv')
train_632.columns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.157022918084865, 'fraction_numerical': 0.05150896301338779, 'mean_word_length': 7.693293885601578, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '2675', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3966, 'n_tokens_neox': 3678, 'n_words': 486, 'path': 'cervical-cancer-screening/data/cc_feature_selection.ipynb', 'repo_name': 'Ravirajadrangi/projects6', 'size': '8814'}