<jupyter_start><jupyter_text>------ **Dementia Patients -- Analysis and Prediction** ***Author : <NAME>*** ****Date : August, 2019****  ***Result Plots*** - 0. Setup     - 0.1. Load libraries     - 0.2. Define paths - 1. Data Preparation       - 1.1. Read Data      - 1.2. Prepare data      - 1.3. Prepare target     - 1.4. Removing Unwanted Features     - 2. Data Analysis     - 2.1. Feature      - 2.2. Target      - 3. Data Preparation and Vector Transformation- 4. Analysis and Imputing Missing Values - 5. Feature Analysis     - 5.1. Correlation Matrix    - 5.2. Feature and target     - 5.3. Feature Selection Models     - 6.Machine Learning -Classification Model   0. Setup   0.1 Load libraries  Loading Libraries<jupyter_code>import sys
sys.path.insert(1, '../preprocessing/')
import numpy as np
import pickle
import scipy.stats as spstats
import matplotlib.pyplot as plt
import seaborn as sns
import pandas_profiling
from sklearn.datasets.base import Bunch
from data_transformation_cls import FeatureTransform
from ast import literal_eval
import plotly.figure_factory as ff
import plotly.offline as py

import pandas as pd
pd.set_option('display.max_columns', None)  
pd.set_option('display.max_rows', None)
pd.set_option('display.max_colwidth', -1)


%matplotlib inline<jupyter_output><empty_output><jupyter_text>0.2 Define paths<jupyter_code># data_path
data_path = '../../../datalcdem/data/optima/dementia_18July/class_fast_normal_slow/'
result_path = '../../../datalcdem/data/optima/dementia_18July/class_fast_normal_slow/results/'<jupyter_output><empty_output><jupyter_text>1. Data Preparation    1.1. Read Data<jupyter_code># Reading Data

#patients data
patient_df = pd.read_csv(data_path+'patients.csv')
print (patient_df.dtypes)
# change dataType if there is something
for col in patient_df.columns: 
    if 'Date' in col:
        patient_df[col] = pd.to_datetime(patient_df[col]) 
 
display(patient_df.head(5))
# From API Data- comorbidities and treatments

# Treatment data
treatmnt_df = pd.read_csv(data_path+'Treatments.csv')
display(treatmnt_df.head(5))
treatmnt_dict = dict(zip(treatmnt_df['name'], treatmnt_df['CUI_ID']))
print ('\n Unique Treatment data size: {}\n'.format(len(treatmnt_dict)))

# Patient treatment data
patient_treatmnt_df = pd.read_csv(data_path+'patient_Treatments.csv')
patient_treatmnt_df['drugName_cui'] = patient_treatmnt_df['drugName'].apply(lambda x: treatmnt_dict[x])
# patient_treatmn_list_df = patient_treatmnt_df.groupby(by='patient_id', as_index=False).agg(lambda x: x.tolist()) # equivalent to below\n",
patient_treatmn_list_df = patient_treatmnt_df.groupby(by='patient_id', as_index=False).agg(lambda x: x.tolist())
display(patient_treatmn_list_df.head(5))

# Patient comorbidities data list
patient_treatmn_list = patient_treatmnt_df['drugName_cui'].unique()
print ('\n Unique Treatment data size: {}, Any NULL Value: {}\n'.format(patient_treatmn_list.size, pd.isnull(patient_treatmn_list).any()))

# Comorbidities data
comorb_df = pd.read_csv(data_path+'comorbidities.csv')
display(comorb_df.head(5))
comorb_dict = dict(zip(comorb_df['name'], comorb_df['CUI_ID']))
print ('\n Unique Comorbidities data size: {}\n'.format(len(comorb_dict)))

# Patient comorbidities data
patient_combor_df = pd.read_csv(data_path+'patient_Comorbidities.csv')
patient_combor_df['disorderName_cui'] = patient_combor_df['disorderName'].apply(lambda x: comorb_dict[x])
patient_combor_list_df = patient_combor_df.groupby(by=['patient_id'], as_index=False).agg(lambda x: x.tolist())
display(patient_combor_list_df.head(5))

# Patient comorbidities data list
patient_combor_list = patient_combor_df['disorderName_cui'].unique()
print ('\n Unique Comorbidities data size: {}, Any NULL Value: {}\n'.format(patient_combor_list.size, pd.isnull(patient_combor_list).any()))
#Preparation Features from Raw data

# Patient Comorbidities data
patient_com_raw_df = pd.read_csv(data_path + 'optima_patients_comorbidities.csv').groupby(by=['patient_id', 'EPISODE_DATE'], as_index=False).agg(lambda x: x.tolist())[['patient_id', 'EPISODE_DATE', 'Comorbidity_cui']]
display(patient_com_raw_df.head(5))
patient_com_raw_df['EPISODE_DATE'] = pd.to_datetime(patient_com_raw_df['EPISODE_DATE'])


# Patient Treatment data
patient_treat_raw_df = pd.read_csv(data_path + 'optima_patients_treatments.csv').groupby(by=['patient_id', 'EPISODE_DATE'], as_index=False).agg(lambda x: x.tolist())[['patient_id', 'EPISODE_DATE', 'Medication_cui']]
display(patient_treat_raw_df.head(5))
patient_treat_raw_df['EPISODE_DATE'] = pd.to_datetime(patient_treat_raw_df['EPISODE_DATE'])

# Join Patient Treatment and Comorbidities data
patient_com_treat_raw_df = pd.merge(patient_com_raw_df, patient_treat_raw_df,on=['patient_id', 'EPISODE_DATE'], how='outer')
patient_com_treat_raw_df.sort_values(by=['patient_id', 'EPISODE_DATE'],axis=0, inplace=True, ascending=True)
patient_com_treat_raw_df.reset_index(drop=True, inplace=True)
patient_com_treat_raw_df.head(5)


#Saving data
patient_com_treat_raw_df.to_csv(data_path + 'patient_com_treat_episode_df.csv', index=False)

# Extracting selected features from Raw data
df_datarequest = pd.read_excel(data_path+'../../optima_excel/' +'Data_Request_Jan_2019_final.xlsx')
display(df_datarequest.head(1))
df_datarequest_features = df_datarequest[['GLOBAL_PATIENT_DB_ID', 'Age At Episode', 'EPISODE_DATE', 'CAMDEX SCORES: MINI MENTAL SCORE']]
df_datarequest_features.rename(columns={'GLOBAL_PATIENT_DB_ID':'patient_id', 'Age At Episode':'Age_At_Episode'}, inplace=True)
display(df_datarequest_features.head(5))

# Merging Join Patient Treatment, Comorbidities and selected features from raw data
patient_com_treat_raw_df['EPISODE_DATE'] = pd.to_datetime(patient_com_treat_raw_df['EPISODE_DATE'])
patient_com_treat_fea_raw_df = pd.merge(patient_com_treat_raw_df,df_datarequest_features,on=['patient_id', 'EPISODE_DATE'], how='left')
patient_com_treat_fea_raw_df.sort_values(by=['patient_id', 'EPISODE_DATE'],axis=0, inplace=True, ascending=True)
patient_com_treat_fea_raw_df.reset_index(inplace=True, drop=True)
display(patient_com_treat_fea_raw_df.head(5))

# Filling misssing MMSE value with patient group Average
patient_com_treat_fea_raw_df['CAMDEX SCORES: MINI MENTAL SCORE']\
                                        = patient_com_treat_fea_raw_df.groupby(by=['patient_id'])['CAMDEX SCORES: MINI MENTAL SCORE'].transform(lambda x: x.fillna(x.mean()))
display(patient_com_treat_fea_raw_df.head(5))

#  19<=Mild<=24 , 14<=Moderate<=18 , Severe<=13 
def change_minimentalscore_to_category(df):
    df.loc[df['CAMDEX SCORES: MINI MENTAL SCORE']>24,'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY'] = 'Normal'
    df.loc[(df['CAMDEX SCORES: MINI MENTAL SCORE']<=24) & (df['CAMDEX SCORES: MINI MENTAL SCORE']>=19),
           'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY'] = 'Mild'
    df.loc[(df['CAMDEX SCORES: MINI MENTAL SCORE']<=18) & (df['CAMDEX SCORES: MINI MENTAL SCORE']>=14),
           'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY'] = 'Moderate'
    df.loc[df['CAMDEX SCORES: MINI MENTAL SCORE']<=13,'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY'] = 'Severe'
    
    return df

patient_com_treat_fea_raw_df = change_minimentalscore_to_category(patient_com_treat_fea_raw_df)

# saving file
patient_com_treat_fea_raw_df.to_csv(data_path + 'patient_com_treat_fea_episode_raw_without_expand_df.csv', index=False)

# Set line number for treatment line
def setLineNumber(lst):
    lst_dict = {ide:0 for ide in lst}
    lineNumber_list = []
    
    for idx in lst:
        if idx in lst_dict:
           lst_dict[idx] = lst_dict[idx] + 1 
           lineNumber_list.append(lst_dict[idx])
            
    return lineNumber_list

patient_com_treat_fea_raw_df['lineNumber'] = setLineNumber(patient_com_treat_fea_raw_df['patient_id'].tolist())
display(patient_com_treat_fea_raw_df.head(5))

# Extend episode data into columns
def extend_episode_data(df):
    id_dict = {i:0 for i in df['patient_id'].tolist()}
    for x in df['patient_id'].tolist():
        if x in id_dict:
            id_dict[x]=id_dict[x]+1
    
    line_updated = [int(j) for i in id_dict.values() for j in range(1,i+1)]
    # print (line_updated[0:10])
    df.update(pd.Series(line_updated, name='lineNumber'),errors='ignore')
    print ('\n----------------After creating line-number for each patients------------------')
    display(df.head(20))
    
    # merging episodes based on id and creating new columns for each episode
    r = df['lineNumber'].max()
    print ('Max line:',r)
    l = [df[df['lineNumber']==i] for i in range(1, int(r+1))]
    print('Number of Dfs to merge: ',len(l))
    df_new = pd.DataFrame()
    tmp_id = []
    for i, df_l in enumerate(l):
        df_l = df_l[~df_l['patient_id'].isin(tmp_id)]
        for j, df_ll in enumerate(l[i+1:]):
            #df_l = df_l.merge(df_ll, on='id', how='left', suffix=(str(j), str(j+1))) #suffixe is not working
            #print (j)
            df_l = df_l.join(df_ll.set_index('patient_id'), on='patient_id', rsuffix='_'+str(j+1))
        tmp_id = tmp_id + df_l['patient_id'].tolist()
        #display(df_l)
        df_new = df_new.append(df_l, ignore_index=True, sort=False)
        return df_new
    


patient_com_treat_fea_raw_df['lineNumber'] = setLineNumber(patient_com_treat_fea_raw_df['patient_id'].tolist())
#patient_com_treat_fea_raw_df = patient_com_treat_fea_raw_df.drop_duplicates(subset=['patient_id', 'EPISODE_DATE'])  # reason for 130 patients
patient_com_treat_fea_raw_df = extend_episode_data(patient_com_treat_fea_raw_df)
display(patient_com_treat_fea_raw_df.head(2))


#Saving extended episode of each patients
patient_com_treat_fea_raw_df.to_csv(data_path + 'patient_com_treat_fea_episode_raw_df.csv', index=False)


# Take only 10 episodes of the patients
fltr_linnum = ['_'+str(i) for i in range(10, 27)]
print (fltr_linnum)
patient_com_treat_fea_raw_df.drop(columns=[col for col in patient_com_treat_fea_raw_df.columns for i in fltr_linnum if i in col], inplace=True)
patient_com_treat_fea_raw_df = patient_com_treat_fea_raw_df.drop([col for col in patient_com_treat_fea_raw_df.columns if 'lineNumber' in col], axis=1).reset_index(drop=True)
#Saving extended episode of each patients
patient_com_treat_fea_raw_df.to_csv(data_path + 'patient_com_treat_fea_episode_0_9_raw_df.csv', index=False)



patient_com_treat_fea_raw_df[[col for col in patient_com_treat_fea_raw_df.columns if 'EPISODE_DATE' in col]].head(10)<jupyter_output><empty_output><jupyter_text>1.2. Prepare data<jupyter_code># Merging Treatment and comorbidities dataframe from API to Patients
'''patient_df = patient_df.merge(patient_combor_list_df[['patient_id', 'disorderName_cui']], on='patient_id', how='left')
patient_df = patient_df.merge(patient_treatmn_list_df[['patient_id', 'drugName_cui']], on='patient_id', how='left')
display(patient_df.head(5))

# Converting all columns containing list data of Patients into separte columns
columns_list = ['disorderName_cui', 'drugName_cui']
classeslist = [patient_combor_list, patient_treatmn_list]
featuretransform = FeatureTransform(patient_df)
patient_df = [featuretransform.apply_feature_transform(column_name=col, func_type='multiLabel', classes=clss, data_type_func='ordinal', literal=False) for col, clss in zip(columns_list, classeslist)][-1]

patient_df.head(1)'''


# Merging selected features from raw data dataframe to Patients
patient_df = patient_df.merge(patient_com_treat_fea_raw_df, on='patient_id', how='left')
display(patient_df.head(5))


patient_df.shape
# Transform list data to individual columns
featuretransform = FeatureTransform(patient_df)
classes_list = [list(featuretransform.extract_value_frm_list(patient_df[col].dropna().tolist())) 
                for col in patient_df.columns if 'Comorbidity_cui' in col or 'Medication_cui' in col]
columns_list = [col for col in patient_df.columns if 'Comorbidity_cui' in col or 'Medication_cui' in col]

patient_df = [featuretransform.apply_feature_transform(column_name=col, func_type='multiLabel', classes=clss, data_type_func='ordinal', literal=False) 
 for col, clss in zip(columns_list, classes_list)][-1]

print (patient_df.shape)

#drop _TF_ columns
patient_df.drop(columns=[col for col in patient_df.columns if '_TF_' in col], inplace=True)
print (patient_df.shape)

# Fill Comorbidity_cui and Medication_cui NaN values with zeros
# patient_df[[col for col in patient_df.columns if 'Comorbidity_cui_TFV' in col or 'Medication_cui_TFV' in col]] = patient_df[[col for col in patient_df.columns if 'Comorbidity_cui_TFV' in col or 'Medication_cui_TFV' in col]].fillna(value=0.0, axis=1)
# print (patient_df.shape)

# Calculate maximum episodes for patients
patient_df['Max_EPISODE']  = np.argmax(np.asarray([((patient_df[col] - patient_df['EPISODE_DATE'])/np.timedelta64(1, 'D'))/365.25
     for col in patient_df.columns if 'EPISODE_DATE' in col]), axis=0)

# Calculate Final Episode Date-First Episode date difference
patient_df['EPISODE_Duration(Years)']  = np.nanmax(np.asarray([((patient_df[col] - patient_df['EPISODE_DATE'])/np.timedelta64(1, 'D'))/365.25
     for col in patient_df.columns if 'EPISODE_DATE' in col]), axis=0)


# Calculate last episode MMSE Category
x_all_mmmse_ep_array = np.asarray([patient_df[col] for col in patient_df.columns if 'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY' in col])

patient_df['Last_EPISODE_MMSE_Category']  = x_all_mmmse_ep_array[[r-1 for r in patient_df['Max_EPISODE']],[c for c in range(x_all_mmmse_ep_array.shape[1])]]

# saving complete prepocessed file
patient_df.to_csv(data_path+'patient_df_complete.csv', index=False)

patient_df_complete = patient_df
#x,y = zip(patient_df['EPISODE_Duration(Years)'], patient_df['Max_EPISODE'])
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt 
ax = sns.scatterplot(x="EPISODE_Duration(Years)", y="Max_EPISODE", data=patient_df[['EPISODE_Duration(Years)', 'Max_EPISODE']])
plt.savefig(result_path+'ep_years.png', dpi=120)<jupyter_output><empty_output><jupyter_text>1.3. Prepare target<jupyter_code>def calculate_class(patient_df, mmseInicialDiscretized, mmseFinalDiscretized, time1, time2, label):
    #return np.where((patient_df['mmseInicialDiscretized']==mmseInicialDiscretized) & (patient_df['mmseFinalDiscretized']==mmseFinalDiscretized) 
    #                & (patient_df['epDateFinal-epDateInicial(year)']>=time1) & (patient_df['epDateFinal-epDateInicial(year)']<time2), label, 9)  # largest value for class+1 = 3
    patient_df.loc[(patient_df['CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY']==mmseInicialDiscretized) 
                   & (patient_df['Last_EPISODE_MMSE_Category']==mmseFinalDiscretized) 
                   & (patient_df['EPISODE_Duration(Years)']>=time1) 
                   & (patient_df['EPISODE_Duration(Years)']<time2), 'patient_class_cat'] = label

# Intialize with 'Unknown'
patient_df['patient_class_cat'] = 'Unknown'

# normal, mild, moderate, severe
calculate_class(patient_df, 'Mild', 'Moderate', 0, 2, 'Fast')
calculate_class(patient_df, 'Mild', 'Moderate', 2, 4, 'Normal')
calculate_class(patient_df, 'Mild', 'Moderate', 4, max(patient_df['EPISODE_Duration(Years)'])+1, 'Slow')
calculate_class(patient_df, 'Moderate', 'Severe', 0, 3, 'Fast')
calculate_class(patient_df, 'Moderate', 'Severe', 3, 5, 'Normal')
calculate_class(patient_df, 'Moderate', 'Severe', 5, max(patient_df['EPISODE_Duration(Years)'])+1, 'Slow')
calculate_class(patient_df, 'Mild', 'Severe', 0, 5, 'Fast_MiS')
calculate_class(patient_df, 'Mild', 'Severe', 5, 9, 'Normal_MiS')
calculate_class(patient_df, 'Mild', 'Severe', 9, max(patient_df['EPISODE_Duration(Years)'])+1, 'Slow_MiS')

# drop columns Which has class
patient_df = patient_df[patient_df['patient_class_cat']!='Unknown']
print (patient_df.shape)
print (patient_df.columns.tolist())<jupyter_output>(130, 1233)
['patient_id', 'gender', 'dementia', 'smoker', 'alcohol', 'education', 'bmi', 'weight', 'ageAtFirstEpisode', 'ageAtLastEpisode', 'malignantComorbidity', 'systemicComorbidity', 'cardiovascularComorbidity', 'metabolicComorbidity', 'infectiousComorbidity', 'neoplasticComorbidity', 'respiratoryComorbidity', 'neurologicalComorbidity', 'inflammatoryComorbidity', 'apoe', 'epDateInicial', 'mmseInicial', 'mmseInicialDiscretized', 'epDateFinal', 'mmseFinal', 'mmseFinalDiscretized', 'epDateToSevereData', 'epDateToMildData', 'epDateToModerateData', 'EPISODE_DATE', 'Comorbidity_cui', 'Comorbidity_cui_TFV_C0002871', 'Comorbidity_cui_TFV_C0006267', 'Comorbidity_cui_TFV_C0002965', 'Comorbidity_cui_TFV_C0521542', 'Comorbidity_cui_TFV_C0024437', 'Comorbidity_cui_TFV_C0033860', 'Comorbidity_cui_TFV_C0014544', 'Comorbidity_cui_TFV_C0040145', 'Comorbidity_cui_TFV_C0035439', 'Comorbidity_cui_TFV_C0013806', 'Comorbidity_cui_TFV_C0010054', 'Comorbidity_cui_TFV_C0020473', 'Comorbidity_cui_TFV_C0010[...]<jupyter_text>1.4. Removing Unwanted Features<jupyter_code># Drop unwanted feature
selected_columns_list = ['gender', 'dementia', 'smoker', 'alcohol', 'education', 'bmi', 'weight', 'apoe', 
                    'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY']\
                   + [col for col in patient_df.columns if '_TFV_' in col or 
                      'Age_At_Episode' in col] + ['patient_class_cat']

print (selected_columns_list)
patient_df = patient_df[selected_columns_list]
patient_df.dropna(axis=1, how='all', inplace=True)  # droping columns having only NULL values
patient_df.to_csv(data_path+'patient_df.csv', index=False)
print (patient_df.shape)
print (patient_df.columns.tolist())<jupyter_output>['gender', 'dementia', 'smoker', 'alcohol', 'education', 'bmi', 'weight', 'apoe', 'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY', 'Comorbidity_cui_TFV_C0002871', 'Comorbidity_cui_TFV_C0006267', 'Comorbidity_cui_TFV_C0002965', 'Comorbidity_cui_TFV_C0521542', 'Comorbidity_cui_TFV_C0024437', 'Comorbidity_cui_TFV_C0033860', 'Comorbidity_cui_TFV_C0014544', 'Comorbidity_cui_TFV_C0040145', 'Comorbidity_cui_TFV_C0035439', 'Comorbidity_cui_TFV_C0013806', 'Comorbidity_cui_TFV_C0010054', 'Comorbidity_cui_TFV_C0020473', 'Comorbidity_cui_TFV_C0010055', 'Comorbidity_cui_TFV_C0442874', 'Comorbidity_cui_TFV_C0003504', 'Comorbidity_cui_TFV_C0041296', 'Comorbidity_cui_TFV_C0428796', 'Comorbidity_cui_TFV_C0007570', 'Comorbidity_cui_TFV_C0014038', 'Comorbidity_cui_TFV_C1384641', 'Comorbidity_cui_TFV_C0041466', 'Comorbidity_cui_TFV_C1260873', 'Comorbidity_cui_TFV_C0271650', 'Comorbidity_cui_TFV_C0034735', 'Comorbidity_cui_TFV_C0026267', 'Comorbidity_cui_TFV_C0152025', 'Comorbidity_cui_TFV_C0012634', 'Comorbid[...]<jupyter_text>2. Data Analysis  2.1. Feature<jupyter_code># For Selected columns only
no_analysis_cols_list = list(set([col for col in patient_df.columns if '_TFV_' in col or 
                      'Age_At_Episode' in col])-set(['Age_At_Episode'] ))

# Generate Profiling 
patient_df_wthot = patient_df.drop(columns=no_analysis_cols_list).reset_index(drop=True)


# Categorical data to Labels for Data Analysis
featuretransform_patient_df_wthot = FeatureTransform(patient_df_wthot)
order ={'mild_drinking':0,'extreme_drinking':1}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='alcohol', func_type='ordinaltolabel', order=order)
order = {'basic':0, 'medium':1, 'higher':2}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='education', func_type='ordinaltolabel', order=order)
order = {'Male':0, 'Female':1}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='gender', func_type='ordinaltolabel', order=order)
order = {'no_smoker':0, 'smoker':1}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='smoker', func_type='ordinaltolabel', order=order)
order = {'E2E3':0, 'E2E4':1, 'E3E3':2, 'E3E4':3, 'E4E4':4}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='apoe', func_type='ordinaltolabel', order=order)
order = {'Mild':0, 'Moderate':1}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY', func_type='ordinaltolabel', order=order)
order = {'Slow':0, 'Slow_MiS':1, 'Normal':2, 'Normal_MiS':3, 'Fast':4, 'Fast_MiS':5}
featuretransform_patient_df_wthot.apply_feature_transform(column_name='patient_class_cat', func_type='ordinaltolabel', order=order)

profile = patient_df_wthot.profile_report(title='Dementia Profiling Report', style={'full_width':True})
profile.to_file(output_file= result_path + "dementia_data_profiling_report_output.html")
# profile = pandas_profiling.ProfileReport(patient_df_wthot)
profile


# # Generate Profiling for whole data
# patient_df_wth = patient_df.reset_index(drop=True)
# profile_with = patient_df_wth.profile_report(title='Dementia Profiling Report', style={'full_width':True})
# profile_with.to_file(output_file= result_path + "dementia_data_profiling_report_output_all.html")

## complete data profile with selected columns 
selected_columns_list = ['gender', 'dementia', 'smoker', 'alcohol', 'education', 'bmi', 'weight', 'apoe', 
                    'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY']\
                   + [col for col in patient_df.columns if '_TFV_' in col or 
                      'Age_At_Episode' in col] + ['patient_class_cat']

print (selected_columns_list)
patient_df_complete = patient_df_complete[selected_columns_list]

no_analysis_cols_list = list(set([col for col in patient_df_complete.columns if '_TFV_' in col or 
                      'Age_At_Episode' in col])-set(['Age_At_Episode'] ))

# Generate Profiling
patient_df_wthot_com = patient_df_complete.drop(columns=no_analysis_cols_list).reset_index(drop=True)


# Categorical data to Labels for Data Analysis
featuretransform_patient_df_wthot_com = FeatureTransform(patient_df_wthot_com)
order ={'mild_drinking':0,'extreme_drinking':1}
featuretransform_patient_df_wthot_com.apply_feature_transform(column_name='alcohol', func_type='ordinaltolabel', order=order)
order = {'basic':0, 'medium':1, 'higher':2}
featuretransform_patient_df_wthot_com.apply_feature_transform(column_name='education', func_type='ordinaltolabel', order=order)
order = {'Male':0, 'Female':1}
featuretransform_patient_df_wthot_com.apply_feature_transform(column_name='gender', func_type='ordinaltolabel', order=order)
order = {'no_smoker':0, 'smoker':1}
featuretransform_patient_df_wthot_com.apply_feature_transform(column_name='smoker', func_type='ordinaltolabel', order=order)
order = {'E2E3':0, 'E2E4':1, 'E3E3':2, 'E3E4':3, 'E4E4':4}
featuretransform_patient_df_wthot_com.apply_feature_transform(column_name='apoe', func_type='ordinaltolabel', order=order)
# order = {'Mild':0, 'Moderate':1}
# featuretransform_patient_df_wthot.apply_feature_transform(column_name='CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY', func_type='ordinaltolabel', order=order)
# order = {'Slow':0, 'Slow_MiS':1, 'Normal':2, 'Normal_MiS':3, 'Fast':4, 'Fast_MiS':5}
# featuretransform_patient_df_wthot.apply_feature_transform(column_name='patient_class_cat', func_type='ordinaltolabel', order=order)

profile = patient_df_wthot_com.profile_report(title='Dementia Profiling Report', style={'full_width':True})
profile.to_file(output_file= result_path + "dementia_data_profiling_report_output_all_patients.html")
'''# Pairplot 
g = sns.pairplot(patient_df_wthot, hue="patient_class_cat", palette="husl")
g.savefig(result_path+"pair_plot_output.png")'''<jupyter_output><empty_output><jupyter_text>2.2. Target<jupyter_code>def plot_distribution(data_select, size_bin) :  
    # 2 datasets
    tmp1 = patient_df_wthot.dropna()[data_select]
    tmp2 = patient_df_wthot.dropna()[data_select]
    hist_data = [tmp1, tmp2]
    #  'Slow', 'Normal_MiS', 'Fast_MiS', 'Slow_MiS'
    group_labels = ['Normal','Fast']
    colors = ['#FFD700', '#7EC0EE']

    fig = ff.create_distplot(hist_data, group_labels, colors = colors, show_hist = True, bin_size = size_bin, curve_type='kde')
    
    fig['layout'].update(title = data_select)

    py.iplot(fig, filename = 'Density plot')

#plot_distribution('weight', 0)
# Using ggplot2 style visuals 
'''plt.style.use('ggplot') 

f, ax = plt.subplots(figsize=(11, 15))

ax.set_facecolor('#fafafa')
ax.set(xlim=(-.05, 200))
plt.ylabel('Variables')
plt.title("Overview Data Set")
ax = sns.boxplot(data = patient_df_wthot, 
  orient = 'h', 
  palette = 'Set2')

plt.savefig(result_path+'ggplot.png')'''<jupyter_output><empty_output><jupyter_text>3. Data Preparation and Vector Transformation<jupyter_code># set patient_df
featuretransform.set_data_frame(patient_df)

# Nominal Transfromation
columns_nominal = ['gender', 'dementia', 'smoker', 'alcohol', 'education', 'apoe', 'CAMDEX SCORES: MINI MENTAL SCORE_CATEGORY']


# featuretransform.apply_feature_transform(column_name='mmseFinalDiscretized', func_type='oneHot', data_type_func='nominal')
patient_df = [featuretransform.apply_feature_transform(column_name=col, func_type='oneHot', data_type_func='nominal') for col in columns_nominal][-1]

class_names_dict = {'Slow':0, 'Slow_MiS':1, 'Normal':2, 'Normal_MiS':3, 'Fast':4, 'Fast_MiS':5}
featuretransform.apply_feature_transform(column_name='patient_class_cat', func_type='ordinaltolabel', order=class_names_dict)

# drop Unwanted created columns
patient_df.drop(columns=columns_nominal + [col+'_TF_' for col in columns_nominal], inplace=True)

print (patient_df.shape, patient_df.columns)<jupyter_output>[array(['Female', 'Male'], dtype=object)]
[array([False,  True])]
boolean data
[array(['no_smoker', 'smoker'], dtype=object)]
[array(['extreme_drinking', 'mild_drinking'], dtype=object)]
[array(['basic', 'higher', 'medium'], dtype=object)]
[array(['E2E3', 'E2E4', 'E3E3', 'E3E4', 'E4E4'], dtype=object)]
[array(['Mild', 'Moderate'], dtype=object)]<jupyter_text>4. Analysis and Impute Missing Values<jupyter_code># Total Missing Values
percent_missing = patient_df.isnull().sum().sum() * 100 / (patient_df.shape[0]*patient_df.iloc[:,0:-2].shape[1])
print (percent_missing, patient_df.shape, patient_df.columns.tolist())

# Drop columns having more than 50% nan values

select_filled_columns = patient_df.isnull().sum()[patient_df.isnull().sum()<patient_df.shape[0]/2.0].index.tolist()
patient_df = patient_df[select_filled_columns].reset_index(drop=True)
patient_df.dropna(axis=1, thresh=patient_df.shape[0]/2.0, inplace=True)

print (patient_df.shape, patient_df.columns.tolist(), select_filled_columns)
print (set(select_filled_columns).intersection(set(patient_df.columns.tolist())))
print (set(patient_df.columns.tolist()).intersection(set(select_filled_columns)))
target_real = patient_df['patient_class_cat-Label'].values
target = patient_df['patient_class_cat-Label'].values
target_names = patient_df['patient_class_cat'].values

data = patient_df.iloc[:,0:-2].values    # drop stages columns for this particular case
class_names = ['Slow', 'Slow_MiS', 'Normal', 'Normal_MiS', 'Fast', 'Fast_MiS'] # Need to change, it is in list to have certain order for the class eg. Slow_MiS=0------------------------------
# class_names = ['mild', 'moderate', 'severe']
patient_data = {'data':data, 'target':target, 'target_names':target_names, 'target_real':target_real, 'class_names':class_names }
pickle.dump( patient_data, open(data_path + 'patient_data.pkl', "wb"))
d = pickle.load(open(data_path + 'patient_data.pkl', 'rb'))

patient_data =  Bunch(data=d['data'], target=d['target'], target_name=d['target_names'], target_real=d['target_real'], class_names=d['class_names'])

data_p = patient_data.data
target_p = patient_data.target
target_real_p = patient_data.target_real
target_names_p = patient_data.target_name
class_names_p = patient_data.class_names
display(data_p.shape, target_p.shape, target_real_p.shape, target_names_p.shape, data_p[0:5], target_p[0:5], target_real_p[0:5], target_names_p[0:5], class_names_p)
# Imputing missing values

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.ensemble import ExtraTreesRegressor

simple_imputer =  SimpleImputer(strategy='mean')  # Needs to modify
data_p_s = simple_imputer.fit_transform(data_p)
print (data_p.shape, np.count_nonzero(np.isnan(data_p)))

iterative_imputer =  IterativeImputer(random_state=0, estimator=ExtraTreesRegressor(n_estimators=10, random_state=0))
data_p_i = iterative_imputer.fit_transform(data_p)
print (data_p_i.shape, np.count_nonzero(np.isnan(data_p_i)))<jupyter_output><empty_output><jupyter_text>5. Feature Analysis  5.1. Correlation Matrix<jupyter_code>data_target = np.append(data_p_i, target_p.reshape(target_p.shape[0],1), axis=1)
print (data_target[:,-1])
fea_columns = patient_df.columns.tolist()[0:-2]
class_column = patient_df.columns.tolist()[-1]
total_columns = fea_columns[:]
total_columns.append(class_column)
df_corr = pd.DataFrame(data_target, columns=total_columns)
cor = df_corr.corr()
cor_target = abs(cor["patient_class_cat-Label"])

# Selecting highly correlated features
relevant_features = cor_target[cor_target > 0.17]
print(relevant_features.sort_values(ascending=False))
print("{}".format('-'*80))<jupyter_output><empty_output><jupyter_text>5.2. Feature and target<jupyter_code># Data Class Imbalance handling
from imblearn.over_sampling import SMOTE

smote = SMOTE(ratio={1:10}, k_neighbors=1)
data_p_i, target_p_i = smote.fit_sample(data_p_i, target_p)

smote = SMOTE(ratio={1:10}, k_neighbors=1)
data_p_s, target_p_s = smote.fit_sample(data_p_s, target_p)<jupyter_output><empty_output><jupyter_text>5.3. Feature Selection Models<jupyter_code># RFECV

import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification

X, y = data_p_i, target_p_i

# Create the RFE object and compute a cross-validated score.
svc = SVC(kernel="linear")
# The "accuracy" scoring is proportional to the number of correct
# classifications
rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(10),
              scoring='accuracy')
rfecv.fit(X, y)
print("Optimal number of features : %d" % rfecv.n_features_)

# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.savefig(result_path + 'RFECV_kerlinear', dpi=120, pad_inch=20)
plt.show()


# get important Feature List
important_feature_list = patient_df.columns[0:-2][rfecv.support_]
print (rfecv.support_[0:14])
print (patient_df.columns[0:14])
print (important_feature_list[0:14])

# patient_df after feature selection
data_target = np.append(data_p_s, target_p_s.reshape(target_p_s.shape[0],1), axis=1)
print (data_target[:,-1])
fea_columns = patient_df.columns.tolist()[0:-2]
class_column = patient_df.columns.tolist()[-1]
total_columns = fea_columns[:]
total_columns.append(class_column)
df_fea = pd.DataFrame(data_target, columns=total_columns)

patient_df_after_fea_selection = df_fea[df_fea.columns[0:-1][rfecv.support_]]
patient_df_after_fea_selection.head(10)<jupyter_output><empty_output><jupyter_text>6.Machine Learning -Classification Model<jupyter_code>## Cross Validation Random Forest

from sklearn.ensemble import RandomForestClassifier
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score, cross_validate, cross_val_predict
from sklearn.metrics import classification_report


X, y = data_p_s[:,rfecv.support_], target_p_s
print (X.shape, y.shape)
clf = RandomForestClassifier(n_estimators=100)
print (cross_validate(clf, X, y, scoring=['recall_macro', 'precision_macro', 'f1_macro', 'accuracy'], cv=5) )
y_pred = cross_val_predict(clf,X, y, cv=5 )
print(classification_report(y, y_pred, target_names=class_names_p))
## Cross Validation SVC
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score, cross_validate, cross_val_predict
from sklearn.metrics import classification_report


X, y = data_p_s[:,rfecv.support_], target_p_s
clf = svm.SVC(gamma='scale', random_state=0)
print (cross_validate(clf, X, y, scoring=['recall_macro', 'precision_macro', 'f1_macro', 'accuracy',], cv=5) )
y_pred = cross_val_predict(clf,X, y, cv=5 )
print(classification_report(y, y_pred, target_names=class_names_p))
# Grid Search
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
#from fastai.imports import *
#from fastai.structured import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.tree import export_graphviz
import numpy as np 
import pandas as pd

%load_ext autoreload
%autoreload 2
%matplotlib inline 
pd.options.mode.chained_assignment = None


data_p_grid = data_p_s[:,rfecv.support_]

rf_param_grid = {
                 'max_depth' : [4, 6, 8,10],
                 'n_estimators': range(1,100),
                 'max_features': ['sqrt', 'auto', 'log2'],
                 'min_samples_split': [2, 3, 10,20],
                 'min_samples_leaf': [1, 3, 10,18],
                 'bootstrap': [True, False],
                 
                 }
m = RandomForestClassifier()
m_r = RandomizedSearchCV(param_distributions=rf_param_grid, 
                                    estimator = m, scoring = "f1_macro", 
                                    verbose = 0, n_iter = 100, cv = 5)

m_r.fit(data_p_grid, target_p_s)

#m_r.fit(data_p, patient_df['patient_class_cat-Label'])

rf_bp = m_r.best_params_
print (rf_bp)

rf_classifier=RandomForestClassifier(n_estimators=rf_bp["n_estimators"],
                                     min_samples_split=rf_bp['min_samples_split'],
                                     min_samples_leaf=rf_bp['min_samples_leaf'],
                                     max_features=rf_bp['max_features'],
                                     max_depth=rf_bp['max_depth'],
                                     bootstrap=rf_bp['bootstrap'])

rf_classifier.fit(data_p_grid, target_p_s)


importances = rf_classifier.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf_classifier.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

#print (indices)
# Print the feature ranking
print("Feature ranking:")
fea_list = important_feature_list # taken from RFECV
for f in range(15):
    print("%d. feature: %s (%f)" % (f + 1, fea_list[indices[f]], importances[indices[f]]))

# plot randomForest
estimator_id = 1  #need to change
estimator = rf_classifier.estimators_[1]

feature_names = fea_list
target_p_s_cat = pd.Series(data=target_p_s).replace({i:j for i, j in enumerate(class_names_p)})
#y_train_str =  class_names_p 
#target_p_s.astype('str') #target_p_s_cat.astype('str')

export_graphviz(estimator, out_file=result_path+'tree.dot', 
                feature_names = feature_names,
                class_names = class_names_p,
                rounded = True, proportion = False, 
                label='root',
                precision = 2, filled = True)

from subprocess import call
call(['dot', '-Tpng', result_path+'tree.dot', '-o', result_path+'tree.png', '-Gdpi=600'])
call(['dot', '-Tpdf', result_path+'tree.dot', '-o', result_path+'tree.pdf', '-Gdpi=600'])

from IPython.display import Image
Image(filename = result_path+'tree.png')
## Cross Validation Random Forest

from sklearn.ensemble import RandomForestClassifier
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score, cross_validate, cross_val_predict
from sklearn.metrics import classification_report


X, y = data_p_grid, target_p_s
print (X.shape, y.shape)
#print (cross_validate(rf_classifier, X, y, scoring=['recall_macro', 'precision_macro', 'f1_macro', 'accuracy'], cv=5) )
y_pred = cross_val_predict(rf_classifier,X, y, cv=5 )
print(classification_report(y, y_pred, target_names=class_names_p))
# For Data for particular patient
st_ix_p = 13
end_ix_p = 14
p = data_p_grid[st_ix_p:end_ix_p]
t = target_p_s[st_ix_p:end_ix_p]

print ('Mean Accuracy: ', rf_classifier.score(data_p_grid, target_p_s)*100)
print ('Predict Probability: ', rf_classifier.predict_proba(p)*100)
print ('Prediction: ', rf_classifier.predict(p))
print ('Target: ', t)

print('\n #########  Estimator ##################\n')
print ('Predict Probability Estimator', estimator.predict_proba(p)*100)
print ('Prediction Estimator: ', estimator.predict(p))
print ('Decision Path Estimator',estimator.decision_path(p).toarray())


importances = estimator.feature_importances_
indices = np.argsort(importances)[::-1]

print ('\nEstimator feature importance')
for f in range(15):
    print("%d. feature: %s (%f)" % (f + 1, fea_list[indices[f]], importances[indices[f]]))
    
def get_branch(clf, patient, target_names, estimator_id=1):
    branchs = []
    for j, tree in enumerate(clf.estimators_):
        #rule = static
        feature = tree.tree_.feature
        #print (feature)
        threshold = tree.tree_.threshold
        node_indicator = tree.decision_path(patient)
        #print ()
        # Similarly, we can also have the leaves ids reached by each sample.
        leave_id = tree.apply(patient)

        sample_id = 0
        node_index = node_indicator.indices[node_indicator.indptr[0]:
        node_indicator.indptr[1]]
        line = ''
        for node_id in node_index:
            #print (node_id)
            if leave_id[sample_id] == node_id:
                continue
            if (patient[sample_id][feature[node_id]] <= threshold[node_id]):
                threshold_sign = "<="
            else:
                threshold_sign = ">"
            line+= 'decision id node '+ str(node_id) + ':' + str(sample_id) + '( = )' + str(feature_names[feature[node_id]]) +') '+ str(patient[sample_id][feature[node_id]])+' '+str(threshold_sign) + " " + str(threshold[node_id])+'\n'
        #print (target_names[tree.predict(patient)[sample_id].astype('int32')])
        line+='classification: '+ str(target_names[tree.predict(patient)[sample_id].astype('int32')])
        branchs.append(line)
    return branchs[estimator_id]


print ('\n For Estimator: {} \n {}: '.format(estimator_id, get_branch(rf_classifier, p, class_names_p, 1)))
display(df_fea.iloc[st_ix_p:end_ix_p, :])
# Important feature for a particular class for a patients
from itertools import chain
import collections

def get_branch_features(clf, patient, target_names, target_class, estimator_id=1):
    #branchs = []
    feature_names_list = []
    class_list= []
    for j, tree in enumerate(clf.estimators_):
        #rule = static
        feature = tree.tree_.feature
        #print (feature)
        threshold = tree.tree_.threshold
        node_indicator = tree.decision_path(patient)
        #print ()
        # Similarly, we can also have the leaves ids reached by each sample.
        leave_id = tree.apply(patient)

        sample_id = 0
        node_index = node_indicator.indices[node_indicator.indptr[0]:
        node_indicator.indptr[1]]
        line = ''
        feature_names_list_local=[]
        for node_id in node_index:
            #print (node_id)
            if leave_id[sample_id] == node_id:
                continue
            if (patient[sample_id][feature[node_id]] <= threshold[node_id]):
                threshold_sign = "<="
            else:
                threshold_sign = ">"
            feature_names_list_local.append(feature_names[feature[node_id]])
            #print ()
            #line+= 'decision id node '+ str(node_id) + ':' + str(sample_id) + '( = )' + str(feature_names[feature[node_id]]) +') '+ str(patient[sample_id][feature[node_id]])+' '+str(threshold_sign) + " " + str(threshold[node_id])+'\n'
        #print (target_names[tree.predict(patient)[sample_id].astype('int32')])
        #line+='classification: '+ str(target_names[tree.predict(patient)[sample_id].astype('int32')])
        #branchs.append(line)
        if (target_class==tree.predict(patient)[sample_id].astype('int32')):
            class_list.append(tree.predict(patient)[sample_id].astype('int32'))
            feature_names_list.append(feature_names_list_local)
    return feature_names_list


#print ('\n For Estimator: {} \n {}: '.format(estimator_id, get_branch(rf_classifier, p, class_names_p, 2, 1)))


p_classess_list = [[] for i in range(7)]
branch_important_feature = []
branch_important_feature_keys = []

for p,t in zip(p,[t]):
    p = p.reshape(1,-1)
    if t == 0:
        p_classess_list[0].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 1:
        p_classess_list[1].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 2:
        p_classess_list[2].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 3:
        p_classess_list[3].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 4:
        p_classess_list[4].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 5:
        p_classess_list[5].append(get_branch_features(rf_classifier, p, class_names_p, t))
    else:
        p_classess_list[6].append(get_branch_features(rf_classifier, p, class_names_p, t))
    
print ([len(x) for x in p_classess_list])             
    
def change_feature_name(x):
    if 'Age_At_Episode' in x:
        x = 'Age'
    elif 'Medication_cui_' in x:
        x = x.split('_')[-1]
    elif 'Comorbidity_cui_'   in x:
        x = x.split('_')[-1]
    return x

def d3_list_d1(l):
    ini_list = l
    flatten_list1 = list(chain.from_iterable(ini_list)) 
    flatten_list2 = list(chain.from_iterable(flatten_list1))
    return flatten_list2


for clss, p_class_list in zip(class_names_p, p_classess_list):
    p_class_flat_list = d3_list_d1(p_class_list)
    for n, x in enumerate(p_class_flat_list):
        p_class_flat_list[n] = change_feature_name(x)
    counter=collections.Counter(p_class_flat_list)
    print ('*'*20, clss, '*'*20)
    print(counter.most_common())
    print (branch_important_feature.append(list(counter.keys())))
    
display (patient_df_after_fea_selection[list(set([col for col in patient_df_after_fea_selection.columns for x in branch_important_feature[t[0]] if x in col]))][st_ix_p:end_ix_p])
# Important feature for a particular class for all patients

p_classess_list = [[] for i in range(len(class_names_p))]
branch_important_feature = []
    
for p,t in zip(data_p_grid, target_p_s):
    p = p.reshape(1,-1)
    if t == 0:
        p_classess_list[0].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 1:
        p_classess_list[1].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 2:
        p_classess_list[2].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 3:
        p_classess_list[3].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 4:
        p_classess_list[4].append(get_branch_features(rf_classifier, p, class_names_p, t))
    elif t == 5:
        p_classess_list[5].append(get_branch_features(rf_classifier, p, class_names_p, t))
    else:
        p_classess_list[6].append(get_branch_features(rf_classifier, p, class_names_p, t))
    
print ([len(x) for x in p_classess_list])             
    
def change_feature_name(x):
    if 'Age_At_Episode' in x:
        x = 'Age'
    elif 'Medication_cui_' in x:
        x = x.split('_')[-1]
    elif 'Comorbidity_cui_'   in x:
        x = x.split('_')[-1]
    return x

def d3_list_d1(l):
    ini_list = l
    flatten_list1 = list(chain.from_iterable(ini_list)) 
    flatten_list2 = list(chain.from_iterable(flatten_list1))
    return flatten_list2


for clss, p_class_list in zip(class_names_p, p_classess_list):
    p_class_flat_list = d3_list_d1(p_class_list)
    for n, x in enumerate(p_class_flat_list):
        p_class_flat_list[n] = change_feature_name(x)
    counter=collections.Counter(p_class_flat_list)
    print ('*'*20, clss, '*'*20)
    print(counter.most_common())
    print (branch_important_feature.append(list(counter.keys())))

print ('\n\n\n')
print ('Most important features for all classes', set.intersection(*[set(i) for i in branch_important_feature]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12988918229625115, 'fraction_numerical': 0.01862008260545738, 'mean_word_length': 5.785298621745788, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '280755', 'license': 'MIT', 'n_tokens_mistral': 16699, 'n_tokens_neox': 15829, 'n_words': 2836, 'path': 'dementia_optima/models/.ipynb_checkpoints/data_kernel_sl_nr_ft-checkpoint.ipynb', 'repo_name': 'SDM-TIB/dementia_mmse', 'size': '44314'}