<jupyter_start><jupyter_text>Read combined dataset to pandas dataframe<jupyter_code>df = pd.read_csv("../data/combinedData.csv")
provider = pd.read_csv("../data/train.csv")
df = df.merge(provider,on="Provider", how="left")
df.shape
df.groupby("PotentialFraud").size()
df.head()
df.shape
df["Provider"].nunique()
provider.head()
df.shape<jupyter_output><empty_output><jupyter_text>Creating df1 with new column "conditions"<jupyter_code>df1 = df
illness = df1[["ClaimID",
           "Depression",
           "Alzheimer",
           "RheumatoidArthritis",        
           "Diabetes",
           "KidneyDisease",
           "IschemicHeart",
           "Stroke",
           "Osteoporasis",
           "ObstrPulmonary",
           "HeartFailure",
           "Cancer",
#            "ClmAdmitDiagnosisCode",
#            "ClmDiagnosisCode_1",
#            "ClmDiagnosisCode_2",
#            "ClmDiagnosisCode_3",
#            "ClmDiagnosisCode_4",
#            "ClmDiagnosisCode_5",
#            "ClmDiagnosisCode_6",
#            "ClmDiagnosisCode_7",
#            "ClmDiagnosisCode_8",
#            "ClmDiagnosisCode_9",
#            "ClmDiagnosisCode_10",
#            "ClmProcedureCode_1",
#            "ClmProcedureCode_2",
#            "ClmProcedureCode_3",
#            "ClmProcedureCode_4",
#            "ClmProcedureCode_5",
#            "ClmProcedureCode_6"
           ]]
illness.head()
for col in illness.columns:
    illness[col].replace(1,col, inplace=True)
illness.head()
cols = [   "Depression",
           "Alzheimer",
           "RheumatoidArthritis",        
           "Diabetes",
           "KidneyDisease",
           "IschemicHeart",
           "Stroke",
           "Osteoporasis",
           "ObstrPulmonary",
           "HeartFailure",
           "Cancer"
           ]

illness["preconditions"] = illness[cols].values.tolist()
illness["preconditions"] = illness["preconditions"].apply(lambda x: list(filter(lambda a: a != 0, x)))
illness["preconditions"] = illness["preconditions"].apply(lambda x: tuple(x))
illness.drop(cols, axis=1, inplace=True)
illness.head()
df1 = df1.merge(illness, on="ClaimID")
df1.head()<jupyter_output><empty_output><jupyter_text>Extract ClaimID and boolean columns for diagnosed illness<jupyter_code>diag = df[["ClaimID",
           "Depression",
           "Alzheimer",
           "RheumatoidArthritis",        
           "Diabetes",
           "KidneyDisease",
           "IschemicHeart",
           "Stroke",
           "Osteoporasis",
           "ObstrPulmonary",
           "HeartFailure",
           "Cancer",
           ]]
diag.head()
diag = diag.fillna(0).set_index('ClaimID')
diag.head()
diag.shape<jupyter_output><empty_output><jupyter_text>Most Frequent Items Visualization<jupyter_code>count = diag.sum(axis = 0).sort_values(ascending=False)
count.plot(kind="bar")
import squarify
plt.rcParams['figure.figsize']=(8,8)
Items = count.to_frame()
size = Items[0].values
lab = Items.index
color = plt.cm.Blues(np.linspace(0,2,22))
squarify.plot(sizes=size, label=lab, alpha = 0.7, color=color)
#plt.title('Tree map of Most Frequent Items')
#plt.axis('off')
#plt.show()<jupyter_output><empty_output><jupyter_text>Run Apriori Algorithm and Association Rules<jupyter_code>frequent_itemsets = apriori(diag, min_support=0.1, use_colnames=True)
rules = association_rules(frequent_itemsets, metric="lift", min_threshold=1)
rules.head()
rules.sort_values("lift")
#rules = rules[(rules['lift'] >= 1)&(rules['confidence'] >= 0.8)].sort_values("lift",ascending=False)
rules = rules[(rules['support'] >= 0.3)&(rules['lift'] >= 1)&(rules['confidence'] >= 0.7)].sort_values("lift",ascending=False)
rules.shape<jupyter_output><empty_output><jupyter_text>Create New Column with a Set of Antecedents and Consequents Rules - To merge with original Dataset<jupyter_code>rules["ante_set"] = rules["antecedents"].apply(lambda x: tuple(x))
rules["con_set"] = rules["consequents"].apply(lambda x: tuple(x))
rules.head()
rules['preconditions'] = rules['ante_set'] + rules['con_set']
#rules['conditions'] = rules.apply(lambda x: x['ante_set'].union(x['con_set']), axis=1)
rules.shape
rules.head()<jupyter_output><empty_output><jupyter_text>Combining df1 with Rules<jupyter_code>df1.shape
df1.head()
df2 = df1
df3 = df1
df3.shape
df2.head()
setlist = [("Alzheimer", "IschemicHeart"),
("Depression", "Diabetes"),
("Depression", "Diabetes", "IschemicHeart"),
("Depression", "IschemicHeart"),
("Diabetes", "IschemicHeart"),
("KidneyDisease", "HeartFailure"),
("KidneyDisease", "IschemicHeart")]

df2["preconditions"].isin(setlist).mean()
df3.head()
df4 = df3.groupby(["Provider","preconditions"])
df4_unstack = df4.size().unstack()
df4_unstack.head()
provider_norule = df4_unstack.sum(axis=1)
provider_rule = df4_unstack[[("Alzheimer", "IschemicHeart"),
("Depression", "Diabetes"),
("Depression", "Diabetes", "IschemicHeart"),
("Depression", "IschemicHeart"),
("Diabetes", "IschemicHeart"),
("KidneyDisease", "HeartFailure"),
("KidneyDisease", "IschemicHeart")]].sum(axis=1)
provider_above5 = provider_rule/provider_norule
provider_above5 = provider_above5.reset_index()
provider_above5.head()
provider_above5_summary = provider_above5[provider_above5.iloc[:,1] >= 0.045]
provider_below5_summary = provider_above5[provider_above5.iloc[:,1] < 0.045]
provider_above5_summary = provider_above5_summary.merge(provider, on="Provider")
provider_below5_summary = provider_below5_summary.merge(provider, on="Provider")
provider_above5_summary
provider_below5_summary
provider_above5_summary.groupby("PotentialFraud")["Provider"].count()
# ~9%
provider_below5_summary.groupby("PotentialFraud")["Provider"].count()
# ~9.54%
rules.shape
df2.shape
df1_rules = rules.merge(df2, on="preconditions", how="inner")
df1_rules.shape
33563/693603
df1_rules.head()
df1_rules.groupby("preconditions")["ClaimID"].count()
df1_rules_summary = df1_rules.groupby(["preconditions","PotentialFraud"]).size().unstack()
df1_rules_summary["total"] = df1_rules_summary["No"] + df1_rules_summary["Yes"]
df1_rules_summary["No"] = df1_rules_summary["No"]/df1_rules_summary["total"] *100
df1_rules_summary["Yes"] = df1_rules_summary["Yes"]/df1_rules_summary["total"] * 100
df1_rules_summary.head()
preconditions = df1_rules[["ClaimID","preconditions"]]
preconditions["count"] = 1
preconditions.head()
preconditions = preconditions.pivot_table(values='count', index="ClaimID", columns='preconditions', aggfunc='first')
preconditions = preconditions.fillna(0)
preconditions<jupyter_output><empty_output><jupyter_text>Analyse Results with Diagnosis and Procedure Codes<jupyter_code>df_ip = pd.read_csv("../data/code_specialty_ip.csv")
df_op = pd.read_csv("../data/code_specialty_op.csv")
new_df = df1_rules
df_ipop = pd.concat([df_ip,df_op],join='inner').merge(provider, on="Provider").merge(new_df,on="ClaimID")
df_ipop.shape
df_ipop.head()
code_analysis = df_ipop[["ClaimID",
                   'cardiology',
                   'urology',
                   'endocrinology',
                   'emergency',
                   'general',
                   'infectious',
                   'oncology',
                   'hematology',
                   'psychiatry',
                   'neurology',
                   'pulmonology',
                   'gastroenterology',
                   'ob-gyn',
                   'dermatology',
                   'orthopedics',
                   'congenital',
                   'neonatology']]
new = df_ipop[['ClaimID','ChronicCond_Alzheimer',
                    'ChronicCond_Heartfailure',
                    'ChronicCond_KidneyDisease',
                    'ChronicCond_Cancer',
                    'ChronicCond_ObstrPulmonary',
                    'ChronicCond_Depression',
                    'ChronicCond_Diabetes',
                    'ChronicCond_IschemicHeart',
                    'ChronicCond_Osteoporasis',
                    'ChronicCond_rheumatoidarthritis',
                    'ChronicCond_stroke']]
new.replace(2, 0,inplace=True)
new.columns = ['ClaimID','Alzheimer',
                    'HeartFailure',
                    'KidneyDisease',
                    'Cancer',
                    'ObstrPulmonary',
                    'Depression',
                    'Diabetes',
                    'IschemicHeart',
                    'Osteoporasis',
                    'Rheumatoidarthritis',
                    'Stroke']
new.head()
code_analysis = code_analysis.merge(new,on="ClaimID")
code_analysis.head()
code_analysis.replace(2, 1,inplace=True)
code_analysis.replace(3, 1,inplace=True)
code_analysis.replace(4, 1,inplace=True)
code_analysis.replace(5, 1,inplace=True)
code_analysis.replace(6, 1,inplace=True)
code_analysis.replace(7, 1,inplace=True)
code_analysis.replace(8, 1,inplace=True)
code_analysis.replace(9, 1,inplace=True)
code_analysis.replace(10, 1,inplace=True)
code_analysis.replace(11, 1,inplace=True)
code_analysis.replace(12, 1,inplace=True)
code_analysis.replace(13, 1,inplace=True)
code_analysis.replace(14, 1,inplace=True)
code_analysis.replace(15, 1,inplace=True)
code_analysis = code_analysis.fillna(0).set_index('ClaimID')
code_analysis.head()
code_analysis.shape<jupyter_output><empty_output><jupyter_text>Run Apriori Algorithm and Association Rules<jupyter_code>merged = preconditions.merge(code_analysis, on="ClaimID")
merged.head()
frequent_itemsets1 = apriori(merged, min_support=0.2, use_colnames=True)
rules1 = association_rules(frequent_itemsets1, metric="lift", min_threshold=1)
#rules1 = association_rules(frequent_itemsets1, metric="lift", min_threshold=0)
rules1.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14573336115167954, 'fraction_numerical': 0.019507615272272065, 'mean_word_length': 3.748390292223873, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '331780', 'license': 'MIT', 'n_tokens_mistral': 3405, 'n_tokens_neox': 3162, 'n_words': 435, 'path': 'deb/.ipynb_checkpoints/Market Basket Analysis - illness3-checkpoint.ipynb', 'repo_name': 'DebLeong/HealthCareCapstone', 'size': '9586'}