<jupyter_start><jupyter_text>Objective Packages<jupyter_code>import pandas as pd 
import numpy as np 
import scipy.stats as stats
import seaborn as sns
import matplotlib.pyplot as plt
from datetime import datetime
from itertools import compress
from matplotlib.ticker import PercentFormatter
from matplotlib_venn import venn3, venn2, venn2_circles
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Datasets<jupyter_code>record = pd.read_pickle('../primary_care/records2.pkl')
pri_cli = pd.read_csv('../primary_care/gp_clinical.txt', sep = '\t', encoding='ISO-8859-1')
diag = pd.read_excel('../primary_care/diagnosis_cvd.xlsx')<jupyter_output><empty_output><jupyter_text>Specifying dataset<jupyter_code>rec = record[(record['discrepancy'] == False) & (record['event_int'] > 0)]
patients = list(rec['eid'].values)
pri = pri_cli[pri_cli['eid'].isin(patients)].reset_index()<jupyter_output><empty_output><jupyter_text>Diagnosis<jupyter_code>diag
diag['diagnosis'].unique()
diagnosis_codes = {
    'hyperlipidaemia': {
        'read2': list(diag[diag['diagnosis'] == 'hyperlipidaemia']['READV2_CODE']),
        'read3': list(diag[diag['diagnosis'] == 'hyperlipidaemia']['READV3_CODE'])
    },
    'hypertension': {
        'read2': list(diag[diag['diagnosis'] == 'hypertension']['READV2_CODE']),
        'read3': list(diag[diag['diagnosis'] == 'hypertension']['READV3_CODE'])
    }, 
    'PAD': {
        'read2': list(diag[diag['diagnosis'] == 'PAD']['READV2_CODE']),
        'read3': list(diag[diag['diagnosis'] == 'PAD']['READV3_CODE'])
    },
    'CKD': {
        'read2': list(diag[diag['diagnosis'] == 'CKD']['READV2_CODE']),
        'read3': list(diag[diag['diagnosis'] == 'CKD']['READV3_CODE'])
    },
    'diabetes': {
        'read2': list(diag[diag['diagnosis'] == 'diabetes']['READV2_CODE']),
        'read3': list(diag[diag['diagnosis'] == 'diabetes']['READV3_CODE'])
    }, 
    'all': {
        'read2': list(diag['READV2_CODE']),
        'read3': list(diag['READV3_CODE'])
    }
}
diag_pri = pri[pri['read_2'].isin(diagnosis_codes['all']['read2'])| pri['read_3'].isin(diagnosis_codes['all']['read3'])]
diag_pri.drop('index', axis = 1, inplace = True)
diag_pri.drop('event_dt', axis = 1, inplace = True)
diag_pri.drop_duplicates(keep = 'first', inplace = True)

new_diag_pri = diag_pri.groupby('eid').agg(list).reset_index()
new_diag_pri
type(new_diag_pri.iloc[0]['read_2'][0])<jupyter_output><empty_output><jupyter_text>diabetes<jupyter_code>diabetes = pri[(pri['read_2'].isin(diagnosis_codes['diabetes']['read2']))|(pri['read_3'].isin(diagnosis_codes['diabetes']['read3']))]
diabetes_df = diabetes.groupby('eid').agg(list).reset_index()
diabetics = list(diabetes_df.eid)<jupyter_output><empty_output><jupyter_text>CKD<jupyter_code>CKD = pri[(pri['read_2'].isin(diagnosis_codes['CKD']['read2']))|(pri['read_3'].isin(diagnosis_codes['CKD']['read3']))]

CKD_df = CKD.groupby('eid').agg(list).reset_index()
chronic = list(CKD_df.eid)<jupyter_output><empty_output><jupyter_text>PAD<jupyter_code>PAD = pri[(pri['read_2'].isin(diagnosis_codes['PAD']['read2']))|(pri['read_3'].isin(diagnosis_codes['PAD']['read3']))]

PAD_df = PAD.groupby('eid').agg(list).reset_index()
peripheral = list(PAD_df.eid)<jupyter_output><empty_output><jupyter_text>Hypertension<jupyter_code>hypertension = pri[(pri['read_2'].isin(diagnosis_codes['hypertension']['read2']))|(pri['read_3'].isin(diagnosis_codes['hypertension']['read3']))]

 hypertension_df = hypertension.groupby('eid').agg(list).reset_index()
hypertensives = list(hypertension_df.eid)<jupyter_output><empty_output><jupyter_text>Hyperlipidaemia<jupyter_code>hyperlipid = pri[(pri['read_2'].isin(diagnosis_codes['hyperlipidaemia']['read2']))|(pri['read_3'].isin(diagnosis_codes['hyperlipidaemia']['read3']))]

hyperlipid_df = hyperlipid.groupby('eid').agg(list).reset_index()
hyperchol= list(hyperlipid_df.eid)<jupyter_output><empty_output><jupyter_text>Gathering<jupyter_code>rec['diabetic'] = ['diabetic' if x in diabetics else "" for x in rec['eid'] ]
rec['CKD'] = ['CKD' if x in chronic else "" for x in rec['eid']]
rec['PAD'] = ['peripheral' if x in peripheral else "" for x in rec['eid']]
rec['hypertension'] = ['hypertensives' if x in hypertensives else "" for x in rec['eid']]
rec['hyperlipidaemia'] = ['hyperchol' if x in hyperchol else "" for x in rec['eid']]
concern = ['diabetic', 'CKD', 'PAD', 'hypertension', 'hyperlipidaemia']
for x in concern: 
    print(rec[x].value_counts())
rec['noconcern'] = rec['diabetic'] + rec['CKD'] + rec['PAD'] + rec['hypertension'] + rec['hyperlipidaemia']
temp = pd.DataFrame(rec.noconcern.value_counts())
temp.to_csv('../primary_care/rec_diagnosis.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19859215017064846, 'fraction_numerical': 0.013438566552901024, 'mean_word_length': 7.830508474576271, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '63174', 'license': 'MIT', 'n_tokens_mistral': 1896, 'n_tokens_neox': 1737, 'n_words': 220, 'path': 'notebooks/diagnosis.ipynb', 'repo_name': 'michelle-holmusk/biobank', 'size': '4688'}