<jupyter_start><jupyter_text>Extract Time Series DataWith this notebook the TimeSeries for predefined FIBER Conditions can be extracted.<jupyter_code>from typing import Optional
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import venn
from fiber import Cohort
from fiber.condition import MRNs
from fiber.config import OCCURRENCE_INDEX, VERBOSE
from fiber.dataframe import (
    column_threshold_clip,
    create_id_column,
    merge_to_base,
    time_window_clip
)
from fiber.plots.distributions import bars
from fiberutils.time_series_utils import ssr_transform
from fiber.condition import Patient, MRNs
from fiber.condition import Diagnosis
from fiber.condition import Measurement, Encounter, Drug, TobaccoUse,LabValue, VitalSign, Procedure
from fiber.storage import yaml as fiberyaml
#from fiberutils.cohort_utils import get_time_series, pivot_time_series
import pandas as pd
import pyarrow.parquet as pq
import numpy as np
import os
import matplotlib.pyplot as plt
from functools import reduce
import json
import math
import inspect
import json
import sys
from pydoc import locate
import requests
##### REQUIRES THE DATAFRAME FOLDER TO BE NAMED 'Cohorts', WHICH INCLUDES ALL PRECOMPUTED DATAFRAMES #####
from typing import Optional

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import venn
from fiber import Cohort
from fiber.condition import MRNs
from fiber.config import OCCURRENCE_INDEX, VERBOSE
from fiber.dataframe import (
    column_threshold_clip,
    create_id_column,
    merge_to_base,
    time_window_clip
)
from fiber.plots.distributions import bars

from fiberutils.time_series_utils import ssr_transform
from fiber.condition import Patient, MRNs
from fiber.condition import Diagnosis
from fiber.condition import Measurement, Encounter, Drug, TobaccoUse,LabValue, VitalSign, Procedure
from fiber.storage import yaml as fiberyaml
#from fiberutils.cohort_utils import get_time_series, pivot_time_series
import pandas as pd
import pyarrow.parquet as pq
import numpy as np
import os
import matplotlib.pyplot as plt
from functools import reduce
import json
import math
import inspect
import json
import sys
from pydoc import locate

#import request
#generic function get cohort from mrns
def df_to_cohort(df):
    #mrns = list(df.index.values)
    #mrns = list(map(str, mrns))
    p_condition = MRNs(df)
    print(MRNs(df))
    cohort = Cohort(p_condition)
    #print(mrns)
    return cohort
#MRN Data frame : 
sample = pq.read_table('Cohort/Feature_Extraction/ALL_HF_cohort_unsupervised_only_all_count_all_mean_preprocessed.parquet').to_pandas()
sample['medical_record_number']=sample.index
# Saving Hf_Onset as age_in_days for PIVOT config necessary and for the function time series for
sample = sample["HF_Onset_age_in_days"]
sample = sample.to_frame()
sample.reset_index(level=0, inplace=True)
sample.rename(columns = {"HF_Onset_age_in_days": "age_in_days"}, inplace = True)
#create a small sample
#sample = sample.head(10) #to try with small sample
sample
#convert df to fiber cohort 
cohort_EF_Baseline = df_to_cohort(sample)
cohort_EF_Baseline
#Functions to get detailed information of the FIBER Conditiond to be able to extract the right data
def rename_columns( df,col_ignore):
    replacements = {}
    for column in df.columns:
        replacements[column] = get(column,col_ignore)
    return replacements

def get(feature,col_ignor):
    #context first empty : 
    context=""
    code=""
    
    # does not support time window information inside feature name yet
    if feature.startswith(col_ignor):
        return feature.replace('_', ' ').replace('.', '|')

    split_name = feature.split('__')
    if split_name[1] in [
        i[0]
        for i in inspect.getmembers(
            sys.modules['fiber.condition'],
            inspect.isclass
        )
    ]:
        aggregation = split_name[0]
        split_name = split_name[1:]
    else:
        aggregation = "time series"
        

    if len(split_name) == 3:
        class_name, context, code = split_name
        condition_class = locate(f'fiber.condition.{class_name}')
        description = get_description(condition_class, code, context)
    else:
        class_name, description = split_name
        
    return f'{class_name} | {code }|{description.capitalize()} ({aggregation}) | {context}' 

def get_description( condition_class, code, context):
    return condition_class(
        code=code,
        context=context
    ).patients_per(
        condition_class.description_column
    )[
        condition_class.description_column.name.lower()
    ].iloc[0]
#columns that should be ignored for the timeseries extraction
col_ignore=('age_in_days', 'gender', 'religion', 'race','medical_record_number','HF_Onset_age_in_days','patient_ethnic_group','deceased_indicator','mother_account_number','address_zip','marital_status_code')
col_ignore<jupyter_output><empty_output><jupyter_text>Create Feature look up table with human readable names<jupyter_code>col=['original_feature_name','human_readable']
feature_look_up=pd.DataFrame(columns=col)
feature_look_up<jupyter_output><empty_output><jupyter_text>Drug Dictionary<jupyter_code>#non drug columns:
col_for_dropping=['age_in_days',
 'gender',
 'religion',
 'race',
 'medical_record_number',
 'patient_ethnic_group',
 'deceased_indicator',
 'mother_account_number',
 'address_zip',
 'date_of_birth',
 'month_of_birth',
 'marital_status_code']
# load drug dataframe: 
#get Medical Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Drug_after_onset_HF_ALL_mmm_0_2'
drug = pq.read_table(df_path).to_pandas()
#print(drug)
drug=drug.drop(col_for_dropping,axis=1)
drug_renamed_column=rename_columns(drug,col_ignore)
for n in drug_renamed_column.items(): 
    parts=n[1].split('|')
    original_feature_name=n[0]
    human_readable=str(parts[2].replace(' (time series) ','')).strip()
    feature_look_up=feature_look_up.append({'original_feature_name':original_feature_name,'human_readable':human_readable},ignore_index=True)
feature_look_up<jupyter_output><empty_output><jupyter_text>Diagnosis Dictionary<jupyter_code># load Diagnosis dataframe: 
#get Medical Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Diagnosis_after_onset_HF_ALL_mmm_0_2_cleaned'
diagnosis = pq.read_table(df_path).to_pandas()
diagnosis=diagnosis.drop(col_for_dropping,axis=1)
#create diagnosis dictionary
diagnosis_renamed_column=rename_columns(diagnosis,col_ignore)

for n in diagnosis_renamed_column.items(): 
    parts=n[1].split('|')
    original_feature_name=n[0]
    human_readable=str(parts[2].replace(' (time series) ','')).strip()
    feature_look_up=feature_look_up.append({'original_feature_name':original_feature_name,'human_readable':human_readable},ignore_index=True)
feature_look_up<jupyter_output><empty_output><jupyter_text>Procedures Dictionary<jupyter_code># load Procedure dataframe: 
#get Procedure Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Procedure_after_onset_HF_ALL_mmm_0_6'
procedures = pq.read_table(df_path).to_pandas()
procedures=procedures.drop(col_for_dropping,axis=1)
procedures_renamed_column=rename_columns(procedures,col_ignore)
for n in procedures_renamed_column.items(): 
    parts=n[1].split('|')
    original_feature_name=n[0]
    human_readable=str(parts[2].replace(' (time series) ','')).strip()
    feature_look_up=feature_look_up.append({'original_feature_name':original_feature_name,'human_readable':human_readable},ignore_index=True)
feature_look_up<jupyter_output><empty_output><jupyter_text>Vital Sign<jupyter_code>col_for_dropping=[]
# load Vital Sign  dataframe: 
#get Vital Sign Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/VitalSign_after_onset_HF_ALL_mmm_0_6_cleaned'
vital = pq.read_table(df_path).to_pandas()
vital=vital.drop(col_for_dropping,axis=1)
#create Vital Signs dictionary
vital_renamed_column=rename_columns(vital,col_ignore)
vital_renamed_column
for n in vital_renamed_column.items(): 
    parts=n[1].split('|')
    original_feature_name=n[0]
    if original_feature_name !='medical_record_number':
        human_readable=str(parts[2].replace(' (time series) ','')).strip()
        feature_look_up=feature_look_up.append({'original_feature_name':original_feature_name,'human_readable':human_readable},ignore_index=True)
feature_look_up<jupyter_output><empty_output><jupyter_text>Lab Values<jupyter_code># load Lab Values dataframe: 
#get Lab Values Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/LabValue_after_onset_HF_ALL_mmm_0_8_cleaned'
lab = pq.read_table(df_path).to_pandas()
lab=lab.drop(col_for_dropping,axis=1)
#create lab values dictionary
lab_renamed_column=rename_columns(lab,col_ignore)
for n in lab_renamed_column.items(): 
    parts=n[1].split('|')
    original_feature_name=n[0]
    if original_feature_name !='medical_record_number':
        human_readable=str(parts[2].replace(' (time series) ','')).strip()
        feature_look_up=feature_look_up.append({'original_feature_name':original_feature_name,'human_readable':human_readable},ignore_index=True)
feature_look_up
feature_look_up.to_parquet('Cohort/feature_look_up.parquet')
vital<jupyter_output><empty_output><jupyter_text>Inpatient<jupyter_code>#non drug columns:
col_for_dropping=['age_in_days',
 'gender',
 'religion',
 'race',
 'medical_record_number',
 'patient_ethnic_group',
 'deceased_indicator',
 'mother_account_number',
 'address_zip',
 'date_of_birth',
 'month_of_birth',
 'marital_status_code']
conditions=Encounter(description='%Inpatient%')
inpatient_time_series=cohort_EF_Baseline.get(conditions)
inpatient_time_series
inpatient_time_series.to_parquet('Cohort/Time_Series/Inpatient_Events_raw.parquet')<jupyter_output><empty_output><jupyter_text>Ejection Fraction<jupyter_code>#Define ejection fraction condition: 
conditions=(LabValue('%ejection%'))
ej_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
ej_time_series
ej_time_series.to_parquet('Cohort/Time_Series/Ejection_Fraction_raw.parquet')<jupyter_output><empty_output><jupyter_text>Drugs<jupyter_code>#non drug columns:
col_for_dropping=['age_in_days',
 'gender',
 'religion',
 'race',
 'medical_record_number',
 'patient_ethnic_group',
 'deceased_indicator',
 'mother_account_number',
 'address_zip',
 'date_of_birth',
 'month_of_birth',
 'marital_status_code']
# load drug dataframe: 
#get Medical Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Drug_after_onset_HF_ALL_mmm_0_2'
drug = pq.read_table(df_path).to_pandas()
#print(drug)
drug=drug.drop(col_for_dropping,axis=1)
drug
#create drug dictionary
drug_renamed_column=rename_columns(drug,col_ignore)

#get Drug Features for Time series
'''structure for drug time series feature: 
conditions=(Drug(context='TDS',
            code='01060-ACETAMINOPHEN TABS',
            description='Acetaminophen tabs',
            data_columns=['medical_record_number','age_in_days', 'context_name',
            'MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|)

'''
for n in drug_renamed_column.items(): 
    parts=n[1].split('|')
    drug_statement='Drug(context=\''+str(parts[3]).strip()+'\',code=\''+str(parts[1]).strip()+'\',description=\''+str(parts[2].replace(' (time series) ','')).strip()+'\',data_columns=[\'medical_record_number\',\'age_in_days\', \'context_name\',\'MATERIAL_NAME\',\'GENERIC_NAME\',\'BRAND1\',\'numeric_value\'])|'
    #drug_statement='a'+n[1]
    print(drug_statement)
    #print(parts[3])

#final Format Drugs Copied from the output above 
conditions=(Drug(context='COMPURECORD',code='Cefazolin',description='Cefazolin',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='COMPURECORD',code='Fentanyl',description='Fentanyl',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='COMPURECORD',code='Midazolam',description='Midazolam',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='COMPURECORD',code='Propofol',description='Propofol',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='Cardiac Cath',code='Antiplatelets+Aspirin',description='Aspirin',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='Cardiac Cath',code='Diuretic+Lasix',description='Lasix',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC IMMUNIZATION',code='51',description='Influenza vaccine',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='10113',description='Potassium chloride 10 meq/100 ml intravenous piggyback',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='10168',description='Potassium chloride er 20 meq tablet,extended release(part/cryst)',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='10196',description='Famotidine 20 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='10292',description='Magnesium oxide 400 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='12097',description='Bisacodyl 10 mg rectal suppository',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='12204',description='Pantoprazole 40 mg tablet,delayed release',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='12421',description='Ferrous sulfate 325 mg (65 mg iron) tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='1311',description='Sennosides 8.6 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='13116',description='Lisinopril 10 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='13136',description='Insulin regular human 100 unit/ml injection solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='13455',description='Warfarin 5 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='14455',description='Fentanyl (pf) 50 mcg/ml injection solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='1451',description='Acetaminophen 325 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='1454',description='Carvedilol 6.25 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='14861',description='Glucagon (human recombinant) 1 mg solution for injection',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='14951',description='Docusate sodium 100 mg capsule',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='15419',description='Aspirin 81 mg chewable tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='16470',description='Lisinopril 5 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='16639',description='Furosemide 20 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='16822',description='Sodium chloride 0.9 % intravenous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='16843',description='Metoprolol succinate er 25 mg tablet,extended release 24 hr',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='17270',description='Insulin lispro 100 unit/ml subcutaneous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='19073',description='Heparin (porcine) 5,000 unit/ml injection solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='19767',description='Electrolyte-a intravenous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='19813',description='Ipratropium bromide 0.02 % solution for inhalation',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='2063',description='Insulin glargine 100 unit/ml subcutaneous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='21305',description='Famotidine (pf) 20 mg/50 ml in 0.9 % nacl (iso) intravenous piggyback',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='2424',description='Albuterol sulfate 2.5 mg/3 ml (0.083 %) solution for nebulization',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='25410',description='Dextrose 50 % in water (d50w) intravenous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='25735',description='Morphine 2 mg/ml injection syringe',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='2767',description='Polyethylene glycol 3350 17 gram oral powder packet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='31504',description='Metoprolol tartrate 25 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='37111',description='Vancomycin 1 gram/200 ml in dextrose 5 % intravenous piggyback',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='400001',description='Zz ims template',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='400284',description='Sodium chloride 0.9 % iv bolus',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='421',description='Furosemide 40 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='5171',description='Dextrose 40 % oral gel',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='5511',description='Sodium chloride 0.45 % intravenous solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='65572',description='Heparin (porcine) 25,000 unit/250 ml (100 unit/ml) in dextrose 5 % iv',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='689',description='Furosemide 10 mg/ml injection solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='7045',description='Clopidogrel 75 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='7307',description='Amlodipine 5 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='74103',description='Ondansetron hcl (pf) 4 mg/2 ml injection solution',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='76121',description='Magnesium sulfate 1 gram/50 ml in 0.9 % sodium chloride iv piggyback',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='7735',description='Aspirin 81 mg tablet,delayed release',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='7769',description='Perflutren lipid microspheres 1.1 mg/ml intravenous suspension',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='8098',description='Spironolactone 25 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='8267',description='Carvedilol 3.125 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='9554',description='Oxycodone-acetaminophen 5 mg-325 mg tablet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='95567',description='Glucagon (human recombinant) 1 mg/ml solution for injection',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='EPIC MEDICATION',code='9940',description='Potassium chloride 20 meq oral packet',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='TDS ALLERGY',code='12336-MEDS',description='Meds',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value'])|
Drug(context='TDS',code='01060-ACETAMINOPHEN TABS',description='Acetaminophen tabs',data_columns=['medical_record_number','age_in_days', 'context_name','MATERIAL_NAME','GENERIC_NAME','BRAND1','numeric_value']))
#extract the time series data
drug_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
# unique values and feature names: 
drug_time_series['material_name'].unique()
drug_time_series.to_parquet('Cohort/Time_Series/Drug_02.parquet')<jupyter_output><empty_output><jupyter_text>Diagnosis<jupyter_code>#non drug columns:
col_for_dropping=['age_in_days',
 'gender',
 'religion',
 'race',
 'medical_record_number',
 'patient_ethnic_group',
 'deceased_indicator',
 'mother_account_number',
 'address_zip',
 'date_of_birth',
 'month_of_birth',
 'marital_status_code']
# load Diagnosis dataframe: 
#get Medical Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Diagnosis_after_onset_HF_ALL_mmm_0_2_cleaned'
diagnosis = pq.read_table(df_path).to_pandas()
diagnosis=diagnosis.drop(col_for_dropping,axis=1)
diagnosis
#create diagnosis dictionary
diagnosis_renamed_column=rename_columns(diagnosis,col_ignore)

diagnosis_renamed_column
#get Drug Features for Time series
'''structure for diagnosis time series feature: 
conditions=(Diagnosis(context='ICD-10',
            code='D64.9',
            data_columns=['medical_record_number','description','age_in_days', 
            'context_name', 'context_diagnosis_code', 'numeric_value'])|)

'''
for n in diagnosis_renamed_column.items(): 
    parts=n[1].split('|')
    diagnosis_statement='Diagnosis(context=\''+str(parts[3]).strip()+'\',code=\''+str(parts[1]).strip()+'\',description=\''+str(parts[2].replace(' (time series) ','')).strip()+'\',data_columns=[\'medical_record_number\',\'age_in_days\',\'description\', \'context_name\',\'context_diagnosis_code\',\'numeric_value\'])|'
    #drug_statement='a'+n[1]
    print(diagnosis_statement)
    #print(parts[3])
#final Format Diagnosis Copied from the output above 
conditions=(Diagnosis(context='APRDRG MDC',code='005',description='Circulatory system diseases',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='D64.9',description='Anemia, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='E11.9',description='Type 2 diabetes mellitus without complications',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='E78.5',description='Hyperlipidemia, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I10',description='Essential (primary) hypertension',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I25.10',description='Atherosclerotic heart disease of native coronary artery without angina pectoris',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I48.91',description='Unspecified atrial fibrillation',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I50.20',description='Unspecified systolic (congestive) heart failure',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I50.22',description='Chronic systolic (congestive) heart failure',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='I50.9',description='Heart failure, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='N17.9',description='Acute kidney failure, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='R06.02',description='Shortness of breath',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='R07.9',description='Chest pain, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='R69',description='Illness, unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-10',code='Z23',description='Encounter for immunization',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='250.00',description='Diabetes mellitus without mention of complication type ii or unspecified type not stated as uncontrolled',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='272.0',description='Pure hypercholesterolemia',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='272.4',description='Other and unspecified hyperlipidemia',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='285.9',description='Anemia unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='401.9',description='Unspecified essential hypertension',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='403.90',description='Hypertensive chronic kidney disease unspecified with chronic kidney disease stage i through stage iv or unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='414.00',description='Coronary atherosclerosis of unspecified type of vessel native or graft',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='414.01',description='Coronary atherosclerosis of native coronary artery',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='416.8',description='Other chronic pulmonary heart diseases',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='424.0',description='Mitral valve disorders',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='425.4',description='Other primary cardiomyopathies',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='427.31',description='Atrial fibrillation',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='428.0',description='Congestive heart failure unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='428.20',description='Unspecified systolic heart failure',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='428.22',description='Chronic systolic heart failure',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='486',description='Pneumonia organism unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='584.9',description='Acute kidney failure unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='585.9',description='Chronic kidney disease unspecified',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='780.79',description='Other malaise and fatigue',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='786.05',description='Shortness of breath',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='786.50',description='Unspecified chest pain',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='I10',description='Not available',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='V04.81',description='Need for prophylactic vaccination and inoculation against influenza',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='V14',description='Personal history of allergy to medicinal agents',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='ICD-9',code='V15.82',description='Personal history of tobacco use',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='10026',description='Shortness of breath',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='113033',description='Chf (congestive heart failure)',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='114039',description='Hyperlipidemia',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='144791',description='Essential hypertension',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='154288',description='Htn (hypertension)',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='154321',description='Cad (coronary artery disease)',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='158530',description='Hypertension',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='163845',description='Essential (primary) hypertension',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='5107',description='Atrial fibrillation',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value'])|
Diagnosis(context='IMO',code='94485',description='Diabetes mellitus',data_columns=['medical_record_number','age_in_days','description', 'context_name','context_diagnosis_code','numeric_value']))
#extract the time series data
diagnosis_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
diagnosis_time_series
# unique values and feature names: 
diagnosis_time_series['description'].unique()
diagnosis_time_series.to_parquet('Cohort/Time_Series/Diagnosis_02.parquet')<jupyter_output><empty_output><jupyter_text>Procedures<jupyter_code>#non Procedure columns:
col_for_dropping=['age_in_days',
 'gender',
 'religion',
 'race',
 'medical_record_number',
 'patient_ethnic_group',
 'deceased_indicator',
 'mother_account_number',
 'address_zip',
 'date_of_birth',
 'month_of_birth',
 'marital_status_code']
# load Procedure dataframe: 
#get Procedure Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/Procedure_after_onset_HF_ALL_mmm_0_6'
procedures = pq.read_table(df_path).to_pandas()
procedures=procedures.drop(col_for_dropping,axis=1)
procedures
#create procedure dictionary
procedures_renamed_column=rename_columns(procedures,col_ignore)

procedures_renamed_column
#get Drug Features for Time series
'''structure for diagnosis time series feature: 
conditions=(Diagnosis(context='ICD-10',
            code='D64.9',
            data_columns=['medical_record_number','description','age_in_days', 
            'context_name', 'context_diagnosis_code', 'numeric_value'])|)

'''
for n in procedures_renamed_column.items(): 
    parts=n[1].split('|')
    procedures_statement='Procedure(context=\''+str(parts[3]).strip()+'\',code=\''+str(parts[1]).strip()+'\',description=\''+str(parts[2].replace(' (time series) ','')).strip()+'\',data_columns=[\'medical_record_number\',\'age_in_days\', \'context_name\',\'PROCEDURE_DESCRIPTION\',\'numeric_value\'])|'
    #drug_statement='a'+n[1]
    print(procedures_statement)
    #print(parts[3])
#Final Procedure Format, copied from above 
conditions=(Procedure(context='CAREFUSION LAB',code='AMY',description='Hemoglobin a1c',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='BMP',description='Basic metabolic panel',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='C&P',description='Cbc+platelet',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='CMP',description='Comprehensive metabolic panel',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='CPD',description='Cbc+plt+diff',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='CTP',description='Cbc+platelet',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='PO4',description='Phosphorus-blood',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='PRO',description='Prothrombin time',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='PT',description='Prothrombin time',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='PTPTT',description='Pt and aptt',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='TROI',description='Troponin i',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='TS-G',description='Type and screen',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CAREFUSION LAB',code='UCHEM',description='Urinalysis, routine',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='71010',description='Radiologic examination, chest; single view, frontal',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='71020',description='Radiologic examination, chest, 2 views, frontal and lateral;',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='80048',description='Basic metabolic panel (calcium, total) this panel must include the following: calcium, total (82310), carbon dioxide (82374), chloride (82435), creatinine (82565), glucose (82947), potassium (84132), sodium (84295), urea nitrogen (bun) (84520)',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='80053',description='Comprehensive metabolic panel this panel must include the following: albumin (82040), bilirubin, total (82247), calcium, total (82310), carbon dioxide (bicarbonate) (82374), chloride (82435), creatinine (82565), glucose (82947), phosphatase, alkaline (84075), potassium (84132), protein, total (84155), sodium (84295), transferase, alanine amino (alt) (sgpt) (84460), transferase, aspartate amino (ast) (sgot) (84450), urea nitrogen (bun) (84520)',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='81003',description='Urinalysis, by dip stick or tablet reagent for bilirubin, glucose, hemoglobin, ketones, leukocytes, nitrite, ph, protein, specific gravity, urobilinogen, any number of these constituents; automated, without microscopy',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='83735',description='Magnesium',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='84100',description='Phosphorus inorganic (phosphate);',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='84484',description='Troponin, quantitative',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='85025',description='Blood count; complete (cbc), automated (hgb, hct, rbc, wbc and platelet count) and automated differential wbc count',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='85027',description='Blood count; complete (cbc), automated (hgb, hct, rbc, wbc and platelet count)',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='85610',description='Prothrombin time;',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='85730',description='Thromboplastin time, partial (ptt); plasma or whole blood',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='CPT-4',code='93000',description='Electrocardiogram, routine ecg with at least 12 leads; with interpretation and report',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EKG',code='93000-ELECTROCARDIOGRAM, COMPLETE',description='Electrocardiogram, complete',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC CPT-4',code='100516',description='Type and screen',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='(RESP)',description='Respirations',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='(T-O)',description='Temperature, oral',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='(T-T)',description='Temperature, tympanic',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='119049',description='Type and screen',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6029',description='Basic metabolic panel',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6032',description='Comprehensive metabolic panel',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6104',description='Urinalysis, routine',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='64302',description='Pt and aptt',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6432',description='Phosphorus-blood',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='64914',description='X-ray chest 1 view portable',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='64924',description='X-ray chest pa and lateral only',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6509',description='Troponin i',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6550',description='Cbc+plt+diff',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6551',description='Cbc+platelet',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='6624',description='Prothrombin time',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='7509',description='Electrocardiogram, complete',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='BP SITE',description='Bp site',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='DBP',description='Diastolic blood pressure',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='HEIGHT',description='Height',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='PAIN SCALE',description='Pain scale',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='PULSE',description='Pulse',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='PULSE OXIMETRY',description='O2 saturation',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='R APACHE TEMPERATURE',description='R apache temperature',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='R IP PAIN SCALE ADULT',description='R ip pain scale adult',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='SBP',description='Systolic blood pressure',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='TEMPERATURE',description='Temperature',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])|
Procedure(context='EPIC',code='WEIGHT',description='Weight',data_columns=['medical_record_number','age_in_days', 'context_name','PROCEDURE_DESCRIPTION','numeric_value'])
)
#extract the time series data
procedures_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
procedures_time_series
procedures_time_series['procedure_description'].unique()
procedures_time_series.to_parquet('Cohort/Time_Series/Procedures_06.parquet')<jupyter_output><empty_output><jupyter_text>Vital Signs<jupyter_code>#non vital signs  columns:
col_for_dropping=[
 'medical_record_number',
    'median__VitalSign__EPIC__(RESP)',
'median__VitalSign__EPIC__(T-O)',
'median__VitalSign__EPIC__(T-T)',
'median__VitalSign__EPIC__DBP',
'median__VitalSign__EPIC__PULSE',
'median__VitalSign__EPIC__PULSE OXIMETRY',
'median__VitalSign__EPIC__SBP',
'median__VitalSign__EPIC__TEMPERATURE',
'min__VitalSign__EPIC__(RESP)',
'min__VitalSign__EPIC__(T-O)',
'min__VitalSign__EPIC__(T-T)',
'min__VitalSign__EPIC__DBP',
'min__VitalSign__EPIC__PULSE',
'min__VitalSign__EPIC__PULSE OXIMETRY',
'min__VitalSign__EPIC__SBP',
'min__VitalSign__EPIC__TEMPERATURE'
 ]
# load Vital Sign  dataframe: 
#get Vital Sign Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/VitalSign_after_onset_HF_ALL_mmm_0_6_cleaned'
vital = pq.read_table(df_path).to_pandas()
vital=vital.drop(col_for_dropping,axis=1)
vital
further_col_drop=[]
for c in vital.columns: 
    if ('max' not in c) : 
        print(c)
        further_col_drop.append(c)
further_col_drop
#create Vital Signs dictionary
vital_renamed_column=rename_columns(vital,col_ignore)

vital_renamed_column
#get Vital Signs Features for Time series
'''structure for diagnosis time series feature: 
conditions=(Diagnosis(context='ICD-10',
            code='D64.9',
            data_columns=['medical_record_number','description','age_in_days', 
            'context_name', 'context_diagnosis_code', 'numeric_value'])|)

'''
for n in vital_renamed_column.items(): 
    parts=n[1].split('|')
    vital_statement='VitalSign(context=\''+str(parts[3]).strip()+'\',code=\''+str(parts[1]).strip()+'\',description=\''+str(parts[2].replace(' (max) ','')).strip()+'\')|'
    #drug_statement='a'+n[1]
    print(vital_statement)
    #print(parts[3])
#Final Vital Signs Format copied from above
conditions=(
VitalSign(context='EPIC',code='(RESP)',description='Respirations')|
VitalSign(context='EPIC',code='(T-O)',description='Temperature, oral')|
VitalSign(context='EPIC',code='(T-T)',description='Temperature, tympanic')|
VitalSign(context='EPIC',code='DBP',description='Diastolic blood pressure')|
VitalSign(context='EPIC',code='PULSE',description='Pulse')|
VitalSign(context='EPIC',code='PULSE OXIMETRY',description='O2 saturation')|
VitalSign(context='EPIC',code='SBP',description='Systolic blood pressure')|
VitalSign(context='EPIC',code='TEMPERATURE',description='Temperature') )
#extract the time series data
vital_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
vital_time_series
# unique values and feature names: 
vital_time_series['context_procedure_code'].unique()
vital_time_series.loc[vital_time_series['medical_record_number']=='1000133708']
vital_time_series.to_parquet('Cohort/Time_Series/VitalSign_06.parquet')<jupyter_output><empty_output><jupyter_text>LabValues<jupyter_code>#non LabValues columns:
col_for_dropping=[
 'median__LabValue__ALBUMIN, BLD',
 'median__LabValue__ALK PHOSPHATASE, BLD',
 'median__LabValue__ALT(SGPT)',
 'median__LabValue__APTT',
 'median__LabValue__AST (SGOT)',
 'median__LabValue__ATRIAL RATE',
 'median__LabValue__BASOPHIL %',
 'median__LabValue__BILIRUBIN TOTAL',
 'median__LabValue__CALCIUM, BLD',
 'median__LabValue__CARBON DIOXIDE-BLD',
 'median__LabValue__CHLORIDE-BLD',
 'median__LabValue__CREATININE-SERUM',
 'median__LabValue__EOSINOPHIL #',
 'median__LabValue__EOSINOPHIL %',
 'median__LabValue__GLUCOSE',
 'median__LabValue__HEMATOCRIT',
 'median__LabValue__HEMOGLOBIN',
 'median__LabValue__INR',
 'median__LabValue__LYMPHOCYTE #',
 'median__LabValue__LYMPHOCYTE %',
 'median__LabValue__MAGNESIUM-BLD',
 'median__LabValue__MEAN CORP. HGB',
 'median__LabValue__MEAN CORP. HGB CONC.',
 'median__LabValue__MEAN CORP. VOLUME',
 'median__LabValue__MEAN PLT VOLUME',
 'median__LabValue__MONOCYTE #',
 'median__LabValue__MONOCYTE %',
 'median__LabValue__NEUTROPHIL #',
 'median__LabValue__NEUTROPHIL %',
 'median__LabValue__P AXIS',
 'median__LabValue__P-R INTERVAL',
 'median__LabValue__PLATELET',
 'median__LabValue__POTASSIUMBLD',
 'median__LabValue__PRO TIME',
 'median__LabValue__PROTEIN TOTAL-BLD',
 'median__LabValue__QRS DURATION',
 'median__LabValue__QT',
 'median__LabValue__QTC',
 'median__LabValue__R AXIS',
 'median__LabValue__RBC BLOOD CELL',
 'median__LabValue__RED DISTRIB. WIDTH',
 'median__LabValue__SODIUM-BLD',
 'median__LabValue__T AXIS',
 'median__LabValue__UREA NITROGEN-BLD',
 'median__LabValue__VENTRICULAR RATE',
 'median__LabValue__WHITE BLOOD CELL',
 'min__LabValue__ALBUMIN, BLD',
 'min__LabValue__ALK PHOSPHATASE, BLD',
 'min__LabValue__ALT(SGPT)',
 'min__LabValue__APTT',
 'min__LabValue__AST (SGOT)',
 'min__LabValue__ATRIAL RATE',
 'min__LabValue__BILIRUBIN TOTAL',
 'min__LabValue__CALCIUM, BLD',
 'min__LabValue__CARBON DIOXIDE-BLD',
 'min__LabValue__CHLORIDE-BLD',
 'min__LabValue__CREATININE-SERUM',
 'min__LabValue__GLUCOSE',
 'min__LabValue__HEMATOCRIT',
 'min__LabValue__HEMOGLOBIN',
 'min__LabValue__INR',
 'min__LabValue__LYMPHOCYTE #',
 'min__LabValue__LYMPHOCYTE %',
 'min__LabValue__MAGNESIUM-BLD',
 'min__LabValue__MEAN CORP. HGB',
 'min__LabValue__MEAN CORP. HGB CONC.',
 'min__LabValue__MEAN CORP. VOLUME',
 'min__LabValue__MEAN PLT VOLUME',
 'min__LabValue__MONOCYTE #',
 'min__LabValue__MONOCYTE %',
 'min__LabValue__NEUTROPHIL #',
 'min__LabValue__NEUTROPHIL %',
 'min__LabValue__P AXIS',
 'min__LabValue__P-R INTERVAL',
 'min__LabValue__PLATELET',
 'min__LabValue__POTASSIUMBLD',
 'min__LabValue__PRO TIME',
 'min__LabValue__PROTEIN TOTAL-BLD',
 'min__LabValue__QRS DURATION',
 'min__LabValue__QT',
 'min__LabValue__QTC',
 'min__LabValue__R AXIS',
 'min__LabValue__RBC BLOOD CELL',
 'min__LabValue__RED DISTRIB. WIDTH',
 'min__LabValue__SODIUM-BLD',
 'min__LabValue__T AXIS',
 'min__LabValue__UREA NITROGEN-BLD',
 'min__LabValue__VENTRICULAR RATE',
 'min__LabValue__WHITE BLOOD CELL',
 'medical_record_number'
 ]
# load Lab Values dataframe: 
#get Lab Values Features for Timeseries Extraction: 
df_path='Cohort/Feature_Extraction/Unsupervised_ALL_HF/LabValue_after_onset_HF_ALL_mmm_0_8_cleaned'
lab = pq.read_table(df_path).to_pandas()
lab=lab.drop(col_for_dropping,axis=1)
lab
further_col_drop=[]
for c in lab.columns: 
    if ('max' not in c) : 
        print(c)
        further_col_drop.append(c)
further_col_drop
#create lab values dictionary
lab_renamed_column=rename_columns(lab,col_ignore)

lab_renamed_column
#get lab Values Features for Time series
'''structure for diagnosis time series feature: 
conditions=(Diagnosis(context='ICD-10',
            code='D64.9',
            data_columns=['medical_record_number','description','age_in_days', 
            'context_name', 'context_diagnosis_code', 'numeric_value'])|)

'''
for n in lab_renamed_column.items(): 
    parts=n[1].split('|')
    lab_statement='LabValue(\''+str(parts[2].replace(' (max) ','')).strip()+'\')|'
    #drug_statement='a'+n[1]
    print(lab_statement)
    #print(parts[3])
#Final Format for LabValues Copied from output above
conditions=(LabValue('Albumin, bld')|
LabValue('Alk phosphatase, bld')|
LabValue('Alt(sgpt)')|
LabValue('Aptt')|
LabValue('Ast (sgot)')|
LabValue('Atrial rate')|
LabValue('Basophil #')|
LabValue('Basophil %')|
LabValue('Bilirubin total')|
LabValue('Calcium, bld')|
LabValue('Carbon dioxide-bld')|
LabValue('Chloride-bld')|
LabValue('Creatinine-serum')|
LabValue('Eosinophil #')|
LabValue('Eosinophil %')|
LabValue('Glucose')|
LabValue('Hematocrit')|
LabValue('Hemoglobin')|
LabValue('Inr')|
LabValue('Lymphocyte #')|
LabValue('Lymphocyte %')|
LabValue('Magnesium-bld')|
LabValue('Mean corp. hgb')|
LabValue('Mean corp. hgb conc.')|
LabValue('Mean corp. volume')|
LabValue('Mean plt volume')|
LabValue('Monocyte #')|
LabValue('Monocyte %')|
LabValue('Neutrophil #')|
LabValue('Neutrophil %')|
LabValue('P axis')|
LabValue('P-r interval')|
LabValue('Platelet')|
LabValue('Potassiumbld')|
LabValue('Pro time')|
LabValue('Protein total-bld')|
LabValue('Qrs duration')|
LabValue('Qt')|
LabValue('Qtc')|
LabValue('R axis')|
LabValue('Rbc blood cell')|
LabValue('Red distrib. width')|
LabValue('Sodium-bld')|
LabValue('T axis')|
LabValue('Urea nitrogen-bld')|
LabValue('Ventricular rate')|
LabValue('White blood cell'))
#conditions=(LabValue('White blood cell'))
#extract the time series data
lab_time_series=cohort_EF_Baseline.time_series_for(
        conditions
    )
lab_time_series
for n in lab_time_series['test_name'].unique(): 
    print(n)
lab_time_series.to_parquet('Cohort/Time_Series/LabValues_08.parquet')
# unique values and feature names: 
t=lab_time_series['test_name'].unique()
for n in t : 
    print(n)<jupyter_output><empty_output><jupyter_text>Snips<jupyter_code>import math
import os
import sys

import fiber
import numpy as np
import pandas as pd
import pyarrow.parquet as pq

# from fiber import fiber
from fiber.cohort import Cohort
from fiber.condition import (
    Diagnosis,
    Drug,
    Encounter,
    LabValue,
    Measurement,
    MRNs,
    Patient,
    Procedure,
    TobaccoUse,
    VitalSign,
)
from fiber.storage import yaml as fiberyaml
from fiber.utils import Timer
from fiberutils import cohort_utils


from typing import Optional
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import venn
from fiber import Cohort
from fiber.condition import MRNs
from fiber.config import OCCURRENCE_INDEX, VERBOSE
from fiber.dataframe import (
    column_threshold_clip,
    create_id_column,
    merge_to_base,
    time_window_clip
)
from fiber.plots.distributions import bars
from fiberutils.time_series_utils import ssr_transform
from fiber.condition import Patient, MRNs
from fiber.condition import Diagnosis
from fiber.condition import Measurement, Encounter, Drug, TobaccoUse,LabValue, VitalSign, Procedure
from fiber.storage import yaml as fiberyaml
#from fiberutils.cohort_utils import get_time_series, pivot_time_series
import pandas as pd
import pyarrow.parquet as pq
import numpy as np
import os
import matplotlib.pyplot as plt
from functools import reduce
import json
import math
import inspect
import json
import sys
from pydoc import locate
import requests
#read dataframe
mrn_table = pq.read_table(
    "Cohort/Feature_Extraction/ALL_HF_cohort_unsupervised_only_after_onset_HF_ALL_all_any_all_mean_small_cleaned.parquet").to_pandas()
mrn_table.reset_index(level=0, inplace=True)
mrn = mrn_table[["medical_record_number", "HF_Onset_age_in_days"]]
# mrn_cond = MRNs(mrns=(mrn))
# cohort = Cohort(mrn_cond)
mrn=mrn.head(10000)
mrn<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15020308692120227, 'fraction_numerical': 0.018147847278635255, 'mean_word_length': 13.172461432189731, 'pattern_counts': {'":': 1, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '491025', 'license': 'MIT', 'n_tokens_mistral': 21887, 'n_tokens_neox': 19907, 'n_words': 2600, 'path': 'Feature_Extraction/Time_Series/Extract_Time_Series_Data.ipynb', 'repo_name': 'Melhac/Subtyping_of_HF_Master_Thesis', 'size': '61550'}