<jupyter_start><jupyter_text>ECG Patient Medications Goals:    - pull PatientIDs and medications from EDW    - compare with medications from internal database    - pull medications for all ECG patients from internal database<jupyter_code>import os
import glob
import numpy as np

import pandas as pd
from ehr.edw import Epic
from ehr.edw import ExternalIdentity
from dotenv import load_dotenv

from ehr.ehr_db import EhrDb

%load_ext autoreload
%autoreload 2
seed = 123

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

# SQL Alchemy to access local database
from sqlalchemy import create_engine
# Files and directories
data_root = os.path.normpath('/mnt/obi0/andreas/data/ecg')
parquet_dir = os.path.join(data_root, 'parquet')

# Database access credentials
dotenv_file = os.path.normpath('/mnt/obi0/andreas/config/credentials.env')
load_dotenv(dotenv_file)

mgh_idtype = ExternalIdentity.MGHMRN

# Epic instance
def create_epic():
    epic = Epic(edw_user=os.environ['EDW_USER'],
                edw_password=os.environ['EDW_PASSWORD'],
                db='PHS',
                out_dir=data_root)
    return epic
# Concatenate all MGH ECG files
concatenated_parquet_file_name = 'MGH_RAW_FROM2001.parquet'
file_list = sorted(glob.glob(os.path.join(parquet_dir, 'MGH_RAW_meta_2*.parquet')))
def concatenate_parquet(file_list):
    df_list = []
    for f, file in enumerate(file_list):
        print(f'Loading file {f+1} of {len(file_list)}: {os.path.basename(file)}')
        df_list.append(pd.read_parquet(file))
    df = pd.concat(df_list, ignore_index=True).reset_index(drop=True)
    return df

# Load the data
#df = concatenate_parquet(file_list)
#df.to_parquet(os.path.join(parquet_dir, concatenated_parquet_file_name))
df = pd.read_parquet(os.path.join(parquet_dir, concatenated_parquet_file_name))
print(f'Loaded {len(df.file.unique())} ECGs from {len(df.PatientID.unique())} mrns.')
# Rename PatientID column: it is really an mrn
df = df.rename(columns={'PatientID': 'mrn'})
df.head()
# Get a sample
np.random.seed(seed)
df_sample = df.sample(n=50).reset_index(drop=True)
sample_mrn_list = list(df_sample.mrn.unique())
print(*sample_mrn_list)
epic = create_epic()
df_id = epic.patientids_from_external(external_patient_list=sample_mrn_list, 
                                      external_identity=mgh_idtype)
epic.close()
patient_id_list = list(df_id.PatientID.unique())
df_id.head(2)
chunk_sizes = {'patientids': 1000,
               'medications': 1000}

epic = create_epic()
fetch_dict = epic.fetch(query_ids=patient_id_list,
                        chunk_sizes=chunk_sizes)
epic.close()
# Development database
db = EhrDb(user = os.environ['EHR_DEV_USER'],
           password = os.environ['EHR_DEV_PASSWORD'],
           host='obi-cpu8',
           port='5432',
           db='ehr_dev_andreas')
# Load some data into the test database
chunk_sizes = {'patientids': 1000,
               'demographics': 1000,
               'medications': 1000}
               
epic = create_epic()
db.import_epic(name='MGH ECG Patient Ids',
               description='MGH MRNs mapped to PatientIDs for the MGH ECG dataset (/mnt/obi0/phi/ecg/MGH_RAW_all_mrn.parquet)',
               protocol='Novel Phenotyping',
               query_ids=patient_id_list,
               chunk_sizes=chunk_sizes,
               epic=epic)
epic.close()
chunk_sizes = {'reference_medication': 1000}
epic = Epic(edw_user=os.environ['EDW_USER'],
            edw_password=os.environ['EDW_PASSWORD'],
            db='PHS',
            out_dir=data_root)
df = epic.reference_medication()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11284153005464481, 'fraction_numerical': 0.015027322404371584, 'mean_word_length': 3.836195508586526, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '231381', 'license': 'MIT', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1254, 'n_words': 245, 'path': 'notebooks/ecg_patient_med.ipynb', 'repo_name': 'awerdich/edwdb', 'size': '3660'}