<jupyter_start><jupyter_text>Exploratory Data Analysis and Data Preparation Medinfo 2019 Data Science Workshop August 26, 2019 Content Development: <NAME>, <NAME>, <NAME>, <NAME><jupyter_code># Import required modules
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
import matplotlib as mplot
%matplotlib inline
import IPython
from IPython.core.display import HTML
from IPython.core.debugger import set_trace
from distutils.version import StrictVersion
print("numpy version:  %s" % np.__version__)
print("pandas version:  %s" % pd.__version__)
print("matplotlib version:  %s" % mplot.__version__)
print("IPython version:  %s" % IPython.__version__)
print("seaborn version:  %s" % sns.__version__)

if StrictVersion(np.__version__) >= StrictVersion('1.13.0') and \
   StrictVersion(pd.__version__) >= StrictVersion('0.20.0') and \
   StrictVersion(mplot.__version__) >= StrictVersion('2.0.0') and \
   StrictVersion(IPython.__version__) >= StrictVersion('5.5.0') and \
   StrictVersion(sns.__version__) >= StrictVersion('0.7.0'):
    print('\nCongratulations, your environment is setup correctly!')
else:
    print('\nEnvironment is NOT setup correctly!')

# Try to install the Excel reader library (its not pre-installed on Colab)
try:
    import xlrd
    print('The Excel library is installed.')
except ImportError:
    print('Installing the Excel library')
    !pip install xlrd
    import xlrd
# Try to install the Bokeh library (its not pre-installed on Colab)
try:
    import bokeh
    print('The Bokeh library is installed.')
except ImportError:
    print('Installing the Bokeh library')
    !pip install bokeh
    import bokeh<jupyter_output><empty_output><jupyter_text>1.1 Check data directorySee if the data exists.  If not, try to download it from github.  1.0 SetupEnsure that your Jupyter environment is setup correctly and import all of the data science libraries that we will need.  If some modules are missing, we will attempt to install the library but it is usually a better practice to install it in your environment directly.Also, if the data is missing, we will attempt to download it (from github) and put it in the "/data_oh" subdirectory of your current working directory (Ohio synthetic data).<jupyter_code># Find the data directory and download data if it is missing

import os, shutil
cwd = os.getcwd()
datadir = cwd + '/data_oh'

print('Data directory is: {}'.format(datadir))
# See if the data exists.  If not, try to download it from github.
if not os.path.exists(datadir+'/patients.csv'):
    print("Data directory doesn't exist!")
    print("Checking out the data from github...")

    !git clone https://github.com/joh06288/Medinfo2019.git
        
    # Move the checked-out files into the /data directory
    files = os.listdir('Medinfo2019')
    for f in files:
        print('Moving %s...' % (f,))
        try:
            shutil.move('Medinfo2019/'+f,'.')
        except:
            print("    Unable to move %s" % (f,))
            
    try:
        shutil.rmtree('Medinfo2019')  # Remove the version control (git) information
    except:
        pass  # Ignore errors.  On Windows, this sometimes fails and leaves the .git directory
print('Data directory contains:\n',os.listdir(datadir))<jupyter_output><empty_output><jupyter_text>2.0 Read in the dataNow that we have all the libraries installed and the data is available, lets try to read it into Pandas DataFrames.  The first thing we will do is define a Data Dictionary (dd) that describes our expectations for the data.  It includes data types for the columns as well is information about whether the column is required or optional.  The data is read into a dictionary of Dataframes (data) and also assigned to  variables (patients, encounters, etc) for convenience.We will convert dates and other fields to the proper format when later when we do data preparation.The dataset is synthetic data generated by the Synthea project (https://github.com/synthetichealth/synthea).  Synthea creates realistic (but not real) EHR-like data that we can use for demonstrating the techniques of data science.<jupyter_code># Read in the data
dd = {}

dd['patients'] = {'pat_id':     {'type': np.str, 'required':True},  
                  'birth_date': {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':True},
                  'death_date': {'type': np.datetime64, 'format': '%Y-%m-%d' }, 
                  'ssn':        {'type': np.str},
                  'drivers':    {'type': np.str},
                  'passport':   {'type': np.str},
                  'prefix':     {'type': np.str},
                  'first':      {'type': np.str, 'required':True},
                  'last':       {'type': np.str, 'required':True},
                  'suffix':     {'type': np.str},
                  'maiden':     {'type': np.str},
                  'marital':    {'type': np.str},
                  'race':       {'type': np.str},
                  'ethnicity':  {'type': np.str},
                  'gender':     {'type': np.str, 'required':True},
                  'birthplace': {'type': np.str},
                  'address':    {'type': np.str, 'required':True},
                  'prior_opioid_abuse_diag': {'type': np.int}
                  }
dd['encounters'] = {'enc_id':                 {'type': np.str, 'required':True}, 
                    'enc_date':               {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':True},
                    'enc_pat_id':             {'type': np.str, 'required':True},
                    'enc_code':               {'type': np.str, 'required':True},
                    'enc_description':        {'type': np.str, 'required':True},
                    'enc_reason_code':        {'type': np.str},
                    'enc_reason_description': {'type': np.str}
                   }
dd['observations'] = {'obs_date':        {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':True},
                      'obs_pat_id':      {'type': np.str, 'required':True},
                      'obs_enc_id':      {'type': np.str, 'required':True},
                      'obs_code':        {'type': np.str, 'required':True},
                      'obs_description': {'type': np.str, 'required':True},
                      'obs_value':       {'type': np.str},
                      'obs_units':       {'type': np.str}
                     }
dd['medications'] = {'med_start_date':         {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':True},
                     'med_stop_date':          {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':False},
                     'med_pat_id':             {'type': np.str, 'required':True},
                     'med_enc_id':             {'type': np.str, 'required':True},
                     'med_code':               {'type': np.str, 'required':True},
                     'med_description':        {'type': np.str, 'required':True},
                     'med_reason_code':        {'type': np.str},
                     'med_reason_description': {'type': np.str},
                     'med_days_supply':        {'type': np.int}
                     }
dd['conditions'] =  {'cond_start_date':         {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':True},
                     'cond_stop_date':          {'type': np.datetime64, 'format': '%Y-%m-%d', 'required':False},
                     'cond_pat_id':             {'type': np.str, 'required':True},
                     'cond_enc_id':             {'type': np.str, 'required':True},
                     'cond_code':               {'type': np.str, 'required':True},
                     'cond_description':        {'type': np.str, 'required':True}
                     }


# Display the data dictionary
# Use HTML to make it look a little nicer

for name, tbl_dd in dd.items():
    display(HTML('<h2>{}</h2>'.format(name)))
    display(pd.DataFrame(tbl_dd).fillna('').T)
# Loop through each of the file defintions in our data dictionary
data = {}
for f in dd:
    m = dd[f]
    col_names = list(m.keys())
    data[f] = pd.read_csv(datadir + '/{}.csv'.format(f), dtype=str, index_col=False, header=0, \
                          names=col_names, keep_default_na=False)

# Assign data to local variables for convenience
patients = data['patients']
encounters = data['encounters']
observations = data['observations']
medications = data['medications']
conditions = data['conditions']
# Display the first 3 patients
patients.head(3)<jupyter_output><empty_output><jupyter_text>Exercise: Display the last 3 patientsHint: the "tail" method displays the last part of a dataframe<jupyter_code># Display the last 3 patients
#???<jupyter_output><empty_output><jupyter_text>2.1 Load the list of opioid medicationsIt will be important to know which of the medications that are prescribed are considered opioids.  The UMLS VSAC maintains value set lists of which medications are considered opioids.  You can download the current list by going to https://vsac.nlm.nih.gov/ and searching for opioid value sets.  We have downloaded the list called "All prescribable opioids used for pain control including Inactive Medications" (oid 1.3.6.1.4.1.6997.4.1.2.234.999.3.2) into the data directory.  We will read the Excel file and pull out the codes from the second sheet in the file.<jupyter_code># Get Opioid code list from VSAC
# oid 1.3.6.1.4.1.6997.4.1.2.234.999.3.2
xl = pd.ExcelFile(datadir + '/AllPrescribableOpioidsUsedForPainControlIncludingInactiveMedications.xlsx')
df = xl.parse("Code List", skiprows=12)
display(df.head(10))
opioids_rxnorm = list(df['Code'].astype(np.str))  # Make sure the codes are treated as strings<jupyter_output><empty_output><jupyter_text>3.0 Exploratory Data Analysis - Part 1Start by looking at the data to see what types of values are in each variables, the relationships and get a feel for the data.  3.1 Start by displaying the data as DataFramesDisplaying the first few rows of the data is a good way to look for obvious issues before working with the data in more detail.<jupyter_code># Loop through each of the file defintions in our data dictionary
for f in dd:
    m = dd[f]
    display(HTML('<h3>{}, {} records</h3>'.format(f,len(data[f]))))
    display(data[f].head(5))
# You can also display the last few rows using the .tail() function.
patients.tail(5)<jupyter_output><empty_output><jupyter_text>3.2 Categorical Variables<jupyter_code># Look at the categorical variables

sns.countplot(x='race', data=patients)
plt.title('Race')
plt.show()

sns.countplot(x='obs_code',data=observations)
plt.title('Observation Codes')
plt.show()
# The observations are hard to read, lets try a bar plot
c = observations['obs_description'].value_counts(ascending=True)
fig = plt.figure(figsize=(8,24))
c.plot(kind="barh")<jupyter_output><empty_output><jupyter_text>3.3 Continuous Variables<jupyter_code># We can get statistics for the continuous variables using the .describe() function
patients.describe()
# For continuous variables, we can graph the distribution

w = observations[observations['obs_code']=='29463-7']   # Find all of the "weight" observations
weights = w['obs_value'].astype(np.float).dropna()
mean = np.mean(weights)
print('Average weight: ',mean)

# Plot the distribution
sns.kdeplot(weights)
plt.xlabel("WEIGHT (kg)")
plt.show()   # If you don't explicitly "show" the plot, Jupyter will automatically show the last plot<jupyter_output><empty_output><jupyter_text>Use Sort and DisplayThis graph looks a little odd.  We have 2 peaks which are expected.  This corresponds to the average weight of a child vs average weight of an adult.  However, we have some values that seem to go all the way to 1400 Kg.  That doesn't seem right.  Let's take a closer look.<jupyter_code># Show the highest 20 weights
print(weights.sort_values(ascending=False).head(20))<jupyter_output><empty_output><jupyter_text>Drop outliersIt doesn't seem reasonable that someone weighs 1321.8 Kg.  So let's drop anything above 500 Kg.  We will use the Pandas Boolean Indexing to create a filter.  You can specify an arbitrary condition which is applied to each row in the DataFrame and only returns those rows that satisfy the conditional.<jupyter_code>weights = weights[weights <= 500]
print(weights.sort_values(ascending=False).head(20))
# Let's re-plot the distribution
sns.kdeplot(weights, shade=True, color='blue')
plt.xlabel("WEIGHT (kg)")
plt.show()<jupyter_output><empty_output><jupyter_text>Exercise:  Graph the Height of Patients and color it RedHint: LOINC code for height is 8302-2<jupyter_code># As an Exercise, graph the Height of patients

h = observations[observations['obs_code']=='xxxx']   
heights = h['obs_value'].astype(np.float).dropna()
#sns.kdeplot(heights,shade=True,color='???')
plt.show()<jupyter_output><empty_output><jupyter_text>4.0 Data Preparation Now that we know a little about our data, we can begin to prepare it for analysis.  Understanding Data Quality is important before you can start working with your data. Data Quality Dimensions- Correctness – Does the data represent a truth about the world? - Birth_date = “1/3/1990”- Completeness – Is data missing? - Death_date = NULL- Consistency – Does data conform to expectations / rules? - Birth_date = “1/3/1990”, death_date = “1/2/1990”  - Rule: birth_date must be <= death_date- Currency – Is the data timely and up-to-date? - Data updated at end of shift?  Nightly load into CDR?  Document data expectations  Required data quality depends on fitness-for-use- Count patients vs count diabetic patients with controlled A1c Dealing with Missing Data Decide on consistent way to handle missing data- Remove “rows” with missing values- Fill in missing values - Constant (i.e. ”0”) - Use the Mean - Multiple Imputation / Regression - Last value- Missingness may be meaningful - MCAR – missingness unrelated to the variable  - The lab lost a sample so results aren’t reported - MAR – missingness related to another variable  - Female patients in pain may under report vs Male patients - MNAR – missingness related to the variable  - Patients with severe depression may not answer survey depression questions For this workshop, we will address Data Quality issues by:1. Find data that is not formatted correctly2. Deal with missing dataIn all of these cases, we will have to decide what to do with the bad data.  We can:1. Delete the data2. Impute a reasonable value for the missing data<jupyter_code># Find data that is not formatted correctly

def parse_date(dt,fmt):
    if type(dt) == str and (dt == ''):
        return dt
    try:
        return pd.to_datetime(dt,format=fmt)
    except:
        return np.datetime64('NaT')

def parse_int(num):
    if type(num) == str and (num == ''):
        return num
    try:
        return int(num)
    except:
        return np.nan

# Loop through our Data Dictionary
for name, tbl_dd in dd.items():
    display(HTML('<h2>{}</h2>'.format(name)))
    d = data[name]
    for field_name, field in tbl_dd.items():
        col = d[field_name]
        field['DQ'] = {}
        field['DQ']['missing'] = len(np.where(col == '')[0])

        if field['type'] == np.datetime64:
            if 'format' in field:
                fmt = field['format']
            else:
                fmt = '%Y-%m-%d'   # Default date format if not specified
                
            d[field_name] = col.apply(lambda x: parse_date(x,fmt))
            field['DQ']['format_errors'] = col.isnull().sum()
        elif field['type'] == np.int:
            d[field_name] = col.apply(lambda x: parse_int(x))
            field['DQ']['format_errors'] = col.isnull().sum()
        elif field['type'] == np.str:
            pass # Everything is valid syntax
            field['DQ']['format_errors'] = 0
            
    # Show the Data Quality information
    display(pd.DataFrame(dd[name]))<jupyter_output><empty_output><jupyter_text>4.2 Drop unwanted columns<jupyter_code># Let's remove some of the data that we won't need to use for the workshop
# It will make some of the screens easier to read
# Put it in a try block in case we've already dropped the columns
try:
    patients.drop(['maiden','passport','drivers','prefix','suffix','ssn','first','last'],axis=1,inplace=True)
except:
    pass
display(patients.head(1))<jupyter_output><empty_output><jupyter_text>4.3 Remove rows with missing dataFor this workshop, we will only deal with missing data by dropping it.  If a row has missing or bad formatted data and the field is required, we will drop the entire row.An alternative is to try to fix the data by imputing a reasonable value for it, such as the column .mean().<jupyter_code># The .isnull() functions are used to find bad data
# The .any() function returns the columns that contain any True values

display(patients.isnull().head(5))
display(patients.isnull().any())

# Let's get a list of all of the columns with some missing data 
missing_cols=patients.columns[patients.isnull().any()]
print(missing_cols)

# We can see how many cells have missing data for each column
patients[missing_cols].isnull().sum()

# Drop missing or incorrectly formatted data for the required patient data fields

# Get the row numbers (index) of each of the rows with missing data
missing = patients[patients.isnull().any(axis=1)].index
print("Missing = ",missing)
print('Before patients shape = ',patients.shape)
patients.drop(missing,inplace=True)

# Make sure the rows are gone
missing = patients[patients.isnull().any(axis=1)].index
print("Missing = ",missing)
print('After patients shape = ',patients.shape)<jupyter_output><empty_output><jupyter_text>4.4 Transform the DataUse the power of Pandas Dataframes to transform the data.  Add new columns as calculations from existing columns, join the data together and get it into the format you need for analysis.<jupyter_code># Create the working dataframe

df = patients
df['age'] = round((pd.Timestamp.today() - pd.to_datetime(patients['birth_date'])).dt.days/365)
df['adult'] = np.where(df['age'] >= 18, 1, 0)

# Determine which patients have ever overdosed
# Use a set to eliminate duplicates
patients_that_overdosed = set(encounters[encounters['enc_reason_code']=='55680006']['enc_pat_id'])  # Overdose
df['overdose'] = np.where(df['pat_id'].isin(patients_that_overdosed), 1, 0)

# Determine which patients were ever prescribed opioids
patients_prescribed_opioids = set(medications[medications['med_code'].isin(opioids_rxnorm)]['med_pat_id'])  # Opioids
df['prescribed_opioids'] = np.where(df['pat_id'].isin(patients_prescribed_opioids), 1, 0)

print('Num patients prescribed opioids = {}, Num overdoses = {}'
         .format(len(patients_prescribed_opioids),len(patients_that_overdosed)))
# Determine which patients have died from an overdose
# Uses binary indexing
obs = observations[(observations['obs_code'] == '69453-9') &   # Death
                                (observations['obs_value'].str.contains('overdose'))]
print('Example of an overdose death observation:')
display(pd.DataFrame(obs.iloc[0,:]))
patients_overdose_deaths = set(obs['obs_pat_id'])
print('Number of overdose deaths = {}'.format(len(patients_overdose_deaths)))
print('Here are the overdose deaths:')
display(df[df['pat_id'].isin(patients_overdose_deaths)])<jupyter_output><empty_output><jupyter_text>4.5 Compute the days_supply variableWe want to compute how many days supply of a medication a patient was prescribed at discharge.  The approach we will use is that for each encounter, we will find all of the medications associated with the encounter.  We will look for medications that are opioids and find the largest days supply for that encounter and store the result in the 'opioid_discharge_days_supply' column.This is most easily accomlished using a function that defines the logic and the ".apply" DataFrame function the will iterate over each row in a DataFrame, call the function and store the result back in the DataFrame.<jupyter_code># Define the function that will perform to logic of compute the discharge opioid days supply

def get_days_supply(pat_id):
    enc_meds = medications[medications['med_pat_id'] == pat_id]
    enc_opioid_meds = enc_meds[enc_meds['med_code'].isin(opioids_rxnorm)]
    max = 0
    if len(enc_opioid_meds) > 0:
        try:
            max = int(enc_opioid_meds['med_days_supply'].max())
        except ValueError:
            max = 0
    return int(max)

# Apply the function to each row (Note: this can take a little while to finish)
df['opioid_discharge_days_supply'] = df.apply(lambda x: get_days_supply(x['pat_id']), axis=1)

# Display the first 5 entries that have a non-zero days supply, just to check our logic
df[df['opioid_discharge_days_supply'] > 0].head(5)<jupyter_output><empty_output><jupyter_text>4.6 Save our Clean DataIt is a good practice to save your clean data so that you don't have to keep re-cleaning it everytime you want to use it.  Pandas (and Python) provides a function to save a variable in a special format that can be easily recreated later.  This is called 'pickling' in Python.  So we will save our cleaned DataFrame, df, to be used by subsequent notebooks.<jupyter_code># Pickle the data 
df.to_pickle(datadir+'/data_cleaned_oh.pkl')<jupyter_output><empty_output><jupyter_text>5.0 Explore the Data - Part 2  Visualizations Visualization Best Practices Tufte's 6 Principles of Graphical Integrity- Representation of numbers should match the true proportions.- Labeling should be clear and detailed.- Design should not vary for some ulterior motive, show only data variation.- To represent money, well known units are best.- The number of dimensions represented should be the same as the number of dimensions in the data.- Representations should not imply unintended context.Source: Tufte ER. The visual display of quantitative information. Cheshire, CT: Graphics press; 2001 Jan Types of Visualizations![Types](images/TypesOfVisualizations.png)Source:  https://extremepresentation.typepad.com/files/choosing-a-good-chart-09.pdf Tell a Story- Use color, size,shape, proportion, progression and text callouts to tell the story- Uses maps to show relationships- Mix different visualizations to show data in the most appropriate manner![](images/TellAStory.png)Source: https://www.tableau.com Python Visualization Tools- Matplotlib – Powerful, complex visualization package - https://matplotlib.org/2.1.1/gallery/index.html- Seaborn – Simplifies Matplotlib, default styles- Bokeh – Interactive visualization in a browser Custom Visualizations d3 – Custom visualizations using JavaScript - https://d3js.org/  , https://bl.ocks.org/mbostock![](images/CustomForceDirected-d3.png)  Build your own in Python ![](images/CustomTimeline.png)  5.1 Outcome variableNow that we have created some new variables, lets take a look at how the outcome variable is associated with our predictors.<jupyter_code># See who overdosed from prescribed opioids by computing the intersection

overlap = patients_that_overdosed.intersection(patients_prescribed_opioids)

print('Num that overdose = {}, Num that were prescribed opioids = {}, overlap = {}'.format(\
    len(patients_that_overdosed),len(patients_prescribed_opioids),len(overlap)))<jupyter_output><empty_output><jupyter_text>How many patients overdosed?Since we store 'overdose' as a 0 or 1, we can just use the mean function to compute what percent of the population overdosed<jupyter_code>overdose = df[df['pat_id'].isin(patients_prescribed_opioids)]
display(overdose['overdose'].value_counts())
print('Percent that overdosed: {0:.2f}%'.format(overdose['overdose'].mean()*100))<jupyter_output><empty_output><jupyter_text>5.2 Crosstabs<jupyter_code># What was the mean number of days_supply for patients that overdosed and were prescribed opioids?

ct = pd.crosstab(df['prescribed_opioids'],df['opioid_discharge_days_supply'])
display(ct)<jupyter_output><empty_output><jupyter_text>Multiple graphs using FacetgridWe can also see multiple graphs at the same time using the Seaborn Facetgrid function.  It lets you see the same graph split by up to 2 variables.  For example, we can look at how overdose is related to age and gender and whether the patient was ever prescribed opioids.<jupyter_code>p = sns.FacetGrid(df, row='overdose', col='gender', hue='prescribed_opioids', sharey=False)
p.map(plt.hist, 'age')
p.add_legend()
plt.show()<jupyter_output><empty_output><jupyter_text>5.3 Graph the patient variables against the outcomeLet's see if gender, race and age are associated with the outcome<jupyter_code>pd.crosstab(patients['gender'],patients['overdose']).plot(kind='bar')
plt.title('Overdose by Gender')
plt.xlabel('Gender')
plt.ylabel('Count')
pd.crosstab(patients['race'],patients['overdose']).plot(kind='bar')
plt.title('Overdose by Race')
plt.xlabel('Race')
plt.ylabel('Count')<jupyter_output><empty_output><jupyter_text>We have a pretty uniform distribution of ages in the patient data<jupyter_code># Histograms are easy to create
patients['age'].hist()
plt.title('Histogram of Age')
plt.xlabel('Age')
plt.ylabel('Frequency')<jupyter_output><empty_output><jupyter_text>5.4 Grouping by a variableWe often want to group related rows together and then count the number rows of each type or find the mean of a variable for each row type.For example, lets count how many encounters each patient has over the timeframe of the data.  We will use the `groupby` function to group on a set of variables.  The operation returns a `groupby` object which doesn't actually group the data but instead acts like a set of instructions telling the DataFrame how to group itself.  We need to apply another function, such as size(), mean() or sum(), to the groups to yield a result.<jupyter_code># How many encounters does each patient have?

encs = encounters.groupby(['enc_pat_id']).size()

# We can store that information directly into the patients DataFrame since `encs` is indexed by the pat_id
patients = patients.set_index('pat_id')
patients['num_encounters'] = encs
patients = patients.reset_index()

# Visualize the number of encounters as a patient ages, use color to highlight the gender difference
sns.lmplot(data=patients,x='age',y='num_encounters',hue='gender')
# We can also apply a function to the groups
df.groupby('overdose').mean()<jupyter_output><empty_output><jupyter_text>5.5 For those that overdose, what is the days_supply?<jupyter_code>display(df[df['overdose']==1].mean())
display(df[df['prescribed_opioids']==1].mean())<jupyter_output><empty_output><jupyter_text>5.6 What are the primary reasons for visit for patients that ever overdosed?<jupyter_code>encounters[encounters['enc_pat_id'].isin(patients_that_overdosed)]['enc_reason_description'].value_counts(sort=True)<jupyter_output><empty_output><jupyter_text>5.7 Who overdosed?<jupyter_code># Who were the patients that overdosed?

pt = df[df['pat_id'].isin(patients_that_overdosed)]

pt.head(5)<jupyter_output><empty_output><jupyter_text>5.8 Get an idea of how a patient progresses through their healthcareWhen exploring the data, it helps to visualize what is happening across time.  You can create small functions within the Jupyter notebook and reuse them further down in the notebook.  In this case, we are looping through the encounter data for a patient and print all of the medications and labs (observations) that are associated with the patient.  The function `display_trajectory` is passed the id for a patient and then prints the information.  We can use this later to further examine data or debug things we don't understand.First, we will define a simple function that prints the details.  You can create little helper functions like this that are reusable and help you get familiar with the data.<jupyter_code>def display_trajectory(df,pt_id):
    pt = patients[patients['pat_id']==pt_id]

    display(pt)
    encs = encounters[encounters.enc_pat_id == pt_id]
    #print(encs.shape)
    for i, e in encs.iterrows():
        #dt = df[df['pat_id']==e['enc_pat_id']].iloc[0]
        print('  {:%Y-%m-%d}: {} ({}) ({})'.format(e['enc_date'], e['enc_description'], \
                         e['enc_code'], e['enc_reason_description']))
        meds = medications[medications['med_enc_id'] == e['enc_id']]
        for j, m in meds.iterrows():
            print('     MED: {:%Y-%m-%d}: {} ({}) days_supply={}'.format(m['med_start_date'],  \
                                            m['med_description'], m['med_code'], m['med_days_supply']))
        labs = observations[observations['obs_enc_id'] == e['enc_id']]
        for k, l in labs.iterrows():
            print('     LAB: {:%Y-%m-%d %H:%M}: {} ({}) {} {}'.format(l['obs_date'], l['obs_description'], l['obs_code'], l['obs_value'], l['obs_units']))
            
            
# Display the trajectory of one of the overdose patients
pt_id = list(patients_that_overdosed)[1]
display_trajectory(df,pt_id)<jupyter_output><empty_output><jupyter_text>5.81 Graphical TimelineWe can get a higher-level overview of the data by drawing a graphical timeline using Matplotlib.<jupyter_code># Function to display a timeline for an encounter

def plot_timeline(pt_id):
  import matplotlib.dates as mdates
  from datetime import datetime

  pt = patients[patients['pat_id']==pt_id]
  encs = encounters[encounters.enc_pat_id == pt_id]

  dates = []
  names = []
  for i, e in encs.iterrows():
      names.append(e['enc_description'])
      dates.append(e['enc_date'])
  levels = np.array([-5, 5, -4, 4, -3, 3, -2, 2, -1, 1])
  fig, ax = plt.subplots(figsize=(12, 10))

  # Create the base line
  ax.plot((min(dates), max(dates)), (0, 0), 'k', alpha=.5)

  # Iterate through encounters and plot the event name
  for i, (iname, idate) in enumerate(zip(names, dates)):
      level = levels[i % len(levels)]
      vert = 'top' if level < 0 else 'bottom'
      ax.scatter(idate, 0, s=100, facecolor='b', edgecolor='k', zorder=9999)
      # Plot a line up to the text
      ax.plot((idate, idate), (0, level), c='b')
      # Give the text a faint background and align it properly
      ax.text(idate, level, iname,
              horizontalalignment='right', verticalalignment=vert, fontsize=10)
  ax.set(title="Timeline for Patient: %s" % (pt_id,))
  # 3 month intervals
  ax.get_xaxis().set_major_locator(mdates.MonthLocator(interval=3))
  ax.get_xaxis().set_major_formatter(mdates.DateFormatter("%b %Y"))
  fig.autofmt_xdate()
  # Remove the y-axis labels
  plt.setp((ax.get_yticklabels() + ax.get_yticklines() + list(ax.spines.values())), visible=False)
  plt.show()
# Plot the first patient's timeline (you can even plot multiple patients)
plot_timeline(list(patients_that_overdosed)[1])<jupyter_output><empty_output><jupyter_text>5.9 Geographic MappingWe can also easily visualize our data geographically.  All of our patients have addresses.  We can use a library called Bokeh and Google Maps API to quickly visualize where are patients come from.<jupyter_code>from bokeh.io import output_notebook, show, reset_output
from bokeh.models import (
  GMapPlot, GMapOptions, ColumnDataSource, Circle, Range1d, PanTool, WheelZoomTool
)

# Read in the Latitude/Longitude of the patients
patlocations = pd.read_csv(datadir + '/patlocations.csv', dtype=str, index_col=False, header=0, keep_default_na=False)
display(patlocations.head(5))

reset_output()
plot = GMapPlot(
    x_range=Range1d(), y_range=Range1d(), 
    map_options=GMapOptions(lat=40, lng=-83, map_type="roadmap", zoom=6)
)
plot.title.text = "Patient Locations"

# For GMaps to function, Google requires you obtain and enable an API key:
#
#     https://developers.google.com/maps/documentation/javascript/get-api-key
#
# Replace the value below with your personal API key:
plot.api_key = "<KEY>"

circle = Circle(x="lng", y="lat", size=15, fill_color="green", fill_alpha=1.0, line_color=None)
src=ColumnDataSource(patlocations)
plot.add_glyph(src, circle)

plot.add_tools(PanTool(), WheelZoomTool())
# Now lets show the plot, which is interactive
output_notebook()
show(plot)<jupyter_output><empty_output><jupyter_text>Show where the patients that overdosed liveWe can use color to highlight patients in different categories.  For example, we can show the patients that overdosed in red to see if there is a pattern to where they live.<jupyter_code># Add 2 new columns to the DataFrame to indicate which patients overdosed and what color should be displayed
patlocations['overdosed'] = np.where(patlocations['pat_id'].isin(patients_that_overdosed), 1, 0)
patlocations['color'] = np.where(patlocations['overdosed'], 'red', 'green')

display(patlocations[patlocations['overdosed'] == 1].head(10))
# Display the graph
reset_output()

plot2 = GMapPlot(
    x_range=Range1d(), y_range=Range1d(), 
    map_options=GMapOptions(lat=40, lng=-83, map_type="roadmap", zoom=6)
)
plot2.title.text = "Patient Locations (Overdose)"
plot2.api_key = "<KEY>"
circle = Circle(x="lng", y="lat", size=15, fill_color="color", fill_alpha=1.0, line_color=None)
src=ColumnDataSource(patlocations)
plot2.add_glyph(src, circle)

plot2.add_tools(PanTool(), WheelZoomTool())

# Now show the graph
output_notebook()
show(plot2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11461854176726219, 'fraction_numerical': 0.011316996619990343, 'mean_word_length': 4.054453935326419, 'pattern_counts': {'":': 0, '<': 140, '<?xml version=': 0, '>': 145, 'https://': 9, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '362748', 'license': 'MIT', 'n_tokens_mistral': 10388, 'n_tokens_neox': 9780, 'n_words': 3306, 'path': 'Medinfo2019_Data_Prep.ipynb', 'repo_name': 'nkbds/medinfo2019', 'size': '33236'}