<jupyter_start><jupyter_text>code for generating ATC codes from rxcui and NDC<jupyter_code>import requests
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import psycopg2
import collections
import asyncio
import getpass
import re
from datetime import datetime as dt
import os,sys,re
import urllib3
import prettytable
from collections import Counter
import seaborn as sns
import random

from sklearn.externals.joblib import Memory
memory = Memory(cachedir='/tmp', verbose=0)
<EMAIL> above any def fxn.

%matplotlib inline
plt.style.use('ggplot')

from notebook.services.config import ConfigManager
cm = ConfigManager()
cm.update('livereveal', {
        'width': 1024,
        'height': 768,
        'scroll': True,
})

%load_ext autotime
import pandas as pd<jupyter_output><empty_output><jupyter_text>Fetch all responses within one Client session first<jupyter_code>#I need help understanding what this is doing, can you elaborate?
#what are the fetch and run functions? This is used in pinging a api, right?
import asyncio
from aiohttp import ClientSession
import time
import json

async def fetch(url, session):
    async with session.get(url) as response:
        return await response.read()

async def run(url_lst):
    tasks = []

    # Fetch all responses within one Client session,
    # keep connection alive for all requests.
    start = time.perf_counter()
    
    async with ClientSession() as session:
        for url in url_lst:
            task = asyncio.ensure_future(fetch(url, session))
            tasks.append(task)

        responses = await asyncio.gather(*tasks)
        # you now have all response bodies in this variable
#         print(responses)

    duration = time.perf_counter() - start
    print('total time: {}'.format(duration))
    return responses<jupyter_output><empty_output><jupyter_text>Access MIMIC database and convert it to dataframe in Pandas<jupyter_code># note, all server information is stored in a config.py file that is present in the .gitignore
import config 
conn = psycopg2.connect(dbname=config.dbname, user=config.user, host=config.host, port=config.port,password=<PASSWORD>)
cur=conn.cursor()

query_schema = 'SET search_path to ' + "mimiciii" + ';'<jupyter_output>Password:········
time: 2.01 s<jupyter_text>Prescription<jupyter_code>#code for generating ATC codes from rxcui and NDC
cur.execute('Select * from mimiciii.prescriptions')
rows = cur.fetchall()
prescriptionTable = pd.DataFrame(rows,columns=col_name)
col_name = []
for i in range(len(cur.description)):
    col_name.append(cur.description[i][0])
print (col_name)
prescriptionTable.head(10)<jupyter_output><empty_output><jupyter_text>Convert NDC to Rxcui and add it in to prescription table<jupyter_code>ndc_lst = prescriptionTable['ndc'].unique()
url_lst = ['http://rxnav.nlm.nih.gov/REST/rxcui.json?idtype=NDC&id={0}'.format(ndc)
           for ndc in ndc_lst]

loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(url_lst))
responses = loop.run_until_complete(future)
def get_rxnormId_by_ndc(response):
    try:
        #class_lst = [x['rxclassMinConceptItem']['classId'] for x in json.loads(response)['rxclassDrugInfoList']['rxclassDrugInfo']]
        return json.loads(response)['idGroup']['rxnormId'][0]
    except:
        return ''
Rx_code_lst = [get_rxnormId_by_ndc(resp) for resp in responses]
ndc_rc_code_lst_map = {ndc: rx
                          for ndc, rx in zip(ndc_lst, Rx_code_lst)}
NDC_RX = pd.DataFrame({'NDC': ndc_lst,
              'Rx_code': Rx_code_lst})
prescriptionTable0 = prescriptionTable
prescriptionTable = prescriptionTable.assign(rxcui = prescriptionTable['ndc'].map(ndc_rc_code_lst_map))
prescriptionTable.head(20)<jupyter_output><empty_output><jupyter_text>Convert Rxcui to ADC and check whether the drug is Anibiotics, and add it into table<jupyter_code>rxcui_lst = prescriptionTable['rxcui'].unique()
url_lst = ['https://rxnav.nlm.nih.gov/REST/rxclass/class/byRxcui.json?rxcui={}&relaSource=ATC'.format(rxcui)
           for rxcui in rxcui_lst]

loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(url_lst))
responses = loop.run_until_complete(future)
def get_ATC_code_lst(response):
    try:
        class_lst = [x['rxclassMinConceptItem']['classId'] for x in json.loads(response)['rxclassDrugInfoList']['rxclassDrugInfo']]
        return class_lst
    except:
        return []

ATC_code_lst_lst = [get_ATC_code_lst(resp) for resp in responses]
pd.DataFrame({'rxcui': rxcui_lst,
                   'ATC_code_lst': ATC_code_lst_lst})
ab_lst = [any(map(lambda x: x.startswith('J01'), ATC_code_lst)) 
           for ATC_code_lst in ATC_code_lst_lst]
pd.DataFrame({'rxcui': rxcui_lst,
                   'ATC_code_lst': ATC_code_lst_lst,
                   'ab': ab_lst})
ab_map = {rxcui: ab for rxcui, ab in zip(rxcui_lst, ab_lst)}
prescriptionTable = prescriptionTable.assign(Antibiotics = prescriptionTable['rxcui'].map(ab_map))<jupyter_output><empty_output><jupyter_text>The table with anibiotics It seems sth different with PostDoc's result. His prescription table has 3500 unique Rxcui code, while mine is only 1800 or so. Garrett, can you help me check the first 500 rows manually. Coz, I can't tell whether a drug is AB or not from its name<jupyter_code>prescriptionTable.head(500)
prescriptionTable['Antibiotics'] = 'True'
newTable = prescriptionTable.loc[prescriptionTable['Antibiotics'] == True, 'drug']
pd.DataFrame(pd.DataFrame(newTable)['drug'].unique()).to_csv('ab.csv')
prescriptionTable.to_csv('newPrescription.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10666187825462381, 'fraction_numerical': 0.00682348716107021, 'mean_word_length': 4.906680805938494, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '615466', 'license': 'MIT', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1836, 'n_words': 459, 'path': 'notebooks/PIPELINE/01-Prescription_table.ipynb', 'repo_name': 'geickelb/mimiciii-antibiotics-modeling', 'size': '5581'}