<jupyter_start><jupyter_text>Notes- Up to 5 discharge reports by HADM_ID   - Up to 39 ICD codes by HADM_ID<jupyter_code>import numpy as np
import pandas as pd
import re, nltk, string
import ipyleaflet as ipyl
import ipywidgets as ipyw
from sklearn.model_selection import train_test_split
import ggplot, datetime, time
import matplotlib
from matplotlib import pyplot as plt
matplotlib.style.use('ggplot')
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Loading dataset<jupyter_code># Loading the original datasets
#tot_nrows = 91691299
#notes = pd.read_csv('../data/NOTEEVENTS.csv', nrows =  50000)
#diagnoses = pd.read_csv('../data/DIAGNOSES_ICD.csv')
df = pd.read_csv('../data/dis_notes_icd9.csv',
                 names = ['HADM_ID', 'SUBJECT_ID', 'DATE', 'TEXT', 'ICD9'])
df
df.shape<jupyter_output><empty_output><jupyter_text>Completeness<jupyter_code>(notes.count(axis=0)/notes.shape[0]).sort_values().plot.barh(figsize = (10, 5))
diagnoses.HADM_ID.value_counts()
notes.HADM_ID.value_counts()<jupyter_output><empty_output><jupyter_text>Clean up<jupyter_code>df.TEXT[0]
def cleantext(x):
    c = x.replace("\n", " ") # remove the lines
    c = re.sub('\[\*\*.*?\*\*\]', '', c) # remove the identifiers and dates
    #c = c.lower() # remove capital letters
    #c = c.translate(str.maketrans("","", string.punctuation)) # remove punctuation
    return c
cleantext(df.TEXT[0])
#notes.TEXT = notes.TEXT.apply(lambda x: cleantext(x))
df.TEXT = df.TEXT.apply(lambda x: cleantext(x))<jupyter_output><empty_output><jupyter_text>TF IDF test I used this: http://www.cs.duke.edu/courses/spring14/compsci290/assignments/lab02.html<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(stop_words='english')
%time tfs = tfidf.fit_transform(notes.TEXT)
print(tfs[0])<jupyter_output>(0, 18640)	0.0396177522116
  (0, 44937)	0.0791848109342
  (0, 49121)	0.0397542696787
  (0, 122543)	0.039792451741
  (0, 18143)	0.176215457895
  (0, 112728)	0.423062199275
  (0, 128107)	0.214573154981
  (0, 74240)	0.122863171978
  (0, 36136)	0.0507474620755
  (0, 43632)	0.273111252863
  (0, 40480)	0.125105183253
  (0, 34686)	0.244874474077
  (0, 81531)	0.210437209238
  (0, 81270)	0.09785608019
  (0, 83831)	0.0800192377036
  (0, 23433)	0.143662615432
  (0, 40939)	0.16011325334
  (0, 74962)	0.09780680023
  (0, 108760)	0.11007913149
  (0, 137697)	0.227706125694
  (0, 88923)	0.0807446919881
  (0, 124182)	0.176869879632
  (0, 104988)	0.0861996947701
  (0, 53331)	0.07961027272
  (0, 67098)	0.14673453495
  :	:
  (0, 77304)	0.122258020085
  (0, 68201)	0.102146212833
  (0, 85342)	0.0970019817485
  (0, 53257)	0.112997652272
  (0, 96569)	0.0554196045707
  (0, 74900)	0.108279411499
  (0, 101122)	0.0441297270068
  (0, 85979)	0.0437769977008
  (0, 69210)	0.0403866077089
  (0, 16662)	0.151648659824
[...]<jupyter_text>Name Entity Recognition Found here https://github.com/AnthonyMRios/pymetamap   Need to download this: https://metamap.nlm.nih.gov/MainDownload.shtml   and before that need to get authorization -> takes 3 days<jupyter_code>from pymetamap import MetaMap
mm = MetaMap.get_instance('/Users/Guillaume/public_mm/bin/metamap16')
%time concepts, error = mm.extract_concepts([df.TEXT[0]])
concepts
def metamap_ner(text):
    concepts, error = mm.extract_concepts([text])
    l = [w[4] for w in concepts] # extract the CUI
    return l
%time metamap_ner(df.TEXT[0])
for i in range(10):
    %time df.TEXT[i:i+1].apply(lambda x: metamap_ner(x))
df.TEXT[0:10].to_csv('../data/metamap_test.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14249578414839797, 'fraction_numerical': 0.2034851039910062, 'mean_word_length': 4.971476510067114, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '315623', 'license': 'Apache-2.0', 'n_tokens_mistral': 1967, 'n_tokens_neox': 1537, 'n_words': 294, 'path': 'notes_exploration/Exploration.ipynb', 'repo_name': 'letslego/W266-ProjectMIMIC', 'size': '3558'}