<jupyter_start><jupyter_text>Summer Hackathon - Covid19 EDA<jupyter_code>import pandas as pd

%matplotlib inline
import numpy as np 
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

# data processing
import pandas as pd 

# data visualization
import seaborn as sns
%matplotlib inline
from matplotlib import pyplot as plt
from matplotlib import style

import numpy as np
import pandas as pd
# For visualizations
import matplotlib.pyplot as plt
# For regular expressions
import re
# For handling string
import string
# For performing mathematical operations
import math

import pandas as pd
from collections import Counter
import pandas as pd
from nltk import tokenize 
from nltk.tokenize import TweetTokenizer
from nltk.tokenize import word_tokenize
import re
import matplotlib.pyplot as plt
from nltk import tokenize 

#sentence split all documents
df = pd.read_excel("/Users/lolajohnston/Desktop/covid-19-data-master/Final_Hackathon_1082Patients.xlsx")
df.columns = ['id','note']
# check for missing data
df.isnull().sum()
# Expand Contractions

# Dictionary of English Contractions
contractions_dict = { "ain't": "are not","'s":" is","aren't": "are not",
                     "can't": "cannot","can't've": "cannot have",
                     "'cause": "because","could've": "could have","couldn't": "could not",
                     "couldn't've": "could not have", "didn't": "did not","doesn't": "does not",
                     "don't": "do not","hadn't": "had not","hadn't've": "had not have",
                     "hasn't": "has not","haven't": "have not","he'd": "he would",
                     "he'd've": "he would have","he'll": "he will", "he'll've": "he will have",
                     "how'd": "how did","how'd'y": "how do you","how'll": "how will",
                     "I'd": "I would", "I'd've": "I would have","I'll": "I will",
                     "I'll've": "I will have","I'm": "I am","I've": "I have", "isn't": "is not",
                     "it'd": "it would","it'd've": "it would have","it'll": "it will",
                     "it'll've": "it will have", "let's": "let us","ma'am": "madam",
                     "mayn't": "may not","might've": "might have","mightn't": "might not", 
                     "mightn't've": "might not have","must've": "must have","mustn't": "must not",
                     "mustn't've": "must not have", "needn't": "need not",
                     "needn't've": "need not have","o'clock": "of the clock","oughtn't": "ought not",
                     "oughtn't've": "ought not have","shan't": "shall not","sha'n't": "shall not",
                     "shan't've": "shall not have","she'd": "she would","she'd've": "she would have",
                     "she'll": "she will", "she'll've": "she will have","should've": "should have",
                     "shouldn't": "should not", "shouldn't've": "should not have","so've": "so have",
                     "that'd": "that would","that'd've": "that would have", "there'd": "there would",
                     "there'd've": "there would have", "they'd": "they would",
                     "they'd've": "they would have","they'll": "they will",
                     "they'll've": "they will have", "they're": "they are","they've": "they have",
                     "to've": "to have","wasn't": "was not","we'd": "we would",
                     "we'd've": "we would have","we'll": "we will","we'll've": "we will have",
                     "we're": "we are","we've": "we have", "weren't": "were not","what'll": "what will",
                     "what'll've": "what will have","what're": "what are", "what've": "what have",
                     "when've": "when have","where'd": "where did", "where've": "where have",
                     "who'll": "who will","who'll've": "who will have","who've": "who have",
                     "why've": "why have","will've": "will have","won't": "will not",
                     "won't've": "will not have", "would've": "would have","wouldn't": "would not",
                     "wouldn't've": "would not have","y'all": "you all", "y'all'd": "you all would",
                     "y'all'd've": "you all would have","y'all're": "you all are",
                     "y'all've": "you all have", "you'd": "you would","you'd've": "you would have",
                     "you'll": "you will","you'll've": "you will have", "you're": "you are",
                     "you've": "you have"}

# Regular expression for finding contractions
contractions_re=re.compile('(%s)' % '|'.join(contractions_dict.keys()))

# Function for expanding contractions
def expand_contractions(text,contractions_dict=contractions_dict):
  def replace(match):
    return contractions_dict[match.group(0)]
  return contractions_re.sub(replace, text)

# Expanding contractions
df['cleaned'] = df['note'].apply(lambda x: expand_contractions(x))

# Lower case
df['cleaned']=df['cleaned'].apply(lambda x: x.lower())

# Removing extra spaces
df['cleaned']=df['cleaned'].apply(lambda x: re.sub(' +',' ',x))

# Count length of each document (clinical note)
df['totalwords'] = df['cleaned'].str.split().str.len()

# Remove punctuations
df['cleaned']=df['cleaned'].apply(lambda x: re.sub('[%s]' % re.escape(string.punctuation), '', x))
# Importing spacy
import spacy
import en_core_web_sm
from sklearn.feature_extraction.text import CountVectorizer

# Import english package of spacy
import en_core_web_sm
nlp = en_core_web_sm.load()
# Lemmatization with stopwords removal
df['lemmatized']=df['cleaned'].apply(lambda x: ' '.join([token.lemma_ for token in list(nlp(x)) if (token.is_stop==False)]))<jupyter_output><empty_output><jupyter_text>Features: - Treat unnamed column as the patient id   Inclusion criteria: - Tested positive for covid-19 (bool) - Age (int) - BMI (float) - Sex (int) - Given written informed consent (bool) - No known allergies to drugs (bool)<jupyter_code>'''
# phrasing
covid_phrasing = ['covid','covid19']
age_phrasing = ['45 female', '45 male', 'age 45', '45 year old', '45yo', '45 yo']
sex_phrasing = ['female','male','woman','man','m', 'f']
consent_phrasing = ['consent','agree']
'''<jupyter_output><empty_output><jupyter_text>INITIAL EDA ON SEX & AGE<jupyter_code># check for females

ftest = sum(list(df['cleaned'].map(lambda s: re.findall(r"female\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
ftest = pd.DataFrame(ftest, columns = ['age_text'])
ftest['age_num'] = ftest.age_text.str.extract('(\d+)')
ftest = ftest.dropna(subset=['age_num'])
ftest['age_num'] = pd.to_numeric(ftest['age_num'] )


bins = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
#labels = ["< 10","10","20", "30", "40", "50", "60", "70", "80", "90"]
ftest['binned'] = pd.cut(ftest['age_num'], bins)
print (ftest)

sns.catplot(x="binned", kind="count", palette="ch:.25", data=ftest, height=5, aspect=8/5).set(title = 'Females - Age')

# check for males
testing = sum(list(df['cleaned'].map(lambda s: re.findall(r"male\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
testing = pd.DataFrame(testing, columns = ['age_text'])
testing['age_num'] = testing.age_text.str.extract('(\d+)')
testing = testing.dropna(subset=['age_num'])
testing['age_num'] = pd.to_numeric(testing['age_num'] )
bins = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
#labels = ["< 10","10","20", "30", "40", "50", "60", "70", "80", "90"]
testing['binned'] = pd.cut(testing['age_num'], bins)


testing['binned'].value_counts()


sns.catplot(x="binned", kind="count", palette="ch:.25", data=testing,height=5, aspect=8/5).set(title = 'Males - Age')

# plotting together

ftest['sex']= 'female'
testing['sex'] = 'male'
df_age = pd.concat([ftest,testing])


sns.set(style="whitegrid")
g = sns.catplot(x="binned", hue="sex", data=df_age,
                height=6, aspect = 8/5, kind="count", palette=["m", "g"])
g.despine(left=True)
g.set_ylabels("Count of Age")
ftest = sum(list(df['cleaned'].map(lambda s: re.findall(r"female\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

# get the sex split percentages

female_percentage = len(ftest.sex)/1113
male_percentage = len(testing.sex)/1113
print("percentage of female: " + str((round(female_percentage, 4))*100))
print("percentage of male: " + str((round(male_percentage, 2))*100))
# create column for male, first create female column by extracting female


# column for female
df['female'] = df['lemmatized'].map(lambda x: True if ' female ' in x else False)
df['f'] = df['cleaned'].map(lambda s: re.findall(r"female\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

df['female'].value_counts(dropna= False)
# column for covid
df['covid'] = df['lemmatized'].map(lambda x: True if 'covid' in x else False)

# replace covid19 with covid for digit pulling

df['lemmatized'] = df['lemmatized'].str.replace('covid19', 'covid')

#df['male_test'] = df['lemmatized'].map(lambda x: re.findall(r" male\S*(?:\s\S+)?\S*(?:\s\S+)?", x),)
df
df['age_v1'] = df['lemmatized'].str.extract('(\d\d)', expand=True)
df['age_v1'] = df['age_v1'].astype(float)

print("average age: " + str(df.age_v1.sum()/1081))
print("min age: " + str(df.age_v1.min()))
print("max age: " + str(df.age_v1.max()))
# check what the age looks like
df['age_v1'].value_counts(dropna=False) # 136 unsure
# reset under 20 to zero and try another method to extract age

df['age_v1'] = np.where(df['age_v1']< 20, 0, df['age_v1'])
(df)

# DROP COLUMN

#df_lem['wrong_age'] = df['age'] > 100 | (df['age'] % 10 == 0)
#df.drop(['age_under20'], axis=1, inplace=True)

# check unsure age

unsure_age = df['age_v1']  == 0
unsure_age = df[unsure_age]
under_age

# find age using 'yo'
unsure_age['age_yo'] = unsure_age['lemmatized'].map(lambda x: re.findall(r"\S*(?:\d+) yo ",x), )
unsure_age['age_yo'] = unsure_age['age_yo'].astype(str)
unsure_age['age_yo'] = unsure_age.age_yo.str.extract('(\d+)')

unsure_age.age_yo.value_counts(dropna = False)
# try with no space yo

unsure_age['age_ns_yo'] = unsure_age['lemmatized'].map(lambda x: re.findall(r"\S*(?:\d+)yo ",x), )
unsure_age['age_ns_yo'] = unsure_age['age_ns_yo'].astype(str)
unsure_age['age_ns_yo'] = unsure_age.age_ns_yo.str.extract('(\d+)')
unsure_age.age_ns_yo.value_counts(dropna = False)
# try with woman

#age_1 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
#age_2 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
unsure_age['age_w'] = unsure_age['lemmatized'].map(lambda x: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",x), )                                                                      
unsure_age['age_w'] = unsure_age['age_w'].astype(str)
unsure_age['age_w'] = unsure_age.age_w.str.extract('(\d+)')
unsure_age.age_w.value_counts(dropna = False)
# try with man

#age_1 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
#age_2 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
unsure_age['age_m'] = unsure_age['lemmatized'].map(lambda x: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",x), )                                                                      
unsure_age['age_m'] = unsure_age['age_m'].astype(str)
unsure_age['age_m'] = unsure_age.age_m.str.extract('(\d+)')
unsure_age.age_w.value_counts(dropna = False)
# fill na with zeros
unsure_age.fillna(0)
df
unsure_age
# create a new dataframe of the ages you found from unsure age

final_age = {'id':[], 'age':[]}

for age1, age2, age3, age4, patient in zip(unsure_age['age_yo'], unsure_age['age_ns_yo'], unsure_age['age_w'], unsure_age['age_m'], unsure_age['id']):
    if (not pd.isna(age1)) and (age1 != 0):
        final_age['id'].append(patient)
        final_age['age'].append(age1)
    elif (not pd.isna(age2)) and (age2 != 0):
        final_age['id'].append(patient)
        final_age['age'].append(age2)
    elif (not pd.isna(age3)) and (age3 != 0):
        final_age['id'].append(patient)
        final_age['age'].append(age3)
    elif (not pd.isna(age4)) and (age4 != 0):
        final_age['id'].append(patient)
        final_age['age'].append(age4)
    else:
        final_age['id'].append(patient)
        final_age['age'].append(0)
        
final_age = pd.DataFrame(final_age)
final_age
df['age'] = df['age_v1']
df_final = df
df_final
final_age
# merge data frames
merged_left = pd.merge(left=df_final, right=final_age, how='left', left_on='id', right_on='id')
merged_left
merged_left = merged_left.fillna(0)

merged_left['age_x'] = merged_left['age_x'].astype(int)
merged_left['age_y'] = merged_left['age_y'].astype(int)
merged_left['age'] = merged_left['age_x']+ merged_left['age_y']

merged_left['age'].value_counts(dropna=False) # down to 86
df10 = merged_left
def do_that_thing(x):
    if (not x.age_yo == 0):
        return(x.age_yo)
    elif (not x.age_ns_yo == 0):
        return(x.age_ns_yo)
    elif ((not pd.isna(x.age_w)) & (not x.age_w == 0)):
        return(x.age_w)
    elif ((not pd.isna(x.age_m)) & (not x.age_m == 0)):
        return(x.age_m)
    else:
        return(0)
unsure_age.assign(final_age = lambda x: do_that_thing(x))
# drop unnecessary age column agev1
df_final.drop(['age_v1'], axis=1, inplace=True)
# Age phrasing

age_1 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_2 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_3 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\s\S+) male \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_4 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\s\S+) female\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # japan hokkaido female 20s symptom onset
age_5 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\d+) yo \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # 46yo
age_6 = sum(list(df['lemmatized'].map(lambda s: re.findall(r"\S*(?:\d+)yo \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # 46 yo

print(len(age_1))
print(len(age_2))
print(len(age_3))
print(len(age_4))
print(len(age_5))
print(len(age_6))

'''

print((age_1))
print((age_2))
print((age_3))
print((age_4))
print((age_5))
print((age_6))
'''
s_woman = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
s_man = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
s_male = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) male \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
s_female = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) female \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

print(len(s_woman))
print(len(s_man))
print(len(s_male))
print(len(s_female))
s_he = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) he \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
s_she = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) she \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

print(len(s_he))
print(len(s_she))
df['f_age'] = df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)\S*(?:\s\S+) female \S*(?:\s\S+)?\S*(?:\s\S+)?",s))
df['f_age'] = df['f_age'].astype(str)
df['age_num'] = df.f_age.str.extract('(\d+)')
df['m_age'] = df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)\S*(?:\s\S+) male \S*(?:\s\S+)?\S*(?:\s\S+)?",s))
df['m_age'] = df['m_age'].astype(str)
df['m_age_num'] = df.m_age.str.extract('(\d+)')
#df['age_num'] = df['age_num'].str.replace('', 0)
df['age_num'] = df['age_num'].fillna(0)

df['m_age_num'] = df['m_age_num'].fillna(0)
df['age_num'] = df['age_num'].astype(int)
df['m_age_num'] = df['m_age_num'].astype(int)

df['age'] = df['age_num'] + df['m_age_num']
df.head()
df_final<jupyter_output><empty_output><jupyter_text>Allergies<jupyter_code>df = df10
aller = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)\S*(?:\s\S+) aller\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
no_known_aller = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)\S*(?:\s\S+) no known aller\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
print((len(aller)))
print((len(no_known_aller)))
df['no_known_allergies'] = df['cleaned'].map(lambda x: True if 'no known allergies to drugs' in x else False)
df['no_known_allergies'].value_counts()<jupyter_output><empty_output><jupyter_text>Age phrasing<jupyter_code># Age phrasing

age_1 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) woman \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_2 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) man \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_3 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) male \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
age_4 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) female\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # japan hokkaido female 20s symptom onset
age_5 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\d+) yo \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # 46yo
age_6 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\d+)yo \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), []) # 46 yo

print(len(age_1))
print(len(age_2))
print(len(age_3))
print(len(age_4))
print(len(age_5))
print(len(age_6))

'''

print((age_1))
print((age_2))
print((age_3))
print((age_4))
print((age_5))
print((age_6))
'''

#print((age_1))
#print((age_2))
#print((age_3))
#print((age_4))
# print((age_5))
#print((age_6))
df<jupyter_output><empty_output><jupyter_text>Wordclouds<jupyter_code>from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

# Start with one review:
text = df.iloc[0,4]

# Create and generate a word cloud image:
wordcloud = WordCloud().generate(text)

# Display the generated image:
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
# lower max_font_size, change the maximum number of word and lighten the background:
wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(text)
plt.figure
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
text = " ".join(review for review in df.cleaned)
print ("There are {} words in the combination of all review.".format(len(text)))
# Create stopword list:
stopwords = set(STOPWORDS)
stopwords.update(["patient", "day", "history", "one", "tablet", "daily", "mg", 
                  "continue", "right","doctor", "disease", "demonstrate", "normal",
                  "transplant","pulse","ultrasound","kidney","stage","surgery","end",
                 "artery","dialysis","follow","blood"]) # added these words after multiple iterations of the wordcloud

# Generate a word cloud image
wordcloud = WordCloud(stopwords=stopwords, background_color="white").generate(text)

# Display the generated image:
# the matplotlib way:
plt.figure(figsize=[12,12])
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

from textblob import TextBlob
df['polarity']=df['lemmatized'].apply(lambda x:TextBlob(x).sentiment.polarity)
df.iloc[30,0]
print("3 Random NOTES with Highest Polarity:")
for index,review in enumerate(df.iloc[df['polarity'].sort_values(ascending=False)[:3].index]['cleaned']): 
    print('NOTES {}:\n'.format(index+1),review)
max_length = max(df['totalwords'])
min_length = min(df['totalwords'])

print(max_length)
print(min_length)
tf1 = (df['lemmatized'][1:2]).apply(lambda x: pd.value_counts(x.split(" "))).sum(axis = 0).reset_index()
tf1.columns = ['words','tf']
tf1<jupyter_output><empty_output><jupyter_text>Tested positive for covid-19 (bool)<jupyter_code>df.drop(['age_v1'], axis=1, inplace=True)
df.drop(['age_x'], axis=1, inplace=True)
df.drop(['age_y'], axis=1, inplace=True)
df
# covid
df['covid_mention'] = df['cleaned'].map(lambda x: True if 'covid19' in x else False)

df['covid_mention'].value_counts()

df = df.sort_values(by= 'covid_mention', ascending = True )
df['covid_mention'].value_counts()
df = df.sort_values(by= 'covid_mention', ascending = True )<jupyter_output><empty_output><jupyter_text>Things to note: - Index 4: Two patient profiles, could be clerical error. starts out describing male patient, actually female patient  Consent<jupyter_code>consent = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) consent\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
print(len(consent))

# 1 word before & 1 word after
consent_agree = sum(list(df['cleaned'].map(lambda x: re.findall(r"\S*(?:\s\S+) agree\S*(?:\s\S+)?\S*(?:\s\S+)?",x))), [])
print(len(consent_agree))
df['consent'] = df['cleaned'].map(lambda x: True if 'consent' in x else False)<jupyter_output><empty_output><jupyter_text>HIV<jupyter_code>df_test = df
df_test
hiv = df_test['cleaned'].map(lambda x: re.findall(r"\S*(?:\s\S+) hiv \S*(?:\s\S+)?\S*(?:\s\S+)?",x))



df_test
#unsure_age['age_yo'] = unsure_age['age_yo'].astype(str)
#unsure_age['age_yo'] = unsure_age.age_yo.str.extract('(\d+)')

df_test['hiv1'] = df_test.hiv.str.extract('hiv')
df_test['hiv'].value_counts(dropna=False)

#merged_left['age'].value_counts(dropna=False) # down to 86
hiv_pos = sum(list(df_test['cleaned'].map(lambda x: re.findall(r"\S*(?:\s\S+) hiv positive\S*(?:\s\S+)?\S*(?:\s\S+)?",x))), [])

hiv_pos<jupyter_output><empty_output><jupyter_text>BMI<jupyter_code>obese = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) obese \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
average = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)diabetes\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
healthy = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) healthy \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
heavy = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) heavy \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

print(len(healthy))
print(len(obese))
print(len(heavy))

#print(average)
bmi_test = sum(list(df['cleaned'].map(lambda s: re.findall(r"bmi\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

print((len(bmi_test)))
ht_1 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) bmi \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
ht_2 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) height \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
ht_3 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) inches \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
ht_4 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) feet \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
ht_4 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) feet \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

#ht_5 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+)  \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])

#ht_5 = sum(
list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) in \S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])


print(len(ht_1))
print(len(ht_2))
print(len(ht_3))
print(len(ht_4))
#print(len(ht_5))

print((ht_1))
#print((ht_2))
#print((ht_3))
#print((ht_4))
print((ht_5))
wt_1 = sum(list(df['cleaned'].map(lambda s: re.findall(r"weigh\S*(?:\s\S+)?\S*(?:\d+)?",s))), []) # weight 210, weighing 210, weighs 210
wt_2 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) kg\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
wt_3 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) lb\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
wt_4 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) kilogram\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
wt_5 = sum(list(df['cleaned'].map(lambda s: re.findall(r"\S*(?:\s\S+) wt\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])


print(len(wt_1))
print(len(wt_2))
print(len(wt_3))
print(len(wt_4))
#print(len(wt_5))
print((wt_4))
# check if in physical exam section?

physical_exam = sum(list(df['cleaned'].map(lambda s: re.findall(r"physical exam\S*(?:\s\S+)?\S*(?:\s\S+)?\S*(?:\s\S+)?",s))), [])
print((len(physical_exam)))
print(physical_exam)
super_df = df_test
new = super_df[['id', 'covid', 'age','female','consent','no_known_allergies']].copy()
new
new['id'] = (new['id'])
new['c1_re'] = (new['covid'])
new['c2_re'] = ((new['age']) >= 35) & ((new['age']) <= 75)
new['c3_re'] = ((new['female']) == False)
new['c4_re']= new['consent']
new['c8_re']= new['no_known_allergies']
re_df = new[['id','c1_re', 'c2_re', 'c3_re','c4_re','c8_re']].copy()
re_df
pwd
re_df.to_pickle('/Users/lolajohnston/Documents/GitHub/msca-hackathon/lola/regex_df.pkl')

re_df["totes"] = re_df.sum(axis=1)
print(re_df)
re_df.sort_values(by= 'totes')
re_df.totes.value_counts(dropna=False)
unsure_age.age_yo.value_counts(dropna = False)
final_peeps = re_df.totes == 5
final_peeps = re_df[final_peeps]
final_peeps<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.25194255725967213, 'fraction_numerical': 0.01480818518367845, 'mean_word_length': 4.833412434741338, 'pattern_counts': {'":': 106, '<': 45, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '230489', 'license': 'MIT', 'n_tokens_mistral': 10538, 'n_tokens_neox': 9900, 'n_words': 1750, 'path': '01_regex_2.ipynb', 'repo_name': 'lolajohnston/deep-shallow-nlp-ensemble', 'size': '24581'}