<jupyter_start><jupyter_text>Load Packages and Data<jupyter_code>from IPython.display import HTML
hidecode=HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')
hidecode
%matplotlib inline
import numpy as np
import pandas as pd
from scipy import stats
from matplotlib import pyplot as plt
import seaborn as sns
import nltk
import re
from ipywidgets import interact, interactive, fixed, HBox, widgets
from collections import Counter
import operator
df = pd.read_csv('webmd1.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Maninpulating the Data<jupyter_code>hidecode
otc = df[df.Type == ' otc']
cols = ['Type','Indication','Condition','Drug','BrandName','GenName','Use','HowtoUse','Precautions','AvoidUse',\
              'Allergies','Interactions','Sides','Effectiveness','EaseofUse','Satisfaction',\
              'Review','Form','Dosage','PkgCount','EstimatedPrice']
otc = otc[cols]
otc.rename(columns = {'Review':'Reviews'}, inplace=True) # rename to Reviews
otc.reset_index(drop=True, inplace=True)
ind = otc.loc[(otc.Reviews == 'Be the first to review it') &\
        ((otc.Effectiveness != ' ') | (otc.EaseofUse != ' ') | (otc.Satisfaction != ' '))].index
otc.loc[ind,'Reviews'] = 1 # fix rows with ratings but no reviews

ind2 = otc.loc[otc.Reviews == 'Be the first to review it'].index
otc.loc[ind2, 'Reviews'] = 0 # change rows with no reviews to value 0

ind3 = otc.loc[(otc.Reviews != 0)&(otc.Reviews.str.contains('User Reviews'))].index
otc.loc[ind3, 'Reviews'] = otc.loc[ind3].Reviews.str.replace('User Reviews', '') # remove text from reviews

otc.Reviews = pd.to_numeric(otc.Reviews) # convert to numeric
otc.loc[otc.Reviews == 0,['Effectiveness','EaseofUse','Satisfaction']] = np.nan

otc.loc[otc.loc[otc.BrandName == ' '].index,'BrandName'] = ''
otc.loc[otc.loc[otc.GenName == ' '].index, 'GenName'] = ''
otc.loc[otc.loc[otc.Dosage == ' '].index, 'Dosage'] = ''
otc.loc[otc.loc[otc.Use == ' '].index, 'Use'] = ''
otc.loc[otc.loc[otc.HowtoUse == ' '].index, 'HowtoUse'] = ''
otc.loc[otc.loc[otc.Precautions == ' '].index, 'Precautions'] = ''
otc.loc[otc.loc[otc.Interactions == ' '].index, 'Interactions'] = ''
otc.loc[otc.loc[otc.Sides == ' '].index, 'Sides'] = ''
otc.loc[otc.loc[otc.Allergies == ' '].index, 'Allergies'] = ''
otc.loc[otc.loc[otc.AvoidUse == ' '].index, 'AvoidUse'] = ''
otc.loc[otc.loc[(otc.Form == ' ') & (otc.PkgCount == ' ') & (otc.EstimatedPrice == ' ')].index,\
       ['Form','PkgCount','EstimatedPrice']] = np.nan
otc.Satisfaction = pd.to_numeric(otc.Satisfaction)
otc.EaseofUse = pd.to_numeric(otc.EaseofUse)
otc.Effectiveness = pd.to_numeric(otc.Effectiveness)
otc.EstimatedPrice = pd.to_numeric(otc.EstimatedPrice)
otc.PkgCount = pd.to_numeric(otc.PkgCount)
otc = otc.assign(UnitPrice = otc.EstimatedPrice/otc.PkgCount)
otc = otc.assign(Overall = (otc.Effectiveness+otc.EaseofUse+otc.Satisfaction)/3)
otc.dropna().sort_values('Drug').head()<jupyter_output><empty_output><jupyter_text>DATA VISULIZATOIN<jupyter_code>hidecode
colFun = {'Drug':['size'],'Reviews':['max','median','sum'],\
                                'EaseofUse':['max','median'],'Satisfaction':['max','median'],\
                               'Effectiveness':['max','median'],'EstimatedPrice':['mean']}

group = otc.groupby(['Condition']).agg(colFun)
groupcols = ['Drug','Effectiveness','EaseofUse','Satisfaction','Reviews','EstimatedPrice']
#group.loc[group.Drug.sort_values('size',ascending=False).index][groupcols]
group1 = group.loc[group['Reviews'].sort_values('sum',ascending=False).index][groupcols]
group1.head()<jupyter_output><empty_output><jupyter_text>Forms of Drugs<jupyter_code>pd.options.display.max_columns = 100
otc_form = otc[['Form','PkgCount','UnitPrice','EstimatedPrice','EaseofUse','Effectiveness','Satisfaction','Overall','Indication']]
formind = otc_form.sort_values('Form').Form.value_counts().index[0:13] # create a list of top drug forms
otc_form = otc_form.reset_index(drop=True).set_index('Form').loc[formind].reset_index() #select top 

forms = pd.DataFrame(otc.sort_values('Form').Form.value_counts()).rename(columns={'Form':'count'})
forms = forms.assign(percentage = forms[['count']]/forms[['count']].sum())
sizes = [x*100 for x in forms['percentage'].tolist()[0:13]]
labels = forms.index.tolist()[0:13] + ['other']
fig = plt.figure(figsize=(8,8))
pieWedgesCollection = plt.pie(sizes + [100 - sum(sizes)],labels=(labels),
                              colors=(sns.color_palette("Set3")+sns.color_palette("Set2")+sns.color_palette("Set1")),
                              startangle=90,
                              autopct='%1.1f%%',
                              shadow=True,
                              radius = 1.2,
                              pctdistance=1.1, labeldistance=1.2)
print 'Top %d out of 51 forms of drugs makes up of %s all OTC drugs' %(len(sizes),'{:.1%}'.format(forms.loc[formind]['percentage'].sum()))
Top 13 out of 51 forms of drugs makes up of 95.1% all OTC drugs
def formplot(x):
    otc_ = otc_form
    return otc_.groupby('Form')[[x]].agg('mean').sort_values(x,ascending=False).plot.barh(figsize=(9,5),
                                                                                         color='pink').legend(loc='upper right')
interact(formplot,
        x= widgets.Dropdown(
        options=['PkgCount','EstimatedPrice','EaseofUse','Effectiveness','Satisfaction','UnitPrice'],
        value='EstimatedPrice',
        description='Form vs:',
        disabled=False,
        button_style='info')
        )
<matplotlib.legend.Legend at 0x10fca908>
<function __main__.formplot>
plt.figure(figsize=(9,5))
sns.boxplot(x='Form',y='Effectiveness', data=otc_form)

grps = pd.unique(otc_form.Form.values)
d_data = {grp:otc_form.dropna()['Effectiveness'][otc_form.Form == grp] \
    for grp in pd.unique(otc_form.Form.values)}
 
k = len(pd.unique(otc_form.Form))  # number of conditions
N = len(otc_form.values)  # conditions times participants
n = otc_form.groupby('Form').size()[0] #Participants in each condition
print 'Null Hypothesis: The mean effectiveness are equal across all forms of drugs'
print stats.bartlett(d_data['tablet'],d_data['liquid'],d_data['capsule'],d_data['cream'],d_data['tab'],d_data['drops'],
                     d_data['lotion'],d_data['oral'],d_data['solution'],d_data['syrup'],d_data['powd'],d_data['spray'],
                     d_data['oint'])
print stats.f_oneway(d_data['tablet'],d_data['liquid'],d_data['capsule'],d_data['cream'],d_data['tab'],d_data['drops'],
                     d_data['lotion'],d_data['oral'],d_data['solution'],d_data['syrup'],d_data['powd'],d_data['spray'],
                     d_data['oint']) 
print '\nThe Bartlett\'s Test tells us our variances are not the same, which invalidates the ANOVA test\n'
print 'ANOVA returns a pvalue of ~ 1.6e-13, but we cannot accept the results of this test\n'
print '\nNull Hypothesis: The median effectiveness of all forms of drugs are equal\n'
print stats.kruskal(d_data['tablet'],d_data['liquid'],d_data['capsule'],d_data['cream'],d_data['tab'],d_data['drops'],
                     d_data['lotion'],d_data['oral'],d_data['solution'],d_data['syrup'],d_data['powd'],d_data['spray'],
                     d_data['oint'])
print '\nUsing a Kruskal test, we reject the null hypothesis in favor of the alternative hypothesis (unequal medians)\n'
Null Hypothesis: The mean effectiveness are equal across all forms of drugs
BartlettResult(statistic=150.35231224453142, pvalue=4.8094147391152934e-26)
F_onewayResult(statistic=7.3077598764218541, pvalue=1.6657279383534421e-13)

The Bartlett's Test tells us our variances are not the same, which invalidates the ANOVA test

ANOVA returns a pvalue of ~ 1.6e-13, but we cannot accept the results of this test


Null Hypothesis: The median effectiveness of all forms of drugs are equal

KruskalResult(statistic=83.854120137436084, pvalue=7.5552550543296734e-13)

Using a Kruskal test, we reject the null hypothesis in favor of the alternative hypothesis (unequal medians)
otclog = otc_form[['Form','Indication','EstimatedPrice']]
otclog.loc[:,'EstimatedPrice'] = np.log(otclog.EstimatedPrice)
C:\Users\jason\Anaconda2\lib\site-packages\pandas\core\indexing.py:465: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s
plt.figure(figsize=(9,5))
sns.boxplot(x='Form',y='EstimatedPrice', data=otclog)
<matplotlib.axes._subplots.AxesSubplot at 0x112b1ef0><jupyter_output><empty_output><jupyter_text>Brand Name vs Generic<jupyter_code>plt.figure(figsize=(9,5))
sns.boxplot(x='Indication',y='Effectiveness', data=otc_form)
<matplotlib.axes._subplots.AxesSubplot at 0x116382e8>
plt.figure(figsize=(9,5))
sns.boxplot(x='Indication',y='EstimatedPrice', data=otclog)
<matplotlib.axes._subplots.AxesSubplot at 0x1110b2b0><jupyter_output><empty_output><jupyter_text>Ease of Use vs Effectiveness to User Satisfaction<jupyter_code>def jointplot(x,color='g'):
    otcjoint = otc[['EaseofUse', 'Satisfaction', 'Effectiveness', 'Reviews', 'EstimatedPrice']]
    otcjoint = otcjoint[otcjoint.apply(lambda x: np.abs(x - x.mean()) / x.std() < 3).all(axis=1)]
    return sns.jointplot(x, otc.Satisfaction,color=color,joint_kws={'alpha':0.1})
plt.figure(figsize=(9,5))

jointplot(otc.EaseofUse,'r')
jointplot(otc.Effectiveness)
<seaborn.axisgrid.JointGrid at 0x10b20208>
<matplotlib.figure.Figure at 0x10b2b550>
otc[['EaseofUse','Effectiveness','Satisfaction','Overall']].corr()<jupyter_output><empty_output><jupyter_text>Other PlotsReviews<jupyter_code>def Histo_Reviews(x,y):
    otchist = otc[['Reviews']].dropna()
    otchist.loc['Reviews'] = np.log(otchist.Reviews)
    otchist = otchist.loc[(otchist.Reviews >= x)&(otchist.Reviews <= y)]
    otchist = otchist[otchist.apply(lambda x: np.abs(x - x.mean()) / x.std() < 3).all(axis=1)]
    plt.figure(figsize=(9,5))
    return sns.distplot(otchist.Reviews,color='y')

# Histo_Reviews()
interact(Histo_Reviews,
    x= widgets.IntSlider(
    min=0,
    value = 5,
    step = 10,
    max=200,
    description='Minimum Reviews:'),
    y= widgets.IntSlider(
    min=0,
    value = 200,
    step = 10,
    max=390,
    description='Max Reviews:')
         )
<matplotlib.axes._subplots.AxesSubplot at 0x13224208>
<function __main__.Histo_Reviews><jupyter_output><empty_output><jupyter_text>Reviews vs Price<jupyter_code>def Reviews_vs_Price(x,y):
    otcscatter = otc[['Reviews','EstimatedPrice']]
    otcscatter = otcscatter.loc[(otcscatter.Reviews > x[0])&(otcscatter.Reviews < x[1])&
                                (otcscatter.EstimatedPrice > y[0])&(otcscatter.EstimatedPrice < y[1])]
    otcscatter = otcscatter[otcscatter.apply(lambda x: np.abs(x - x.mean()) / x.std() < 3).all(axis=1)]
    return otcscatter.plot.scatter('Reviews','EstimatedPrice',figsize=(9,5), color = 'blue')
   
interact(Reviews_vs_Price,
    x= widgets.IntRangeSlider(
    min=0,
    value = [0,390],
    step = 10,
    max=390,
    description='Reviews:'),
    y= widgets.IntRangeSlider(
    min=0,
    value = [0,150],
    step = 5,
    max=150,
    description='Price:')
        )    
<matplotlib.axes._subplots.AxesSubplot at 0x11d5b5f8>
<function __main__.Reviews_vs_Price><jupyter_output><empty_output><jupyter_text>All Continous Vars<jupyter_code>g = sns.PairGrid(otc[['Effectiveness','EaseofUse','Satisfaction','Reviews','EstimatedPrice','UnitPrice']])
g.map(plt.scatter)
# g.map_lower(plt.kdeplot)
# g.map_upper(plt.hist)
<seaborn.axisgrid.PairGrid at 0x131eb320><jupyter_output><empty_output><jupyter_text>Robot Pharamist OTC Drug Finder<jupyter_code>hidecode

#['Use','HowtoUse','Precautions','AvoidUse','Allergies','Interactions','Sides''BrandName','GenName',] # columns removed
def search_conditions(string, tg2, tg3, age, dd1, tg1, na, conds, form, count, revrange, currmed, health):
    search = ['Indication','Condition','Drug',\
              'Effectiveness','EaseofUse','Satisfaction',\
              'Reviews','Form','PkgCount','EstimatedPrice']
    index = []
    a1index = []
    df = otc
    if na:
        df = df.loc[(df.EaseofUse >= 0)&(df.EstimatedPrice >= 0)].reset_index(drop=True)
    
    if revrange:    
        df = df.loc[(df.Reviews >= revrange)].reset_index(drop=True)
    
    if form != 'All':
        df = df.loc[df.Form == form].reset_index(drop=True)
        
    if conds != '':
        conds = '(' + ')|('.join(conds) + ')'
    for i in range(len(df)):
        if re.search(conds.lower(), df.Allergies[i].lower()):
            a1index += [i]
        else:
            continue
    df = df[~df.index.isin(a1index)].reset_index(drop=True)
                    
    if tg2:
        for i in range(len(df)):
            if re.search('^'+string.lower()+'$', df.Condition[i].lower()):
                index += [i]
            else:
                continue
        return df[search].loc[index].sort_values(dd1, ascending=tg1).head(count).set_index('Drug')
    
    if tg3:
        regex = '^(?=.*\\b'+'\\b)(?=.*\\b'.join(string.lower().split())+'\\b).*$'
        for i in range(len(df)):
            if re.search(regex, df.Condition[i].lower()):
                index += [i]
            else:
                continue
        return df[search].loc[index].sort_values(dd1, ascending=tg1).head(count).set_index('Drug')
    
    else:
        for i in range(len(df)):
            if re.search(string.lower(), df.Condition[i].lower()):
                index += [i]
            else:
                continue
        return df[search].loc[index].sort_values(dd1, ascending=tg1).head(count).set_index('Drug') 
    
# search_conditions('cough',False,'False','','Effectiveness',False,True,'All',5) # for debugging only<jupyter_output><empty_output><jupyter_text>Widget Creation<jupyter_code>allergstr = []
reg = re.compile(r"[\d+\w+]{2,}")
for i in range(len(otc.Allergies)):
    allergstr += nltk.word_tokenize(otc.Allergies[i])
allergstr = [x for x in list(set(allergstr)) if reg.match(x)]
allergstr = []
reg = re.compile(r"[\d+\w+]{2,}")
for i in range(len(otc.Allergies)):
    allergstr += nltk.word_tokenize(otc.Allergies[i])
allergstr = [x for x in list(set(allergstr)) if reg.match(x)]
tg1 = widgets.ToggleButton(
    value=False,
    description='Ascending/Descending',
    disabled=False,
    button_style='success', # 'success', 'info', 'warning', 'danger' or ''
    tooltip='Description',
    icon='check'
)

tg2 = widgets.ToggleButton(
    value=False,
    description='Match Exact',
    disabled=False,
    button_style='warning', # 'success', 'info', 'warning', 'danger' or ''
    tooltip='Description',
    icon='check'
)

tg3 = widgets.ToggleButton(
    value=False,
    description='Match Term',
    disabled=False,
    button_style='success', # 'success', 'info', 'warning', 'danger' or ''
    tooltip='Description',
    icon='check'
)

dd1 = widgets.Dropdown(
    options={'Estimated Price': 'EstimatedPrice', 'Reviews': 'Reviews', 'Ease of Use': 'EaseofUse',\
            'Satisfaction': 'Satisfaction', 'Effectiveness': 'Effectiveness'},
    value='Reviews',
    description='Sort by:',
    disabled=False,
    button_style='info' # 'success', 'info', 'warning', 'danger' or ''
)

str1 = widgets.Text(
    value='Cough',
    placeholder='',
    description='Search:',
    disabled=False
)

int1 = widgets.IntSlider(
    value = 5,
    step = 1,
    min = 1,
    max = 10,
    description = 'Show:'
)

age1 = widgets.IntText(
    value = 35,
    step = 1,
    min = 1,
    max = 100,
    description = 'Age:'
)

na1 = widgets.ToggleButton(
    value=True,
    description='Remove NA',
    disabled=False,
    button_style='warning', # 'success', 'info', 'warning', 'danger' or ''
    tooltip='Description',
    icon='check'
)

for1 = widgets.Dropdown(
    options=['All'] + otc.Form.unique().tolist(),
    value='All',
    description='Form:',
    disabled=False,
    button_style='info' # 'success', 'info', 'warning', 'danger' or ''
)

revrange = widgets.IntSlider(
    value= 5,
    min=0,
    max= 50,
    step=5,
    description='Minimum Reviews:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='i',
    slider_color='white',
    color='black'
)

a= widgets.SelectMultiple(
    options=['NONE'] +sorted(allergstr),
    value=['NONE'],
    description='Allergies:',
    disabled=False
)

currmed = widgets.Text(
    value='',
    placeholder='',
    description='Current Meds:',
    disabled=False
)

health = widgets.Text(
    value='',
    placeholder='',
    description='Known Illnesses:',
    disabled=False
)<jupyter_output><empty_output><jupyter_text>Drug Finder<jupyter_code>interactive(search_conditions,
            string = str1,
            tg1 = tg1,
            tg3 = tg3,
            conds = a,
            age = age1,
            dd1 = dd1,
            na = na1,
            tg2 = tg2,
            form = for1,
            count = int1,
            revrange = revrange,
            currmed = currmed,
            health = health,
            __manual=False)
hidecode<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16960503720663994, 'fraction_numerical': 0.02083571837435604, 'mean_word_length': 4.013199426111909, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '299768', 'license': 'MIT', 'n_tokens_mistral': 6492, 'n_tokens_neox': 6181, 'n_words': 1053, 'path': 'webmd.ipynb', 'repo_name': 'ksu-hmi/The-Right-Remedy', 'size': '17470'}