<jupyter_start><jupyter_text>Captsone project for the IBM Data Science Professional Certificate (https://www.coursera.org/professional-certificates/ibm-data-science?courses)<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
print("Import Completed")
# The code was removed by Watson Studio for sharing.<jupyter_output>CSV read into dataframe<jupyter_text>1. Data Exploration  A. Find the Nulls<jupyter_code>data1 = []
for i in df.columns:
    data1.append([i, df[i].nunique(), df[i].unique(), df[i].dtypes, df[i].isnull().sum(), (df[i].isnull().sum() / len(df)) * 100])
    
df1 = pd.DataFrame(data1)
df1.columns = ['FEATURE', '# OF UNIQUE', 'UNIQUE VALUES AS AN ARRAY', 'DATA TYPE', '# OF NULLS', '% OF VALUES THAT ARE NULLS']
df1<jupyter_output><empty_output><jupyter_text>Clean Data Based on *B. Investigate the Nulls* Results  Drop Features<jupyter_code># Drop the following features: INTKEY, EXCEPTRSNCODE, EXCEPTRSNDESC, SDOTCOLNUM
# axis = 1 means drop the whole column
dfb = df.drop(['INTKEY', 'EXCEPTRSNCODE', 'EXCEPTRSNDESC', 'SDOTCOLNUM'], axis = 1)
dfb.columns<jupyter_output><empty_output><jupyter_text>Replace NANs with 'N'<jupyter_code># Replace NANs with 'N' for the following features: INATTENTIONIND, PEDROWNOTGRNT, SPEEDING
dfb1 = dfb
dfb1['INATTENTIONIND'] = dfb1['INATTENTIONIND'].replace(np.nan, 'N')
dfb1['PEDROWNOTGRNT'] = dfb1['PEDROWNOTGRNT'].replace(np.nan, 'N')
dfb1['SPEEDING'] = dfb1['SPEEDING'].replace(np.nan, 'N')
dfb1['INATTENTIONIND'].unique()
dfb1['PEDROWNOTGRNT'].unique()
dfb1['SPEEDING'].unique()
dfb1.columns<jupyter_output><empty_output><jupyter_text>C. Investigate the Remaining Features with Nulls<jupyter_code>data2 = []

for i in dfb1.columns:
    data2.append([i, dfb1[i].nunique(), dfb1[i].unique(), dfb1[i].dtypes, dfb1[i].isnull().sum(), (dfb1[i].isnull().sum() / len(dfb1)) * 100])
    
df2 = pd.DataFrame(data2)
df2.columns = ['FEATURE', '# OF UNIQUE', 'UNIQUE VALUES AS AN ARRAY', 'DATA TYPE', '# OF NULLS', '% OF VALUES THAT ARE NULLS']
df2<jupyter_output><empty_output><jupyter_text>Clean Data Based on *C. Investigate the Remaining Features with Nulls* Results  Drop Rows with Null Values<jupyter_code># Drop rows with null values for the following features: ADDRTYPE, COLLISIONTYPE, JUNCTIONTYPE, UNDERINFL, WEATHER, ROADCOND, LIGHTCOND
# axis = 0 means to drop rows where NANs are the values for the abovementioned features
dfc = dfb1.dropna(subset=['ADDRTYPE', 'COLLISIONTYPE', 'JUNCTIONTYPE', 'UNDERINFL', 'WEATHER', 'ROADCOND', 'LIGHTCOND'], axis = 0)

nvs = dfc.isnull().sum()
nvs_per = (nvs / dfc['SEVERITYCODE'].count()) * 100

print(dfc.columns)
print(nvs_per)<jupyter_output>Index(['SEVERITYCODE', 'X', 'Y', 'OBJECTID', 'INCKEY', 'COLDETKEY', 'REPORTNO',
       'STATUS', 'ADDRTYPE', 'LOCATION', 'SEVERITYCODE.1', 'SEVERITYDESC',
       'COLLISIONTYPE', 'PERSONCOUNT', 'PEDCOUNT', 'PEDCYLCOUNT', 'VEHCOUNT',
       'INCDATE', 'INCDTTM', 'JUNCTIONTYPE', 'SDOT_COLCODE', 'SDOT_COLDESC',
       'INATTENTIONIND', 'UNDERINFL', 'WEATHER', 'ROADCOND', 'LIGHTCOND',
       'PEDROWNOTGRNT', 'SPEEDING', 'ST_COLCODE', 'ST_COLDESC', 'SEGLANEKEY',
       'CROSSWALKKEY', 'HITPARKEDCAR'],
      dtype='object')
SEVERITYCODE      0.000000
X                 1.546242
Y                 1.546242
OBJECTID          0.000000
INCKEY            0.000000
COLDETKEY         0.000000
REPORTNO          0.000000
STATUS            0.000000
ADDRTYPE          0.000000
LOCATION          0.128489
SEVERITYCODE.1    0.000000
SEVERITYDESC      0.000000
COLLISIONTYPE     0.000000
PERSONCOUNT       0.000000
PEDCOUNT          0.000000
PEDCYLCOUNT       0.000000
VEHCOUNT          0.000000
INCDATE          [...]<jupyter_text>Drop Features Deemed Unnecessary or Redundant<jupyter_code># Drop the following features: X, Y, LOCATION, ST_COLCODE, ST_COLDESC
dfc1 = dfc.drop(['X', 'Y', 'LOCATION', 'ST_COLCODE', 'ST_COLDESC'], axis = 1)
dfc1.columns<jupyter_output><empty_output><jupyter_text>Replacing 0s with Ns and 1s with Ys<jupyter_code>dfc2 = dfc1
dfc2['UNDERINFL'] = dfc2['UNDERINFL'].replace('0', 'N')
dfc2['UNDERINFL'] = dfc2['UNDERINFL'].replace('1', 'Y')
dfc2['UNDERINFL'].unique()<jupyter_output><empty_output><jupyter_text>Removing Rows with Unknown as their values<jupyter_code># Cleaning WEATHER rows
indexWeather = dfc2[dfc2['WEATHER'] == 'Unknown'].index
dfc3 = dfc2.drop(indexWeather, axis = 0)
# Cleaning ROADCOND rows
indexRoad = dfc3[dfc3['ROADCOND'] == 'Unknown'].index
dfc3 = dfc3.drop(indexRoad, axis = 0)
# Cleaning LIGHTCOND rows
indexLight = dfc3[dfc3['LIGHTCOND'] == 'Unknown'].index
dfc3 = dfc3.drop(indexLight, axis = 0)
# verifyin that the rows have been removed
print("WEATHER uniques: ", dfc3['WEATHER'].unique())
print("ROADCOND uniques: ", dfc3['ROADCOND'].unique())
print("LIGHTCOND uniques: ", dfc3['LIGHTCOND'].unique())
# Examining the results of preliminary cleaning based on 
# nulls, improperly labeled values, and useless values
print(dfc3.shape)
print(df.shape)
print("Percent of features: ", len(dfc3.columns) / len(df.columns) * 100)
print("Percent of rows: ", len(dfc3) / len(df) * 100)
print("Reduction of features: ", len(df.columns) - len(dfc3.columns), ", ", 100 - (len(dfc3.columns) / len(df.columns) * 100))
print("Reduction of rows: ", len(df) - len(dfc3), ", ", 100 - (len(dfc3) / len(df) * 100))
# The amount of features has been reduced by 9, or 23.68%.
# The amount of observations has been reduced by 26,828, or 13.78%.<jupyter_output>(167845, 29)
(194673, 38)
Percent of features:  76.31578947368422
Percent of rows:  86.21894150704001
Reduction of features:  9 ,  23.68421052631578
Reduction of rows:  26828 ,  13.781058492959986<jupyter_text>D. Examining the Remaining Features for Relevenacy<jupyter_code>data3 = []
ds = dfc3
for i in ds.columns:
    data3.append([i, ds[i].nunique(), ds[i].unique(), ds[i].dtypes, ds[i].isnull().sum(), (ds[i].isnull().sum() / len(ds)) * 100])
    
df3 = pd.DataFrame(data3)
df3.columns = ['FEATURE', '# OF UNIQUE', 'UNIQUE VALUES AS AN ARRAY', 'DATA TYPE', '# OF NULLS', '% OF VALUES THAT ARE NULLS']
df3<jupyter_output><empty_output><jupyter_text>Clean Data Based on *D. Examining the Remaining Features for Relevenacy* Results  Drop Features Deemed Unnecessary<jupyter_code># Drop the following features:
dfd1 = dfc3.drop(['OBJECTID', 'INCKEY', 'COLDETKEY', 'REPORTNO', 'STATUS', 'SEVERITYCODE.1', 'SEVERITYDESC', 'INCDATE', 'INCDTTM', 'SDOT_COLCODE', 'SDOT_COLDESC', 'SEGLANEKEY', 'CROSSWALKKEY'], axis = 1)<jupyter_output><empty_output><jupyter_text>E. Examine Remaining Features<jupyter_code># Keep the following features:
# ADDRTYPE, COLLISIONTYPE, PERSONCOUNT, PEDCOUNT, PEDCYLCOUNT, VEHCOUNT, JUNCTIONTYPE, INATTENTIONIND, UNDERINFL, WEATHER, ROADCOND, LIGHTCOND, PEDROWNOTGRNT, SPEEDING, HITPARKEDCAR
ds = dfd1
data5 = []
for i in ds.columns:
    data5.append([i, ds[i].nunique(), ds[i].unique(), ds[i].dtypes, ds[i].isnull().sum(), (ds[i].isnull().sum() / len(ds)) * 100])
    
df5 = pd.DataFrame(data5)
df5.columns = ['FEATURE', '# OF UNIQUE', 'UNIQUE VALUES AS AN ARRAY', 'DATA TYPE', '# OF NULLS', '% OF VALUES THAT ARE NULLS']
df5
# this is a great place to start, so let's start plotting to futher examine the features we have<jupyter_output><empty_output><jupyter_text>F. Explore Remaining Features Through Plots<jupyter_code>import seaborn as sns
# convert the int64 features to objects for plotting
dfep = dfd1.astype({"PERSONCOUNT": object, "PEDCOUNT": object, "PEDCYLCOUNT": object, "VEHCOUNT": object})
# drop problematic features for plotting
dfep1 = dfep.drop(['PERSONCOUNT', 'PEDCOUNT', 'PEDCYLCOUNT', 'VEHCOUNT'], axis = 1)
dfep1.columns<jupyter_output><empty_output><jupyter_text>WORKING<jupyter_code>dfep1.columns = ['Severity Code', 'Address Type', 'Collision Type', 'Junction Type', 'Inattention', 'Under Influence', 'Weather', 'Road Conditions', 'Lighting Conditions', 'Pedestrian Right of Way Granted', 'Speeding', 'Hit a Parked Car']
itercolumns = iter(dfep1.columns)
next(itercolumns)

for p in itercolumns:
    # 3.2 plot severity vs lighting conditions
    # prepare the data to be plotted
    dflc = (dfep1
        .groupby([p, 'Severity Code'])
        .size()
        .reset_index()
        .replace({'Severity Code': {1:"prop. damage", 2:"minor injury"}})
        .replace({'Y':"Yes", 'N':"No"})
        .rename(columns={0:"count"}))

    # calculate survival % per town of embarkation    
    dflc["percent"] = (dflc
                     .groupby(p)
                     .apply(lambda x: x["count"] / x["count"].sum()).values)

    # sort the dataframe to match the drawing order    
    dflc.sort_values(by=['Severity Code', p], inplace = True)

    # plot severity vs lighting conditions
    plt.style.use('ggplot')
    fig = sns.catplot(
        x="count", y=p, hue='Severity Code',
        kind="bar", data=dflc, height=10, aspect=2, legend_out = False, orient = "h")

    for i, bar in enumerate(fig.ax.patches):
        fig.ax.annotate(
            # reference the pre-calculated row in the dataframe
            f"{dflc.iloc[i, 3] :.0%}",
            #xycoords="data",
            #xytext=(30, -15),
            fontsize = 18,
            #textcoords="offset points",
            xy=(bar.get_width(), bar.get_y() + (bar.get_height() / 2)),
            #horizontalalignment='center', 
            verticalalignment='center'
            )
    
    # make space for annonations
    plt.margins(x=0.2)

    # make final adjustments
    plt.title(p + " vs. Number of Collisions by Severity", fontsize = 30)
    plt.ylabel(p , fontsize = 24)
    plt.xlabel('Number of Collisions', fontsize = 24)
    plt.tick_params(labelsize=18)
    plt.legend(fontsize = 18, title = "Severity", loc = 'lower right', title_fontsize = 18, shadow = True)
    plt.show()<jupyter_output><empty_output><jupyter_text>WORKING<jupyter_code># 3.10 plot severity vs number of cyclists
indvar = 'PEDCYLCOUNT'
depvar = 'Number of Cyclists'

# prepare the data to be plotted
dflc = (dfep
    .groupby([indvar, 'SEVERITYCODE'])
    .size()
    .reset_index()
    .replace({'SEVERITYCODE': {1:"prop. damage", 2:"minor injury"}})
    .rename(columns={0:"count"}))

# calculate survival % per town of embarkation    
dflc["percent"] = (dflc
                 .groupby(indvar)
                 .apply(lambda x: x["count"] / x["count"].sum()).values)

# sort the dataframe to match the drawing order    
dflc.sort_values(by=['SEVERITYCODE', indvar], inplace = True)

new_row = {'PEDCYLCOUNT':2, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
dflc = dflc.append(new_row, ignore_index = True)

# plot severity vs lighting conditions
plt.style.use('ggplot')
fig = sns.catplot(
    x="count", y=indvar, hue='SEVERITYCODE',
    kind="bar", data=dflc, height=10, aspect=2, legend_out = False, orient = "h")

for i, bar in enumerate(fig.ax.patches):
   
    height = bar.get_height()
    fig.ax.annotate(
        # reference the pre-calculated row in the dataframe
        f"{dflc.iloc[i, 3] :.0%}",
        #xycoords="data",
        #xytext=(30, -15),
        fontsize = 18,
        #textcoords="offset points",
        xy=(bar.get_width(), bar.get_y() + (bar.get_height() / 2)),
        #horizontalalignment='center', 
        verticalalignment='center'
        )
    
# make space for annonations
plt.margins(x=0.2)

# make final adjustments
plt.title(depvar + ' vs. Number of Collisions by Severity', fontsize = 30)
plt.xlabel('Number of Collisions', fontsize = 24)
plt.ylabel(depvar, fontsize = 24)
plt.tick_params(labelsize=18)
plt.legend(fontsize = 18, title = "Severity", loc = 'lower right', title_fontsize = 18, shadow = True)
plt.show()
# 3.10 plot severity vs number of cyclists
indvar = 'PEDCOUNT'
depvar = 'Number of Pedestrians'

# prepare the data to be plotted
dflc = (dfep
    .groupby([indvar, 'SEVERITYCODE'])
    .size()
    .reset_index()
    .replace({'SEVERITYCODE': {1:"prop. damage", 2:"minor injury"}})
    .rename(columns={0:"count"}))

# calculate survival % per town of embarkation    
dflc["percent"] = (dflc
                 .groupby(indvar)
                 .apply(lambda x: x["count"] / x["count"].sum()).values)

# sort the dataframe to match the drawing order    
dflc.sort_values(by=['SEVERITYCODE', indvar], inplace = True)

new_row = {'PEDCOUNT':4, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
new_row1 = {'PEDCOUNT':5, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
new_row2 = {'PEDCOUNT':6, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
dflc = dflc.append(new_row, ignore_index = True)
dflc = dflc.append(new_row1, ignore_index = True)
dflc = dflc.append(new_row2, ignore_index = True)

# plot severity vs lighting conditions
plt.style.use('ggplot')
fig = sns.catplot(
    x="count", y=indvar, hue='SEVERITYCODE',
    kind="bar", data=dflc, height=10, aspect=2, legend_out = False, orient = "h")

for i, bar in enumerate(fig.ax.patches):
   
    height = bar.get_height()
    fig.ax.annotate(
        # reference the pre-calculated row in the dataframe
        f"{dflc.iloc[i, 3] :.0%}",
        #xycoords="data",
        #xytext=(30, -15),
        fontsize = 18,
        #textcoords="offset points",
        xy=(bar.get_width(), bar.get_y() + (bar.get_height() / 2)),
        #horizontalalignment='center', 
        verticalalignment='center'
        )
    
# make space for annonations
plt.margins(x=0.2)

# make final adjustments
plt.title(depvar + ' vs. Number of Collisions by Severity', fontsize = 30)
plt.xlabel('Number of Collisions', fontsize = 24)
plt.ylabel(depvar, fontsize = 24)
plt.tick_params(labelsize=18)
plt.legend(fontsize = 18, title = "Severity", loc = 'lower right', title_fontsize = 18, shadow = True)
plt.show()
# 3.10 plot severity vs number of cyclists
indvar = 'VEHCOUNT'
depvar = 'Number of Vehicles'

# prepare the data to be plotted
dflc = (dfep
    .groupby([indvar, 'SEVERITYCODE'])
    .size()
    .reset_index()
    .replace({'SEVERITYCODE': {1:"prop. damage", 2:"minor injury"}})
    .rename(columns={0:"count"}))

# calculate survival % per town of embarkation    
dflc["percent"] = (dflc
                 .groupby(indvar)
                 .apply(lambda x: x["count"] / x["count"].sum()).values)

new_row = {'VEHCOUNT':12, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row1 = {'VEHCOUNT':10, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
dflc = dflc.append(new_row, ignore_index = True)
dflc = dflc.append(new_row1, ignore_index = True)

# sort the dataframe to match the drawing order    
dflc.sort_values(by=['SEVERITYCODE', indvar], inplace = True)

# plot severity vs lighting conditions
plt.style.use('ggplot')
fig = sns.catplot(
    x="count", y=indvar, hue='SEVERITYCODE',
    kind="bar", data=dflc, height=10, aspect=2, legend_out = False, orient = "h")

for i, bar in enumerate(fig.ax.patches):
   
    height = bar.get_height()
    fig.ax.annotate(
        # reference the pre-calculated row in the dataframe
        f"{dflc.iloc[i, 3] :.0%}",
        #xycoords="data",
        #xytext=(30, -15),
        fontsize = 18,
        #textcoords="offset points",
        xy=(bar.get_width(), bar.get_y() + (bar.get_height() / 2)),
        #horizontalalignment='center', 
        verticalalignment='center'
        )
    
# make space for annonations
plt.margins(x=0.2)

# make final adjustments
plt.title(depvar + ' vs. Number of Collisions by Severity', fontsize = 30)
plt.xlabel('Number of Collisions', fontsize = 24)
plt.ylabel(depvar, fontsize = 24)
plt.tick_params(labelsize=18)
plt.legend(fontsize = 18, title = "Severity", loc = 'lower right', title_fontsize = 18, shadow = True)
plt.show()
# 3.10 plot severity vs number of cyclists
indvar = 'PERSONCOUNT'
depvar = 'Number of People'

# prepare the data to be plotted
dflc = (dfep
    .groupby([indvar, 'SEVERITYCODE'])
    .size()
    .reset_index()
    .replace({'SEVERITYCODE': {1:"prop. damage", 2:"minor injury"}})
    .rename(columns={0:"count"}))

# calculate survival % per town of embarkation    
dflc["percent"] = (dflc
                 .groupby(indvar)
                 .apply(lambda x: x["count"] / x["count"].sum()).values)

new_row = {'PERSONCOUNT':21, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row1 = {'PERSONCOUNT':26, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row2 = {'PERSONCOUNT':31, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row3 = {'PERSONCOUNT':35, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row4 = {'PERSONCOUNT':36, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row5 = {'PERSONCOUNT':41, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row6 = {'PERSONCOUNT':43, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row7 = {'PERSONCOUNT':44, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row8 = {'PERSONCOUNT':47, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row9 = {'PERSONCOUNT':53, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
new_row10 = {'PERSONCOUNT':57, 'SEVERITYCODE':'minor injury', 'count':0, 'percent':0.000000}
dflc = dflc.append(new_row, ignore_index = True)
dflc = dflc.append(new_row1, ignore_index = True)
dflc = dflc.append(new_row2, ignore_index = True)
dflc = dflc.append(new_row3, ignore_index = True)
dflc = dflc.append(new_row4, ignore_index = True)
dflc = dflc.append(new_row5, ignore_index = True)
dflc = dflc.append(new_row6, ignore_index = True)
dflc = dflc.append(new_row7, ignore_index = True)
dflc = dflc.append(new_row8, ignore_index = True)
dflc = dflc.append(new_row9, ignore_index = True)
dflc = dflc.append(new_row10, ignore_index = True)
new_row11 = {'PERSONCOUNT':39, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
new_row12 = {'PERSONCOUNT':48, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
new_row13 = {'PERSONCOUNT':54, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
new_row14 = {'PERSONCOUNT':81, 'SEVERITYCODE':'prop. damage', 'count':0, 'percent':0.000000}
dflc = dflc.append(new_row11, ignore_index = True)
dflc = dflc.append(new_row12, ignore_index = True)
dflc = dflc.append(new_row13, ignore_index = True)
dflc = dflc.append(new_row14, ignore_index = True)

# sort the dataframe to match the drawing order    
dflc.sort_values(by=['SEVERITYCODE', indvar], inplace = True)

# plot severity vs lighting conditions
plt.style.use('ggplot')
fig = sns.catplot(
    x="count", y=indvar, hue='SEVERITYCODE',
    kind="bar", data=dflc, height=30, aspect=2, legend_out = False, orient = "h")

for i, bar in enumerate(fig.ax.patches):
   
    height = bar.get_height()
    fig.ax.annotate(
        # reference the pre-calculated row in the dataframe
        f"{dflc.iloc[i, 3] :.0%}",
        #xycoords="data",
        #xytext=(30, -15),
        fontsize = 18,
        #textcoords="offset points",
        xy=(bar.get_width(), bar.get_y() + (bar.get_height() / 2)),
        #horizontalalignment='center', 
        verticalalignment='center'
        )
    
# make space for annonations
plt.margins(x=0.2)

# make final adjustments
plt.title(depvar + ' vs. Number of Collisions by Severity', fontsize = 30)
plt.xlabel('Number of Collisions', fontsize = 24)
plt.ylabel(depvar, fontsize = 24)
plt.tick_params(labelsize=18)
plt.legend(fontsize = 18, title = "Severity", loc = 'lower right', title_fontsize = 18, shadow = True)
plt.show()<jupyter_output><empty_output><jupyter_text>Transform Data<jupyter_code>ds = dfd1
data5 = []
for i in ds.columns:
    data5.append([i, ds[i].nunique(), ds[i].unique(), ds[i].dtypes, ds[i].isnull().sum(), (ds[i].isnull().sum() / len(ds)) * 100])
    
df5 = pd.DataFrame(data5)
df5.columns = ['FEATURE', '# OF UNIQUE', 'UNIQUE VALUES AS AN ARRAY', 'DATA TYPE', '# OF NULLS', '% OF VALUES THAT ARE NULLS']
df5
Z = dfd1.drop(['SEVERITYCODE'], axis = 1)
# We have to subtract 1 on SEVERITYCODE to get it ready for the logistic regression
# here 1 means a severity code of 2 (minor injury) and 
# 0 means a severity code of 1 (only property damage)
Y = dfd1['SEVERITYCODE'] - 1
# We have to get dummies for the following:
# ADDRTYPE, COLLISIONTYPE, JUNCTIONTYPE, WEATHER, ROADCOND, & LIGHTCOND
dummy_addrtype = pd.get_dummies(dfd1["ADDRTYPE"])
dummy_collisiontype = pd.get_dummies(dfd1["COLLISIONTYPE"])
dummy_junctiontype  = pd.get_dummies(dfd1["JUNCTIONTYPE"])
dummy_weather = pd.get_dummies(dfd1["WEATHER"])
dummy_roadcond = pd.get_dummies(dfd1["ROADCOND"])
dummy_lightcond = pd.get_dummies(dfd1["LIGHTCOND"])

# merging data from dummies into the dataframe
Z = pd.concat([Z, dummy_addrtype], axis = 1)
Z = pd.concat([Z, dummy_collisiontype], axis = 1)
Z = pd.concat([Z, dummy_junctiontype], axis = 1)
Z = pd.concat([Z, dummy_weather], axis = 1)
Z = pd.concat([Z, dummy_roadcond], axis = 1)
Z = pd.concat([Z, dummy_lightcond], axis = 1)

# dropping original categorical variable columns
Z.drop("ADDRTYPE", axis = 1, inplace = True)
Z.drop("COLLISIONTYPE", axis = 1, inplace = True)
Z.drop("JUNCTIONTYPE", axis = 1, inplace = True)
Z.drop("WEATHER", axis = 1, inplace = True)
Z.drop("ROADCOND", axis = 1, inplace = True)
Z.drop("LIGHTCOND", axis = 1, inplace = True)
Z.shape
# We have to convert yes-no binary features into 0-and-1 binary features:
# INATTENTIONIND, UNDERINFL, PEDROWNOTGRNT, SPEEDING, HITPARKEDCAR
Z = Z.replace({'Y':1, 'N':0})<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>import itertools
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix
from sklearn.metrics import jaccard_similarity_score
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score
from sklearn.metrics import log_loss
import statsmodels.api as sm<jupyter_output><empty_output><jupyter_text>Each Independant Variable vs. Dependent Variable (SEVERITYCODE)<jupyter_code>for IV in dfd1.columns:
    Y1 = Y
    Z1 = dfd1[IV]
    Z1.reshape(-1, 1)
    X_trainLR, X_testLR, y_trainLR, y_testLR = train_test_split(Z1, Y1, test_size=0.2, random_state=4)
    LR = LogisticRegression(C=0.01, solver='liblinear').fit(X_trainLR, y_trainLR.values.ravel())
    log_reg = sm.Logit(y_trainLR, X_trainLR).fit(method='bfgs')
    print(IV, ": ", log_reg.summary())
Zaddrtype = Z['Intersection', 'Block', 'Alley']
X_trainLR, X_testLR, y_trainLR, y_testLR = train_test_split(Zaddrtype, Y, test_size=0.2, random_state=4)
LR = LogisticRegression(C=0.01, solver='liblinear').fit(X_trainLR, y_trainLR.values.ravel())
log_reg = sm.Logit(y_trainLR, X_trainLR).fit(method='bfgs')
print("Address Type: ", log_reg.summary())<jupyter_output><empty_output><jupyter_text>Final Logistic Regression with correlated Independant Variables<jupyter_code>X_trainLR, X_testLR, y_trainLR, y_testLR = train_test_split(Z, Y, test_size=0.2, random_state=4)
print ('Train set:', X_trainLR.shape,  y_trainLR.shape)
print ('Test set:', X_testLR.shape,  y_testLR.shape)
LR = LogisticRegression(C=0.01, solver='liblinear').fit(X_trainLR, y_trainLR.values.ravel())
yhatLR = LR.predict(X_testLR)
yhat_probLR = LR.predict_proba(X_testLR)
jaccard_similarity_score(y_testLR, yhatLR)
print("F1 Score: ", f1_score(y_testLR, yhatLR, average='weighted')) 
print("Jaccard Score: ", jaccard_similarity_score(y_testLR, yhatLR))
print("Log Loss: ", log_loss(y_testLR, yhat_probLR))
# define function to plot a confusion matrix
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
# Compute confusion matrix
cnf_matrix = confusion_matrix(y_testLR, yhatLR, labels=[1,0])
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=['severity=2','severity=1'],normalize= False,  title='Confusion matrix')
log_reg = sm.Logit(y_trainLR, X_trainLR).fit(method='bfgs')
print(log_reg.summary())<jupyter_output>/opt/conda/envs/Python36/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in greater
  return (self.a < x) & (x < self.b)
/opt/conda/envs/Python36/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in less
  return (self.a < x) & (x < self.b)
/opt/conda/envs/Python36/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:1831: RuntimeWarning: invalid value encountered in less_equal
  cond2 = cond0 & (x <= self.a)<jupyter_text>ROC Curve<jupyter_code>yhat_probsLR = yhat_probLR[:, 1]
print(yhat_probsLR)
aucLR = roc_auc_score(y_testLR, yhat_probsLR)
print('AUC: %.2f' % aucLR)
fprLR, tprLR, thresholdsLR = roc_curve(y_testLR, yhat_probsLR, pos_label = 1)
plt.plot(fprLR, tprLR, color='#87ceeb', label='Logistic Regression')
plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic (ROC) Curve')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15188762071992976, 'fraction_numerical': 0.03366797724930336, 'mean_word_length': 4.537518495032763, 'pattern_counts': {'":': 5, '<': 82, '<?xml version=': 0, '>': 78, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '294958', 'license': 'MIT', 'n_tokens_mistral': 10231, 'n_tokens_neox': 9473, 'n_words': 2170, 'path': 'Captsone (v 2).ipynb', 'repo_name': 'mjf4010/IBM-Professional-Data-Science-Certificate', 'size': '26197'}