<jupyter_start><jupyter_text>https://www.kaggle.com/benoit72/uk-accidents-10-years-history-with-many-variablesData is a conglomerate of data for a 10 year stretch of UK car accidents<jupyter_code>#import TensorFlow and Keras API
import tensorflow as tf
from tensorflow import keras

#other imports
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os
import cv2
#define functions

#Create even amounts of each category to reduce bias
def BalanceInputData(df, x_cols, y_col):
    #where x_col is list of strings and y_col is catagorical integer
    data = []
    #shuffle df before taking samples
    df = df.sample(frac=1).reset_index(drop=True)
    #Find limiting catagory
    max_sample = df[y_col].value_counts().min()
    limit_val = df[y_col].value_counts().index[-1]
    cat_count = len(df[y_col].value_counts())
    non_limits = df[y_col].value_counts().index[0:cat_count-1]
    #store x and y as arrays
    X = np.array(df[x_cols])
    Y = np.array(df[y_col])
    #create empty arrays to return balanced shuffled data
    x = []
    y = []
    #loop through y values and get up to max_sample (X, Y) pairs
    #from each catagory in non_limits
    for val in non_limits:
        sample_count = 0
        for i in range(len(Y)):
            if Y[i] == val and sample_count < max_sample:
                data.append([X[i], Y[i]])
                sample_count = sample_count+1
    #Get all limit values as (X, Y) pairs
    for i in range(len(Y)):
        if Y[i] == limit_val:
            data.append([X[i], Y[i]])
    #Shuffle data before returning
    np.random.shuffle(data)
    #populate x,y with balanced shuffled data
    for i in range(len(data)):
        x.append(data[i][0])
        y.append(data[i][1])    
    #return x and y
    x = np.array(x)
    y = np.array(y)
    return(x,y)

def PlotTrainingData(path, file_name):
    path, file_name = path, file_name
    path = os.path.join(path,file_name)
    epoch_data = pd.read_csv(path)
    epoch_data.drop(columns=['epoch'],inplace=True)
    display(epoch_data)
    plt.figure(figsize=(10,7))
    plt.tight_layout()
    #Plot accuracy and val_accuracy
    plt.subplot(2,2,1)
    acc_data = epoch_data[['accuracy','val_accuracy']]
    plt.xlabel('epoch')
    plt.ylabel('accuracy')
    plt.title('Model Accuracy')
    plt.plot(acc_data)
    plt.legend(['train','test'])
    #Plot loss and val_loss
    plt.subplot(2,2,2)
    loss_data = epoch_data[['loss','val_loss']]
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.title('Model Loss')
    plt.plot(loss_data)
    plt.legend(['train','test'])
    #Print max val_acc and min val_loss
    max_acc = epoch_data['val_accuracy'].max()
    min_loss = epoch_data['val_loss'].min()
    print('Max validation accuracy: {}\nMin validation loss: {}'.format(max_acc,min_loss))
#Import data
raw_data = pd.read_csv('Accidents0514.csv')
display(raw_data.head())
print(raw_data.columns)
#Reduce data to a few fields. Goal is to predict Accident_Severity
data = raw_data[['Accident_Severity','Day_of_Week', 'Road_Type', 'Speed_limit', 'Junction_Control', 'Light_Conditions', 'Weather_Conditions', 'Road_Surface_Conditions']]
#Look at reduced data set
data.describe()
#Most data is coded. Bring in coded sheets as pandas dfs
file = 'Road-Accident-Safety-Data-Guide.xls'

Accident_Severity = pd.read_excel(file, sheet_name = 'Accident Severity')
Day_of_Week = pd.read_excel(file, sheet_name = 'Day of Week')
Road_Type = pd.read_excel(file, sheet_name = 'Road Type')
Junction_Control = pd.read_excel(file, sheet_name = 'Junction Control')
Light_Conditions = pd.read_excel(file, sheet_name = 'Light Conditions')
Weather_Conditions = pd.read_excel(file, sheet_name = 'Weather')
Road_Surface_Conditions = pd.read_excel(file, sheet_name = 'Road Surface')

codes = [Accident_Severity, Day_of_Week, Road_Type, Junction_Control,
         Light_Conditions, Weather_Conditions, Road_Surface_Conditions]


# #Run this to see all codes
# for code in codes:
#     display(code)
#Check nulls
null = pd.DataFrame(data.isnull().sum())
display(null)
#Create histograms of day of week
display(Day_of_Week)
name = 'Day_of_Week'
data[name].hist()
plt.title(name)
#Create histograms of roadtype
display(Road_Type)
name = 'Road_Type'
data[name].hist()
plt.title(name)
#Speed limit hist
name = 'Speed_limit'
data[name].hist()
plt.title(name)
#Junction_Control hist
display(Junction_Control)
name = 'Junction_Control'
data[name].hist()
plt.title(name)
#Light_Conditions hist
display(Light_Conditions)
name = 'Light_Conditions'
data[name].hist()
plt.title(name)
#Weather_Conditions hist
display(Weather_Conditions)
name = 'Weather_Conditions'
data[name].hist()
plt.title(name)
#Road_Surface_Conditions hist
display(Road_Surface_Conditions)
name = 'Road_Surface_Conditions'
data[name].hist()
plt.title(name)
#Data is very unbalanced
Sev_Group = data.groupby(['Accident_Severity']).count()
display(Sev_Group)
#Make catagoricals binary sparse comulns
cols = ['Day_of_Week', 'Road_Type', 'Junction_Control', 
        'Light_Conditions', 'Weather_Conditions', 'Road_Surface_Conditions']

preproc_data = pd.get_dummies(data, columns=cols)

#Scale ordinals
preproc_data['Speed_limit'] = preproc_data['Speed_limit']/preproc_data['Speed_limit'].max()

#Balance data
#define x and y cols
x_cols = list(preproc_data.columns)
x_cols = x_cols[1:]

y_col = 'Accident_Severity'

bal_x, bal_y = BalanceInputData(df=preproc_data, x_cols=x_cols, y_col=y_col)
print(len(bal_x))
print(len(bal_y))
#Turn y into catagorical output
from tensorflow.keras.utils import to_categorical
bal_y = to_categorical(bal_y)
#Setting up the NN
from tensorflow.keras.layers import Dense, Activation

#initiate the model
model = keras.Sequential()

#Add layers
model.add(Dense(128, activation='relu'))

model.add(Dense(256, activation='relu'))

model.add(Dense(512, activation='relu'))

model.add(Dense(256, activation='relu'))

model.add(Dense(128, activation='relu'))

model.add(Dense(4, activation='softmax'))
#Create early stop callback
from tensorflow.keras.callbacks import EarlyStopping

stop_on_val_loss = EarlyStopping(
    monitor='val_loss', 
    min_delta=0.0001, patience=5)
#Callback to store epoch data for graphing
from tensorflow.keras.callbacks import CSVLogger

file_name = 'CarCrash_logs.csv'
csv_logger = CSVLogger(filename=file_name, separator=',', append=False)
#Define loss function
loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True)
#compile model
model.compile(optimizer='SGD',
              loss=loss_fn,
              metrics=['accuracy'])
#Train the model
model.fit(bal_x,bal_y, batch_size=256, epochs = 256,
         validation_split=0.25, callbacks=[csv_logger, stop_on_val_loss])
#Plot accuracy and loss
path = 'C:/Users/<NAME>/Documents/School/IE 4910 Python ML/Assignments/Project 3 Neural Networks'

PlotTrainingData(path=path, file_name=file_name)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12309699869508482, 'fraction_numerical': 0.009424387414818037, 'mean_word_length': 4.915951972555746, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '145862', 'license': 'MIT', 'n_tokens_mistral': 2505, 'n_tokens_neox': 2409, 'n_words': 555, 'path': 'Projects/Project-3-NNs/Project 3_Neural Networks - Car Crashes.ipynb', 'repo_name': 'spacemuggle/Python-ML-Class-2020', 'size': '6902'}