<jupyter_start><jupyter_text>Insurance MLpredict risk of accidents<jupyter_code>try:
  # %tensorflow_version only exists in Colab.
  %tensorflow_version 2.x
except Exception:
  pass

import tensorflow as tf
tf.__version__
tf.random.set_seed(42)
import numpy as np
np.__version__
import matplotlib.pyplot as plt
%matplotlib inline

import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (20, 8)
mpl.rcParams['axes.titlesize'] = 24
mpl.rcParams['axes.labelsize'] = 20
# !pip install -q dtreeviz
# https://github.com/parrt/dtreeviz
import dtreeviz
dtreeviz.__version__
# https://github.com/AndreasMadsen/python-lrcurve
# !pip install -q lrcurve
from lrcurve import KerasLearningCurve
# XXX: THIS IS VERY GENERAL AND CAN BE USED PRETTY MUCH ANYWHERE

from dtreeviz import clfviz

def plot_decision_boundaries(model, X, y_true, x1_range=None, x2_range=None):
  _, ax = plt.subplots(figsize=(8,4), dpi=300)


  ranges = None
  if x1_range and x2_range:
    ranges=(x1_range, x2_range)

  clfviz(
              model, X, y_true, 
             show=['instances', 'boundaries', 'probabilities', 'misclassified'],
             markers=['v', '^', 'd'],
             ntiles=50, 
             ax=ax,
            ranges=ranges,
             tile_fraction=1.0, 
             boundary_markersize=1.0,
             feature_names=["Age", "Max Speed"],
             colors={'class_boundary': 'black',
                     'tile_alpha': 0.5,
                    #  'warning' : 'yellow',
                     'classes':
                         [None,  # 0 classes
                          None,  # 1 class
                          None,  # 2 classes
                          ['#FF8080', '#FFFF80', '#8080FF'],  # 3 classes
                          ]
                     }
               )<jupyter_output><empty_output><jupyter_text>Step 1: Loading and exploring our data setThis is a database of customers of an insurance company. Each data point is one customer. Risk is expressed as a number between 0 and 1. 1 meaning highest and 0 meaning lowerst risk of having an accident.<jupyter_code># XXX: why would everyone need to know where the data is being loadded from and what if that changes? also: how to even do that?
import pandas as pd

# df = pd.read_csv('https://raw.githubusercontent.com/DJCordhose/insurance-ml/main/data/insurance-customers-risk-1500.csv')
df = pd.read_csv('../data/insurance-customers-risk-1500.csv')
# XXX: Loading is mandatory, but why analysis of the data in a training notebook?
df.head()
df.describe()
features = ['speed', 'age', 'miles']
import seaborn as sns
# XXX: COLORS ARE WEIRD

plt.figure(figsize=(10, 10))

cm = df.corr()
cm3 = cm.iloc[:3, :3]

hm = sns.heatmap(cm3,
                cbar=True,
                annot=True,
                square=True,
#                 cmap='Blues', 
                fmt='.2f',
                yticklabels=features,
                xticklabels=features)<jupyter_output><empty_output><jupyter_text>Step 2: Training a neural network on 2 dimensions of the data<jupyter_code>y = df['group'].values
# add more columns to list to have fewer features to train on
X = df.drop(['risk', 'group', 'miles'], axis='columns').values
# reorder, first age, then speed to match plotting
X = pd.DataFrame(np.array([X[:, 1], X[:, 0]]).T)
X.shape, y.shape
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)
X_train.shape, X_val.shape, y_train.shape, y_val.shape
### XXX: THERE IS SO MUCH ROOM FOR EXPERIMENT AND MAKING COPIES HERE

from tensorflow.keras.layers import InputLayer, Dense, Dropout, \
                                    BatchNormalization, Activation

num_features = X.shape[1]
dropout = 0.6
model = tf.keras.Sequential()

model.add(InputLayer(name='input', input_shape=(num_features,)))

# model.add(Dense(500, name='hidden1'))
# model.add(Activation('relu'))
# model.add(BatchNormalization())
# model.add(Dropout(dropout))

# model.add(Dense(500, name='hidden2'))
# model.add(Activation('relu'))
# model.add(BatchNormalization())
# model.add(Dropout(dropout))

# model.add(Dense(500, name='hidden3'))
# model.add(Activation('relu'))
# model.add(BatchNormalization())
# model.add(Dropout(dropout))

model.add(Dense(name='output', units=3, activation='softmax'))

model.summary()
%%time

# XXX: this cries for a function with some parameters

BATCH_SIZE = 32
EPOCHS = 50

model.compile(loss='sparse_categorical_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

history = model.fit(X_train, y_train,
                    validation_data=(X_val, y_val),
                    epochs=EPOCHS, 
                    batch_size=BATCH_SIZE, 
                    callbacks=[KerasLearningCurve()],
                    verbose=0)
# XXX: getting final metrics is very common
train_loss, train_metric = model.evaluate(X_train, y_train, batch_size=BATCH_SIZE)
train_loss, train_metric
test_loss, test_metric = model.evaluate(X_val, y_val, batch_size=BATCH_SIZE)
test_loss, test_metric
# XXX: those plots are happning all the time
plt.yscale('log')
plt.ylabel("loss")
plt.xlabel("epochs")
plt.title('Loss over epochs')

plt.plot(history.history['loss']);
plt.plot(history.history['val_loss']);

plt.legend(['Training', 'Validation']);
plt.ylabel("accuracy")
plt.xlabel("epochs")
plt.title('Accuracy over epochs')

plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
# XXX: those are plausibility checks and should be regression tests on quality of the model
model.predict([[48, 100]])
# this should be low risk (group 2)
model.predict([[48, 100]]).argmax()
# assert model.predict([[48, 100]]).argmax() == 2
model.predict([[30, 150]])
# high risk expected
model.predict([[30, 150]]).argmax()
# XXX version without boundaries is straight forward, but one with ranges: which ranges make sense and why?
# plot_decision_boundaries(model, X, y, x1_range=(10, 150), x2_range=(50, 250))
plot_decision_boundaries(model, X, y)
# model.save?
# XXX: loading and saving of model are one-liners, but there are different formats and they are hard to remember
model.save('classifier.h5', save_format='h5')
model.save('classifier', save_format='tf')
!ls -l
!ls -l classifier/
!tar czvf classifier.tgz ./classifier
!ls -l<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1274076413009157, 'fraction_numerical': 0.021313545942532364, 'mean_word_length': 3.423882681564246, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '314950', 'license': 'MIT', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2045, 'n_words': 533, 'path': 'workspace/insurance_ml.ipynb', 'repo_name': 'DJCordhose/insurance-ml', 'size': '6334'}