<jupyter_start><jupyter_text>Demo: Convolutional neural networks on the “slash” dataset==========================================================*Fraida Fund* In this demo, we’l look at an example of a task that is difficult for“classical” machine learning models, and difficult for fully connectedneural networks, but easy for convolutional neural networks.<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
import keras
import numpy as np
import pandas as pd
import scipy

from sklearn.model_selection import train_test_split
from sklearn import ensemble, neighbors, linear_model, svm

from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, Activation, Conv2D, Flatten, BatchNormalization, InputLayer, AvgPool2D, MaxPool2D, GlobalAvgPool2D
import tensorflow.keras.backend as K
from keras.utils.vis_utils import plot_model<jupyter_output><empty_output><jupyter_text>The slash dataset-----------------The “slash” dataset, developed by [SophieSearcy](https://soph.info/slash-data), is a set of images, each of whichincludes a “slash” on a background of random noise. The data is dividedinto two classes according to whether the slash is downward facing orupward facing.<jupyter_code>def gen_example(size=20, label=0):

    max_s_pattern = int(size // 4)
    s_pattern = 4
    pattern = 1- np.eye(s_pattern)
    if label:
        pattern = pattern[:, ::-1]
    ex = np.ones((size,size))
    point_loc = np.random.randint(0, size - s_pattern + 1,
                                  size=(2, ))  # random x,y point
    ex[point_loc[0]:point_loc[0] + s_pattern, point_loc[1]:point_loc[1] +
       s_pattern] = pattern  # set point to
    ex = ex + .5*(np.random.rand(size, size) - .5)
    np.clip(ex,0.,1., out=ex)
    return ex
examples = []

n_side = 30
n_ex = 500 #number of examples in each class

for i in range(n_ex):
    examples.append(gen_example(size=n_side, label=0))
    examples.append(gen_example(size=n_side, label=1))
    
y = np.array([0,1]*n_ex)
x = np.stack(examples)
plt.figure(figsize=(18,4))

n_print = 10 # number of examples to show

ex_indices = np.random.choice(len(y), n_print, replace=False)
for i, index in enumerate(ex_indices):
    plt.subplot(1, n_print, i+1, )
    plt.imshow(x[index,...], cmap='gray')
    plt.title(f"y = {y[index]}")<jupyter_output><empty_output><jupyter_text>We’l prepare training and test data in two formats:-   “flat” for traditional ML models and fully connected neural    networks, which don’t care about the spatial arrangement of the    features.-   “image” for convolutional neural networks.<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(x, y, stratify=y, test_size=0.25)

x_train_flat = x_train.reshape(x_train.shape[0], -1)
x_test_flat = x_test.reshape(x_test.shape[0], -1)

x_train_img = x_train[...,np.newaxis]
x_test_img = x_test[...,np.newaxis]
print("Flat data shape:  ", x_train_flat.shape)
print("Image data shape: ", x_train_img.shape)<jupyter_output><empty_output><jupyter_text>The feature data is in the range 0 to 1:<jupyter_code>x.min(), x.max()<jupyter_output><empty_output><jupyter_text>Train logistic regression, random forest, KNN, SVM models--------------------------------------------------------- Next, we’l try to train some classic ML models on this dataset.<jupyter_code>models = {
    "Logistic\n Regression": linear_model.LogisticRegression(),
    "KNN-1": neighbors.KNeighborsClassifier(n_neighbors=1),
    "KNN-3": neighbors.KNeighborsClassifier(n_neighbors=3),
    "Random\n Forest": ensemble.RandomForestClassifier(n_estimators=100),
    "SVM -\n Linear": svm.SVC(kernel="linear"),
    "SVM -\n RBF kernel": svm.SVC(kernel="rbf")
}
results = []

for model_name in models.keys():    
    model = models[model_name]
    model.fit(x_train_flat, y_train)
    
    train_score = model.score(x_train_flat, y_train)
    test_score = model.score(x_test_flat, y_test)   
    
    results.append({"model": model_name, "train_score": train_score, "test_score": test_score})
results_df = pd.DataFrame(results)

plt.figure(figsize =(10,10));

plt.subplot(2,1,1)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['train_score']);
plt.ylim(0,1);
plt.xlabel("")

plt.subplot(2,1,2)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['test_score']);
plt.ylim(0,1);<jupyter_output><empty_output><jupyter_text>Are these the results we expected? Why or why not? Do *any* of these models do a good job of learning whether a slash isforward-facing or backward-facing? Train a fully connected neural network--------------------------------------<jupyter_code>nin = x_train_flat.shape[1]
nh1 = 64
nh2 = 64
nh3 = 64  
nout = 1 
model_fc = Sequential()
model_fc.add(Dense(units=nh1, input_shape=(nin,), activation='relu', name='hidden1'))
model_fc.add(Dense(units=nh2, input_shape=(nh1,), activation='relu', name='hidden2'))
model_fc.add(Dense(units=nh3, input_shape=(nh2,), activation='relu', name='hidden3'))
model_fc.add(Dense(units=nout, activation='sigmoid', name='output'))

model_fc.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

model_fc.summary()
hist = model_fc.fit(x_train_flat, y_train, epochs=100, 
     validation_split=0.25,  callbacks=[
        keras.callbacks.ReduceLROnPlateau(factor=.5, patience=2, verbose=1),
        keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True, verbose=1)
    ])
train_score = model_fc.evaluate(x_train_flat, y_train)[1]
test_score = model_fc.evaluate(x_test_flat, y_test)[1]
results.append({"model": 'FC Neural Net', "train_score": train_score, "test_score": test_score})
results_df = pd.DataFrame(results)

plt.figure(figsize =(11,10));

plt.subplot(2,1,1)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['train_score']);
plt.ylim(0,1);
plt.xlabel("")

plt.subplot(2,1,2)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['test_score']);
plt.ylim(0,1);<jupyter_output><empty_output><jupyter_text>Train a convolutional neural network------------------------------------<jupyter_code>filters = 10
model_conv = Sequential()
model_conv.add(InputLayer(input_shape=x_train_img.shape[1:]))
model_conv.add(Conv2D(filters, kernel_size=3, padding="same", activation="relu", use_bias=False ))
model_conv.add(MaxPool2D(pool_size=(2, 2)))
model_conv.add(BatchNormalization())
model_conv.add(Conv2D(filters, kernel_size=3, padding="same", activation="relu", use_bias=False ))
model_conv.add(GlobalAvgPool2D())
model_conv.add(Dense(1, activation="sigmoid"))

model_conv.summary()

model_conv.compile("adam", loss="binary_crossentropy", metrics=["accuracy"])
hist = model_conv.fit(x_train_img, y_train, epochs=100, 
     validation_split=0.25,  callbacks=[
        keras.callbacks.ReduceLROnPlateau(factor=.5, patience=2, verbose=1),
        keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True, verbose=1)
    ])

train_score = model_conv.evaluate(x_train_img, y_train)[1]
test_score = model_conv.evaluate(x_test_img, y_test)[1]
results.append({"model": 'ConvNet', "train_score": train_score, "test_score": test_score})
results_df = pd.DataFrame(results)

plt.figure(figsize =(12,10));

plt.subplot(2,1,1)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['train_score']);
plt.ylim(0,1);
plt.xlabel("")

plt.subplot(2,1,2)
sns.barplot(x=results_df.sort_values('test_score')['model'], y=results_df.sort_values('test_score')['test_score']);
plt.ylim(0,1);<jupyter_output><empty_output><jupyter_text>Using the same model on different slashes----------------------------------------- Not only did our convolutional network learn forward and backwardslashes - it can even generalize to slightly different forward andbackward slashes.Let’s generate data with heavier background noise, and longer slashes:<jupyter_code>noise_scale = 0.9
s_pattern = 15
def gen_example_different(size=20, label=0):

    max_s_pattern = int(size // 4)
    pattern = 1- np.eye(s_pattern)
    if label:
        pattern = pattern[:, ::-1]
    ex = np.ones((size,size))
    point_loc = np.random.randint(0, size - s_pattern + 1,
                                  size=(2, ))  # random x,y point
    ex[point_loc[0]:point_loc[0] + s_pattern, point_loc[1]:point_loc[1] +
       s_pattern] = pattern  # set point to
    ex = ex + noise_scale*(np.random.rand(size, size) - .5)
    np.clip(ex,0.,1., out=ex)
    return ex

examples = []

n_side = 30
n_ex = 50 #number of examples in each class

for i in range(n_ex):
    examples.append(gen_example_different(size=n_side, label=0))
    examples.append(gen_example_different(size=n_side, label=1))
    
y_new = np.array([0,1]*n_ex)
x_new = np.stack(examples)

plt.figure(figsize=(18,4))

n_print = 10 # number of examples to show

ex_indices = np.random.choice(len(y_new), n_print, replace=False)
for i, index in enumerate(ex_indices):
    plt.subplot(1, n_print, i+1, )
    plt.imshow(x_new[index,...], cmap='gray')
    plt.title(f"y = {y_new[index]}")
plt.figure(figsize=(18,4))

for i, index in enumerate(ex_indices):
    plt.subplot(1, n_print, i+1, )
    plt.imshow(x_new[index,...], cmap='gray')
    plt.title("yhat =  %0.2f" % model_conv.predict(x_new[index].reshape((1,30,30,1))))
new_test_score = model_conv.evaluate(x_new[...,np.newaxis], y_new)[1]<jupyter_output><empty_output><jupyter_text>What about forward and backward slashes at different angles?<jupyter_code>max_rot = 10
def gen_example_rotated(size=20, label=0):

    max_s_pattern = int(size // 4)
    s_pattern = 15
    pattern = 1- np.eye(s_pattern)
    if label:
        pattern = pattern[:, ::-1]
    ex = np.ones((size,size))
    point_loc = np.random.randint(0, size - s_pattern + 1,   size=(2, ))  
    ex[point_loc[0]:point_loc[0] + s_pattern, point_loc[1]:point_loc[1] + s_pattern] = pattern  
    rot_angle = np.random.uniform(-max_rot, max_rot)
    ex = scipy.ndimage.rotate(ex, angle=rot_angle, cval=1, reshape = False)
    ex = ex + noise_scale*(np.random.rand(size, size) - .5)

    np.clip(ex,0.,1., out=ex)
    return ex

examples = []

n_side = 30
n_ex = 50 #number of examples in each class

for i in range(n_ex):
    examples.append(gen_example_rotated(size=n_side, label=0))
    examples.append(gen_example_rotated(size=n_side, label=1))
    
y_new = np.array([0,1]*n_ex)
x_new = np.stack(examples)

plt.figure(figsize=(18,4))

n_print = 10 # number of examples to show

ex_indices = np.random.choice(len(y_new), n_print, replace=False)
for i, index in enumerate(ex_indices):
    plt.subplot(1, n_print, i+1, )
    plt.imshow(x_new[index,...], cmap='gray')
    plt.title(f"y = {y_new[index]}")
plt.figure(figsize=(18,4))

for i, index in enumerate(ex_indices):
    plt.subplot(1, n_print, i+1, )
    plt.imshow(x_new[index,...], cmap='gray')
    plt.title("yhat =  %0.2f" % model_conv.predict(x_new[index].reshape((1,30,30,1))))
new_test_score = model_conv.evaluate(x_new[...,np.newaxis], y_new)[1]<jupyter_output><empty_output><jupyter_text>Visualizing what the network learns-----------------------------------<jupyter_code>from ipywidgets import interactive
from ipywidgets import Layout
import ipywidgets as widgets

def plot_layer(test_idx, layer_idx):
  convout1_f = K.function(model_conv.inputs, [model_conv.layers[layer_idx].output])
  convolutions = np.squeeze(convout1_f(x[test_idx].reshape((1,30,30,1))))
  if (len(convolutions.shape)) > 1:
    m = convolutions.shape[2]
    n = int(np.ceil(np.sqrt(m)))

    # Visualization of each filter of the layer
    fig = plt.figure(figsize=(15,12))
    print(model_conv.layers[layer_idx].name)
    for i in range(m):
        ax = fig.add_subplot(n,n,i+1)
        ax.imshow(convolutions[:,:,i], cmap='gray')
        ax.set_title(i)
  else:
    print(model_conv.layers[layer_idx].name)
    plt.imshow(convolutions.reshape(1, convolutions.shape[0]), cmap='gray');
    plt.yticks([])
    plt.xticks(range(convolutions.shape[0]))

style = {'description_width': 'initial'}
layout = Layout(width="800px")
test_idx = widgets.IntSlider(min=0, max=len(x)-1, value=0, style=style, layout=layout)
layer_idx = widgets.IntSlider(min=0, max=len(model_conv.layers)-2, value=0, style=style, layout=layout)
interactive(plot_layer, test_idx=test_idx, layer_idx=layer_idx)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16601076218777608, 'fraction_numerical': 0.022568468396112762, 'mean_word_length': 4.797020484171322, 'pattern_counts': {'":': 15, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '620452', 'license': 'MIT', 'n_tokens_mistral': 4674, 'n_tokens_neox': 4489, 'n_words': 869, 'path': 'notebooks/8-slash-dataset.ipynb', 'repo_name': 'ffund/ml-notebooks', 'size': '12451'}