<jupyter_start><jupyter_text>model = keras.Sequential() Bottleneck to keep parameters lowmodel.add(keras.layers.Conv2D(24, kernel_size = (1, 1),  squeeze                              activation = 'relu',                              bias_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None),                              kernel_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None),                              input_shape = [size, size, 3]))model.add(keras.layers.Conv2D(24, kernel_size = (3, 3),  map                               strides = (2, 2),                              activation = 'relu',                              bias_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None),                              kernel_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None)))model.add(keras.layers.Conv2D(32, kernel_size = (1, 1),  expand                             activation = 'relu',                             bias_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None),                              kernel_initializer = keras.initializers.RandomUniform(minval = 0.001, maxval = 0.4, seed = None)))model.add(keras.layers.BatchNormalization(momentum = 0.99))model.add(keras.layers.SpatialDropout2D(rate = 0.25))model.add(keras.layers.MaxPooling2D(pool_size = (2, 2)))model.add(keras.layers.Flatten())model.add(keras.layers.Dense(Y.shape[1] * 2, activation = 'tanh'))model.add(keras.layers.Dropout(0.1))model.add(keras.layers.Dense(Y.shape[1], activation = 'softmax'))model.compile(loss = keras.losses.categorical_crossentropy,             optimizer = keras.optimizers.Adam(lr = 0.0001, amsgrad = True),             metrics = ['accuracy'])model.summary() model = keras.Sequential()model.add(keras.layers.Conv2D(32, kernel_size = (1, 1),                             activation = 'relu',                             input_shape = [size, size, 3]))model.add(keras.layers.SpatialDropout2D(rate = 0.25))model.add(keras.layers.Conv2D(32, kernel_size = (3, 3),                             strides = (2, 2),                             activation = 'relu',                             input_shape = [size, size, 3]))model.add(keras.layers.SpatialDropout2D(rate = 0.25))model.add(keras.layers.Conv2D(64, kernel_size = (1, 1),                             activation = 'relu'))model.add(keras.layers.SpatialDropout2D(rate = 0.25))model.add(keras.layers.normalization.BatchNormalization(momentum = 0.99))model.add(keras.layers.MaxPooling2D(pool_size = (2, 2)))model.add(keras.layers.Flatten())model.add(keras.layers.Dense(Y.shape[1] * 4, activation = 'tanh'))model.add(keras.layers.Dense(Y.shape[1] * 3, activation = 'tanh'))model.add(keras.layers.Dense(Y.shape[1] * 2, activation = 'tanh'))model.add(keras.layers.Dense(Y.shape[1], activation = 'softmax'))model.compile(loss = keras.losses.categorical_crossentropy,             optimizer = keras.optimizers.Adam(lr = 0.0001, amsgrad = True),              metrics = ['accuracy'])model.summary() net_in = keras.layers.Input(shape = [size, size, 3]) Send input to two different convolving layers: one vertical and one horizontalconv_horiz = keras.layers.Conv2D(16, kernel_size = (4, 2), strides = (2, 2))(net_in)conv_verti = keras.layers.Conv2D(16, kernel_size = (2, 4), strides = (2, 2))(net_in) NormalizebNorm_horiz = keras.layers.normalization.BatchNormalization()(conv_horiz)bNorm_verti = keras.layers.normalization.BatchNormalization()(conv_verti) Activationsconv_horiz_activ = keras.layers.ReLU()(bNorm_horiz)conv_verti_activ = keras.layers.ReLU()(bNorm_verti) Dropoutsconv_horiz_dropout = keras.layers.SpatialDropout2D(0.25)(conv_horiz_activ)conv_verti_dropout = keras.layers.SpatialDropout2D(0.25)(conv_verti_activ) Convolve into square shapeconv_horiz_reshaped = keras.layers.Conv2D(32, kernel_size = (2, 4), strides = (2, 2))(conv_horiz_activ)conv_verti_reshaped = keras.layers.Conv2D(32, kernel_size = (4, 2), strides = (2, 2))(conv_verti_activ) Normalize againbNorm_horiz_reshaped = keras.layers.normalization.BatchNormalization()(conv_horiz_reshaped)bNorm_verti_reshaped = keras.layers.normalization.BatchNormalization()(conv_verti_reshaped) Activations againconv_horiz_reshaped_activ = keras.layers.ReLU()(bNorm_horiz_reshaped)conv_verti_reshaped_activ = keras.layers.ReLU()(bNorm_verti_reshaped) Dropouts againconv_horiz_reshaped_dropout = keras.layers.SpatialDropout2D(0.25)(conv_horiz_reshaped_activ)conv_verti_reshaped_dropout = keras.layers.SpatialDropout2D(0.25)(conv_verti_reshaped_activ) Mergemerged = keras.layers.concatenate([conv_horiz_reshaped_activ, conv_verti_reshaped_activ]) Max pooling and flattenpooled = keras.layers.MaxPooling2D(2, 2)(merged)flat = keras.layers.Flatten()(pooled) Outputnet_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(flat) Create model and compilemodel = keras.Model(net_in, net_out)model.compile(loss = keras.losses.categorical_crossentropy,             optimizer = keras.optimizers.Adam(lr = 0.001, amsgrad = True),             metrics = ['accuracy'])model.summary() img_in = keras.layers.Input(shape = [size, size, 3]) Split and convolve'''conv_horiz = keras.layers.Conv2D(16, kernel_size = (4, 2), padding = 'same')(img_in)conv_horiz = keras.layers.LeakyReLU()(conv_horiz)conv_horiz = keras.layers.Conv2D(32, kernel_size = (4, 2), padding = 'same')(conv_horiz)conv_horiz = keras.layers.LeakyReLU()(conv_horiz)conv_horiz = keras.layers.SpatialDropout2D(0.1)(conv_horiz)conv_verti = keras.layers.Conv2D(16, kernel_size = (2, 4), padding = 'same')(img_in)conv_verti = keras.layers.LeakyReLU()(conv_verti)conv_verti = keras.layers.Conv2D(32, kernel_size = (2, 4), padding = 'same')(conv_verti)conv_verti = keras.layers.LeakyReLU()(conv_verti)conv_verti = keras.layers.SpatialDropout2D(0.1)(conv_verti)'''conv_small = keras.layers.Conv2D(16, kernel_size = (1, 1))(img_in)conv_small = keras.layers.Conv2D(16, kernel_size = (2, 2), padding = 'same')(conv_small)conv_small = keras.layers.LeakyReLU()(conv_small)conv_small = keras.layers.Conv2D(32, kernel_size = (1, 1))(conv_small)conv_small = keras.layers.Conv2D(32, kernel_size = (2, 2), padding = 'same')(conv_small)conv_small = keras.layers.LeakyReLU()(conv_small)conv_small = keras.layers.SpatialDropout2D(0.1)(conv_small)conv_large = keras.layers.Conv2D(16, kernel_size = (1, 1))(img_in)conv_large = keras.layers.Conv2D(16, kernel_size = (4, 4), padding = 'same')(conv_large)conv_large = keras.layers.LeakyReLU()(conv_large)conv_large = keras.layers.Conv2D(32, kernel_size = (1, 1))(conv_large)conv_large = keras.layers.Conv2D(32, kernel_size = (4, 4), padding = 'same')(conv_large)conv_large = keras.layers.LeakyReLU()(conv_large)conv_large = keras.layers.SpatialDropout2D(0.1)(conv_large) Merge, dropout, and poolconv_merged = keras.layers.concatenate([conv_small, conv_large])conv_merged = keras.layers.SpatialDropout2D(0.5)(conv_merged)conv_merged = keras.layers.MaxPooling2D(2, 2)(conv_merged) Dense hidden and network outputdense = keras.layers.Flatten()(conv_merged)dense = keras.layers.Dropout(0.25)(dense)dense = keras.layers.Dense(Y.shape[1] * 3, activation = 'tanh')(dense)net_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(dense) Create model and compilemodel = keras.Model(img_in, net_out)model.compile(loss = keras.losses.categorical_crossentropy,              optimizer = keras.optimizers.Adam(lr = 0.001, amsgrad = True),              metrics = ['accuracy'])model.summary() img_in = keras.layers.Input(shape = [size, size, 3]) Convolutions of different sizesconv_small = keras.layers.Conv2D(16, kernel_size = (4, 4), padding = 'same')(img_in)conv_small = keras.layers.LeakyReLU()(conv_small)conv_small = keras.layers.SpatialDropout2D(0.25)(conv_small)conv_large = keras.layers.Conv2D(16, kernel_size = (8, 8), padding = 'same')(img_in)conv_large = keras.layers.LeakyReLU()(conv_large)conv_large = keras.layers.SpatialDropout2D(0.25)(conv_large) Merge, dropout, poolconv_merged = keras.layers.concatenate([conv_small, conv_large])conv_merged = keras.layers.SpatialDropout2D(0.5)(conv_merged)conv_merged = keras.layers.MaxPooling2D(2, 2)(conv_merged) Split and convolve, but with smaller filtersconv_small = keras.layers.Conv2D(32, kernel_size = (2, 2), padding = 'same')(img_in)conv_small = keras.layers.LeakyReLU()(conv_small)conv_small = keras.layers.SpatialDropout2D(0.25)(conv_small)conv_large = keras.layers.Conv2D(32, kernel_size = (4, 4), padding = 'same')(img_in)conv_large = keras.layers.LeakyReLU()(conv_large)conv_large = keras.layers.SpatialDropout2D(0.25)(conv_large) Merge, dropout, poolconv_merged = keras.layers.concatenate([conv_small, conv_large])conv_merged = keras.layers.SpatialDropout2D(0.5)(conv_merged)conv_merged = keras.layers.MaxPooling2D(2, 2)(conv_merged) Dense hiddendense = keras.layers.Flatten()(conv_merged)dense = keras.layers.Dense(Y.shape[1] * 4, activation = 'tanh')(dense)net_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(dense) Create model and compilemodel = keras.Model(img_in, net_out)model.compile(loss = keras.losses.categorical_crossentropy,              optimizer = keras.optimizers.Adam(lr = 0.001, amsgrad = True),              metrics = ['accuracy'])model.summary() img_in = keras.layers.Input(shape = [size, size, 3])conv_tiny = keras.layers.Conv2D(16, kernel_size = (1, 1))(img_in)conv_tiny = keras.layers.ReLU()(keras.layers.normalization.BatchNormalization()(conv_tiny))conv_small = keras.layers.Conv2D(16, kernel_size = (2, 2), padding = 'same')(keras.layers.Conv2D(16, kernel_size = (1, 1))(img_in))conv_small = keras.layers.ReLU()(keras.layers.normalization.BatchNormalization()(conv_small))conv_large = keras.layers.Conv2D(16, kernel_size = (4, 4), padding = 'same')(keras.layers.Conv2D(16, kernel_size = (1, 1))(img_in))conv_large = keras.layers.ReLU()(keras.layers.normalization.BatchNormalization()(conv_large)) img_in = keras.layers.Input(shape = [size, size, 3])conv_large = keras.layers.Conv2D(24, kernel_size = (4,  4), strides = (2, 2), activation = 'relu')(img_in)conv_small = keras.layers.Conv2D(24, kernel_size = (2, 2), activation = 'relu')(img_in)spDropout_large = keras.layers.SpatialDropout2D(0.5)(conv_large)spDropout_small = keras.layers.SpatialDropout2D(0.5)(conv_small)bNorm_large = keras.layers.normalization.BatchNormalization()(spDropout_large)bNorm_small = keras.layers.normalization.BatchNormalization()(spDropout_small)pool_large = keras.layers.MaxPooling2D(2, 2)(bNorm_large)pool_small = keras.layers.MaxPooling2D(2, 2)(bNorm_small)flat_large = keras.layers.Flatten()(pool_large)flat_small = keras.layers.Flatten()(pool_small)dense_large = keras.layers.Dense(Y.shape[1], activation = 'tanh')(flat_large)dense_small = keras.layers.Dense(Y.shape[1], activation = 'tanh')(flat_small)combined = keras.layers.concatenate([dense_large, dense_small])net_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(combined)model = keras.Model(img_in, net_out)model.compile(loss = keras.losses.categorical_crossentropy,             optimizer = keras.optimizers.Adam(lr = 0.0001, amsgrad = True),             metrics = ['accuracy'])model.summary() img_in = keras.layers.Input(shape = [size, size, 3])conv = keras.layers.Conv2D(24, kernel_size = (1, 1), activation = 'relu')(img_in)conv = keras.layers.Conv2D(24, kernel_size = (2, 2), activation = 'relu')(conv)conv = keras.layers.Conv2D(48, kernel_size = (1, 1), activation = 'relu')(conv)conv = keras.layers.SpatialDropout2D(0.25)(conv)conv = keras.layers.MaxPooling2D(2, 2)(conv)conv = keras.layers.Conv2D(48, kernel_size = (1, 1), activation = 'relu')(conv)conv = keras.layers.Conv2D(48, kernel_size = (2, 2), activation = 'relu')(conv)conv = keras.layers.Conv2D(96, kernel_size = (1, 1), activation = 'relu')(conv)conv = keras.layers.SpatialDropout2D(0.25)(conv)conv = keras.layers.MaxPooling2D(2, 2)(conv)dense = keras.layers.Flatten()(conv)dense = keras.layers.Dense(Y.shape[1] * 5, activation = 'tanh')(dense)dense = keras.layers.Dense(Y.shape[1] * 3, activation = 'tanh')(dense)net_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(dense)model = keras.Model(img_in, net_out)model.compile(loss = keras.losses.categorical_crossentropy,             optimizer = keras.optimizers.Adam(lr = 0.001, amsgrad = True),             metrics = ['accuracy'])model.summary()<jupyter_code>kernel_size = 3

def to_grayscale(img_tensor):
    # Using luminosity formula, which weights green more heavily since human eyes are more senstive to green
    return (0.21 * img_tensor[:, :, :, :1]) + (0.72 * img_tensor[:, :, :, 1:2]) + (0.07 * img_tensor[:, :, :, -1:])

img_in = keras.layers.Input(shape = [size, size, 3])

# Add noise to the input
img = keras.layers.GaussianNoise(0.3)(img_in)
img = keras.layers.MaxPooling2D(2, 2)(img)

# Grayscale convolutions
conv_gsc = keras.layers.Lambda(to_grayscale)(img) # Convert to grayscale w/ luminosity formula
conv_gsc = keras.layers.Conv2D(32, kernel_size = (kernel_size, kernel_size))(conv_gsc)
conv_gsc = keras.layers.ELU()(conv_gsc)
conv_gsc = keras.layers.Conv2D(32, kernel_size = (kernel_size, kernel_size))(conv_gsc)
conv_gsc = keras.layers.ELU()(conv_gsc)
#conv_gsc = keras.layers.SpatialDropout2D(0.1)(conv_gsc)
conv_gsc = keras.layers.MaxPooling2D(2, 2)(conv_gsc)

# Color convolutions
conv_rgb = keras.layers.Conv2D(32, kernel_size = (kernel_size, kernel_size))(img)
conv_rgb = keras.layers.ELU()(conv_rgb)
conv_rgb = keras.layers.Conv2D(32, kernel_size = (kernel_size, kernel_size))(conv_rgb)
conv_rgb = keras.layers.ELU()(conv_rgb)
#conv_rgb = keras.layers.SpatialDropout2D(0.1)(conv_rgb)
conv_rgb = keras.layers.MaxPooling2D(2, 2)(conv_rgb)

# Mix both together and convolve
conv_merged = keras.layers.concatenate([conv_gsc, conv_rgb])
conv_merged = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_merged)
conv_merged = keras.layers.ELU()(conv_merged)
conv_merged = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_merged)
conv_merged = keras.layers.ELU()(conv_merged)
#conv_merged = keras.layers.SpatialDropout2D(0.1)(conv_merged)
conv_merged = keras.layers.MaxPooling2D(2, 2)(conv_merged)

# Convolve with just grayscale
conv_gsc = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_gsc)
conv_gsc = keras.layers.ELU()(conv_gsc)
conv_gsc = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_gsc)
conv_gsc = keras.layers.ELU()(conv_gsc)
#conv_gsc = keras.layers.SpatialDropout2D(0.1)(conv_gsc)
conv_gsc = keras.layers.MaxPooling2D(2, 2)(conv_gsc)

# Convolve with just color
conv_rgb = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_rgb)
conv_rgb = keras.layers.ELU()(conv_rgb)
conv_rgb = keras.layers.Conv2D(64, kernel_size = (kernel_size, kernel_size))(conv_rgb)
conv_rgb = keras.layers.ELU()(conv_rgb)
conv_rgb = keras.layers.SpatialDropout2D(0.1)(conv_rgb)
conv_rgb = keras.layers.MaxPooling2D(2, 2)(conv_rgb)

# Merge all
conv_merged = keras.layers.concatenate([conv_gsc, conv_rgb, conv_merged])
conv_merged = keras.layers.Conv2D(128, kernel_size = (1, 1))(conv_merged)
conv_merged = keras.layers.SpatialDropout2D(0.25)(conv_merged)

# Dense and out
dense = keras.layers.Flatten()(conv_merged)
dense = keras.layers.Dropout(0.25)(dense)
dense = keras.layers.Dense(256, activation = 'relu')(dense)
dense = keras.layers.Dropout(0.5)(dense)
dense = keras.layers.Dense(64, activation = 'relu')(dense)
dense = keras.layers.Dropout(0.5)(dense)

net_out = keras.layers.Dense(Y.shape[1], activation = 'softmax')(dense)


# Color and grayscale convolutions
model = keras.Model(img_in, net_out)
model.compile(loss = keras.losses.categorical_crossentropy,
             optimizer = keras.optimizers.Adam(lr = 0.0001, amsgrad = True),
             metrics = ['accuracy'])
model.summary()
SVG(model_to_dot(model).create(prog = 'dot', format = 'svg'))
history = model.fit(X_train, Y_train, batch_size = 200, 
                    epochs = 100, verbose = 0,
                    validation_data = (X_test, Y_test),
                    callbacks = [TQDMNotebookCallback()])
print('Accuracy: ', model.evaluate(X_test, Y_test)[1] * 100.0, '%')
plt.figure(1)

# Accuracy
plt.subplot(211)
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc = 'upper left')

# Loss
plt.subplot(212)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc = 'upper left')

plt.tight_layout()
plt.show()
model.save_weights('model_alt.weights')
model.save('model_alt.model')
preds = model.predict(X_test)
img = 10
np.set_printoptions(suppress = True)
print(preds[img] * 100.0)
print(Y_test[img])

plt.imshow(keras.preprocessing.image.array_to_img(X_test[img, :, :, :]))
plt.show()

#for a in Y_train:
#    print(a)
def getIndexOfHighestConfidence(pred):
    highestConfidenceIndex = 0
    for i in range(len(pred)):
        if pred[i] > pred[highestConfidenceIndex]:
            highestConfidenceIndex = i
    return highestConfidenceIndex

def getIndexOfClassification(test):
    classIndex = 0
    for i in range(len(test)):
        if test[i] == 1:
            classIndex = i
            break
    return classIndex

numCorrect = 0
for i in range(len(preds)):
    if getIndexOfHighestConfidence(preds[i]) == getIndexOfClassification(Y_test[i]):
        numCorrect = numCorrect + 1

percentCorrect = numCorrect / len(preds)
print(percentCorrect * 100.0)
# Glass, paper, cardboard, plastic, metal, trash
# For each category, let's see what the network would generally predict when it saw an image of that type
classCounts = [0, 0, 0, 0, 0, 0] # number of images for each category (from testing data)
predStats = [[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], 
             [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]] 

for i in range(len(preds)):
    predictedClass = getIndexOfHighestConfidence(preds[i])
    actualClass = getIndexOfClassification(Y_test[i])
    
    classCounts[actualClass] = classCounts[actualClass] + 1
    predStats[actualClass][predictedClass] = predStats[actualClass][predictedClass] + 1

for i in range(6):
    for j in range(6):
        predStats[i][j] = (predStats[i][j] / classCounts[i]) * 100.0

#for i in range(6):
#    print(str((predStats[4][i] / 205) * 100.0))
classNames = ['Glass', 'Paper', 'Cardboard', 'Plastic', 'Metal', 'Trash']

print('Breakdown of network predictions for testing set:\n------------------------')
for i in range(6):
    print(classNames[i] + ':', classCounts[i], 'images')
    print('Network predictions given an image of', classNames[i] + ':')
    for j in range(6):
        print('\t' + classNames[j] + ':', str(np.round(predStats[i][j], 2)) + '%')
    print('\n')<jupyter_output>Breakdown of network predictions for testing set:
------------------------
Glass: 251 images
Network predictions given an image of Glass:
	Glass: 71.71%
	Paper: 3.59%
	Cardboard: 4.38%
	Plastic: 6.37%
	Metal: 8.76%
	Trash: 5.18%


Paper: 297 images
Network predictions given an image of Paper:
	Glass: 12.46%
	Paper: 76.09%
	Cardboard: 3.03%
	Plastic: 1.68%
	Metal: 5.39%
	Trash: 1.35%


Cardboard: 202 images
Network predictions given an image of Cardboard:
	Glass: 15.84%
	Paper: 5.94%
	Cardboard: 66.34%
	Plastic: 3.96%
	Metal: 5.45%
	Trash: 2.48%


Plastic: 241 images
Network predictions given an image of Plastic:
	Glass: 24.48%
	Paper: 10.37%
	Cardboard: 8.3%
	Plastic: 46.89%
	Metal: 9.13%
	Trash: 0.83%


Metal: 205 images
Network predictions given an image of Metal:
	Glass: 32.2%
	Paper: 9.27%
	Cardboard: 6.34%
	Plastic: 7.32%
	Metal: 42.44%
	Trash: 2.44%


Trash: 69 images
Network predictions given an image of Trash:
	Glass: 34.78%
	Paper: 4.35%
	Cardboard: 7.25%
	Plastic: 8.7%
	Metal[...]
{'fraction_non_alphanumeric': 0.15154096717614632, 'fraction_numerical': 0.03833625657729892, 'mean_word_length': 4.96591928251121, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '22261', 'license': 'MIT', 'n_tokens_mistral': 7913, 'n_tokens_neox': 7346, 'n_words': 1221, 'path': 'Project/project_code.ipynb', 'repo_name': 'CSCI4850/S19-team1-project', 'size': '19955'}