<jupyter_start><jupyter_text>1. Classifying movie reviews: Binary classification<jupyter_code># load pre-processed imdb

from keras.datasets import imdb

(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10_000)
print(f"{len(train_data[0]) = }")
print(f"{train_data[0][:10] = }")
print(f"{train_labels[0] = }")
for i in range(5):
    sample_len = len(train_data[i])
    print(f"{i = }: {sample_len = }")
import numpy as np
from typing import List

def multi_hot_encode(sequences: List[List], dimension: int = 10_000):
    results = np.zeros(shape=(len(sequences), dimension))

    for i, seq in enumerate(sequences):
        results[i, seq] = 1.
    
    return results

x_train = multi_hot_encode(train_data)
x_test = multi_hot_encode(test_data)
# also vectorize labels
y_train = np.asarray(train_labels).astype("float32") 
y_test = np.asarray(test_labels).astype("float32") 
# build 3 layer model
def get_model(*, n_representation_layers=3, n_units=16, intermediate_activation="relu", loss="binary_crossentropy"):
    
    # representation layers
    layer_list = n_representation_layers * [layers.Dense(n_units, activation=intermediate_activation)]

    # final layer
    layer_list.append(layers.Dense(1, activation="sigmoid"))
    
    model = keras.Sequential(layer_list)
    model.compile(
        optimizer="rmsprop",
        loss=loss,
        metrics=["accuracy"]
    )

    return model

model = get_model()
# create validation set

x_val = x_train[:10_000]
partial_x_train = x_train[10_000:]
y_val = y_train[:10_000]
partial_y_train = y_train[10_000:]
history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val),
)
# plot train and val losses
import matplotlib.pyplot as plt
def plot_history(history: keras.callbacks.History, variable_name: str):
    history_dict = history.history

    train_values = history_dict[variable_name]
    val_values = history_dict[f"val_{variable_name}"]
    epochs = range(1, len(train_values) + 1)

    plt.figure()
    plt.plot(epochs, train_values, "bo", label=f"Training {variable_name}")
    plt.plot(epochs, val_values, "b", label=f"Validation {variable_name}")
    plt.title(f"Training and validation {variable_name}")
    plt.xlabel("Epochs")
    plt.ylabel(f"{variable_name}")
    plt.legend()
    plt.show()


plot_history(history, "loss")

plot_history(history, "accuracy")<jupyter_output><empty_output><jupyter_text>model is clearly overfitting. For now, just re-init and train on full data for just 4 epochs<jupyter_code>model = get_model()
model.fit(x_train, y_train, epochs=4, batch_size=512)
results = model.evaluate(x_test, y_test)
print(f"{results = }")<jupyter_output>results = [0.29747629165649414, 0.881879985332489]<jupyter_text>Additional experiments<jupyter_code># mse as loss
model = get_model(loss="mse")

history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val),
)

plot_history(history, 'loss')
plot_history(history, 'accuracy')
# tanh activation
model = get_model(intermediate_activation="tanh")

history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val),
)

plot_history(history, 'loss')
plot_history(history, 'accuracy')<jupyter_output>Epoch 1/20
30/30 [==============================] - 1s 33ms/step - loss: 0.4938 - accuracy: 0.7953 - val_loss: 0.3866 - val_accuracy: 0.8751
Epoch 2/20
30/30 [==============================] - 0s 9ms/step - loss: 0.3146 - accuracy: 0.9038 - val_loss: 0.3168 - val_accuracy: 0.8874
Epoch 3/20
30/30 [==============================] - 0s 10ms/step - loss: 0.2430 - accuracy: 0.9246 - val_loss: 0.2907 - val_accuracy: 0.8875
Epoch 4/20
30/30 [==============================] - 0s 9ms/step - loss: 0.1981 - accuracy: 0.9399 - val_loss: 0.2754 - val_accuracy: 0.8902
Epoch 5/20
30/30 [==============================] - 0s 14ms/step - loss: 0.1661 - accuracy: 0.9507 - val_loss: 0.2708 - val_accuracy: 0.8910
Epoch 6/20
30/30 [==============================] - 0s 11ms/step - loss: 0.1422 - accuracy: 0.9585 - val_loss: 0.2733 - val_accuracy: 0.8892
Epoch 7/20
30/30 [==============================] - 0s 11ms/step - loss: 0.1225 - accuracy: 0.9655 - val_loss: 0.2820 - val_accuracy: 0.8864
Epoch 8/20
30/3[...]<jupyter_text>2. Classifying newswires: Multi-class classification<jupyter_code>from keras.datasets import reuters
(train_data, train_labels), (test_data, test_labels) = reuters.load_data(num_words=10_000)
print(f"{len(train_data) = }")
print(f"{len(test_data) = }")
print(f"{train_data[10] = }")
print(f"{train_labels[10] = }")
# vectorize inputs
x_train = multi_hot_encode(train_data)
x_test = multi_hot_encode(test_data)
y_train = multi_hot_encode(train_labels, dimension=46) # this should be equivalent to one-hot-oncoding
y_test = multi_hot_encode(test_labels, dimension=46)
# check that it's actually equivalent
assert (y_train.sum(axis=1) == 1.).all()
assert (y_test.sum(axis=1) == 1.).all()
# lets use the build-in versions of this instead
from tensorflow.keras.utils import to_categorical

y_train = to_categorical(train_labels)
y_test = to_categorical(test_labels)
# very similar to previous one, just change: 
# - units last layer
# - default n_units (more capacity)
# - loss
def get_model(*, n_representation_layers=3, n_units=64, intermediate_activation="relu", loss="categorical_crossentropy"):
    
    # representation layers
    layer_list = n_representation_layers * [layers.Dense(n_units, activation=intermediate_activation)]

    # final layer
    layer_list.append(layers.Dense(46, activation="softmax"))
    
    model = keras.Sequential(layer_list)
    model.compile(
        optimizer="rmsprop",
        loss=loss,
        metrics=["accuracy"]
    )

    return model

model = get_model()
# validation split
x_val = x_train[:1_000]
partial_x_train = x_train[1_000:]
y_val = y_train[:1_000]
partial_y_train = y_train[1_000:]
# train for 20 epochs
history = model.fit(
    partial_x_train,
    partial_y_train,
    epochs=20,
    batch_size=512,
    validation_data=(x_val, y_val)
)
plot_history(history, "loss")
plot_history(history, "accuracy")
# starts to overfit after 9 epochs. Retrain with full data
model = get_model()
model.fit(x_train, y_train, epochs=9, batch_size=512)
results = model.evaluate(x_test, y_test)
print(f"{results = }")
# how does that compare to random classifier?
import copy

# this is a cool way to ensure random predictions
# with perfect distribution!
random_predictions = copy.copy(test_labels) #not really random just yet
np.random.shuffle(random_predictions) #now they are random
accuracy = 100 * (random_predictions == test_labels).mean()
print(f"{accuracy = :.3f}%")

# we could have used a different encoding for the targets, just an integer tensor
y_train = np.array(train_labels)
y_test = np.array(test_labels)

print(f"{y_train.shape = }")
print(f"{y_test.shape = }")
# we just need to change the loss
model = get_model(loss="sparse_categorical_crossentropy")
model.fit(x_train, y_train, epochs=9, batch_size=512)
results = model.evaluate(x_test, y_test)
print(f"{results = }")<jupyter_output>Epoch 1/9
18/18 [==============================] - 1s 8ms/step - loss: 2.6561 - accuracy: 0.5839
Epoch 2/9
18/18 [==============================] - 0s 8ms/step - loss: 1.4867 - accuracy: 0.7277
Epoch 3/9
18/18 [==============================] - 0s 8ms/step - loss: 1.0473 - accuracy: 0.7956
Epoch 4/9
18/18 [==============================] - 0s 7ms/step - loss: 0.8068 - accuracy: 0.8412
Epoch 5/9
18/18 [==============================] - 0s 7ms/step - loss: 0.6436 - accuracy: 0.8723
Epoch 6/9
18/18 [==============================] - 0s 7ms/step - loss: 0.5240 - accuracy: 0.8980
Epoch 7/9
18/18 [==============================] - 0s 7ms/step - loss: 0.4312 - accuracy: 0.9141
Epoch 8/9
18/18 [==============================] - 0s 7ms/step - loss: 0.3599 - accuracy: 0.9283
Epoch 9/9
18/18 [==============================] - 0s 7ms/step - loss: 0.3036 - accuracy: 0.9360
71/71 [==============================] - 0s 2ms/step - loss: 0.8713 - accuracy: 0.8019
results = [0.8713235259056091, 0.8018699[...]<jupyter_text>3. Predicting house prices: Regression<jupyter_code>from keras.datasets import boston_housing
(train_data, train_labels), (test_data, test_labels) = boston_housing.load_data()
print(f"{train_data.shape = }")
print(f"{train_labels.shape = }")
print(f"{test_data.shape = }")
print(f"{test_labels.shape = }")
print(f"{train_data[0] = }")
print(f"{train_labels[0] = }")
# normalize inputs 

# compute mean and std, apply to train
mean = train_data.mean(axis=0)
train_data -= mean
std = train_data.std(axis=0)
train_data /= std

# apply to test
test_data -= mean
test_data /= std

def get_model(*, n_representation_layers=2, n_units=64, intermediate_activation="relu", loss="mse"):
    
    # representation layers
    layer_list = n_representation_layers * [layers.Dense(n_units, activation=intermediate_activation)]

    # final layer
    layer_list.append(layers.Dense(1))
    
    model = keras.Sequential(layer_list)
    model.compile(
        optimizer="rmsprop",
        loss=loss,
        metrics=["mae"]
    )

    return model

model = get_model()
# lets use k-fold cross validation
k = 4
num_val_samples = len(train_data) // k
num_epochs = 200
all_mae_histories = []

for i in range(k):
    print(f"Processing fold {i}")
    val_start = i * num_val_samples
    val_end = (i + 1) * num_val_samples
    
    val_data = train_data[val_start:val_end]
    val_labels = train_labels[val_start:val_end]

    partial_train_data = np.concatenate((train_data[:val_start], train_data[val_end:]), axis=0)
    partial_train_labels = np.concatenate((train_labels[:val_start], train_labels[val_end:]), axis=0)

    model = get_model()

    history = model.fit(partial_train_data, partial_train_labels, epochs=num_epochs, batch_size=16, verbose=0, 
                        validation_data=(val_data, val_labels))
    all_mae_histories.append(history.history["val_mae"])
average_mae_history = np.stack(all_mae_histories).mean(axis=0)

plt.plot(range(1, len(average_mae_history) + 1), average_mae_history)
plt.xlabel("Epochs")
plt.ylabel("Validation MAE")
plt.show()
# zoom in 
zoom_average_mae_history = average_mae_history[25:]

plt.plot(range(1, len(zoom_average_mae_history) + 1), zoom_average_mae_history)
plt.xlabel("Epochs")
plt.ylabel("Validation MAE")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17397910731244065, 'fraction_numerical': 0.05745489078822412, 'mean_word_length': 4.916292134831461, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '198848', 'license': 'MIT', 'n_tokens_mistral': 4216, 'n_tokens_neox': 3763, 'n_words': 772, 'path': 'my_notebooks/chapter04_getting-started-with-neural-networks.ipynb', 'repo_name': 'eballesteros/deep-learning-with-python-notebooks', 'size': '10530'}