<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import time
import datetime
import tensorflow as tf
from tensorflow import keras
import tensorflow.keras.models as models
# We use models.Model which is the same as keras.Model
import tensorflow.keras.layers as layers
print(tf.__version__)
print(keras.__version__)<jupyter_output>2.4.0
2.4.0<jupyter_text>Data setup<jupyter_code># load data
X_test = np.load("X_test.npy")
y_test = np.load("y_test.npy")
person_train_valid = np.load("person_train_valid.npy")
X_train_valid = np.load("X_train_valid.npy")
y_train_valid = np.load("y_train_valid.npy")
person_test = np.load("person_test.npy")
# adjust label values
y_train_valid -= 769
y_test -= 769
print ('Training/Valid data shape: {}'.format(X_train_valid.shape))
print ('Test data shape: {}'.format(X_test.shape))
print ('Training/Valid target shape: {}'.format(y_train_valid.shape))
print ('Test target shape: {}'.format(y_test.shape))
print ('Person train/valid shape: {}'.format(person_train_valid.shape))
print ('Person test shape: {}'.format(person_test.shape))
# split into train and validation set. Assumes iid
perm = np.random.permutation(X_train_valid.shape[0])
numTrain = int(0.8*X_train_valid.shape[0])
numVal = X_train_valid.shape[0] - numTrain
Xtrain = X_train_valid[perm[0:numTrain]]
ytrain = y_train_valid[perm[0:numTrain]]
Xval = X_train_valid[perm[numTrain: ]]
yval = y_train_valid[perm[numTrain: ]]<jupyter_output><empty_output><jupyter_text>Fully-connected (Dense) and RNN layers. Compiling a model.<jupyter_code>hidden_dim = 20
# Building a Sequential model:
model = models.Sequential()
model.add(layers.Permute((2, 1), input_shape=(22, 1000))) # (batch, 22, 1000) -> (batch, 1000, 22)
model.add(layers.SimpleRNN(hidden_dim)) # (batch, 1000, 22) -> (batch, hidden_dim)
                                        # Use return_sequences=True to use in cascaded RNNs
model.add(layers.Dense(4, activation='softmax')) # (batch, hidden_dim) -> (batch, 4)
model.compile('adam', 'sparse_categorical_crossentropy', metrics=['acc'])\
    # Need to compile model before running
    # use optimizer='sgd' if desired. Seems to be case-insensitive
    # use loss = 'categorical_crossentropy' for one-hot labels
    # see https://www.tensorflow.org/api_docs/python/tf/keras/metrics for other metrics.
    #     Metrics might not have string names.
model.summary()
# Equivalent functional declaration:
inputs = layers.Input(shape=(22, 1000), name='inputs')
    # Needed for first layer. Expects input of (batch, 22, 1000)
    # We can also name our layers. Useful for getting a layer by a string name.
p1 = layers.Permute((2, 1), name='p1')(inputs)
    # Format is ClassName(*args, **kwargs)(upstream_layer)
    # Permute: count batch dimension as 0
rnn = layers.SimpleRNN(hidden_dim, name='rnn')(p1)
outputs = layers.Dense(4, activation='softmax', name='outputs')(rnn)
model = models.Model(inputs=inputs, outputs=outputs, name='functional_model') # or keras.Model(*, **)
    # Need to declare a model specifying input and output layers.
    # Can pass lists of layers instead.
    # We can also name our model.
model.compile('adam', 'sparse_categorical_crossentropy', metrics=['acc']) # Compiling is the same.
rnn_weights = model.get_layer('rnn').get_weights()
# Dense layer: 22->20
# Recurrent weight: 20x20, i.e. hidden_dim x hidden_dim
# bias: 20
for w in rnn_weights:
    print(w.shape)
model.summary()
# If you want an intermediate output, compile another model:
rnn_output = models.Model(inputs=inputs, outputs=model.get_layer('rnn').output)
    # Can pass the object reference (e.g. inputs) or the layer output (model.get_layer('rnn').output)
out = rnn_output(Xval)
print(type(out), out.shape)
# Adding weight and activity regularizers:
from tensorflow.keras.regularizers import L1, L2
layers.Dense(4, activation='softmax', kernel_regularizer=L1(0.01), activity_regularizer=L2(0.01))
# Concatenating layers
inputs = layers.Input(shape=(22, 1000))
d1 = layers.Dense(20, activation='relu', kernel_regularizer=L1(0.01))(inputs)
d2 = layers.Dense(10, activation='relu', kernel_regularizer=L2(0.01), activity_regularizer=L1(0.001))(inputs)
cat1 = layers.Concatenate()([d1, d2])
f1 = layers.Flatten()(cat1)
outputs = layers.Dense(4, activation='softmax')(f1)
model = models.Model(inputs=inputs, outputs=outputs)
model.summary()<jupyter_output>Model: "model_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 22, 1000)]   0                                            
__________________________________________________________________________________________________
dense_5 (Dense)                 (None, 22, 20)       20020       input_2[0][0]                    
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 22, 10)       10010       input_2[0][0]                    
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 22, 30)       0           dense_5[0][0]              [...]<jupyter_text>Training a model:<jupyter_code># For visualization via TensorBoard
logdir = "logs/scalars/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)
# See also: lr_scheduler
loss_hist = model.fit(Xtrain, ytrain, validation_data=(Xval, yval), epochs=5, callbacks=[tensorboard_callback])
# Necessary for %tensorboard ...
%load_ext tensorboard
# Re-run after timeout if error, or find and go to localhost:port
%tensorboard --logdir logs/scalars --port=7000
# To shutdown: replace $pid with the process id and run:
# Unix:
#!kill $pid
# Windows:
#!Taskkill /PID $pid /F
hist = loss_hist.history
list(hist)
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(hist['loss'])
plt.plot(hist['val_loss'])
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'])

plt.subplot(1, 2, 2)
plt.plot(hist['acc'])
plt.plot(hist['val_acc'])
plt.ylabel('acc')
plt.xlabel('epoch')
plt.legend(['train', 'val'])<jupyter_output><empty_output><jupyter_text>Implementing Shallow ConvNet (Schirrmeister et al. 2018)<jupyter_code>def Ksquare(x):
    return tf.pow(x, 2)
def Klog(x):
    return tf.math.log(x)
inputs = layers.Input(shape=(22, 1000))
r1 = layers.Reshape((22, 1000, 1))(inputs)
    # (N, 22, 1000) -> (N, 22, 1000, 1)
c1 = layers.Conv2D(40, (1, 25), activation='elu', data_format='channels_last')(r1)
    # (N, 22, 1000, 1) -> (N, 22, 976, 40), i.e. NHWC -> NHWC. 'channels_last' is default
p1 = layers.Permute((2, 1, 3))(c1)
    # (N, 22, 976, 40) -> (N, 976, 22, 40)
r2 = layers.Reshape((976, 22*40))(p1)
    # (N, 976, 22, 40) -> (N, 976, 22*40)
d1 = layers.Dense(40, activation='elu')(r2)
    # (N, 976, 22*40) -> (N, 976, 40)
    # weight_shape = 22*40 x 40 = 35200
    # bias_shape = 40
sq1 = layers.Activation(Ksquare)(d1)
ap1 = layers.AveragePooling1D(75, strides=15)(sq1)
    # (N, 976, 40) -> (N, 61, 40)
log1 = layers.Activation(Klog)(ap1)
f1 = layers.Flatten()(log1)
    # (N, 61, 40) -> (N, 61*40)
outputs = layers.Dense(4, activation='softmax')(f1)
    # (N, 61*40) -> (N, 4)
model = models.Model(inputs=inputs, outputs=outputs, name='shallow_convnet')
model.compile('adam', 'sparse_categorical_crossentropy', metrics=['acc'])
#model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy(), 
#             metrics=[keras.metrics.SparseCategoricalAccuracy()])
model.summary()
loss_hist = model.fit(Xtrain, ytrain, validation_data = (Xval, yval), epochs = 10, verbose=True)
hist = loss_hist.history

plt.figure(figsize=(15, 7))
plt.subplot(1, 2, 1)
plt.plot(hist['loss'])
plt.plot(hist['val_loss'])
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'])

plt.subplot(1, 2, 2)
plt.plot(hist['acc'])
plt.plot(hist['val_acc'])
plt.ylabel('acc')
plt.xlabel('epoch')
plt.legend(['train', 'val'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15904958176748696, 'fraction_numerical': 0.042671839010789185, 'mean_word_length': 4.956678700361011, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '660114', 'license': 'MIT', 'n_tokens_mistral': 3077, 'n_tokens_neox': 2781, 'n_words': 643, 'path': 'examples/keras_tutorial_2.4.0.ipynb', 'repo_name': 'cmunozcortes/c247-final-project', 'size': '8249'}