title timeseries classification with a transformer model theodoros ntakourishttps github comntakouris date created 20210625 last modified 20210805 description this notebook demonstrates how to do timeseries classification using a transformer model accelerator gpu introduction this is the transformer architecture from attention is all you needhttps arxiv orgabs1706 03762 applied to timeseries instead of natural language this example requires tensorflow 2 4 or higher load the dataset we are going to use the same dataset and preprocessing as the timeseries classification from scratchhttps keras ioexamplestimeseriestimeseriesclassificationfromscratch example build the model our model processes a tensor of shape batch size sequence length features where sequence length is the number of time steps and features is each input timeseries you can replace your classification rnn layers with this one the inputs are fully compatible we include residual connections layer normalization and dropout the resulting layer can be stacked multiple times the projection layers are implemented through keras layers conv1d attention and normalization feed forward part the main part of our model is now complete we can stack multiple of those transformerencoder blocks and we can also proceed to add the final multilayer perceptron classification head apart from a stack of dense layers we need to reduce the output tensor of the transformerencoder part of our model down to a vector of features for each data point in the current batch a common way to achieve this is to use a pooling layer for this example a globalaveragepooling1d layer is sufficient train and evaluate conclusions in about 110120 epochs 25s each on colab the model reaches a training accuracy of 0 95 validation accuracy of 84 and a testing accuracy of 85 without hyperparameter tuning and that is for a model with less than 100k parameters of course parameter count and accuracy could be improved by a hyperparameter search and a more sophisticated learning rate schedule or a different optimizer introduction this is the transformer architecture from attention is all you need https arxiv org abs 1706 03762 applied to timeseries instead of natural language this example requires tensorflow 2 4 or higher load the dataset we are going to use the same dataset and preprocessing as the timeseries classification from scratch https keras io examples timeseries timeseries_classification_from_scratch example build the model our model processes a tensor of shape batch size sequence length features where sequence length is the number of time steps and features is each input timeseries you can replace your classification rnn layers with this one the inputs are fully compatible we include residual connections layer normalization and dropout the resulting layer can be stacked multiple times the projection layers are implemented through keras layers conv1d attention and normalization feed forward part the main part of our model is now complete we can stack multiple of those transformer_encoder blocks and we can also proceed to add the final multi layer perceptron classification head apart from a stack of dense layers we need to reduce the output tensor of the transformerencoder part of our model down to a vector of features for each data point in the current batch a common way to achieve this is to use a pooling layer for this example a globalaveragepooling1d layer is sufficient train and evaluate conclusions in about 110 120 epochs 25s each on colab the model reaches a training accuracy of 0 95 validation accuracy of 84 and a testing accuracy of 85 without hyperparameter tuning and that is for a model with less than 100k parameters of course parameter count and accuracy could be improved by a hyperparameter search and a more sophisticated learning rate schedule or a different optimizer
import numpy as np
import keras
from keras import layers


def readucr(filename):
    data = np.loadtxt(filename, delimiter="\t")
    y = data[:, 0]
    x = data[:, 1:]
    return x, y.astype(int)


root_url = "https://raw.githubusercontent.com/hfawaz/cd-diagram/master/FordA/"

x_train, y_train = readucr(root_url + "FordA_TRAIN.tsv")
x_test, y_test = readucr(root_url + "FordA_TEST.tsv")

x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))

n_classes = len(np.unique(y_train))

idx = np.random.permutation(len(x_train))
x_train = x_train[idx]
y_train = y_train[idx]

y_train[y_train == -1] = 0
y_test[y_test == -1] = 0




def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):
    
    x = layers.MultiHeadAttention(
        key_dim=head_size, num_heads=num_heads, dropout=dropout
    )(inputs, inputs)
    x = layers.Dropout(dropout)(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    res = x + inputs

    
    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation="relu")(res)
    x = layers.Dropout(dropout)(x)
    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)
    x = layers.LayerNormalization(epsilon=1e-6)(x)
    return x + res





def build_model(
    input_shape,
    head_size,
    num_heads,
    ff_dim,
    num_transformer_blocks,
    mlp_units,
    dropout=0,
    mlp_dropout=0,
):
    inputs = keras.Input(shape=input_shape)
    x = inputs
    for _ in range(num_transformer_blocks):
        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)

    x = layers.GlobalAveragePooling1D(data_format="channels_first")(x)
    for dim in mlp_units:
        x = layers.Dense(dim, activation="relu")(x)
        x = layers.Dropout(mlp_dropout)(x)
    outputs = layers.Dense(n_classes, activation="softmax")(x)
    return keras.Model(inputs, outputs)




input_shape = x_train.shape[1:]

model = build_model(
    input_shape,
    head_size=256,
    num_heads=4,
    ff_dim=4,
    num_transformer_blocks=4,
    mlp_units=[128],
    mlp_dropout=0.4,
    dropout=0.25,
)

model.compile(
    loss="sparse_categorical_crossentropy",
    optimizer=keras.optimizers.Adam(learning_rate=1e-4),
    metrics=["sparse_categorical_accuracy"],
)
model.summary()

callbacks = [
    keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)
]

model.fit(
    x_train,
    y_train,
    validation_split=0.2,
    epochs=150,
    batch_size=64,
    callbacks=callbacks,
)

model.evaluate(x_test, y_test, verbose=1)