<jupyter_start><jupyter_text>This Notebook uses a Session Event Dataset from E-Commerce Website (https://www.kaggle.com/mkechinov/ecommerce-behavior-data-from-multi-category-store and https://rees46.com/) to build an Outlier Detection based on an Autoencoder.<jupyter_code>import mlflow
import numpy as np
import os
import shutil
import pandas as pd
import tensorflow as tf
import tensorflow.keras as keras
import tensorflow_hub as hub
from itertools import product

# enable gpu growth if gpu is available
gpu_devices = tf.config.experimental.list_physical_devices('GPU')
for device in gpu_devices:
    tf.config.experimental.set_memory_growth(device, True)

# tf.keras.mixed_precision.set_global_policy('mixed_float16')

tf.config.optimizer.set_jit(True)

%load_ext watermark
%watermark -v -iv<jupyter_output>INFO:tensorflow:Mixed precision compatibility check (mixed_float16): OK
Your GPU will likely run quickly with dtype policy mixed_float16 as it has compute capability of at least 7.0. Your GPU: GeForce RTX 2070 SUPER, compute capability 7.5
numpy            1.19.4
mlflow           1.14.1
tensorflow       2.4.0
tensorflow_hub   0.11.0
pandas           1.1.5
tensorflow.keras 2.4.0
CPython 3.6.9
IPython 7.16.1<jupyter_text>Setting Registry and Tracking URI for MLflow<jupyter_code># Use this registry uri when mlflow is created by docker container with a mysql db backend
#registry_uri = os.path.expandvars('mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@localhost:3306/${MYSQL_DATABASE}')

# Use this registry uri when mlflow is running locally by the command:
# "mlflow server --backend-store-uri sqlite:///mlflow.db --default-artifact-root ./mlruns --host 0.0.0.0"
registry_uri = 'sqlite:///mlflow.db'

tracking_uri = 'http://localhost:5000'

mlflow.tracking.set_registry_uri(registry_uri)
mlflow.tracking.set_tracking_uri(tracking_uri)<jupyter_output><empty_output><jupyter_text>The Data is taken from https://www.kaggle.com/mkechinov/ecommerce-behavior-data-from-multi-category-store and https://rees46.com/ Each record/line in the file has the following fields:1. event_time: When did the event happened (UTC)2. event_type: Event type: one of [view, cart, remove_from_cart, purchase] 3. product_id4. category_id5. category_code: Category meaningful name (if present)6. brand: Brand name in lower case (if present)7. price8. user_id: Permanent user ID9. user_session: User session ID<jupyter_code># Read first 500.000 Rows
for chunk in pd.read_table("2019-Dec.csv",
                           sep=",", header=0,
                           infer_datetime_format=True, low_memory=False, chunksize=500000):
    # Filter out other event types than 'view'
    chunk = chunk[chunk['event_type'] == 'view']
    # Filter out missing 'category_code' rows
    chunk = chunk[chunk['category_code'].isna() == False]
    chunk.reset_index(drop=True, inplace=True)

    # Filter out all Sessions of length 1
    count_sessions = chunk.groupby('user_session').count()
    window_length = count_sessions.max()[0]
    unique_sessions = [count_sessions.index[i] for i in range(
        count_sessions.shape[0]) if count_sessions.iloc[i, 0] == 1]
    chunk = chunk[~chunk['user_session'].isin(unique_sessions)]
    chunk.reset_index(drop=True, inplace=True)

    # Text embedding based on https://tfhub.dev/google/nnlm-en-dim50/2
    last_category = []
    for i, el in enumerate(chunk['category_code']):
        last_category.append(el.split('.')[-1])
    chunk['Product'] = last_category
    embed = hub.load("https://tfhub.dev/google/nnlm-en-dim50/2")
    embeddings = embed(chunk['Product'].tolist())
    for dim in range(embeddings.shape[1]):
        chunk['embedding_'+str(dim)] = embeddings[:, dim]

    # Standardization
    mean = chunk['price'].mean(axis=0)
    print('Mean:', mean)
    std = chunk['price'].std(axis=0)
    print('Std:', std)
    chunk['price_standardized'] = (chunk['price'] - mean) / std

    chunk.sort_values(by=['user_session', 'event_time'], inplace=True)
    chunk['price_standardized'] = chunk['price_standardized'].astype('float32')
    chunk['product_id'] = chunk['product_id'].astype('int32')
    chunk.reset_index(drop=True, inplace=True)

    print('Sessions:', pd.unique(chunk['user_session']).shape)
    print('Unique Products:', pd.unique(chunk['product_id']).shape)
    print('Unique category_code:', pd.unique(chunk['category_code']).shape)

    columns = ['embedding_'+str(i) for i in range(embeddings.shape[1])]
    columns.append('price_standardized')
    columns.append('user_session')
    columns.append('Product')
    columns.append('product_id')
    columns.append('category_code')

    df = chunk[columns]
    break
df<jupyter_output>Mean: 284.7710546866538
Std: 349.46740231584886
Sessions: (61296,)
Unique Products: (38515,)
Unique category_code: (134,)<jupyter_text>Delete Rows with equal or less than 6 Product Occurrences<jupyter_code>count_product_id_mapped = df.groupby('product_id').count()
products_to_delete = count_product_id_mapped.loc[count_product_id_mapped['embedding_0'] <= 6].index
products_to_delete<jupyter_output><empty_output><jupyter_text>Slice Sessions from the Dataframe<jupyter_code>list_sessions = []
list_last_clicked = []
list_last_clicked_temp = []
current_id = df.loc[0, 'user_session']
current_index = 0

columns = ['embedding_'+str(i) for i in range(embeddings.shape[1])]
columns.append('price_standardized')
columns.insert(0, 'product_id')

for i in range(df.shape[0]):
    if df.loc[i, 'user_session'] != current_id:
        list_sessions.append(df.loc[current_index:i-2, columns])
        list_last_clicked.append(df.loc[i-1, 'product_id'])
        list_last_clicked_temp.append(df.loc[i-1, columns])
        current_id = df.loc[i, 'user_session']
        current_index = i<jupyter_output><empty_output><jupyter_text>Delete Sessions with Length larger than 30<jupyter_code>print(len(list_sessions))
list_sessions_filtered = []
list_last_clicked_filtered = []
list_last_clicked_temp_filtered = []

for index, session in enumerate(list_sessions):
    if not (session.shape[0] > 30):
        if not (session['product_id'].isin(products_to_delete).any()):
            list_sessions_filtered.append(session)
            list_last_clicked_filtered.append(list_last_clicked[index])
            list_last_clicked_temp_filtered.append(list_last_clicked_temp[index])
            
len(list_sessions_filtered)<jupyter_output>61295<jupyter_text>Slice Sessions if label and last product from session is the sameExample:- From: session: [ 1506  1506 11410 11410  2826  2826], ground truth: 2826- To: session: [ 1506  1506 11410 11410], ground truth: 2826<jupyter_code>print("Length before", len(list_sessions_filtered))
list_sessions_processed = []
list_last_clicked_processed = []
list_session_processed_autoencoder = []

for i, session in enumerate(list_sessions_filtered):
    if session['product_id'].values[-1] == list_last_clicked_filtered[i]:
        mask = session['product_id'].values == list_last_clicked_filtered[i]
        if session[~mask].shape[0] > 0:
            list_sessions_processed.append(session[~mask])
            list_last_clicked_processed.append(list_last_clicked_filtered[i])
            list_session_processed_autoencoder.append(pd.concat([session[~mask], pd.DataFrame(list_last_clicked_temp_filtered[i]).T],
                                                            ignore_index=True))
    else:
        list_sessions_processed.append(session)
        list_last_clicked_processed.append(list_last_clicked_filtered[i])
        list_session_processed_autoencoder.append(pd.concat([session, pd.DataFrame(list_last_clicked_temp_filtered[i]).T],
                                                            ignore_index=True))

print("Length after", len(list_sessions_processed))<jupyter_output>Length before 44551
Length after 30941<jupyter_text>Create Item IDs starting from value 1 for Embeddings and One Hot Layer<jupyter_code>mapping = pd.read_csv('../ID_Mapping.csv')[['Item_ID', 'Mapped_ID']]
dict_items = mapping.set_index('Item_ID').to_dict()['Mapped_ID']

for index, session in enumerate(list_session_processed_autoencoder):
    session['product_id'] = session['product_id'].map(dict_items)
# Pad all Sessions with 0. Embedding Layer and LSTM will use Masking to ignore zeros.
list_sessions_padded = []
window_length = 31

for df in list_session_processed_autoencoder:
    np_array = df.values
    result = np.zeros((window_length, 1), dtype=np.float32)

    result[:np_array.shape[0],:1] = np_array[:,:1]
    list_sessions_padded.append(result)


# Save the results, because the slicing can take some time
np.save('list_sessions_padded_autoencoder.npy', list_sessions_padded)

sessions_padded = np.array(list_sessions_padded)

n_output_features = int(sessions_padded.max())
n_unique_input_ids = int(sessions_padded.max())
window_length = sessions_padded.shape[1]
n_input_features = sessions_padded.shape[2]
print("n_output_features", n_output_features)
print("n_unique_input_ids", n_unique_input_ids)
print("window_length", window_length)
print("n_input_features", n_input_features)<jupyter_output>n_output_features 9494
n_unique_input_ids 9494
window_length 31
n_input_features 1<jupyter_text>Training: Start here if the preprocessing was already executed<jupyter_code>sessions_padded = np.load('list_sessions_padded_autoencoder.npy')
print(sessions_padded.shape)
n_output_features = int(sessions_padded.max())
n_unique_input_ids = int(sessions_padded.max())
window_length = sessions_padded.shape[1]
n_input_features = sessions_padded.shape[2]<jupyter_output>(30941, 31, 1)<jupyter_text>Grid Search HyperparameterDictionary with different hyperparameters to train on.MLflow will track those in a database.<jupyter_code>grid_search_dic = {'hidden_layer_size': [300],
                   'batch_size': [32],
                   'embedding_dim': [200],
                   'window_length': [window_length],
                   'dropout_fc': [0.0], #0.2
                   'n_output_features': [n_output_features],
                   'n_input_features': [n_input_features]}

# Cartesian product
grid_search_param = [dict(zip(grid_search_dic, v)) for v in product(*grid_search_dic.values())]
grid_search_param<jupyter_output><empty_output><jupyter_text>LSTM Autoencoder in functional API- Input: x rows (time steps) of Item IDs in a Session- Output: reconstructed Session<jupyter_code>def build_autoencoder(window_length=50,
                      units_lstm_layer=100,
                      n_unique_input_ids=0,
                      embedding_dim=200,
                      n_input_features=1,
                      n_output_features=3,
                      dropout_rate=0.1):

    inputs = keras.layers.Input(
        shape=[window_length, n_input_features], dtype=np.float32)

    # Encoder
    # Embedding Layer
    embedding_layer = tf.keras.layers.Embedding(
        n_unique_input_ids+1, embedding_dim, input_length=window_length)  # , mask_zero=True)
    embeddings = embedding_layer(inputs[:, :, 0])

    mask = inputs[:, :, 0] != 0

    # LSTM Layer 1
    lstm1_output, lstm1_state_h, lstm1_state_c = keras.layers.LSTM(units=units_lstm_layer, return_state=True,
                                                                   return_sequences=True)(embeddings, mask=mask)
    lstm1_state = [lstm1_state_h, lstm1_state_c]

    # Decoder
    # input: lstm1_state_c, lstm1_state_h
    decoder_state_c = lstm1_state_c
    decoder_state_h = lstm1_state_h
    decoder_outputs = tf.expand_dims(lstm1_state_h, 1)

    list_states = []
    decoder_layer = keras.layers.LSTM(
        units=units_lstm_layer, return_state=True, return_sequences=True, unroll=False)
    for i in range(window_length):
        decoder_outputs, decoder_state_h, decoder_state_c = decoder_layer(decoder_outputs,
                                                                          initial_state=[decoder_state_h,
                                                                                         decoder_state_c])
        list_states.append(decoder_state_h)
    stacked = tf.stack(list_states, axis=1)

    fc_layer = tf.keras.layers.Dense(
        n_output_features+1, kernel_initializer='he_normal')

    fc_layer_output = tf.keras.layers.TimeDistributed(fc_layer)(
        stacked, mask=mask)

    mask_softmax = tf.tile(tf.expand_dims(mask, axis=2),
                           [1, 1, n_output_features+1])

    softmax = tf.keras.layers.Softmax(axis=2, dtype=tf.float32)(
        fc_layer_output, mask=mask_softmax)

    model = keras.models.Model(inputs=[inputs],
                               outputs=[softmax])
    return model<jupyter_output><empty_output><jupyter_text>Convert Numpy Array to tf.data.Dataset for better training performanceThe function will return a zipped tf.data.Dataset with the following Shapes:- x: (batches, window_length)- y: (batches,)<jupyter_code>def array_to_tf_data_api(train_data_x, train_data_y, batch_size=64, window_length=50,
                         validate=False):
    """Applies sliding window on the fly by using the TF Data API.
    Args:
      train_data_x: Input Data as Numpy Array, Shape (rows, n_features)
      batch_size: Batch Size.
      window_length: Window Length or Window Size.
      future_length: Number of time steps that will be predicted in the future.
      n_output_features: Number of features that will be predicted.
      validate: True if input data is a validation set and does not need to be shuffled
      shift: Shifts the Sliding Window by this Parameter.
    Returns:
      tf.data.Dataset
    """

    X = tf.data.Dataset.from_tensor_slices(train_data_x)
    y = tf.data.Dataset.from_tensor_slices(train_data_y)

    if not validate:
        train_tf_data = tf.data.Dataset.zip((X, y)).cache() \
            .shuffle(buffer_size=200000, reshuffle_each_iteration=True)\
            .batch(batch_size).prefetch(1)
        return train_tf_data
    else:
        return tf.data.Dataset.zip((X, y)).batch(batch_size)\
            .prefetch(1)<jupyter_output><empty_output><jupyter_text>Custom TF Callback to log Metrics by MLflow<jupyter_code>class MlflowLogging(tf.keras.callbacks.Callback):
    def __init__(self, **kwargs):
        super().__init__()  # handles base args (e.g., dtype)

    def on_epoch_end(self, epoch, logs=None):
        keys = list(logs.keys())
        for key in keys:
            mlflow.log_metric(str(key), logs.get(key), step=epoch)
class CustomCategoricalCrossentropy(keras.losses.Loss):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.bce = tf.keras.losses.SparseCategoricalCrossentropy(
            from_logits=False, reduction='sum')

    @tf.function
    def call(self, y_true, y_pred):
        total = 0.0
        for i in tf.range(y_pred.shape[1]):
            loss = self.bce(y_true[:, i, 0], y_pred[:, i, :])
            total = total + loss
        return total

    def get_config(self):
        base_config = super().get_config()
        return {**base_config}

    def from_config(cls, config):
        return cls(**config)
class CategoricalAccuracy(keras.metrics.Metric):
    def __init__(self, name="categorical_accuracy", **kwargs):
        super(CategoricalAccuracy, self).__init__(name=name, **kwargs)
        self.true = self.add_weight(name="true", initializer="zeros")
        self.count = self.add_weight(name="count", initializer="zeros")
        self.accuracy = self.add_weight(name="count", initializer="zeros")

    def update_state(self, y_true, y_pred, sample_weight=None):
        y_true = tf.cast(y_true, "float32")
        y_pred = tf.cast(y_pred, "float32")

        mask = y_true[:, :, 0] != 0
        argmax = tf.cast(tf.argmax(y_pred, axis=2), "float32")
        temp = argmax == y_true[:, :, 0]
        true = tf.reduce_sum(tf.cast(temp[mask], dtype=tf.float32))
        self.true.assign_add(true)
        self.count.assign_add(
            tf.cast(tf.shape(temp[mask])[0], dtype="float32"))

        self.accuracy.assign(tf.math.divide(self.true, self.count))

    def result(self):
        return self.accuracy

    def reset_states(self):
        # The state of the metric will be reset at the start of each epoch.
        self.accuracy.assign(0.0)


class CategoricalSessionAccuracy(keras.metrics.Metric):
    def __init__(self, name="categorical_session_accuracy", **kwargs):
        super(CategoricalSessionAccuracy, self).__init__(name=name, **kwargs)
        self.true = self.add_weight(name="true", initializer="zeros")
        self.count = self.add_weight(name="count", initializer="zeros")
        self.accuracy = self.add_weight(name="count", initializer="zeros")

    def update_state(self, y_true, y_pred, sample_weight=None):
        y_true = tf.cast(y_true, "float32")
        y_pred = tf.cast(y_pred, "float32")

        mask = y_true[:, :, 0] != 0
        argmax = tf.cast(tf.argmax(y_pred, axis=2), "float32")
        temp = argmax == y_true[:, :, 0]
        temp = tf.reduce_all(temp, axis=1)
        true = tf.reduce_sum(tf.cast(temp, dtype=tf.float32))
        self.true.assign_add(true)
        self.count.assign_add(tf.cast(tf.shape(temp)[0], dtype="float32"))

        self.accuracy.assign(tf.math.divide(self.true, self.count))

    def result(self):
        return self.accuracy

    def reset_states(self):
        # The state of the metric will be reset at the start of each epoch.
        self.accuracy.assign(0.0)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>with mlflow.start_run() as parent_run:
    for params in grid_search_param:
        batch_size = params['batch_size']
        window_length = params['window_length']
        embedding_dim = params['embedding_dim']
        dropout_fc = params['dropout_fc']
        hidden_layer_size = params['hidden_layer_size']
        n_output_features = params['n_output_features']
        n_input_features = params['n_input_features']

        with mlflow.start_run(nested=True) as child_run:
            # log parameter
            mlflow.log_param('batch_size', batch_size)
            mlflow.log_param('window_length', window_length)
            mlflow.log_param('hidden_layer_size', hidden_layer_size)
            mlflow.log_param('dropout_fc_layer', dropout_fc)
            mlflow.log_param('embedding_dim', embedding_dim)
            mlflow.log_param('n_output_features', n_output_features)
            mlflow.log_param('n_unique_input_ids', n_unique_input_ids)
            mlflow.log_param('n_input_features', n_input_features)

            model = build_autoencoder(window_length=window_length,
                                             n_output_features=n_output_features,
                                             n_unique_input_ids=n_unique_input_ids,
                                             n_input_features=n_input_features,
                                             embedding_dim=embedding_dim,
                                             units_lstm_layer=hidden_layer_size,
                                             dropout_rate=dropout_fc)

            data = array_to_tf_data_api(sessions_padded,
                                        sessions_padded,
                                        window_length=window_length,
                                        batch_size=batch_size)

            model.compile(loss=CustomCategoricalCrossentropy(),#tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False, reduction='sum'),
                          optimizer=keras.optimizers.Nadam(learning_rate=1e-3),
                          metrics=[CategoricalAccuracy(), CategoricalSessionAccuracy()])

            model.fit(data, shuffle=True, initial_epoch=0, epochs=20,
                      callbacks=[MlflowLogging()])
            
            model.compile()
            model.save("./tmp")
            model.save_weights('weights')

            mlflow.tensorflow.log_model(tf_saved_model_dir='./tmp',
                                        tf_meta_graph_tags='serve',
                                        tf_signature_def_key='serving_default',
                                        artifact_path='saved_model',
                                        registered_model_name='Session Based LSTM Recommender')

            shutil.rmtree("./tmp")<jupyter_output>Epoch 1/20
967/967 [==============================] - 95s 70ms/step - loss: 536.0073 - categorical_accuracy: 0.0037 - categorical_session_accuracy: 0.0000e+00
Epoch 2/20
967/967 [==============================] - 65s 67ms/step - loss: 189.0837 - categorical_accuracy: 0.0097 - categorical_session_accuracy: 4.4663e-05
Epoch 3/20
967/967 [==============================] - 63s 66ms/step - loss: 174.3795 - categorical_accuracy: 0.0173 - categorical_session_accuracy: 3.9590e-04
Epoch 4/20
967/967 [==============================] - 63s 66ms/step - loss: 147.5806 - categorical_accuracy: 0.0341 - categorical_session_accuracy: 0.0023
Epoch 5/20
967/967 [==============================] - 63s 65ms/step - loss: 125.2717 - categorical_accuracy: 0.0621 - categorical_session_accuracy: 0.0089
Epoch 6/20
967/967 [==============================] - 63s 65ms/step - loss: 100.3309 - categorical_accuracy: 0.0977 - categorical_session_accuracy: 0.0234
Epoch 7/20
967/967 [==============================] - 62s [...]
{'fraction_non_alphanumeric': 0.11862017106870947, 'fraction_numerical': 0.02702321646771313, 'mean_word_length': 3.333808553971487, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 52, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '568', 'license': 'MIT', 'n_tokens_mistral': 7165, 'n_tokens_neox': 6654, 'n_words': 1271, 'path': 'outlier_detection/training_outlier_detection.ipynb', 'repo_name': 'felix-exel/kfserving-advanced', 'size': '21278'}