<jupyter_start><jupyter_text>The grand quest: make it actually work (4 points)Your main task is to use some of the tricks you've learned on the network and analyze if you can improve __validation MAE__. Try __at least 3 options__ from the list below for a passing grade. Write a short report about what you have tried. More ideas = more bonus points. __Please be serious:__ " plot learning curves in MAE/epoch, compare models based on optimal performance, test one change at a time. You know the drill :)You can use either pure __tensorflow__ or __keras__. Feel free to adapt the seminar code for your needs.<jupyter_code>import nltk
import keras

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import keras.layers as L
%matplotlib inline

from collections import Counter
from sklearn.feature_extraction import DictVectorizer
from sklearn.model_selection import train_test_split<jupyter_output>Using TensorFlow backend.<jupyter_text>First, let's use some code from the seminar that contains preprocessing.<jupyter_code>data = pd.read_csv("./Train_rev1.csv", index_col=None)

data['Log1pSalary'] = np.log1p(data['SalaryNormalized']).astype('float32')

text_columns = ["Title", "FullDescription"]
categorical_columns = ["Category", "Company", "LocationNormalized", "ContractType", "ContractTime"]
target_column = "Log1pSalary"

data[categorical_columns] = data[categorical_columns].fillna('NaN')

tokenizer = nltk.tokenize.WordPunctTokenizer()

data["FullDescription"] = data["FullDescription"].astype(str).apply(
    lambda x: ' '.join(tokenizer.tokenize(x.lower())), 1)
data["Title"] = data["Title"].astype(str).apply(
    lambda x: ' '.join(tokenizer.tokenize(x.lower())), 1)

tokens = []
for title in data["Title"]:
    tokens += title.split()
for title in data["FullDescription"]:
    tokens += title.split()
    
token_counts = Counter(tokens)

min_count = 10

tokens = [token for token, count in token_counts.items() if count >= min_count]

UNK, PAD = "UNK", "PAD"
tokens = [UNK, PAD] + sorted(tokens)
token_to_id = {token: id for id, token in enumerate(tokens)}

UNK_IX, PAD_IX = map(token_to_id.get, [UNK, PAD])

def as_matrix(sequences, max_len=None):
    """ Convert a list of tokens into a matrix with padding """
    if isinstance(sequences[0], str):
        sequences = list(map(str.split, sequences))
        
    max_len = min(max(map(len, sequences)), max_len or float('inf'))
    
    matrix = np.full((len(sequences), max_len), np.int32(PAD_IX))
    for i,seq in enumerate(sequences):
        row_ix = [token_to_id.get(word, UNK_IX) for word in seq[:max_len]]
        matrix[i, :len(row_ix)] = row_ix
    
    return matrix

top_companies, top_counts = zip(*Counter(data['Company']).most_common(1000))
recognized_companies = set(top_companies)
data["Company"] = data["Company"].apply(lambda comp: comp if comp in recognized_companies else "Other")

categorical_vectorizer = DictVectorizer(dtype=np.float32, sparse=False)
categorical_vectorizer.fit(data[categorical_columns].apply(dict, axis=1))

data_train, data_val = train_test_split(data, test_size=0.2, random_state=42)
data_train.index = range(len(data_train))
data_val.index = range(len(data_val))

def make_batch(data, max_len=None, word_dropout=0):
    """
    Creates a keras-friendly dict from the batch data.
    :param word_dropout: replaces token index with UNK_IX with this probability
    :returns: a dict with {'title' : int64[batch, title_max_len]
    """
    batch = {}
    batch["Title"] = as_matrix(data["Title"].values, max_len)
    batch["FullDescription"] = as_matrix(data["FullDescription"].values, max_len)
    batch['Categorical'] = categorical_vectorizer.transform(data[categorical_columns].apply(dict, axis=1))
    
    if word_dropout != 0:
        batch["FullDescription"] = apply_word_dropout(batch["FullDescription"], 1. - word_dropout)
    
    if target_column in data.columns:
        batch[target_column] = data[target_column].values
    
    return batch

def apply_word_dropout(matrix, keep_prop, replace_with=UNK_IX, pad_ix=PAD_IX,):
    dropout_mask = np.random.choice(2, np.shape(matrix), p=[keep_prop, 1 - keep_prop])
    dropout_mask &= matrix != pad_ix
    return np.choose(dropout_mask, [matrix, np.full_like(matrix, replace_with)])
def iterate_minibatches(data, batch_size=256, shuffle=True, cycle=False, **kwargs):
    """ iterates minibatches of data in random order """
    while True:
        indices = np.arange(len(data))
        if shuffle:
            indices = np.random.permutation(indices)

        for start in range(0, len(indices), batch_size):
            batch = make_batch(data.iloc[indices[start : start + batch_size]], **kwargs)
            target = batch.pop(target_column)
            yield batch, target
        
        if not cycle: break
def print_metrics(model, data, batch_size=batch_size, name="", **kw):
    squared_error = abs_error = num_samples = 0.0
    for batch_x, batch_y in iterate_minibatches(data, batch_size=batch_size, shuffle=False, **kw):
        batch_pred = model.predict(batch_x)[:, 0]
        squared_error += np.sum(np.square(batch_pred - batch_y))
        abs_error += np.sum(np.abs(batch_pred - batch_y))
        num_samples += len(batch_y)
    print("%s results:" % (name or ""))
    print("Mean square error: %.5f" % (squared_error / num_samples))
    print("Mean absolute error: %.5f" % (abs_error / num_samples))
    return squared_error, abs_error<jupyter_output><empty_output><jupyter_text>Then let's define some consts and useful callbacks.<jupyter_code>batch_size = 256
epochs = 100
steps_per_epoch = 100
n_tokens=len(tokens)
n_cat_features=len(categorical_vectorizer.vocabulary_)
hid_size=64

callbacks = [
    # Early stopping callback
    keras.callbacks.EarlyStopping(monitor='val_mean_absolute_error', min_delta=0.0005, patience=3),
    # Tensorboard to visualize learning curves
    keras.callbacks.TensorBoard(log_dir='./logs/')
]<jupyter_output><empty_output><jupyter_text>Now we can take the seminar model and look how strong its baseline is. Ofc we'll put our callbacks inside.<jupyter_code>def build_model(n_tokens=len(tokens), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=64):
    l_title = L.Input(shape=[None], name="Title")
    l_descr = L.Input(shape=[None], name="FullDescription")
    l_categ = L.Input(shape=[n_cat_features], name="Categorical")
    
    emb = L.Embedding(n_tokens, 2 * hid_size)
    
    l_title_emb = emb(l_title)
    l_descr_emb = emb(l_descr)
    
    l_title_conv = L.Convolution1D(hid_size, kernel_size=2, activation='relu')(l_title_emb)
    l_descr_conv = L.Convolution1D(hid_size, kernel_size=5, activation='relu')(l_descr_emb)
    
    l_title_out = L.GlobalMaxPool1D()(l_title_conv)
    l_descr_out = L.GlobalMaxPool1D()(l_descr_conv)
    
    l_categ_out = L.Dense(hid_size, activation='relu')(l_categ)
    
    l_combined = L.Concatenate()([l_title_out, l_descr_out, l_categ_out])
    l_dense_clf = L.Dense(hid_size, activation='relu')(l_combined)
    
    output_layer = L.Dense(1)(l_dense_clf)
    
    model = keras.models.Model(inputs=[l_title, l_descr, l_categ], outputs=[output_layer])
    model.compile('adam', 'mean_squared_error', metrics=['mean_absolute_error'])
    return model
model = build_model()

model.fit_generator(iterate_minibatches(data_train, batch_size, cycle=True, word_dropout=0.05), 
                    epochs=epochs, steps_per_epoch=steps_per_epoch,
                    validation_data=iterate_minibatches(data_val, batch_size, cycle=True),
                    validation_steps=data_val.shape[0] // batch_size,
                    callbacks=callbacks
                   )
print_metrics(model, data_train, name='Train')
print_metrics(model, data_val, name='Val');<jupyter_output>Train results:
Mean square error: 0.03584
Mean absolute error: 0.13863
Val results:
Mean square error: 0.06084
Mean absolute error: 0.18102<jupyter_text>To improve it, let's try to add more neurons in the dense layers and dropout.<jupyter_code>def evaluate_model(model):
    model.fit_generator(iterate_minibatches(data_train, batch_size, cycle=True, word_dropout=0.05), 
                        epochs=epochs, steps_per_epoch=steps_per_epoch,
                        validation_data=iterate_minibatches(data_val, batch_size, cycle=True),
                        validation_steps=data_val.shape[0] // batch_size,
                        callbacks=callbacks
                       )
    
    print_metrics(model, data_train, name='Train')
    print_metrics(model, data_val, name='Val');
l_categ_out = L.Dense(2 * hid_size, activation='relu')(l_categ)
l_categ_out = L.Dropout(0.5)(l_categ_out)

l_combined = L.Concatenate()([l_title_out, l_descr_out, l_categ_out])
l_dense_clf = L.Dense(2 * hid_size, activation='relu')(l_combined)

output_layer = L.Dense(1)(l_dense_clf)

model = keras.models.Model(inputs=[l_title, l_descr, l_categ], outputs=[output_layer])
model.compile('adam', 'mean_squared_error', metrics=['mean_absolute_error'])

evaluate_model(model)<jupyter_output>Epoch 1/100
100/100 [==============================] - 11s 107ms/step - loss: 16.0088 - mean_absolute_error: 2.4636 - val_loss: 0.3215 - val_mean_absolute_error: 0.4403
Epoch 2/100
100/100 [==============================] - 10s 104ms/step - loss: 0.3178 - mean_absolute_error: 0.4424 - val_loss: 0.1914 - val_mean_absolute_error: 0.3346
Epoch 3/100
100/100 [==============================] - 11s 106ms/step - loss: 0.2357 - mean_absolute_error: 0.3774 - val_loss: 0.1632 - val_mean_absolute_error: 0.3085
Epoch 4/100
100/100 [==============================] - 11s 106ms/step - loss: 0.1821 - mean_absolute_error: 0.3300 - val_loss: 0.1257 - val_mean_absolute_error: 0.2671
Epoch 5/100
100/100 [==============================] - 11s 107ms/step - loss: 0.1499 - mean_absolute_error: 0.2975 - val_loss: 0.1107 - val_mean_absolute_error: 0.2490
Epoch 6/100
100/100 [==============================] - 11s 109ms/step - loss: 0.1316 - mean_absolute_error: 0.2763 - val_loss: 0.1014 - val_mean_absolute_error[...]<jupyter_text>Almost no affect. Then we gonna separate convolutional/dense and activation layers and place a batchnorm in the midst.<jupyter_code>l_title_conv = L.Convolution1D(hid_size, kernel_size=2)(l_title_emb)
l_title_conv = L.BatchNormalization()(l_title_conv)
l_title_conv = L.Activation('relu')(l_title_conv)
l_descr_conv = L.Convolution1D(hid_size, kernel_size=5)(l_descr_emb)
l_descr_conv = L.BatchNormalization()(l_descr_conv)
l_descr_conv = L.Activation('relu')(l_descr_conv)

l_title_out = L.GlobalMaxPool1D()(l_title_conv)
l_descr_out = L.GlobalMaxPool1D()(l_descr_conv)

l_categ_out = L.Dense(hid_size, activation='relu')(l_categ)
l_categ_out = L.BatchNormalization()(l_categ_out)
l_categ_out = L.Activation('relu')(l_categ_out)

l_combined = L.Concatenate()([l_title_out, l_descr_out, l_categ_out])

l_dense_clf = L.Dense(hid_size, activation='relu')(l_combined)
l_dense_clf = L.BatchNormalization()(l_dense_clf)
l_dense_clf = L.Activation('relu')(l_dense_clf)

output_layer = L.Dense(1)(l_dense_clf)

model = keras.models.Model(inputs=[l_title, l_descr, l_categ], outputs=[output_layer])
model.compile('adam', 'mean_squared_error', metrics=['mean_absolute_error'])

evaluate_model(model)<jupyter_output>Epoch 1/100
100/100 [==============================] - 12s 121ms/step - loss: 68.5091 - mean_absolute_error: 8.1989 - val_loss: 17.8413 - val_mean_absolute_error: 4.0067
Epoch 2/100
100/100 [==============================] - 12s 118ms/step - loss: 28.1971 - mean_absolute_error: 5.1998 - val_loss: 3.9842 - val_mean_absolute_error: 1.6866
Epoch 3/100
100/100 [==============================] - 12s 120ms/step - loss: 7.3154 - mean_absolute_error: 2.5457 - val_loss: 0.5466 - val_mean_absolute_error: 0.5697
Epoch 4/100
100/100 [==============================] - 12s 118ms/step - loss: 1.1929 - mean_absolute_error: 0.9159 - val_loss: 1.0370 - val_mean_absolute_error: 0.8508
Epoch 5/100
100/100 [==============================] - 12s 119ms/step - loss: 0.3582 - mean_absolute_error: 0.4681 - val_loss: 2.2258 - val_mean_absolute_error: 1.4149
Epoch 6/100
100/100 [==============================] - 12s 117ms/step - loss: 0.2727 - mean_absolute_error: 0.4066 - val_loss: 2.0186 - val_mean_absolute_err[...]<jupyter_text>Or we always can add more layers...<jupyter_code>l_categ_out = L.Dense(hid_size, activation='relu')(l_categ)
l_categ_out = L.Dense(hid_size, activation='relu')(l_categ_out)

l_combined = L.Concatenate()([l_title_out, l_descr_out, l_categ_out])
l_dense_clf = L.Dense(2 * hid_size, activation='relu')(l_combined)
l_dense_clf = L.Dense(2 * hid_size, activation='relu')(l_dense_clf)

output_layer = L.Dense(1)(l_dense_clf)

model = keras.models.Model(inputs=[l_title, l_descr, l_categ], outputs=[output_layer])
model.compile('adam', 'mean_squared_error', metrics=['mean_absolute_error'])

evaluate_model(model)<jupyter_output>Epoch 1/100
100/100 [==============================] - 12s 124ms/step - loss: 7.4275 - mean_absolute_error: 1.5666 - val_loss: 0.6399 - val_mean_absolute_error: 0.7085
Epoch 2/100
100/100 [==============================] - 12s 117ms/step - loss: 0.4719 - mean_absolute_error: 0.5333 - val_loss: 0.6582 - val_mean_absolute_error: 0.7421
Epoch 3/100
100/100 [==============================] - 12s 118ms/step - loss: 0.3756 - mean_absolute_error: 0.4753 - val_loss: 0.7919 - val_mean_absolute_error: 0.8313
Epoch 4/100
100/100 [==============================] - 12s 119ms/step - loss: 0.2137 - mean_absolute_error: 0.3496 - val_loss: 0.2379 - val_mean_absolute_error: 0.4037
Epoch 5/100
100/100 [==============================] - 12s 117ms/step - loss: 0.1518 - mean_absolute_error: 0.2843 - val_loss: 0.1516 - val_mean_absolute_error: 0.2999
Epoch 6/100
100/100 [==============================] - 12s 118ms/step - loss: 0.1468 - mean_absolute_error: 0.2815 - val_loss: 0.1380 - val_mean_absolute_error:[...]
{'fraction_non_alphanumeric': 0.15805361891755912, 'fraction_numerical': 0.052900165313016605, 'mean_word_length': 4.915816326530612, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '585056', 'license': 'MIT', 'n_tokens_mistral': 5471, 'n_tokens_neox': 4868, 'n_words': 985, 'path': 'week02_classification/homework_part2.ipynb', 'repo_name': 'waytobehigh/nlp_course', 'size': '13913'}