<jupyter_start><jupyter_text>Salary prediction, episode II: make it actually work (4 points)Your main task is to use some of the tricks you've learned on the network and analyze if you can improve __validation MAE__. Try __at least 3 options__ from the list below for a passing grade. Write a short report about what you have tried. More ideas = more bonus points. __Please be serious:__ " plot learning curves in MAE/epoch, compare models based on optimal performance, test one change at a time. You know the drill :)You can use either __pytorch__ or __tensorflow__ or any other framework (e.g. pure __keras__). Feel free to adapt the seminar code for your needs. For tensorflow version, consider `seminar_tf2.ipynb` as a starting point.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
# !wget https://ysda-seminars.s3.eu-central-1.amazonaws.com/Train_rev1.zip
# !unzip Train_rev1.zip
data = pd.read_csv("./Train_rev1.csv", index_col=None)
data.shape
data['Log1pSalary'] = np.log1p(data['SalaryNormalized']).astype('float32')

plt.figure(figsize=[8, 4])
plt.subplot(1, 2, 1)
plt.hist(data["SalaryNormalized"], bins=20);

plt.subplot(1, 2, 2)
plt.hist(data['Log1pSalary'], bins=20);

text_columns = ["Title", "FullDescription"]
categorical_columns = ["Category", "Company", "LocationNormalized", "ContractType", "ContractTime"]
TARGET_COLUMN = "Log1pSalary"

data[categorical_columns] = data[categorical_columns].fillna('NaN') # cast missing values to string "NaN"

data.sample(3)
import nltk

from nltk.tokenize import RegexpTokenizer

tokenizer = RegexpTokenizer(r'\w+')
def process_tokens(sentence):
  return ' '.join(tokenizer.tokenize(str(sentence).lower()))
data[['FullDescription', 'Title']].fillna('NA', inplace=True)
data['FullDescriptionTokenized'] = data['FullDescription'].apply(process_tokens)
data['TitleTokenized'] = data['Title'].apply(process_tokens)
data[['FullDescription', 'Title']] = data[['FullDescriptionTokenized', 'TitleTokenized']]
data.drop(['FullDescriptionTokenized', 'TitleTokenized'], axis=1, inplace=True)
from sklearn.feature_extraction import DictVectorizer
from collections import Counter

# we only consider top-1k most frequent companies to minimize memory usage
top_companies, top_counts = zip(*Counter(data['Company']).most_common(1000))
recognized_companies = set(top_companies)
data["Company"] = data["Company"].apply(lambda comp: comp if comp in recognized_companies else "Other")

categorical_vectorizer = DictVectorizer(dtype=np.float32, sparse=False)
categorical_vectorizer.fit(data[categorical_columns].apply(dict, axis=1))
from sklearn.model_selection import train_test_split

data_train, data_val = train_test_split(data, test_size=0.2, random_state=42)
data_train.index = range(len(data_train))
data_val.index = range(len(data_val))

print("Train size = ", len(data_train))
print("Validation size = ", len(data_val))
from sklearn.feature_extraction.text import CountVectorizer
count_vect = CountVectorizer(decode_error='ignore', lowercase=False, min_df=10)
count_vect.fit(data_train.FullDescription + ' ' + data_train.Title)
import torch
import tqdm

BATCH_SIZE = 32
EPOCHS = 10
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def to_tensors(batch, device):
    batch_tensors = dict()
    for key, arr in batch.items():
        if key in ["FullDescription", "Title"]:
            batch_tensors[key] = torch.tensor(arr, device=device, dtype=torch.int64)
        else:
            batch_tensors[key] = torch.tensor(arr, device=device)
    return batch_tensors

def make_batch(data, word_dropout=0, device=DEVICE):
    """
    Creates a keras-friendly dict from the batch data.
    :param word_dropout: replaces token index with UNK_IX with this probability
    :returns: a dict with {'title' : int64[batch, title_max_len]
    """
    batch = {}
    batch["Title"] = count_vect.transform(data.Title).todense()
    batch["FullDescription"] = count_vect.transform(data.FullDescription).todense()
    batch['Categorical'] = categorical_vectorizer.transform(data[categorical_columns].apply(dict, axis=1))
    
    if TARGET_COLUMN in data.columns:
        batch[TARGET_COLUMN] = data[TARGET_COLUMN].values
    
    return to_tensors(batch, device)

def iterate_minibatches(data, batch_size=256, shuffle=True, cycle=False, device=DEVICE, **kwargs):
    """ iterates minibatches of data in random order """
    while True:
        indices = np.arange(len(data))
        if shuffle:
            indices = np.random.permutation(indices)

        for start in range(0, len(indices), batch_size):
            batch = make_batch(data.iloc[indices[start : start + batch_size]], **kwargs)
            yield batch
        
        if not cycle: break

def print_metrics(model, data, batch_size=BATCH_SIZE, name="", **kw):
    squared_error = abs_error = num_samples = 0.0
    model.eval()
    with torch.no_grad():
        for batch in iterate_minibatches(data, batch_size=batch_size, shuffle=False, **kw):
            batch_pred = model(batch)
            squared_error += torch.sum(torch.square(batch_pred - batch[TARGET_COLUMN]))
            abs_error += torch.sum(torch.abs(batch_pred - batch[TARGET_COLUMN]))
            num_samples += len(batch_pred)
    mse = squared_error.detach().cpu().numpy() / num_samples
    mae = abs_error.detach().cpu().numpy() / num_samples
    print("%s results:" % (name or ""))
    print("Mean square error: %.5f" % mse)
    print("Mean absolute error: %.5f" % mae)
    return mse, mae

#Basic version

import torch.nn as nn
import torch.functional as F


class SalaryPredictorWithEmbeddings(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=32):
        super().__init__()
        self.title_embeddings = nn.Embedding(n_tokens, hid_size)
        self.title_conv = nn.Conv1d(hid_size, 10, 3)
        self.description_embeddings = nn.Embedding(n_tokens, hid_size)
        self.description_conv = nn.Conv1d(hid_size, 5, 3)
        self.cat_features_embeddings = nn.Linear(n_cat_features, 5, False)
        self.fc = nn.Linear(20, 1)
        
    def forward(self, batch):
        title = self.title_embeddings(batch['Title'])
        title = self.title_conv(title.permute(0,2,1))
        title = title.max(dim=2)[0]
        description = self.description_embeddings(batch['FullDescription'])
        description = self.description_conv(description.permute(0,2,1))
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)
basic_model = SalaryPredictorWithEmbeddings().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(basic_model.parameters(), lr=1e-4)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    basic_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
        ):
        pred = basic_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    print_metrics(basic_model, data_val)
import gensim.downloader 
embeddings = gensim.downloader.load("glove-wiki-gigaword-100")
def vectorize_sum(sentence):
    """
    implement a function that converts preprocessed comment to a sum of token vectors
    """
    features = []
    
    for word in sentence.split():
      if word in embeddings.vocab:
        features.append(embeddings.get_vector(word))
    features = np.array(features, dtype='float64')
    return features

def pad_sentence_sequence(sentence_seq):
    max_len = 0
    vectorized_sentences = []
    for sentence in sentence_seq:
        vectorized_sentence = vectorize_sum(sentence)
        if vectorized_sentence.size:
            vectorized_sentences.append(vectorized_sentence)
            max_len = max(max_len, vectorized_sentence.shape[0])
        else:
            vectorized_sentences.append(np.zeros((1, embeddings.vector_size)))
    
    max_len = max(3, max_len)

    padded_vectors = []
    for i, vectorized_sentence in enumerate(vectorized_sentences):
        padded_vectors.append(np.pad(vectorized_sentence, ((0, max_len - vectorized_sentence.shape[0]), (0, 0))))
    return padded_vectors


def make_batch(data, word_dropout=0, device=DEVICE):
    """
    Creates a keras-friendly dict from the batch data.
    :param word_dropout: replaces token index with UNK_IX with this probability
    :returns: a dict with {'title' : int64[batch, title_max_len]
    """
    batch = {}
    batch["Title"] = pad_sentence_sequence(data.Title.values)
    batch["FullDescription"] = pad_sentence_sequence(data.FullDescription.values)
    batch['Categorical'] = categorical_vectorizer.transform(data[categorical_columns].apply(dict, axis=1))
    
    if TARGET_COLUMN in data.columns:
        batch[TARGET_COLUMN] = data[TARGET_COLUMN].values
    
    return to_tensors(batch, device)

def to_tensors(batch, device):
    batch_tensors = dict()
    for key, arr in batch.items():
        if key in ["FullDescription", "Title"]:
            batch_tensors[key] = torch.tensor(arr, device=device, dtype=torch.float)
        else:
            batch_tensors[key] = torch.tensor(arr, device=device, dtype=torch.float)
    return batch_tensors
#

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv = nn.Conv1d(hid_size, 10, 3)
        self.description_conv = nn.Conv1d(hid_size, 5, 3)
        self.cat_features_embeddings = nn.Linear(n_cat_features, 5, False)
        self.fc = nn.Linear(20, 1)
        
    def forward(self, batch):
        title = batch['Title']
        title = self.title_conv(title.permute(0,2,1))
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description = self.description_conv(description.permute(0,2,1))
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)
preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
        ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    print_metrics(preembedded_model, data_val)

del basic_model
### adding padding not to loose time dims

BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv = nn.Conv1d(hid_size, 5, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 5, False)
        self.fc = nn.Linear(20, 1)

    def forward(self, batch):
        title = batch['Title']
        title = self.title_conv(title.permute(0,2,1))
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description = self.description_conv(description.permute(0,2,1))
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)
#trying parallel convs
BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv1 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.title_conv2 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv1 = nn.Conv1d(hid_size, 5, 3, padding='same')
        self.description_conv2 = nn.Conv1d(hid_size, 5, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 5, False)
        self.fc = nn.Linear(35, 1)

    def forward(self, batch):
        title = batch['Title']
        title1 = self.title_conv1(title.permute(0,2,1))
        title2 = self.title_conv2(title.permute(0,2,1))
        title = torch.cat([title1, title2], dim=1)
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description1 = self.description_conv1(description.permute(0,2,1))
        description2 = self.description_conv2(description.permute(0,2,1))
        description = torch.cat([description1, description2], dim=1)
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)
#adding more features
BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv1 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.title_conv2 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.description_conv1 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv2 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 10, False)
        self.fc = nn.Linear(70, 1)

    def forward(self, batch):
        title = batch['Title']
        title1 = self.title_conv1(title.permute(0,2,1))
        title2 = self.title_conv2(title.permute(0,2,1))
        title = torch.cat([title1, title2], dim=1)
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description1 = self.description_conv1(description.permute(0,2,1))
        description2 = self.description_conv2(description.permute(0,2,1))
        description = torch.cat([description1, description2], dim=1)
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)
#adding Dropout

BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv1 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.title_conv2 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.description_conv1 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv2 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 10, False)
        self.dropout_layer = nn.Dropout(p=0.01)
        self.fc = nn.Linear(70, 1)

    def forward(self, batch):
        title = batch['Title']
        title1 = self.title_conv1(title.permute(0,2,1))
        title2 = self.title_conv2(title.permute(0,2,1))
        title = torch.cat([title1, title2], dim=1)
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description1 = self.description_conv1(description.permute(0,2,1))
        description2 = self.description_conv2(description.permute(0,2,1))
        description = torch.cat([description1, description2], dim=1)
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        concatenated = torch.cat([title, description, cat_features], dim=1)
        concatenated = self.dropout_layer(concatenated)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)
#adding Batchnorm

BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv1 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.title_conv2 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.description_conv1 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv2 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 10, False)
        self.cat_features_norm = nn.BatchNorm1d(10)
        self.dropout_layer = nn.Dropout(p=0.01)
        self.fc = nn.Linear(70, 1)

    def forward(self, batch):
        title = batch['Title']
        title1 = self.title_conv1(title.permute(0,2,1))
        title2 = self.title_conv2(title.permute(0,2,1))
        title = torch.cat([title1, title2], dim=1)
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description1 = self.description_conv1(description.permute(0,2,1))
        description2 = self.description_conv2(description.permute(0,2,1))
        description = torch.cat([description1, description2], dim=1)
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        cat_features = self.cat_features_norm(cat_features)
        concatenated = torch.cat([title, description, cat_features], dim=1)
        concatenated = self.dropout_layer(concatenated)
        output = self.fc(concatenated)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)

#adding more FCs layers

BATCH_SIZE = 32
EPOCHS = 10

class SalaryPredictorPreEmbedded(nn.Module):
    def __init__(self, n_tokens=len(count_vect.vocabulary_), n_cat_features=len(categorical_vectorizer.vocabulary_), hid_size=embeddings.vector_size):
        super().__init__()
        self.title_conv1 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.title_conv2 = nn.Conv1d(hid_size, 20, 3, padding='same')
        self.description_conv1 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.description_conv2 = nn.Conv1d(hid_size, 10, 3, padding='same')
        self.cat_features_embeddings = nn.Linear(n_cat_features, 10, False)
        self.cat_features_norm = nn.BatchNorm1d(10)
        self.dropout_layer = nn.Dropout(p=0.01)
        self.fc = nn.Linear(70, 20)
        self.fc_norm = nn.BatchNorm1d(20)
        self.fc_droput = nn.Dropout(p=0.01)
        self.fc2 = nn.Linear(20, 1)


    def forward(self, batch):
        title = batch['Title']
        title1 = self.title_conv1(title.permute(0,2,1))
        title2 = self.title_conv2(title.permute(0,2,1))
        title = torch.cat([title1, title2], dim=1)
        title = title.max(dim=2)[0]
        description = batch['FullDescription']
        description1 = self.description_conv1(description.permute(0,2,1))
        description2 = self.description_conv2(description.permute(0,2,1))
        description = torch.cat([description1, description2], dim=1)
        description = description.max(dim=2)[0]
        cat_features = self.cat_features_embeddings(batch['Categorical'])
        cat_features = self.cat_features_norm(cat_features)
        concatenated = torch.cat([title, description, cat_features], dim=1)
        concatenated = self.dropout_layer(concatenated)
        output = self.fc(concatenated)
        output = self.fc_norm(output)
        output = self.fc_droput(output)
        output = self.fc2(output)
        return output.squeeze(dim=1)

preembedded_model = SalaryPredictorPreEmbedded().to(DEVICE)
criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(preembedded_model.parameters(), lr=1e-3)

for epoch in range(EPOCHS):
    print(f"epoch: {epoch}")
    preembedded_model.train()
    for i, batch in tqdm.notebook.tqdm(enumerate(
            iterate_minibatches(data_train, batch_size=BATCH_SIZE, device=DEVICE)),
            total=len(data_train) // BATCH_SIZE
    ):
        pred = preembedded_model(batch)
        loss = criterion(pred, batch[TARGET_COLUMN])/BATCH_SIZE
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print_metrics(preembedded_model, data_val)<jupyter_output>epoch: 0<jupyter_text>A short reportPlease tell us what you did and how did it work.``, i guess...  Recommended options A) CNN architectureAll the tricks you know about dense and convolutional neural networks apply here as well.* Dropout. Nuff said.* Batch Norm. This time it's `nn.BatchNorm*`/`L.BatchNormalization`* Parallel convolution layers. The idea is that you apply several nn.Conv1d to the same embeddings and concatenate output channels.* More layers, more neurons, ya know... B) Play with poolingThere's more than one way to perform pooling:* Max over time (independently for each feature)* Average over time (excluding PAD)* Softmax-pooling:$$ out_{i, t} = \sum_t {h_{i,t} \cdot {{e ^ {h_{i, t}}} \over \sum_\tau e ^ {h_{j, \tau}} } }$$* Attentive pooling$$ out_{i, t} = \sum_t {h_{i,t} \cdot Attn(h_t)}$$, where $$ Attn(h_t) = {{e ^ {NN_{attn}(h_t)}} \over \sum_\tau e ^ {NN_{attn}(h_\tau)}}  $$and $NN_{attn}$ is a dense layer.The optimal score is usually achieved by concatenating several different poolings, including several attentive pooling with different $NN_{attn}$ (aka multi-headed attention).The catch is that keras layers do not inlude those toys. You will have to [write your own keras layer](https://keras.io/layers/writing-your-own-keras-layers/). Or use pure tensorflow, it might even be easier :) C) Fun with wordsIt's not always a good idea to train embeddings from scratch. Here's a few tricks:* Use a pre-trained embeddings from `gensim.downloader.load`. See last lecture.* Start with pre-trained embeddings, then fine-tune them with gradient descent. You may or may not download pre-trained embeddings from [here](http://nlp.stanford.edu/data/glove.6B.zip) and follow this [manual](https://keras.io/examples/nlp/pretrained_word_embeddings/) to initialize your Keras embedding layer with downloaded weights.* Use the same embedding matrix in title and desc vectorizer D) Going recurrentWe've already learned that recurrent networks can do cool stuff in sequence modelling. Turns out, they're not useless for classification as well. With some tricks of course..* Like convolutional layers, LSTM should be pooled into a fixed-size vector with some of the poolings.* Since you know all the text in advance, use bidirectional RNN  * Run one LSTM from left to right  * Run another in parallel from right to left   * Concatenate their output sequences along unit axis (dim=-1)* It might be good idea to mix convolutions and recurrent layers differently for title and description E) Optimizing seriously* You don't necessarily need 100 epochs. Use early stopping. If you've never done this before, take a look at [early stopping callback(keras)](https://keras.io/callbacks/earlystopping) or in [pytorch(lightning)](https://pytorch-lightning.readthedocs.io/en/latest/common/early_stopping.html).  * In short, train until you notice that validation  * Maintain the best-on-validation snapshot via `model.save(file_name)`  * Plotting learning curves is usually a good idea  Good luck! And may the force be with you!<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11804828755999856, 'fraction_numerical': 0.018189036053267892, 'mean_word_length': 3.9561795743158648, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '371128', 'license': 'MIT', 'n_tokens_mistral': 9257, 'n_tokens_neox': 8947, 'n_words': 1867, 'path': 'week02_classification/homework_part2.ipynb', 'repo_name': 'ivkrasovskiy/nlp_course', 'size': '27709'}