<jupyter_start><jupyter_text>Load preprocessed data Run the script that downloads and processes the MovieLens data.Uncomment it to run the download & processing script.<jupyter_code>#!python ../src/download.py
import numpy as np
from sklearn.model_selection import train_test_split
from torch import from_numpy
from torch.utils.data import DataLoader
from torch.utils.data import TensorDataset
from torch.utils.data import BatchSampler
from torch.utils.data import SequentialSampler

fh = np.load('data/dataset.npz')

# We have a bunch of feature columns and last column is the y-target
# Note pytorch is finicky about need int64 types
train_x = fh['train_x'].astype(np.int64)
train_y = fh['train_y']

# We've already split into train & test
X_test = fh['test_x'].astype(np.int64)
Y_test = fh['test_y']

X_train, X_val, Y_train, Y_val = train_test_split(train_x, train_y)


n_user = int(fh['n_user'])
n_item = int(fh['n_item'])
n_occu = int(fh['n_occu'])
n_rank = int(fh['n_ranks'])

# columns are user_id, item_id and other features 
# we won't use the 3rd and 4th columns
print(X_train)
print(' ')
print(Y_train)



def dataloader(*arrs, batch_size=1024):
    dataset = TensorDataset(*arrs)
    arr_size = len(arrs[0])
    bs = BatchSampler(SequentialSampler(range(arr_size)),
                      batch_size=batch_size, drop_last=False)
    return DataLoader(dataset, batch_sampler=bs, shuffle=False)
 
train = dataloader(from_numpy(X_train), from_numpy(Y_train))
test = dataloader(from_numpy(X_test), from_numpy(Y_test))
val = dataloader(from_numpy(X_val), from_numpy(Y_val))
n_user
from abstract_model import AbstractModel
import torch
from torch import nn
import torch.nn.functional as F
import pytorch_lightning as pl


def l2_regularize(array):
    loss = torch.sum(array ** 2.0)
    return loss


def sample_gaussian(mu, log_var):
    var = log_var.mul(0.5).exp_().to(mu.device)
    eps = torch.FloatTensor(var.size()).normal_().to(mu.device)
    return mu + eps * var


def gaussian_kldiv(mu, log_var):
    kld = mu.pow(2).add_(log_var.exp()).mul_(-1).add_(1).add_(log_var)
    kldloss = torch.sum(kld).mul_(-0.5)
    return kldloss


class MF(AbstractModel):
    def __init__(self, n_user, n_item, 
                 k=18, c_vector=1.0, c_bias=1.0,
                 c_kld=1.0, batch_size=128):
        super().__init__()
        self.k = k
        self.n_user = n_user
        self.n_item = n_item
        self.c_bias = c_bias
        self.c_kld = c_kld
        self.batch_size = batch_size
        
        # We've added new terms here:
        self.user_mu = nn.Embedding(n_user, k)
        self.user_lv = nn.Embedding(n_user, k)
        self.item_mu = nn.Embedding(n_item, k)
        self.item_lv = nn.Embedding(n_item, k)
        
        self.bias_user = nn.Embedding(n_user, 1)
        self.bias_item = nn.Embedding(n_item, 1)
        self.bias = nn.Parameter(torch.ones(1))

    def forward(self, inputs):
        user_id = inputs[:, 0]
        item_id = inputs[:, 1]
        
        # *NEW: stochastically-sampled user & item vectors
        vector_user = sample_gaussian(self.user_mu(user_id), self.user_lv(user_id))
        vector_item = sample_gaussian(self.item_mu(item_id), self.item_lv(item_id))
        
        bias_user = self.bias_user(user_id).squeeze()
        bias_item = self.bias_item(item_id).squeeze()
        
        biases = (self.bias + bias_user + bias_item)
        ui_interaction = torch.sum(vector_user * vector_item, dim=1)
        prediction = ui_interaction + biases
        return prediction

    def loss(self, prediction, target):
        # MSE error between target = R_ui and prediction = p_u * q_i
        loss_mse = F.mse_loss(prediction, target.squeeze())
        return loss_mse, {"mse": loss_mse}
    
    def reg(self):
        reg_bias_user =  l2_regularize(self.bias_user.weight) * self.c_bias
        reg_bias_item = l2_regularize(self.bias_item.weight) * self.c_bias
        
        # *NEW: regularization:
        user_kld = gaussian_kldiv(self.user_mu.weight, self.user_lv.weight) * self.c_kld
        item_kld = gaussian_kldiv(self.item_mu.weight, self.item_lv.weight) * self.c_kld
        total = reg_bias_user + reg_bias_item + user_kld + item_kld
        log = dict(reg_bias_user=reg_bias_user, reg_bias_item=reg_bias_item,
                   user_kld=user_kld, item_kld=item_kld)
        return total, log
from pytorch_lightning.loggers.wandb import WandbLogger


k = 128
c_bias = 1e-3
c_vector = 1e-5
c_kld = 1.0
model = MF(n_user, n_item,
           k=k, c_bias=c_bias, c_vector=c_vector,
           c_kld = 1.0,
           batch_size=1024)


# add a logger
logger = WandbLogger(name="07_mf", project="simple_mf")

trainer = pl.Trainer(max_epochs=100, logger=logger,
                     early_stop_callback=True,
                     gpus=1, progress_bar_refresh_rate=1)<jupyter_output>GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0]<jupyter_text>Test model<jupyter_code>trainer.test(model)<jupyter_output>/opt/conda/lib/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:37: UserWarning: you defined a test_step but have no test_dataloader. Skipping test loop
  warnings.warn(*args, **kwargs)
Failed to query for notebook name, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable
[34m[1mwandb[0m: Currently logged in as: [33msf-moody[0m (use `wandb login --relogin` to force relogin)<jupyter_text>Run model<jupyter_code>trainer.fit(model, train, val)<jupyter_output>| Name      | Type      | Params
----------------------------------------
0 | user_mu   | Embedding | 773 K 
1 | user_lv   | Embedding | 773 K 
2 | item_mu   | Embedding | 505 K 
3 | item_lv   | Embedding | 505 K 
4 | bias_user | Embedding | 6 K   
5 | bias_item | Embedding | 3 K   
/opt/conda/lib/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:37: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  warnings.warn(*args, **kwargs)
{'fraction_non_alphanumeric': 0.11179222455234715, 'fraction_numerical': 0.016615583158573963, 'mean_word_length': 3.369274136715997, 'pattern_counts': {'":': 1, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '124716', 'license': 'MIT', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2219, 'n_words': 496, 'path': 'notebooks/07 MF variational.ipynb', 'repo_name': 'cemoody/simple_mf', 'size': '6199'}