<jupyter_start><jupyter_text>MAMO Framework  API setup<jupyter_code>!git clone https://github.com/swisscom/ai-research-mamo-framework.git
%cd ai-research-mamo-framework
import sys
sys.path.insert(0,'.')<jupyter_output><empty_output><jupyter_text>API run for movielens<jupyter_code>!mkdir -p data
!cd data && gdown --id 15KwO7tk9S4M5raro2ndkYswFLh7MpPkt
!cd data && unzip movielens_data.zip
!mkdir -p outputs
import torch
import numpy as np
import os

from dataloader.ae_data_handler import AEDataHandler
from models.multi_VAE import MultiVAE
from loss.vae_loss import VAELoss
from metric.recall_at_k import RecallAtK
from metric.revenue_at_k import RevenueAtK
from trainer import Trainer

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", default="data", help="the path to the directory where the data is stored")
parser.add_argument("--models_dir", default="outputs", help="the path to the directory where to save the models, it must be empty")
args = parser.parse_args(args={})

# get the arguments
dir_path = args.data_dir
save_to_path = args.models_dir

# set up logging
import logging
import sys
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger("main")
logger.setLevel(logging.INFO)

# set cuda if available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_data_path = os.path.join(
    dir_path, "movielens_small_training.npy")
validation_input_data_path = os.path.join(
    dir_path, "movielens_small_validation_input.npy")
validation_output_data_path = os.path.join(
    dir_path, "movielens_small_validation_test.npy")
test_input_data_path = os.path.join(
    dir_path, "movielens_small_test_input.npy")
test_output_data_path = os.path.join(
    dir_path, "movielens_small_test_test.npy")
products_data_path = os.path.join(
    dir_path, "movielens_products_data.npy")

data_handler = AEDataHandler(
    "MovieLensSmall", train_data_path, validation_input_data_path,
    validation_output_data_path, test_input_data_path,
    test_output_data_path)

input_dim = data_handler.get_input_dim()
output_dim = data_handler.get_output_dim()

products_data_np = np.load(products_data_path)
products_data_torch = torch.tensor(
    products_data_np, dtype=torch.float32).to(device)

# create model
model = MultiVAE(params="yaml_files/params_multi_VAE_training.yaml")

correctness_loss = VAELoss()
revenue_loss = VAELoss(weighted_vector=products_data_torch)
losses = [correctness_loss, revenue_loss]

recallAtK = RecallAtK(k=10)
revenueAtK = RevenueAtK(k=10, revenue=products_data_np)
validation_metrics = [recallAtK, revenueAtK]

trainer = Trainer(data_handler, model, losses, validation_metrics, save_to_path)
trainer.train()
print(trainer.pareto_manager._pareto_front)<jupyter_output>INFO:main:Trainer: Started with initializing trainer...
INFO:main:Trainer: Reading trainer parameters.
INFO:main:Trainer: Random seed: 42
INFO:main:Trainer: Normalize gradients: True
INFO:main:Trainer: Learning rate: 0.001000
INFO:main:Trainer: Batch size training: 500
INFO:main:Trainer: Shuffle training: 1
INFO:main:Trainer: Drop last batch training: 1
INFO:main:Trainer: Batch size validation: 500
INFO:main:Trainer: Shuffle validation: 1
INFO:main:Trainer: Drop last batch validation: 0
INFO:main:Trainer: Number of epochs: 50.000000
INFO:main:Trainer: Frank Wolfe max iterations: 100
INFO:main:Trainer: Annealing: True
INFO:main:Trainer: Beta start: 0.000000
INFO:main:Trainer: Beta cap: 0.300000
INFO:main:Trainer: Beta step: 0.000030
INFO:main:Trainer: Losses: 
INFO:main:VAELoss Trainer: VAELoss 
INFO:main:Trainer: Validation metrics: 
INFO:main:Recall at 10 Trainer: Revenue at 10 
INFO:main:Trainer: Saving models to: outputs
INFO:main:Trainer: Training on device: cpu
INFO:main:Trainer: [...]
{'fraction_non_alphanumeric': 0.09894459102902374, 'fraction_numerical': 0.018469656992084433, 'mean_word_length': 7.08315565031983, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '391103', 'license': 'Apache-2.0', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1307, 'n_words': 294, 'path': '_docs/nbs/T671443-MAMO-Framework.ipynb', 'repo_name': 'sparsh-ai/recohut', 'size': '3790'}