<jupyter_start><jupyter_text>First we need to import utils package and tensorflow with addons.<jupyter_code>!wget  https://raw.githubusercontent.com/zombak79/vasp/main/utils.py
from google.colab import drive
drive.mount('/content/drive')
!pip install tensorflow_addons
from utils import *
import tensorflow as tf
import tensorflow_addons as tfa

set_seed(42)<jupyter_output><empty_output><jupyter_text>Next, we define our model. Note that reparametrization trick is done manually - it can be done with tensorflow probability package, which is doing that by itself automatically.<jupyter_code>class DiagonalToZero(tf.keras.constraints.Constraint):
    def __call__(self, w):
        """Set diagonal to zero"""
        q = tf.linalg.set_diag(w, tf.zeros(w.shape[0:-1]), name=None)
        return q

class Sampling(tf.keras.layers.Layer):
    """Uses (z_mean, z_log_var) to sample z, the vector encoding a basket."""

    def call(self, inputs):
        z_mean, z_log_var = inputs
        batch = tf.shape(z_mean)[0]
        dim = tf.shape(z_mean)[1]
        epsilon = tf.keras.backend.random_normal(shape=(batch, dim), stddev=1.)
        return z_mean + tf.exp(0.5 * z_log_var) * epsilon

class VASP(Model):
    class Model(tf.keras.Model):
        def __init__(self, num_words, latent=1024, hidden=1024, items_sampling=1.):
            """
            num_words             nr of items in dataset (size of tokenizer)
            latent                size of latent space
            hidden                size of hidden layers
            items_sampling        Large items datatsets can be very gpu memory consuming in EASE layer.
                                  This coefficient reduces number of ease parametrs by taking only
                                  fraction of items sorted by popularity as input for model.
                                  Note: This coef should be somewhere around coverage@100 achieved by full
                                  size model.
                                  For ML20M this coef should be between 0.4888 (coverage@100 for full model)
                                  and 1.0
                                  For Netflix this coef should be between 0.7055 (coverage@100 for full
                                  model) and 1.0
            """
            super(VASP.Model, self).__init__()

            self.sampled_items = int(num_words * items_sampling)

            assert self.sampled_items > 0
            assert self.sampled_items <= num_words

            self.s = self.sampled_items < num_words

            # ************* ENCODER ***********************
            self.encoder1 = tf.keras.layers.Dense(hidden)
            self.ln1 = tf.keras.layers.LayerNormalization()
            self.encoder2 = tf.keras.layers.Dense(hidden)
            self.ln2 = tf.keras.layers.LayerNormalization()
            self.encoder3 = tf.keras.layers.Dense(hidden)
            self.ln3 = tf.keras.layers.LayerNormalization()
            self.encoder4 = tf.keras.layers.Dense(hidden)
            self.ln4 = tf.keras.layers.LayerNormalization()
            self.encoder5 = tf.keras.layers.Dense(hidden)
            self.ln5 = tf.keras.layers.LayerNormalization()
            self.encoder6 = tf.keras.layers.Dense(hidden)
            self.ln6 = tf.keras.layers.LayerNormalization()
            self.encoder7 = tf.keras.layers.Dense(hidden)
            self.ln7 = tf.keras.layers.LayerNormalization()

            # ************* SAMPLING **********************
            self.dense_mean = tf.keras.layers.Dense(latent,
                                                    name="Mean")
            self.dense_log_var = tf.keras.layers.Dense(latent,
                                                       name="log_var")

            self.sampling = Sampling(name='Sampler')

            # ************* DECODER ***********************
            self.decoder1 = tf.keras.layers.Dense(hidden)
            self.dln1 = tf.keras.layers.LayerNormalization()
            self.decoder2 = tf.keras.layers.Dense(hidden)
            self.dln2 = tf.keras.layers.LayerNormalization()
            self.decoder3 = tf.keras.layers.Dense(hidden)
            self.dln3 = tf.keras.layers.LayerNormalization()
            self.decoder4 = tf.keras.layers.Dense(hidden)
            self.dln4 = tf.keras.layers.LayerNormalization()
            self.decoder5 = tf.keras.layers.Dense(hidden)
            self.dln5 = tf.keras.layers.LayerNormalization()

            self.decoder_resnet = tf.keras.layers.Dense(self.sampled_items,
                                                        activation='sigmoid',
                                                        name="DecoderR")
            self.decoder_latent = tf.keras.layers.Dense(self.sampled_items,
                                                        activation='sigmoid',
                                                        name="DecoderL")

            # ************* PARALLEL SHALLOW PATH *********

            self.ease = tf.keras.layers.Dense(
                self.sampled_items,
                activation='sigmoid',
                use_bias=False,
                kernel_constraint=DiagonalToZero(),  # critical to prevent learning simple identity
            )

        def call(self, x, training=None):
            sampling = self.s
            if sampling:
                sampled_x = x[:, :self.sampled_items]
                non_sampled = x[:, self.sampled_items:] * 0.
            else:
                sampled_x = x

            z_mean, z_log_var, z = self.encode(sampled_x)
            if training:
                d = self.decode(z)
                # Add KL divergence regularization loss.
                kl_loss = 1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var)
                kl_loss = tf.reduce_mean(kl_loss)
                kl_loss *= -0.5
                self.add_loss(kl_loss)
                self.add_metric(kl_loss, name="kl_div")
            else:
                d = self.decode(z_mean)

            if sampling:
                d = tf.concat([d, non_sampled], axis=-1)

            ease = self.ease(sampled_x)

            if sampling:
                ease = tf.concat([ease, non_sampled], axis=-1)

            return d * ease

        def decode(self, x):
            e0 = x
            e1 = self.dln1(tf.keras.activations.swish(self.decoder1(e0)))
            e2 = self.dln2(tf.keras.activations.swish(self.decoder2(e1) + e1))
            e3 = self.dln3(tf.keras.activations.swish(self.decoder3(e2) + e1 + e2))
            e4 = self.dln4(tf.keras.activations.swish(self.decoder4(e3) + e1 + e2 + e3))
            e5 = self.dln5(tf.keras.activations.swish(self.decoder5(e4) + e1 + e2 + e3 + e4))

            dr = self.decoder_resnet(e5)
            dl = self.decoder_latent(x)

            return dr * dl

        def encode(self, x):
            e0 = x
            e1 = self.ln1(tf.keras.activations.swish(self.encoder1(e0)))
            e2 = self.ln2(tf.keras.activations.swish(self.encoder2(e1) + e1))
            e3 = self.ln3(tf.keras.activations.swish(self.encoder3(e2) + e1 + e2))
            e4 = self.ln4(tf.keras.activations.swish(self.encoder4(e3) + e1 + e2 + e3))
            e5 = self.ln5(tf.keras.activations.swish(self.encoder5(e4) + e1 + e2 + e3 + e4))
            e6 = self.ln6(tf.keras.activations.swish(self.encoder6(e5) + e1 + e2 + e3 + e4 + e5))
            e7 = self.ln7(tf.keras.activations.swish(self.encoder7(e6) + e1 + e2 + e3 + e4 + e5 + e6))

            z_mean = self.dense_mean(e7)
            z_log_var = self.dense_log_var(e7)
            z = self.sampling((z_mean, z_log_var))

            return z_mean, z_log_var, z

    def create_model(self, latent=2048, hidden=4096, ease_items_sampling=1., summary=False):
        self.model = VASP.Model(self.dataset.num_words, latent, hidden, ease_items_sampling)
        self.model(self.split.train_gen[0][0])
        if summary:
            self.model.summary()
        self.mc = MetricsCallback(self)

    def compile_model(self, lr=0.00002, fl_alpha=0.25, fl_gamma=2.0):
        """
        lr         learning rate of Nadam optimizer
        fl_alpha   alpha parameter of focal crossentropy
        fl_gamma   gamma parameter of focal crossentropy
        """
        self.model.compile(
            optimizer=tf.keras.optimizers.Nadam(lr),
            loss=lambda x, y: tfa.losses.sigmoid_focal_crossentropy(x, y, alpha=fl_alpha, gamma=fl_gamma),
            metrics=['mse', cosine_loss]
        )

    def train_model(self, epochs=150):
        self.model.fit(
            self.split.train_gen,
            validation_data=self.split.validation_gen,
            epochs=epochs,
            callbacks=[self.mc]
        )<jupyter_output><empty_output><jupyter_text>Now, we can load previously preprocessed dataset. We also load pre-defined train/test/validation split.<jupyter_code>d = '/content/drive/MyDrive/MovieLens/'
dataset = Data(d=d, pruning='u5')
dataset.splits = []
dataset.create_splits(1, 10000, shuffle=False, generators=False)
dataset.split.train_users = pd.read_json(d+"train_users.json").userid.apply(str).to_frame()
dataset.split.validation_users = pd.read_json(d+"val_users.json").userid.apply(str).to_frame()
dataset.split.test_users = pd.read_json(d+"test_users.json").userid.apply(str).to_frame()
dataset.split.generators()
# open('/content/drive/MyDrive/MovieLens/toki.json',"w").write(dataset.toki.to_json())
json_string = open('/content/drive/MyDrive/MovieLens/toki.json').read()
dataset.toki = tf.keras.preprocessing.text.tokenizer_from_json(json_string)<jupyter_output><empty_output><jupyter_text>Now we can create an instance of our model and train it.For validation during training we sample 80% of the user's interactions randomly as input for the model, and then we measure Recall@20, Recall@50 and NDCG@100 for predicted interactions against the remaining 20% of the user's interactions.This method can give different results for different seeds, but since it is used for validation during training only, it's good enough.<jupyter_code>m = VASP(dataset.split, name="VASP_ML20_1")
m.create_model(latent=2048, hidden=4096, ease_items_sampling= 0.5)
m.model.summary()
m.model.load_weights('/content/drive/MyDrive/MovieLens/VASP_ML20_1_best_ncdg_100/VASP_ML20_1')
print("=" * 80)
print("Train for 50 epochs with lr 0.00005")
m.compile_model(lr=0.00005, fl_alpha=0.25, fl_gamma=2.0)
m.train_model(50-33)
print("=" * 80)
print("Than train for 20 epochs with lr 0.00001")
m.compile_model(lr=0.00001, fl_alpha=0.25, fl_gamma=2.0)
m.train_model(20)
print("=" * 80)
print("Than train for 20 epochs with lr 0.000001")
m.compile_model(lr=0.00001, fl_alpha=0.25, fl_gamma=2.0)
m.train_model(20)
# m.model.load_weights("/content/drive/MyDrive/RS/VASP_ML20_1_best_recall_50/VASP_ML20_1")
x,y = dataset.split.validation_gen.__getitem__(2)
x.shape,y.shape
m.model.output_shape<jupyter_output><empty_output><jupyter_text>Dataframe with the details of the training.<jupyter_code>m.mc.get_history_df()<jupyter_output><empty_output><jupyter_text>And the details of the training as a plot.<jupyter_code>m.mc.plot_history()<jupyter_output><empty_output><jupyter_text>For final evaluation we do 5-fold validation on the user's interaction history. We think that this is more objective measure than only hide sampled 20% of the user's interactions randomly.<jupyter_code># 5-fold evaluation on the test set

# test_r20s = []
# test_r50s = []
# test_n100s = []

# for fold in range(1,6):
#     ev=Evaluator(m.split, method=str(fold)+'_20')
#     ev.update(m.model)

#     test_n100s.append(ev.get_ncdg(100))
#     test_r20s.append(ev.get_recall(20))
#     test_r50s.append(ev.get_recall(50))

# print("TEST SET (MEAN)")
# print("5-fold mean NCDG@100", round(sum(test_n100s) / len(test_n100s),3))
# print("5-fold mean Recall@20", round(sum(test_r20s) / len(test_r20s),3))
# print("5-fold mean Recall@50", round(sum(test_r50s) / len(test_r50s),3))
! cp /content/VASP_ML20_1_best_recall_*   /content/drive/MyDrive/RS -r<jupyter_output><empty_output><jupyter_text>Inference<jupyter_code>import tensorflow as tf

class DiagonalToZero(tf.keras.constraints.Constraint):
    def __call__(self, w):
        """Set diagonal to zero"""
        q = tf.linalg.set_diag(w, tf.zeros(w.shape[0:-1]), name=None)
        return q

class Sampling(tf.keras.layers.Layer):
    """Uses (z_mean, z_log_var) to sample z, the vector encoding a basket."""

    def call(self, inputs):
        z_mean, z_log_var = inputs
        batch = tf.shape(z_mean)[0]
        dim = tf.shape(z_mean)[1]
        epsilon = tf.keras.backend.random_normal(shape=(batch, dim), stddev=1.)
        return z_mean + tf.exp(0.5 * z_log_var) * epsilon


class VASP(tf.keras.Model):
    def __init__(self, num_words, latent=1024, hidden=1024, items_sampling=1.):
        """
        num_words             nr of items in dataset (size of tokenizer)
        latent                size of latent space
        hidden                size of hidden layers
        items_sampling        Large items datatsets can be very gpu memory consuming in EASE layer.
                              This coefficient reduces number of ease parametrs by taking only
                              fraction of items sorted by popularity as input for model.
                              Note: This coef should be somewhere around coverage@100 achieved by full
                              size model.
                              For ML20M this coef should be between 0.4888 (coverage@100 for full model)
                              and 1.0
                              For Netflix this coef should be between 0.7055 (coverage@100 for full
                              model) and 1.0
        """
        # super(tf.keras.Model, self).__init__()
        super().__init__()
        self.num_words = num_words
        self.sampled_items = int(num_words * items_sampling)

        assert self.sampled_items > 0
        assert self.sampled_items <= num_words

        self.s = self.sampled_items < num_words

        # ************* ENCODER ***********************
        self.encoder1 = tf.keras.layers.Dense(hidden)
        self.ln1 = tf.keras.layers.LayerNormalization()
        self.encoder2 = tf.keras.layers.Dense(hidden)
        self.ln2 = tf.keras.layers.LayerNormalization()
        self.encoder3 = tf.keras.layers.Dense(hidden)
        self.ln3 = tf.keras.layers.LayerNormalization()
        self.encoder4 = tf.keras.layers.Dense(hidden)
        self.ln4 = tf.keras.layers.LayerNormalization()
        self.encoder5 = tf.keras.layers.Dense(hidden)
        self.ln5 = tf.keras.layers.LayerNormalization()
        self.encoder6 = tf.keras.layers.Dense(hidden)
        self.ln6 = tf.keras.layers.LayerNormalization()
        self.encoder7 = tf.keras.layers.Dense(hidden)
        self.ln7 = tf.keras.layers.LayerNormalization()

        # ************* SAMPLING **********************
        self.dense_mean = tf.keras.layers.Dense(latent,
                                                name="Mean")
        self.dense_log_var = tf.keras.layers.Dense(latent,
                                                    name="log_var")

        self.sampling = Sampling(name='Sampler')

        # ************* DECODER ***********************
        self.decoder1 = tf.keras.layers.Dense(hidden)
        self.dln1 = tf.keras.layers.LayerNormalization()
        self.decoder2 = tf.keras.layers.Dense(hidden)
        self.dln2 = tf.keras.layers.LayerNormalization()
        self.decoder3 = tf.keras.layers.Dense(hidden)
        self.dln3 = tf.keras.layers.LayerNormalization()
        self.decoder4 = tf.keras.layers.Dense(hidden)
        self.dln4 = tf.keras.layers.LayerNormalization()
        self.decoder5 = tf.keras.layers.Dense(hidden)
        self.dln5 = tf.keras.layers.LayerNormalization()

        self.decoder_resnet = tf.keras.layers.Dense(self.sampled_items,
                                                    activation='sigmoid',
                                                    name="DecoderR")
        self.decoder_latent = tf.keras.layers.Dense(self.sampled_items,
                                                    activation='sigmoid',
                                                    name="DecoderL")

        # ************* PARALLEL SHALLOW PATH *********

        self.ease = tf.keras.layers.Dense(
            self.sampled_items,
            activation='sigmoid',
            use_bias=False,
            kernel_constraint=DiagonalToZero(),  # critical to prevent learning simple identity
        )

    def call(self, x, training=None):
        sampling = self.s
        if sampling:
            sampled_x = x[:, :self.sampled_items]
            non_sampled = x[:, self.sampled_items:] * 0.
        else:
            sampled_x = x

        z_mean, z_log_var, z = self.encode(sampled_x)
        if training:
            d = self.decode(z)
            # Add KL divergence regularization loss.
            kl_loss = 1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var)
            kl_loss = tf.reduce_mean(kl_loss)
            kl_loss *= -0.5
            self.add_loss(kl_loss)
            self.add_metric(kl_loss, name="kl_div")
        else:
            d = self.decode(z_mean)

        if sampling:
            d = tf.concat([d, non_sampled], axis=-1)

        ease = self.ease(sampled_x)

        if sampling:
            ease = tf.concat([ease, non_sampled], axis=-1)

        return d * ease

    def decode(self, x):
        e0 = x
        e1 = self.dln1(tf.keras.activations.swish(self.decoder1(e0)))
        e2 = self.dln2(tf.keras.activations.swish(self.decoder2(e1) + e1))
        e3 = self.dln3(tf.keras.activations.swish(self.decoder3(e2) + e1 + e2))
        e4 = self.dln4(tf.keras.activations.swish(self.decoder4(e3) + e1 + e2 + e3))
        e5 = self.dln5(tf.keras.activations.swish(self.decoder5(e4) + e1 + e2 + e3 + e4))

        dr = self.decoder_resnet(e5)
        dl = self.decoder_latent(x)

        return dr * dl

    def encode(self, x):
        e0 = x
        e1 = self.ln1(tf.keras.activations.swish(self.encoder1(e0)))
        e2 = self.ln2(tf.keras.activations.swish(self.encoder2(e1) + e1))
        e3 = self.ln3(tf.keras.activations.swish(self.encoder3(e2) + e1 + e2))
        e4 = self.ln4(tf.keras.activations.swish(self.encoder4(e3) + e1 + e2 + e3))
        e5 = self.ln5(tf.keras.activations.swish(self.encoder5(e4) + e1 + e2 + e3 + e4))
        e6 = self.ln6(tf.keras.activations.swish(self.encoder6(e5) + e1 + e2 + e3 + e4 + e5))
        e7 = self.ln7(tf.keras.activations.swish(self.encoder7(e6) + e1 + e2 + e3 + e4 + e5 + e6))

        z_mean = self.dense_mean(e7)
        z_log_var = self.dense_log_var(e7)
        z = self.sampling((z_mean, z_log_var))

        return z_mean, z_log_var, z

# def create_model(num_words, latent=2048, hidden=4096, ease_items_sampling=1., summary=False):
# num_words = 14409
# model = VASP(num_words, latent, hidden, ease_items_sampling)
# # model(self.split.train_gen[0][0])



# m = VASP()
model = VASP(num_words = 14409, latent=2048, hidden=4096,items_sampling =0.33)
model.load_weights("/content/drive/MyDrive/RS/VASP_ML20_1_best_recall_50/VASP_ML20_1")
import numpy as np
e = np.random.choice([0,1],size =14409 ).reshape(1,-1)
p = model.call(e)
# model.summary()
p
p.shape
p[0].numpy()
import tensorflow as tf
toki = tf.keras.preprocessing.text.Tokenizer()
toki.fit_on_texts("A B C D".split())
toki.texts_to_matrix('A')

!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<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12402743652743653, 'fraction_numerical': 0.029586404586404585, 'mean_word_length': 2.249126891734575, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '333138', 'license': 'MIT', 'n_tokens_mistral': 6702, 'n_tokens_neox': 6311, 'n_words': 1192, 'path': 'src/vasp/VASP.ipynb', 'repo_name': 'blurry-mood/Recommendation-System', 'size': '19536'}