<jupyter_start><jupyter_text>BERT Embedding GenerationThis notebook contains generating BERTBASE embedding on different pooling strategies.<jupyter_code>import os
import urllib

from google.colab import drive, files
from getpass import getpass

from google.colab import drive
ROOT = '/content/drive'
GOOGLE_DRIVE_PATH = 'My Drive/Colab Notebooks/recommender/w266-final'
PROJECT_PATH = os.path.join(ROOT, GOOGLE_DRIVE_PATH)

drive.mount(ROOT)

%cd {PROJECT_PATH}
import os
import sys
import re
import pandas as pd
import numpy as np
import itertools
import pickle
import random
import tensorflow as tf

from commons.store import PickleStore, NpyStore
from tqdm import tqdm
from IPython.core.display import HTML
from importlib import reload

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>1. Load Pre-filtered DatasetLoad the clean pre-processed dataset.<jupyter_code>amazon = False

if amazon:
  input_pkl = '../dataset/25-65_tokens_grouped_Movies_and_TV_v2.pkl'
else:
  input_pkl = '../dataset/25-65_tokens_grouped_yelp.pkl'

pkl_store = PickleStore(input_pkl)

grouped_reviews_df = pkl_store.load(asPandasDF=True, \
                                    columns=['reviewerID', 'asin', 'overall', 'userReviews', 'itemReviews'])
print(len(grouped_reviews_df))
display(HTML(grouped_reviews_df.head(1).to_html()))
#grouped_reviews = grouped_reviews_df[['userReviews', 'itemReviews', 'overall']].to_numpy()
grouped_reviews = grouped_reviews_df.to_numpy()
grouped_reviews[0]
!pip install transformers
import tensorflow as tf

from transformers import BertTokenizer
from transformers import TFBertModel, BertConfig
# Detect hardware
try:
  tpu_resolver = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection
except ValueError:
  tpu_resolver = None
  gpus = tf.config.experimental.list_logical_devices("GPU")

# Select appropriate distribution strategy
if tpu_resolver:
  tf.config.experimental_connect_to_cluster(tpu_resolver)
  tf.tpu.experimental.initialize_tpu_system(tpu_resolver)
  strategy = tf.distribute.experimental.TPUStrategy(tpu_resolver)
  print('Running on TPU ', tpu_resolver.cluster_spec().as_dict()['worker'])
elif len(gpus) > 1:
  strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])
  print('Running on multiple GPUs ', [gpu.name for gpu in gpus])
elif len(gpus) == 1:
  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU
  print('Running on single GPU ', gpus[0].name)
else:
  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU
  print('Running on CPU')
  
print("Number of accelerators: ", strategy.num_replicas_in_sync)<jupyter_output>Running on single GPU  /device:GPU:0
Number of accelerators:  1<jupyter_text>Using BERT huggingface  library to load the BERT tokenizer and model.<jupyter_code>bert_model_name = 'bert-base-uncased'

MAX_LEN = 128

config = BertConfig() 
config.output_hidden_states = True # set to True to obtain hidden states

with strategy.scope():
  tokenizer = BertTokenizer.from_pretrained(bert_model_name, do_lower_case=True)
  user_bert = TFBertModel.from_pretrained(bert_model_name, config=config)
  item_bert = TFBertModel.from_pretrained(bert_model_name, config=config)<jupyter_output><empty_output><jupyter_text>2. Pooling Strategies  2.1. Last Hidden State<jupyter_code>def last_hidden_state_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens)[0]
    item_embedding = item_model(item_tokens)[0]

    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)

    #_embeddings[i] = ((dict(user_embedding=user_embedding, 
    #                      item_embedding=item_embedding)), label)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = last_hidden_state_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)

embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output>Saving ../dataset/embedding/grouped_embedding_0-150000_yelp.npy ...
	... 150000 records
Done!!<jupyter_text>2.2. Sum Last Four Hidden States<jupyter_code>def sum_last_four_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens).hidden_states
    item_embedding = item_model(item_tokens).hidden_states

    # Sum last four hiddden states
    user_embedding = tf.reduce_sum(user_embedding[-4:], axis=0)
    item_embedding = tf.reduce_sum(item_embedding[-4:], axis=0)

    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = sum_last_four_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_sumlastfour_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_sumlastfour_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)
  
embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output>Saving ../dataset/embedding/grouped_embedding_sumlastfour_0-150000_yelp.npy ...
	... 150000 records
Done!!<jupyter_text>2.3. Sum Last Twelve Hidden States<jupyter_code>def sum_last_twelve_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens).hidden_states
    item_embedding = item_model(item_tokens).hidden_states

    # Sum last twelve hiddden states
    user_embedding = tf.reduce_sum(user_embedding[-12:], axis=0)
    item_embedding = tf.reduce_sum(item_embedding[-12:], axis=0)

    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = sum_last_twelve_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_sumlasttwelve_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_sumlasttwelve_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)
  
embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output>Saving ../dataset/embedding/grouped_embedding_sumlasttwelve_0-150000_yelp.npy ...
	... 150000 records
Done!!<jupyter_text>2.4. Second-To-Last Hidden State<jupyter_code>def second_to_last_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens).hidden_states
    item_embedding = item_model(item_tokens).hidden_states
    
    # Second-to-last hiddden states
    user_embedding = user_embedding[-2]
    item_embedding = item_embedding[-2]
    
    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = second_to_last_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_secondtolast_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_secondtolast_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)
  
embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output>Saving ../dataset/embedding/grouped_embedding_secondtolast_0-150000_yelp.npy ...
	... 150000 records
Done!!<jupyter_text>2.5. First Layer Hidden State<jupyter_code>def first_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens).hidden_states
    item_embedding = item_model(item_tokens).hidden_states
    
    # First embedding
    user_embedding = user_embedding[0]
    item_embedding = item_embedding[0]
    
    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = first_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_first_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_first_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)
  
embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output><empty_output><jupyter_text>2.6. Concat Last Four Hidden States<jupyter_code>def concat_last_four_embedding(samples, tokenizer, user_model, item_model, max_len=128):

  def tokenize(reviews):
    return tokenizer(list(reviews), padding='max_length', truncation=True, max_length=max_len, return_tensors='tf')

  total = len(samples)
  _embeddings = np.empty(len(samples), dtype=object)

  for i, reviews in enumerate(samples):

    user_tokens = tokenize(reviews[3])
    item_tokens = tokenize(reviews[4])
    
    reviewerID = reviews[0]
    asin = reviews[1]
    label = reviews[2]
    
    user_embedding = user_model(user_tokens).hidden_states
    item_embedding = item_model(item_tokens).hidden_states

    # Concat last four hiddden states
    user_embedding = tf.concat(user_embedding[-4:], axis=2)
    item_embedding = tf.concat(item_embedding[-4:], axis=2)

    user_embedding = tf.stack([user_embedding])
    item_embedding = tf.stack([item_embedding])

    user_embedding = tf.keras.layers.GlobalAveragePooling2D()(user_embedding)
    item_embedding = tf.keras.layers.GlobalAveragePooling2D()(item_embedding)
    
    _embeddings[i] = dict(reviewerID=reviewerID, asin=asin, 
                            user_embedding=user_embedding, 
                            item_embedding=item_embedding,
                            label=label)

    
    print(f'\rEmbedding... {i+1} of {total} record(s) -- {(i+1)/total*100:.2f}%', end='')
  
  print('\n\tDone!')
  return _embeddings
%%time
start = 0
end = 150000
embeddings = concat_last_four_embedding(grouped_reviews[start:end], tokenizer, user_bert, item_bert)
embedding_dir = '../dataset/embedding/'
if not os.path.exists(embedding_dir):
    os.makedirs(embedding_dir)

if amazon:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_concatlastfour_',str(start),'-',str(end),'_Movies_and_TV.npy'])
else:
  embedding_npy = ''.join([embedding_dir, 'grouped_embedding_concatlastfour_',str(start),'-',str(end),'_yelp.npy'])

if os.path.exists(embedding_npy):
  os.remove(embedding_npy)
  
embedding_store = NpyStore(embedding_npy)
embedding_store.write(embeddings)<jupyter_output>Saving ../dataset/25-65_tokens/embedding/grouped_embedding_concatlastfour_0-50000_Movies_and_TV.npy ...
	... 50000 records
Done!!
{'fraction_non_alphanumeric': 0.12130827434708323, 'fraction_numerical': 0.015987307786185012, 'mean_word_length': 4.63002404671934, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '144269', 'license': 'MIT', 'n_tokens_mistral': 6000, 'n_tokens_neox': 5722, 'n_words': 910, 'path': 'bert_embedding_colab.ipynb', 'repo_name': 'sony-w/RecommendationTransformer', 'size': '16388'}