<jupyter_start><jupyter_text>Getting only the clickout item actions!<jupyter_code>train = train[train['action_type'] == 'clickout item']
train = train[['reference', 'impressions']]<jupyter_output><empty_output><jupyter_text>![alt text](https://cdn-images-1.medium.com/max/800/1*yiH5sZI-IBxDSQMKhvbcHw.png)  Skip-gram for the itemsbased only on the impressions they appear with!  Dataset<jupyter_code>class InputGenerator:
  
  def __init__(self, item_tokenizer):
    self.item_tokenizer = item_tokenizer
    
  def input_generator_gen(self, df):
    def gen():

      for index, row in df.iterrows():
        reference = self.item_tokenizer.transform(row['reference'])
        impressions = self.item_tokenizer.transform(row['impressions'].split('|'))
        
        for impression in impressions:
          if reference == impression:
            continue
          yield reference, [impression]
    return gen, (tf.int32, tf.int32),

i = InputGenerator(item_tokenizer)  
params = {
    'batch_size': 4096,
    'items_vocabulary': len(item_tokenizer.vocabulary),
    'items_embedding': 100,
}
gen, types = i.input_generator_gen(train)
dataset_train = tf.data.Dataset.from_generator(gen, types)
dataset_train = dataset_train.repeat().batch(params['batch_size'])
# batch = dataset_train.make_one_shot_iterator().get_next()

steps_per_epoch = len(train) // params['batch_size']
tf.logging.set_verbosity(tf.logging.INFO)
!rm -r skip-gram-model
# It can only be trained!
def skip_gram_items(
   features, # This is batch_features from input_fn
   labels,   # This is batch_labels from input_fn
   mode,     # An instance of tf.estimator.ModeKeys, see below
   params):  # Additional configuration
  
  reference = features
  impression = labels
  
  # Encoding
  items_embeddings = tf.keras.layers.Embedding(input_dim=params['items_vocabulary'],
                                               output_dim=params['items_embedding'])
  
  # Reverse encoding
  W2 = tf.Variable(tf.random_normal([params['items_embedding'],
                                     params['items_vocabulary']]))
  b2 = tf.Variable(tf.random_normal([params['items_vocabulary']]))
  
  features.set_shape([None, params['items_embedding']])
  embeddings = tf.reshape(items_embeddings(features), [-1, params['items_embedding']])
  
  layer = tf.matmul(embeddings, W2) + b2
  layer = tf.nn.softmax(layer)
  
  loss = tf.losses.sparse_softmax_cross_entropy(labels=impression,
                                                logits=layer)
  loss = tf.reduce_mean(loss)
  optimizer = tf.train.AdamOptimizer(learning_rate=0.1)
  
  train_op = optimizer.minimize(
    loss=loss,
    global_step=tf.train.get_global_step()
  )
  
  return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)


run_config = tf.estimator.RunConfig(log_step_count_steps=100,
                                    # save_summary_steps=20,
                                    save_checkpoints_steps=100,
                                    keep_checkpoint_max=3)

skipgram_estimator = tf.estimator.Estimator(
  model_fn=skip_gram_items,
  model_dir='./skip-gram-model',
  params=params,
  config=run_config
)

skipgram_estimator.train(input_fn=lambda: tf.data.Dataset.from_generator(gen, types),
                   steps=1000 * steps_per_epoch)<jupyter_output><empty_output><jupyter_text>CBOW for the items<jupyter_code>class InputGeneratorCBOW:
  
  def __init__(self, item_tokenizer):
    self.item_tokenizer = item_tokenizer
    
  def input_generator_gen(self, df):
    def gen():

      for index, row in df.iterrows():
        reference = self.item_tokenizer.transform(row['reference'])
        impressions = self.item_tokenizer.transform(row['impressions'].split('|'))
        
        if reference in impressions:
          impressions.remove(reference)
        yield impressions, [reference]
    return gen, (tf.int32, tf.int32), (tf.TensorShape([None]), tf.TensorShape([None]))
  
i_cbow = InputGeneratorCBOW(item_tokenizer)
params = {
    'batch_size': 256,
    'items_vocabulary': len(item_tokenizer.vocabulary),
    'items_embedding': 50,
}

def get_cbow_dataset():
  gen_cbow, types, shapes = i_cbow.input_generator_gen(train)
  dataset_train = tf.data.Dataset.from_generator(gen_cbow, types)
  dataset_train = dataset_train.repeat().padded_batch(params['batch_size'], shapes)
  return dataset_train


batch = get_cbow_dataset().make_one_shot_iterator().get_next()

with tf.Session() as sess:
  sess.run([tf.local_variables_initializer(), tf.tables_initializer()])
  b = sess.run(batch)
b
b[1].shape
# It can only be trained!
def cbow_items(
   features, # This is batch_features from input_fn
   labels,   # This is batch_labels from input_fn
   mode,     # An instance of tf.estimator.ModeKeys, see below
   params):  # Additional configuration

  # In CBOW we predict the target REFERENCE from the context IMPRESSIONS.
  reference = labels
  impressions = features

  # Encoding
  items_embeddings = tf.keras.layers.Embedding(input_dim=params['items_vocabulary'],
                                               output_dim=params['items_embedding'])

  # Reverse encoding
  W2 = tf.Variable(tf.random_normal([params['items_embedding'],
                                     params['items_vocabulary']]))
  b2 = tf.Variable(tf.random_normal([params['items_vocabulary']]))

  embeddings = tf.reduce_mean(items_embeddings(impressions), axis=1)

  layer = tf.matmul(embeddings, W2) + b2
  # layer = tf.nn.softmax(layer)

  loss = tf.losses.sparse_softmax_cross_entropy(labels=tf.reshape(reference, [-1]),
                                                logits=layer)
  loss = tf.reduce_mean(loss)
  optimizer = tf.train.AdamOptimizer(learning_rate=0.0000001)

  train_op = optimizer.minimize(
    loss=loss,
    global_step=tf.train.get_global_step()
  )

  return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)


run_config = tf.estimator.RunConfig(log_step_count_steps=30,
                                  # save_summary_steps=20,
                                  save_checkpoints_steps=100,
                                  keep_checkpoint_max=3)

cbow_estimator = tf.estimator.Estimator(
model_fn=cbow_items,
model_dir='./cbow-model',
params=params,
config=run_config
)

cbow_estimator.train(input_fn=get_cbow_dataset,
                 steps=1000 * steps_per_epoch)
# !rm -r cbow-model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11081629467769627, 'fraction_numerical': 0.011393788044326518, 'mean_word_length': 3.344406779661017, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '529512', 'license': 'MIT', 'n_tokens_mistral': 2138, 'n_tokens_neox': 2060, 'n_words': 342, 'path': 'notebooks/Items_embeddings.ipynb', 'repo_name': 'antonpetkoff/recsys-2019', 'size': '6407'}