<jupyter_start><jupyter_text>Negative Sampling:<jupyter_code>n_sampled=100
with train_graph.as_default():
  softmax_w=tf.Variable(tf.truncated_normal((n_vocab,n_embedding),stddev=0.1))
  softmax_b=tf.Variable(tf.zeros(n_vocab))
  
  loss=tf.nn.sampled_softmax_loss(softmax_w,softmax_b,labels,embed,n_sampled,n_vocab)
  
  cost=tf.reduce_mean(loss)
  optimizer=tf.train.AdamOptimizer().minimize(cost)
with train_graph.as_default():
  valid_size=16
  valid_window=100
  valid_examples=np.array(random.sample(range(valid_window),valid_size//2))
  valid_examples=np.append(valid_examples,random.sample(range(1000,1000+valid_window),valid_size//2))
  valid_dataset=tf.constant(valid_examples,dtype=tf.int32)
  
  norm=tf.sqrt(tf.reduce_sum(tf.square(embedding),1,keepdims=True))
  normalized_embedding=embedding/norm
  valid_embedding=tf.nn.embedding_lookup(normalized_embedding,valid_dataset)
  similarity=tf.matmul(valid_embedding,tf.transpose(normalized_embedding))
!mkdir checkpoints
epochs=10
batch_size=1000
window_size=10

with train_graph.as_default():
  saver=tf.train.Saver()
  
with tf.Session(graph=train_graph) as sess:
  iteration=1
  loss=0
  sess.run(tf.global_variables_initializer())

  for e in range(1,epochs+1):
    batches=get_batches(train_words,batch_size,window_size)
    start=time.time()
    for x,y in batches:
      feed={inputs:x,
           labels:np.array(y)[:,None]}
      train_loss,_=sess.run([cost,optimizer],feed_dict=feed)
      loss+=train_loss
      if iteration % 100 == 0: 
          end = time.time()
          print("Epoch {}/{}".format(e, epochs),
                "Iteration: {}".format(iteration),
                "Avg. Training loss: {:.4f}".format(loss/100),
                "{:.4f} sec/batch".format((end-start)/100))
          loss = 0
          start = time.time()

      if iteration % 1000 == 0:
          # note that this is expensive (~20% slowdown if computed every 500 steps)
          sim = similarity.eval()
          for i in range(valid_size):
              valid_word = int_to_vocab[valid_examples[i]]
              top_k = 8 # number of nearest neighbors
              nearest = (-sim[i, :]).argsort()[1:top_k+1]
              log = 'Nearest to %s:' % valid_word
              for k in range(top_k):
                  close_word = int_to_vocab[nearest[k]]
                  log = '%s %s,' % (log, close_word)
              print(log)

      iteration += 1
    save_path = saver.save(sess, "checkpoints/text8.ckpt")
    embed_mat = sess.run(normalized_embedding)
with train_graph.as_default():
    saver = tf.train.Saver()

with tf.Session(graph=train_graph) as sess:
    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))
    embed_mat = sess.run(embedding)
viz_words = 500
tsne = TSNE()
embed_tsne = tsne.fit_transform(embed_mat[:viz_words, :])
fig, ax = plt.subplots(figsize=(14, 14))
for idx in range(viz_words):
    plt.scatter(*embed_tsne[idx, :], color='steelblue')
    plt.annotate(int_to_vocab[idx], (embed_tsne[idx, 0], embed_tsne[idx, 1]), alpha=0.8)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1369281045751634, 'fraction_numerical': 0.024183006535947713, 'mean_word_length': 3.7457364341085273, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '218019', 'license': 'MIT', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1106, 'n_words': 161, 'path': 'Word Embedding/Embeddings_word_to_vectors.ipynb', 'repo_name': 'CodeLogist/ML-NewBie-s-Heaven', 'size': '3060'}