<jupyter_start><jupyter_text>Architecture - PV-DBOW Distributed Bag of Words version of Paragraph Vector  Adding batchsizes for speedup<jupyter_code>bucket_list = []

def generate_batch_pvdbow(doc_ids, doc_repr, sample_size=20, batch_size=1000, window_size=7):
    global bucket_list

    docs_ids_to_select = list(set(doc_ids) - set(bucket_list))
    
    
    if len(docs_ids_to_select) < batch_size//sample_size:
        bucket_list = []
        docs_ids_to_select = doc_ids
        
    index = 0 
    train_dX = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    train_label = np.ndarray(shape=(batch_size, window_size), dtype=np.int32)
    random_docs = random.sample(docs_ids_to_select, batch_size//sample_size)    # Choose set of random documents 

    bucket_list += random_docs
    
    for id_ in random_docs:
        for j in range(sample_size):                                 # Generating a dataset of sample size 
            random_index = random.randint(0, len(doc_repr[id_]) - window_size)
            sample_window = doc_repr[id_][random_index: random_index + window_size]
            train_dX[index] = id_
            train_label[index] = sample_window
            index += 1
    return train_dX, train_label 
doc_size = len(docs)
embedding_size_d = 100
embedding_size_w = 100
vocab_size = len(word2ids)
window_size = 7
n_neg_samples = 20
learning_rate = 10e-2
epochs = 10001
batch_size=1000
mu=0.99
# Define placeholders for training 
train_dX = tf.placeholder(tf.int32, shape=[batch_size, 1])
train_label = tf.placeholder(tf.int32, shape=[batch_size, window_size])
doc_embedding_np = np.random.randn(doc_size, embedding_size_d)/np.sqrt(doc_size + embedding_size_d)
word_embedding_np = np.random.randn(vocab_size, embedding_size_w)/np.sqrt(vocab_size + embedding_size_w)
# Define matrix for doc_embedding and word_embedding 
doc_embedding = tf.Variable(doc_embedding_np.astype(np.float32), name="doc_embedding")
word_embedding = tf.Variable(word_embedding_np.astype(np.float32),name="word_embedding")
# Define weights for the output unit 
weights = tf.Variable(tf.truncated_normal([vocab_size, embedding_size_d], 
                                       stddev=1.0 / math.sqrt(vocab_size)))
biases = tf.Variable(tf.zeros(vocab_size))
print weights.get_shape()
# Add the doc2vec from the doc_embedding 
embed = tf.nn.embedding_lookup(doc_embedding, train_dX[:, 0])
loss = tf.nn.sampled_softmax_loss(weights=weights, \
                                  biases=biases, \
                                  labels=train_label, \
                                  inputs=embed, \
                                  num_sampled=n_neg_samples, \
                                  num_classes=vocab_size, \
                                  num_true=window_size)
loss = tf.reduce_mean(loss)
optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(loss)
import os
os.system("mkdir /dev/shm/tensorflow_models")
saver = tf.train.Saver()

with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)
    average_loss = 0
    
    for step in range(epochs):
        epoch_error = 0.0
        temp_dX, temp_labels = generate_batch_pvdbow(doc_ids=doc_ids, doc_repr=doc_repr)
        feed_dict = {train_dX : temp_dX,train_label : temp_labels}
        op, l = sess.run([optimizer, loss], 
                                    feed_dict=feed_dict)
        
        epoch_error += l
                
        if step % 1000 == 0:
            print "Error at epoch : ", step, " = ", epoch_error
            
    save_path = saver.save(sess, "/dev/shm/tensorflow_models/model_pvdbow_batch_training.ckpt")
    print("Model saved in file: %s" % save_path)<jupyter_output>Error at epoch :  0  =  2.60651564598
Error at epoch :  1000  =  2.04538917542
Error at epoch :  2000  =  2.05157375336
Error at epoch :  3000  =  2.02576971054
Error at epoch :  4000  =  2.00320339203
Error at epoch :  5000  =  2.07759428024
Error at epoch :  6000  =  1.98165380955
Error at epoch :  7000  =  1.98353922367
Error at epoch :  8000  =  1.98361158371
Error at epoch :  9000  =  1.97825825214
Error at epoch :  10000  =  1.96157824993
Model saved in file: /dev/shm/tensorflow_models/model_pvdbow_batch_training.ckpt<jupyter_text>Evaluation of the representation<jupyter_code>doc_pvdbow = None

with tf.Session() as sess:
    saver = tf.train.Saver()
    saver.restore(sess, "/dev/shm/tensorflow_models/model_pvdbow_batch_training.ckpt")
    print("Model restored.")
    doc2vec = doc_pvdbow = doc_embedding.eval()
    #performanceTest(doc2vec, list(imdb_data['sentiment']), method=None)
from sklearn.manifold import TSNE
model = TSNE(perplexity=50, n_iter=5000)
Z = model.fit_transform(doc_pvdbow) 
df['axis1'] = Z[:, 0]
df['axis2'] = Z[:, 1]
df.reset_index(drop=True, inplace=True)
df.head()
from ggplot import *
ggplot(aes(x='axis1', y='axis2', color='label'), data=df)  + geom_point()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10133765707336846, 'fraction_numerical': 0.050466153222537495, 'mean_word_length': 3.1125, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '228356', 'license': 'MIT', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1708, 'n_words': 328, 'path': 'module3/DBOW/doc2vec PV-DBOW Distributed Bag of Words version of Paragraph Vector.ipynb', 'repo_name': 'kochhar/Representation-Learning-for-NLP', 'size': '4934'}