<jupyter_start><jupyter_text>A TensorFlow Word2Vec Model for Word Similarity Prediction<jupyter_code>import urllib.request
import collections
import math
import os
import random
import zipfile
import datetime as dt
import numpy as np
import tensorflow as tf<jupyter_output><empty_output><jupyter_text>BackgroundWord2Vec is a model that was created by [Mikolov et al.](http://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf). It uses the concept of "word embeddings", which is a way to represent relationships between words using vectors. This makes it a useful tool to find words that are similar to eachother.Here is an example of an embedding matrix taken from the TensorFlow tutorial:![embedding_matrix](https://www.tensorflow.org/images/tsne.png)  DataThe data used here is a cleaned version of the first 10^9 bytes of an English Wikipedia dump performed on Mar. 3, 2006.  See [this site](https://cs.fit.edu/~mmahoney/compression/textdata.html) for more information.<jupyter_code>def maybe_download(filename, url, expected_bytes):
    """Download a file if not present, and make sure it's the right size."""
    if not os.path.exists(filename):
        filename, _ = urllib.request.urlretrieve(url + filename, filename)
    statinfo = os.stat(filename)
    if statinfo.st_size == expected_bytes:
        print('Found and verified', filename)
    else:
        print(statinfo.st_size)
        raise Exception(
            'Failed to verify ' + filename + '. Can you get to it with a browser?')
    return filename
url = 'http://mattmahoney.net/dc/'
filename = maybe_download('text8.zip', url, 31344016)
# Read the data into a list of strings.
def read_data(filename):
    """Extract the first file enclosed in a zip file as a list of words."""
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data
vocabulary = read_data(filename)
print(vocabulary[:7])
['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse']
def build_dataset(words, n_words):
    """Process raw inputs into a dataset."""
    count = [['UNK', -1]]
    count.extend(collections.Counter(words).most_common(n_words - 1))
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    data = list()
    unk_count = 0
    for word in words:
        if word in dictionary:
            index = dictionary[word]
        else:
            index = 0  # dictionary['UNK']
            unk_count += 1
        data.append(index)
    count[0][1] = unk_count
    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return data, count, dictionary, reversed_dictionary
def collect_data(vocabulary_size=10000):
    """Read data and create the dictionary"""
    url = 'http://mattmahoney.net/dc/'
    filename = maybe_download('text8.zip', url, 31344016)
    vocabulary = read_data(filename)
    print(vocabulary[:7])
    data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,
                                                                vocabulary_size)
    del vocabulary  # Hint to reduce memory.
    return data, count, dictionary, reverse_dictionary
data_index = 0
def generate_batch(data, batch_size, num_skips, skip_window):
    """Generate batch data"""
    global data_index
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * skip_window
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    context = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    span = 2 * skip_window + 1  # [ skip_window input_word skip_window ]
    buffer = collections.deque(maxlen=span)
    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    for i in range(batch_size // num_skips):
        target = skip_window  # input word at the center of the buffer
        targets_to_avoid = [skip_window]
        for j in range(num_skips):
            while target in targets_to_avoid:
                target = random.randint(0, span - 1)
            targets_to_avoid.append(target)
            batch[i * num_skips + j] = buffer[skip_window]  # this is the input word
            context[i * num_skips + j, 0] = buffer[target]  # these are the context words
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    # Backtrack a little bit to avoid skipping words in the end of a batch
    data_index = (data_index + len(data) - span) % len(data)
    return batch, context
vocabulary_size = 10000
data, count, dictionary, reverse_dictionary = collect_data(vocabulary_size=vocabulary_size)<jupyter_output>Found and verified text8.zip
['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse']<jupyter_text>TensorFlow Model<jupyter_code>graph = tf.Graph()
batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a context.
# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent.
valid_size = vocabulary_size     # Random set of words to evaluate similarity on.
valid_window = 100  # Only pick dev samples in the head of the distribution.
valid_examples = np.arange(valid_size) # np.random.choice(valid_window, valid_size, replace=False)
num_sampled = 64    # Number of negative examples to sample.<jupyter_output><empty_output><jupyter_text>There is a fast scheme called Noise Contrastive Estimation (NCE).  Instead of taking the probability of the context word compared to all of the possible context words in the vocabulary, this method randomly samples 2-20 possible context words and evaluates the probability only from these.<jupyter_code>with graph.as_default():
    
    # Input data.
    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
    train_context = tf.placeholder(tf.int32, shape=[batch_size, 1])
    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
    
    # Look up embeddings for inputs.
    embeddings = tf.Variable(
      tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)
 
    # Construct the variables for the NCE loss
    nce_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

    nce_loss = tf.reduce_mean(
        tf.nn.nce_loss(weights=nce_weights,
                       biases=nce_biases,
                       labels=train_context,
                       inputs=embed,
                       num_sampled=num_sampled,
                       num_classes=vocabulary_size))

    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(nce_loss)
    
    # Compute the cosine similarity between minibatch examples and all embeddings.
    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(
      normalized_embeddings, valid_dataset)
    similarity = tf.matmul(
      valid_embeddings, normalized_embeddings, transpose_b=True)

    # Add variable initializer.
    init = tf.global_variables_initializer()<jupyter_output><empty_output><jupyter_text>Run the Model<jupyter_code>def train(graph, num_steps):
    with tf.Session(graph=graph) as session:
        with session.as_default():

            # We must initialize all variables before we use them.
            init.run()
            print('Initialized')

            average_loss = 0
            for step in range(num_steps):
                batch_inputs, batch_context = generate_batch(data,
                    batch_size, num_skips, skip_window)
                feed_dict = {train_inputs: batch_inputs, train_context: batch_context}

                # We perform one update step by evaluating the optimizer op (including it
                # in the list of returned values for session.run()
                _, loss_val = session.run([optimizer, nce_loss], feed_dict=feed_dict)
                average_loss += loss_val

                if step % 1000 == 0:
                    if step > 0:
                        average_loss /= 2000
                    # The average loss is an estimate of the loss over the last 2000 batches.
                    print('Average loss at step ', step, ': ', average_loss)
                    average_loss = 0

                # Note that this is expensive (~20% slowdown if computed every 500 steps)
                if step % 1000 == 0:
                    sim = similarity.eval()
                    for i in range(valid_size):
                        valid_word = reverse_dictionary[valid_examples[i]]
                        top_k = 8  # number of nearest neighbors
                        nearest = (-sim[i, :]).argsort()[1:top_k + 1]
                        log_str = 'Nearest to %s:' % valid_word
                        for k in range(top_k):
                            close_word = reverse_dictionary[nearest[k]]
                            log_str = '%s %s,' % (log_str, close_word)
                    print(log_str)
                    
            final_embeddings = normalized_embeddings.eval()
        saver = tf.train.Saver()
        saver.save(session, os.path.join("model.ckpt"))<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>num_steps = 10000
softmax_start_time = dt.datetime.now()
train(graph, num_steps=num_steps)
softmax_end_time = dt.datetime.now()
print("Training took {} minutes to run {} iterations".format(
    (softmax_end_time-softmax_start_time).total_seconds()/60, str(num_steps)))<jupyter_output>Initialized
Average loss at step  0 :  207.810714722
Nearest to gershwin: eminem, armored, disputes, acting, stands, authority, cantor, derivation,
Average loss at step  1000 :  31.9783105853
Nearest to gershwin: acting, authority, motor, rand, press, translation, certain, happy,
Average loss at step  2000 :  9.28103744006
Nearest to gershwin: disputes, dialects, authority, acting, motor, stands, don, elaborate,
Average loss at step  3000 :  4.79677247
Nearest to gershwin: disputes, dialects, authority, stands, acting, motor, feature, don,
Average loss at step  4000 :  3.50414316523
Nearest to gershwin: disputes, authority, dialects, stands, acting, motor, feature, don,
Average loss at step  5000 :  2.88645278847
Nearest to gershwin: disputes, authority, dialects, stands, acting, motor, feature, don,
Average loss at step  6000 :  2.80906002653
Nearest to gershwin: disputes, stands, authority, acting, dialects, feature, motor, translation,
Average loss at step  7000 :  2.6058074379
Near[...]<jupyter_text>Predict similarity<jupyter_code>def predict_sim(input_word, model_path):
    # Reinitialize things
    with graph.as_default(): 
        
        # Input data.
        train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
        train_context = tf.placeholder(tf.int32, shape=[batch_size, 1])
        valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

        # Look up embeddings for inputs.
        embeddings = tf.Variable(
          tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)

        # Compute the cosine similarity between minibatch examples and all embeddings.
        norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / norm
        valid_embeddings = tf.nn.embedding_lookup(
          normalized_embeddings, valid_dataset)
        similarity = tf.matmul(
          valid_embeddings, normalized_embeddings, transpose_b=True)

        # Add variable initializer.
        init = tf.global_variables_initializer()
    
    with tf.Session(graph=graph) as session:
        saver = tf.train.Saver()
        saver.restore(session,
                            os.path.join(model_path, "model.ckpt"))

        sim = similarity.eval()
        if input_word in dictionary:
            idx = dictionary[input_word]
            valid_word = reverse_dictionary[idx]
            top_k = 3  # number of nearest neighbors
            nearest = (-sim[idx, :]).argsort()[1:top_k + 1]
            log_str = 'Nearest to %s:' % valid_word
            for k in range(top_k):
                close_word = reverse_dictionary[nearest[k]]
                log_str = '%s %s' % (log_str, close_word)
            print(log_str)
        else:
            return 'Word not present in dictionary.  Try a different one.'<jupyter_output><empty_output><jupyter_text>Let's test the trained model and see if it can predict similar words.<jupyter_code># Define location of saved model
model_path = os.getcwd()
graph = tf.Graph()
predict_sim('science', model_path)<jupyter_output>INFO:tensorflow:Restoring parameters from /Users/micheleenharris/Documents/bin/github/pybotframework/tutorials/tensorflow_word2vec/model.ckpt
Nearest to science: provides regardless vs
{'fraction_non_alphanumeric': 0.09254498714652956, 'fraction_numerical': 0.02064116134885831, 'mean_word_length': 2.9684968496849686, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '39236', 'license': 'MIT', 'n_tokens_mistral': 4119, 'n_tokens_neox': 3812, 'n_words': 1111, 'path': 'tutorials/tensorflow_word2vec/TensorFlow Word2Vec.ipynb', 'repo_name': 'michhar/pybotframework', 'size': '13226'}