<jupyter_start><jupyter_text>Finding Synonyms and AnalogiesThis notebook is taken from a [PyTorch NLP tutorial](https://github.com/joosthub/pytorch-nlp-tutorial-ny2018/blob/master/day_1/0_Using_Pretrained_Embeddings.ipynb) source: [repository for the training tutorial as the 2018 O'Reilly AI Conference in NYC on April 29 and 30, 2018](https://github.com/joosthub/pytorch-nlp-tutorial-ny2018)Another related sources [chapter 10.6](http://www.d2l.ai/chapter_natural-language-processing/similarity-analogy.html) of [Dive into Deep Learning](http://www.d2l.ai/index.html) and [chapter 10.5](http://www.d2l.ai/chapter_natural-language-processing/glove.html) explaining Word Embedding with Global Vectors (GloVe)<jupyter_code># you might need to install annoy with the following command:
# conda install -c conda-forge python-annoy
from annoy import AnnoyIndex
import numpy as np
import torch
from tqdm import tqdm_notebook
from argparse import Namespace<jupyter_output><empty_output><jupyter_text>Glove embeddings can be downloaded from [GloVe webpage](https://nlp.stanford.edu/projects/glove/).Given the sizes of the files, it is better not to dowload it through WIFI!<jupyter_code># this need to be modified!
args = Namespace(
    glove_filename='/home/lelarge/data/glove/glove.6B.100d.txt'
)
def load_word_vectors(filename):
    word_to_index = {}
    word_vectors = []
    
    with open(filename) as fp:
        for line in tqdm_notebook(fp.readlines(), leave=False):
            line = line.split(" ")
            
            word = line[0]
            word_to_index[word] = len(word_to_index)
            
            vec = np.array([float(x) for x in line[1:]])
            word_vectors.append(vec)
            
    return word_to_index, word_vectors
word_to_index, word_vectors = load_word_vectors(args.glove_filename)
len(word_vectors)
word_vectors[0].shape
word_to_index['beautiful']
class PreTrainedEmbeddings(object):
    def __init__(self, glove_filename):
        self.word_to_index, self.word_vectors = load_word_vectors(glove_filename)
        self.word_vector_size = len(self.word_vectors[0])
        
        self.index_to_word = {v: k for k, v in self.word_to_index.items()}
        self.index = AnnoyIndex(self.word_vector_size, metric='euclidean')
        print('Building Index')
        for _, i in tqdm_notebook(self.word_to_index.items(), leave=False):
            self.index.add_item(i, self.word_vectors[i])
        self.index.build(50)
        print('Finished!')
    
    def get_embedding(self, word):
        return self.word_vectors[self.word_to_index[word]]
    
    def closest(self, word, n=1):
        vector = self.get_embedding(word)
        nn_indices = self.index.get_nns_by_vector(vector, n)
        return [self.index_to_word[neighbor] for neighbor in nn_indices]
    
    def closest_v(self, vector, n=1):
        nn_indices = self.index.get_nns_by_vector(vector, n)
        return [self.index_to_word[neighbor] for neighbor in nn_indices]
    
    def sim(self, w1, w2):
        return np.dot(self.get_embedding(w1), self.get_embedding(w2))

glove = PreTrainedEmbeddings(args.glove_filename)
glove.closest('apple', n=5)
glove.closest('chip', n=5)
glove.closest('baby', n=5)
glove.closest('beautiful', n=5)
def SAT_analogy(w1, w2, w3):
    '''
    Solves problems of the type:
    w1 : w2 :: w3 : __
    '''
    closest_words = []
    try:
        w1v = glove.get_embedding(w1)
        w2v = glove.get_embedding(w2)
        w3v = glove.get_embedding(w3)
        w4v = w3v + (w2v - w1v)
        closest_words = glove.closest_v(w4v, n=5)
        closest_words = [w for w in closest_words if w not in [w1, w2, w3]]
    except:
        pass
    if len(closest_words) == 0:
        print(':-(')
    else:
        print('{} : {} :: {} : {}'.format(w1, w2, w3, closest_words[0]))
SAT_analogy('man', 'he', 'woman')
SAT_analogy('fly', 'plane', 'sail')
SAT_analogy('beijing', 'china', 'tokyo')
SAT_analogy('man', 'woman', 'son')
SAT_analogy('man', 'leader', 'woman')
SAT_analogy('woman', 'leader', 'man')<jupyter_output>woman : leader :: man : leaders
{'fraction_non_alphanumeric': 0.12521440823327615, 'fraction_numerical': 0.018867924528301886, 'mean_word_length': 3.6124293785310733, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '229670', 'license': 'Apache-2.0', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1435, 'n_words': 303, 'path': 'Playing_with_word_embedding.ipynb', 'repo_name': 'lasofivec/dataflowr', 'size': '4081'}