<jupyter_start><jupyter_text>Word2Vec Skipgram impl. in PyTorch["Word2Vec in Pytorch - Continuous Bag of Words and Skipgrams: Pytorch implementation"](https://srijithr.gitlab.io/post/word2vec/)  <NAME> (2018-09-09)which was adapted from<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import numpy as np
import urllib.request

from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk import word_tokenize

import sklearn
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import euclidean_distances
def get_key (word_id):
    for key, val in word_to_ix.items():
        if val == word_id:
            print(key)


def cluster_embeddings (filename, nclusters):
    X = np.load(filename)
    kmeans = KMeans(n_clusters=nclusters, random_state=0).fit(X)
    center = kmeans.cluster_centers_
    distances = euclidean_distances(X, center)

    for i in np.arange(0, distances.shape[1]):
        word_id = np.argmin(distances[:, i])
        print(word_id)
        get_key(word_id)


def read_data (file_path):
    tokenizer = RegexpTokenizer(r'\w+')
    data = urllib.request.urlopen(file_path)
    data = data.read().decode('utf8')
    
    tokenized_data = word_tokenize(data)
    stop_words = set(stopwords.words('english'))
    stop_words.update(['.', ',', ':', ';', '(', ')', '#', '--', '...', '"'])
    
    cleaned_words = [ i for i in tokenized_data if i not in stop_words ]
    return(cleaned_words)
CONTEXT_SIZE = 3
EMBEDDING_DIM = 10

torch.manual_seed(1)

test_sentence = """Empathy for the poor may not come easily to people who never experienced it. They may blame the victims and insist their predicament can be overcome through determination and hard work.
But they may not realize that extreme poverty can be psychologically and physically incapacitating — a perpetual cycle of bad diets, health care and education exacerbated by the shaming and self-fulfilling prophecies that define it in the public imagination.
<NAME> — perhaps more than any artist — saw poverty as “the most savage of all human afflictions” and realized the power of empathy to help us understand it. It was neither an abstract problem nor political symbol, but something he endured growing up destitute in rural Kansas and having spent years documenting poverty throughout the world, including the United States.
That sensitivity informed “Freedom’s Fearful Foe: Poverty,” his celebrated photo essay published in Life magazine in June 1961. He took readers into the lives of a Brazilian boy, <NAME>, and his family, who lived in the ramshackle Catacumba favela in the hills outside Rio de Janeiro. These stark photographs are the subject of a new book, “Gordon Parks: The Flavio Story” (Steidl/The Gordon Parks Foundation), which accompanies a traveling exhibition co-organized by the Ryerson Image Centre in Toronto, where it opens this week, and the J. Paul Getty Museum. Edited with texts by the exhibition’s co-curators, <NAME> and <NAME>, the book also includes a recent interview with Mr. da Silva and essays by <NAME>, <NAME> and <NAME>.
""".split()
# we should tokenize the input, but we will ignore that for now
# build a list of tuples.  Each tuple is ([ word_i-2, word_i-1 ], target word)

#test_sentence = read_data('https://www.gutenberg.org/files/57884/57884-0.txt')

ngrams = []

for i in range(len(test_sentence) - CONTEXT_SIZE):
    tup = [test_sentence[j] for j in np.arange(i + 1 , i + CONTEXT_SIZE + 1) ]
    ngrams.append((test_sentence[i],tup))

# print the first 3, just so you can see what they look like
print(ngrams)
vocab = set(test_sentence)
print("Length of vocabulary", len(vocab))

word_to_ix = {word: i for i, word in enumerate(vocab)}
word_to_ix
class SkipgramModeler (nn.Module):

    def __init__(self, vocab_size, embedding_dim, context_size):
        super(SkipgramModeler, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.linear1 = nn.Linear(embedding_dim, 128)
        self.linear2 = nn.Linear(128, context_size * vocab_size)
        #self.parameters['context_size'] = context_size

        
    def forward (self, inputs):
        embeds = self.embeddings(inputs).view((1, -1))  # -1 implies size inferred for that index from the size of the data
        #print(np.mean(np.mean(self.linear2.weight.data.numpy())))
        
        out1 = F.relu(self.linear1(embeds)) # output of first layer
        out2 = self.linear2(out1)           # output of second layer
        #print(embeds)
        
        log_probs = F.log_softmax(out2, dim=1).view(CONTEXT_SIZE,-1)
        return log_probs

    
    def predict (self, input):
        context_idxs = torch.tensor([word_to_ix[input]], dtype=torch.long)
        res = self.forward(context_idxs)
        res_arg = torch.argmax(res)
        res_val, res_ind = res.sort(descending=True)
        indices = [res_ind[i][0] for i in np.arange(0, 3)]
        
        for arg in indices:
            print( [ (key, val) for key,val in word_to_ix.items() if val == arg ])


    def freeze_layer (self, layer):
        for name, child in model.named_children():
            print(name, child)
            
            if name == layer:
                for names, params in child.named_parameters():
                    print(names, params)
                    print(params.size())
                    params.requires_grad = False

                    
    def print_layer_parameters (self):
        for name, child in model.named_children():
                print(name, child)
                
                for names,params in child.named_parameters():
                    print(names, params)
                    print(params.size())

                    
    def write_embedding_to_file (self, filename):
        for i in self.embeddings.parameters():
            weights = i.data.numpy()
            
        np.save(filename, weights)
from tqdm import tqdm

losses = []
loss_function = nn.NLLLoss()

model = SkipgramModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)
optimizer = optim.SGD(model.parameters(), lr=0.001)

# Freeze embedding layer
#model.freeze_layer('embeddings')

for epoch in tqdm(range(550)):
    total_loss = 0
    
    #------- Embedding layers are trained as well here ----#
    #lookup_tensor = torch.tensor([word_to_ix["poor"]], dtype=torch.long)
    #hello_embed = model.embeddings(lookup_tensor)
    #print(hello_embed)
    # -----------------------------------------------------#

    model.predict('psychologically')

    for context, target in ngrams:
        # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words
        # into integer indices and wrap them in tensors)
        #print(context,target)

        context_idxs = torch.tensor([word_to_ix[context]], dtype=torch.long)
        #print("Context id",context_idxs)

        # Step 2. Recall that torch *accumulates* gradients. Before passing in a
        # new instance, you need to zero out the gradients from the old
        # instance
        model.zero_grad()

        # Step 3. Run the forward pass, getting log probabilities over next
        # words
        log_probs = model(context_idxs)
        #print(log_probs)

        # Step 4. Compute your loss function. (Again, Torch wants the target
        # word wrapped in a tensor)
        target_list = torch.tensor([word_to_ix[w] for w in target], dtype=torch.long)
        loss = loss_function(log_probs, target_list)
        #print(loss)

        # Step 5. Do the backward pass and update the gradient
        loss.backward()
        optimizer.step()

        # Get the Python number from a 1-element Tensor by calling tensor.item()
        total_loss += loss.item()
        
    print(total_loss)
    losses.append(total_loss)
#print(losses)  # The loss decreased every iteration over the training data!

#Print the model layer parameters
#model.print_layer_parameters()

#Predict the next word given n context words
model.predict('psychologically')
model.write_embedding_to_file('embeddings_skipgrams.npy')

cluster_embeddings('embeddings_skipgrams.npy', 5)<jupyter_output>[('and', 72)]
[('physically', 117)]
[('incapacitating', 99)]
103
more
193
where
116
experienced
56
which
88
informed
{'fraction_non_alphanumeric': 0.10178916827852998, 'fraction_numerical': 0.011726305609284332, 'mean_word_length': 3.03955078125, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '299944', 'license': 'MIT', 'n_tokens_mistral': 2590, 'n_tokens_neox': 2445, 'n_words': 808, 'path': 'wip/nn1.ipynb', 'repo_name': 'ArenasGuerreroJulian/kglab', 'size': '8350'}