<jupyter_start><jupyter_text>Word embedding> "Create word embeddings simply from scratch"- comments: true- badges: true- categories: [ai]- publishes: true We will use the following method to build simple word embeddings. We create a matrix where we put the co-occurences of all the words. We factorize that matrix.<jupyter_code>import numpy as np
import scipy as sp
import pandas as pd
sentences = [
    "a dog is a sweet animal",
    "a cat is a mean beast", 
    "a human is a different creature",
    "a cat is a nice pet",
    "a dog is a nice pet also"
]
from collections import defaultdict
from itertools import product, combinations

Nij_counts = defaultdict(int)

N = 1
k = 5 # The window size
window_size = 2

vocab = set()
for sentence in sentences: 
    for idx_a, word_a in enumerate(sentence.split(" ")): 
        start = idx_a - 1 
        stop = idx_a + 2
        for word_b in sentence.split(" ")[start:stop]:
            if word_a == word_b:
              continue
            Nij_counts[(word_a, word_b)] += 1
            N += 1
            vocab.add(word_a)
            vocab.add(word_b)

Ni_counts = defaultdict(int)
Nj_counts = defaultdict(int)
for (i,j), N_ij in Nij_counts.items():
  Ni_counts[ i ] += N_ij
  Nj_counts[ j ] += N_ij


Pi = {k:v/N for k,v in Ni_counts.items()}
Pj = {k:v/N for k,v in Nj_counts.items()}
Pij = {k:v/N for k,v in Nij_counts.items()}
pmi_matrix = np.zeros((len(vocab), len(vocab)))
for i, word_i in enumerate(vocab): 
    for j, word_j in enumerate(vocab):
        pmi_matrix[i][j] = np.log( Pij.get((word_i, word_j), 0) / (Pi[word_i] * Pj[word_j] ))  #- np.log(k)

pmi_matrix[ pmi_matrix < 0] = 0
pd.DataFrame(pmi_matrix, columns=vocab, index=vocab)
U, sigma, Vt = np.linalg.svd(pmi_matrix)
word_embeddings = U * sigma
pd.DataFrame(U * sigma, index=vocab)
U_embeddings = {word: (U * sigma)[index, :] for index, word in enumerate(vocab)}
V_embeddings = {word: Vt[index, :] for index, word in enumerate(vocab)}
(U_embeddings['cat'] @ U_embeddings['dog'],
U_embeddings['cat'] @ U_embeddings['is'],
U_embeddings['cat'] @ U_embeddings['human'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12918434700612919, 'fraction_numerical': 0.004714757190004715, 'mean_word_length': 3.6030368763557483, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '459425', 'license': 'Apache-2.0', 'n_tokens_mistral': 818, 'n_tokens_neox': 790, 'n_words': 234, 'path': '_notebooks/2021-09-19-word-embeddings.ipynb', 'repo_name': 'RCambier/blog', 'size': '2121'}