<jupyter_start><jupyter_text>Lecture 10 - Loading and Visualizing GloVe<jupyter_code>import os
import os.path
import numpy as np

from tqdm import tqdm
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>I'm loading [GloVe](https://nlp.stanford.edu/projects/glove/) from google drive since it is attached to colab. You can find a url to the pre-trained embedding below or go to the GloVe's website to download it.<jupyter_code>from google.colab import drive

drive.mount('/content/drive')
base = """/content/drive/MyDrive/GloVe"""
assert os.path.exists(base)

if not os.path.exists(os.path.join(base, "glove.6B.zip")):
  !wget http://nlp.stanford.edu/data/glove.6B.zip
  !mv glove.6B.zip drive/MyDrive/GloVe/

!unzip -l drive/MyDrive/GloVe/glove.6B.zip<jupyter_output>Mounted at /content/drive
Archive:  drive/MyDrive/GloVe/glove.6B.zip
  Length      Date    Time    Name
---------  ---------- -----   ----
171350079  2014-08-04 20:15   glove.6B.50d.txt
347116733  2014-08-04 20:14   glove.6B.100d.txt
693432828  2014-08-04 20:14   glove.6B.200d.txt
1037962819  2014-08-27 19:19   glove.6B.300d.txt
---------                     -------
2249862459                     4 files<jupyter_text>There are several versions avalible, so I have to choose one.<jupyter_code>n_vocab = 400000
n_dim = 200

import zipfile
import io

idx2voc = list()
GloVe = np.zeros((n_vocab, n_dim))
with zipfile.ZipFile(os.path.join(base, "glove.6B.zip")) as archive:
  with archive.open("glove.6B.%id.txt" % n_dim) as raw:
    for n, line in tqdm(enumerate(io.TextIOWrapper(raw, encoding=None, newline='\n')), 
                     total=n_vocab, desc="Loading vectors"):
      tokens = line.split(' ')
      idx2voc.append(tokens[0])
      GloVe[n, :] = np.asarray(tokens[1:], dtype=np.float32)
vocabulary = set(idx2voc)
voc2idx = {word:n for n, word in enumerate(idx2voc)}
GloVe[voc2idx['silence'], :]<jupyter_output><empty_output><jupyter_text>Word analogies<jupyter_code>from sklearn.metrics import pairwise_distances
vec = lambda token: np.vstack(GloVe[voc2idx[token], :]).T

v = vec('star')

distances = pairwise_distances(GloVe, v, metric='euclidean').ravel()

for idx in np.argsort(distances)[:20]:
  print(idx2voc[idx], '\t', distances[idx])
import random
random.choices(list(vocabulary), k=20)
# v = vec('king')
v = vec('king') - vec('man') + vec('woman')
# v = vec('queen') + vec('man') - vec('woman')

distances = pairwise_distances(GloVe, v, metric='euclidean').ravel()

for idx in np.argsort(distances)[:20]:
  print(idx2voc[idx], '\t', distances[idx])<jupyter_output>king 	 0.7042604306040586
queen 	 5.256927584849657
prince 	 5.415242229239351
monarch 	 5.463253010710028
kingdom 	 5.621511180803411
throne 	 5.808637217409268
reign 	 5.886935461583319
uncle 	 5.982255724499488
ii 	 5.988985645435528
crown 	 6.012373561394796
nephew 	 6.0877090368285165
cousin 	 6.161330447399032
iii 	 6.1692461766340365
brother 	 6.21298971862443
later 	 6.295843975895495
afterwards 	 6.300898549205696
latter 	 6.31596081226383
iv 	 6.322683596804751
mentioned 	 6.3288909280658014
accompanied 	 6.329484272009041<jupyter_text>Load a book<jupyter_code>import nltk
nltk.download('gutenberg')
print(nltk.corpus.gutenberg.fileids())
books = ['carroll-alice.txt']
words = nltk.corpus.gutenberg.words(books)
text = """No more be grieved at that which thou hast done:
Roses have thorns, and silver fountains mud:
Clouds and eclipses stain both moon and sun,
And loathsome canker lives in sweetest bud.
All men make faults, and even I in this,
Authorizing thy trespass with compare,
Myself corrupting, salving thy amiss,
Excusing thy sins more than thy sins are;
For to thy sensual fault I bring in sense,
Thy adverse party is thy advocate,
And ‘gainst myself a lawful plea commence:
Such civil war is in my love and hate,
That I an accessary needs must be,
To that sweet thief which sourly robs from me."""

words = nltk.tokenize.wordpunct_tokenize(text)
words = [word.lower() for word in words if word.isalpha()]

tokencounts = dict()
for word in words:
  if word not in tokencounts.keys():
    tokencounts[word] = 1
  else:
    tokencounts[word] += 1

print("%i tokens in the text(s)" % len(tokencounts.keys()))<jupyter_output>80 tokens in the text(s)<jupyter_text>POS tags for colourWe know that baseline POS taggers do quite well. Let's make one for the visualisation.<jupyter_code>nltk.download('brown')
from nltk.corpus import brown
nltk.download('universal_tagset')

pos = dict()
for sentence in brown.tagged_sents(tagset='universal'):
  for n, (word, tag) in enumerate(sentence):
    word = word.lower()
    if word not in pos.keys():
      pos[word] = {tag: 1}
    elif tag not in pos[word].keys():
      pos[word][tag] = 1
    else:
      pos[word][tag] += 1
for word in pos.keys():
  pos[word] = sorted(pos[word])[0]

missing = set(tokencounts.keys()).difference(set(pos.keys()))
print("Tags are missing for %.1f%% of the words" % (100*np.sum([tokencounts[token] for token in missing])/len(tokencounts.keys())))
from sklearn.decomposition import PCA
pca = PCA(n_components=min(100, GloVe.shape[1]//2))
V_pca = pca.fit_transform(GloVe)

plt.figure()
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.show()
idx_tag = dict()
tag2idx = {tag:n for n, tag in enumerate(set(pos.values()).union({'UNK'}))}
for word in set(tokencounts.keys()).intersection(vocabulary):
  if word in pos:
    idx_tag[voc2idx[word]] = tag2idx[pos[word]]
  else:
    idx_tag[voc2idx[word]] = tag2idx['UNK']

idx = list(idx_tag.keys())
y = np.asarray([idx_tag[i] for i in idx])
s = np.asarray([tokencounts[idx2voc[i]] for i in idx])
assert len(idx) == len(y)
assert len(idx) == len(s)
fig = plt.figure(figsize=(8, 5), dpi=100)
ax = fig.add_subplot(1, 1, 1, projection='3d')
# ax.scatter(V_pca[idx, 0], V_pca[idx, 1], V_pca[idx, 2], c=y, cmap='Pastel1', marker='.', alpha=.8)
ax.scatter(V_pca[idx, 0], V_pca[idx, 1], V_pca[idx, 2], c=y, s=s*20, cmap='tab10', marker='.', alpha=.6)
ax.axis('off')
fig.show()

print("%i points" % len(idx))
sentence = "This parrot is simply pining for the fjords"
# sentence = "machine learning for language technologists"

words =  [word for word in sentence.lower().split() if word in vocabulary]
d = len(words) - len(sentence.split())
if d > 0:
  print("%i word(s) was not in the vocabulary")
path = np.concatenate([np.vstack(V_pca[voc2idx[word]]).T for word in words])

fig = plt.figure(figsize=(8, 5), dpi=100)
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.plot3D(path[:, 0], path[:, 1], path[:, 2], 'o-', alpha=.8)
# # ax.scatter(X_pca[idx, 0], X_pca[idx, 1], X_pca[idx, 2], c='b', marker='.', alpha=.3)
# # ax.scatter(X_pca[:, 0], X_pca[:, 1], X_pca[:, 2], c='b', cmap='Spectral', alpha=.3)
# # ax.view_init(20, 0)
fig.show()
try:
  from MulticoreTSNE import MulticoreTSNE as TSNE
except:
  !pip install MulticoreTSNE
finally:
  from MulticoreTSNE import MulticoreTSNE as TSNE
import os
tsne = TSNE(n_components=3, n_jobs=os.cpu_count(), verbose=True, metric='cosine')
V_tsne = tsne.fit_transform(GloVe[idx, :])
tsne
plt.style.use('dark_background')
fig = plt.figure(figsize=(8, 5), dpi=100)
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.scatter(V_tsne[:, 0], V_tsne[:, 1], V_tsne[:, 2], c=y, s=s*60, cmap='tab10', marker='.', alpha=.5)
# ax.scatter(V_tsne[:, 0], V_tsne[:, 1], V_tsne[:, 2], c=y, s=s, cmap='Spectral', marker='.', alpha=.3)
# ax.view_init(20, 20)
ax.axis('off')
fig.show()
try:
  from umap import UMAP
except:
  !pip install umap-learn
finally:
  from umap import UMAP
umap = UMAP(n_components=3, metric='euclidean')
# umap = UMAP(n_components=3, metric='cosine')
X_umap3 = umap.fit_transform(GloVe[idx, :])
plt.style.use('dark_background')
fig = plt.figure(figsize=(8, 5), dpi=100)
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.scatter(X_umap3[:, 0], X_umap3[:, 1], X_umap3[:, 2], c=y, s=s*60, cmap='tab10', marker='.', alpha=.5)
# ax.scatter(V_tsne[:, 0], V_tsne[:, 1], V_tsne[:, 2], c=y, s=s, cmap='Spectral', marker='.', alpha=.3)
# ax.view_init(20, 20)
ax.axis('off')
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16071428571428573, 'fraction_numerical': 0.07428075396825397, 'mean_word_length': 4.7689556509299, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '170555', 'license': 'MIT', 'n_tokens_mistral': 3649, 'n_tokens_neox': 3206, 'n_words': 763, 'path': 'Lecture 10 - Loading and Visualizing GloVe.ipynb', 'repo_name': 'fredrikwahlberg/5LN708', 'size': '8064'}