<jupyter_start><jupyter_text><jupyter_code>import nltk
from nltk.corpus import reuters
import numpy as np
from sklearn.decomposition import TruncatedSVD
import matplotlib.pyplot as plt
nltk.download('reuters')

START_TOKEN = '<START>'
END_TOKEN = '<END>'
def read_corpus(category="crude"):
  files = reuters.fileids(category)
  return [[START_TOKEN]+[w.lower() for w in list(reuters.words(f))]+[END_TOKEN] for f in files]
def distinct_no_of_words(corpus):
  words = [y for x in corpus for y in x]
  words = sorted(list(set(words)))
  n_words = len(words)
  return words, n_words

def co_occ_compute(corpus, win_size=4):
  words, n_words = distinct_no_of_words(corpus)
  mat = np.zeros((n_words, n_words))
  word2idx = dict(zip(words, range(n_words)))
  for doc in corpus:
    for i in range(len(doc)):
      start = max(i-win_size, 0)
      end = min(i+win_size+1, len(doc))
      context_words = doc[start:i] + doc[i:end]
      for word in context_words:
        mat[word2idx[word], word2idx[doc[i]]] += 1
  return mat, word2idx
def reduce_to_n(mat, n=2):
  iters = 10
  svd = TruncatedSVD(n_components=n, n_iter=iters)
  return svd.fit_transform(mat)
def plot(mat, word2idx, words):
  plt.figure(figsize=(10, 6))
  for word in words:
    i = word2idx[word]
    emb = mat[i]
    x, y = emb[0], emb[1]
    plt.scatter(x, y, marker='x', color='red')
    plt.text(x, y, word, fontsize=18)
  plt.show()
def main():
  corpus = read_corpus()
  mat_co_occ, word2idx_co_occ = co_occ_compute(corpus)
  mat_redu_co_occ = reduce_to_n(mat_co_occ, n=2)
  mat_lengths = np.linalg.norm(mat_redu_co_occ, axis=1)
  mat_norm = mat_redu_co_occ / mat_lengths[:, np.newaxis]

  words = ['barrels', 'bpd', 'energy', 'industry', 'oil', 'petroleum', 'kuwait', 'venezuela', 'ecuador']
  plot(mat_norm, word2idx_co_occ, words)
if __name__ == '__main__':
  main()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14408141403320834, 'fraction_numerical': 0.012854847348687734, 'mean_word_length': 4.8375, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '339418', 'license': 'Apache-2.0', 'n_tokens_mistral': 772, 'n_tokens_neox': 763, 'n_words': 166, 'path': 'Count_Based_Word_Vectors.ipynb', 'repo_name': 'nin-ed/NLP-Algorithms', 'size': '1867'}