<jupyter_start><jupyter_text>Word2Vec Using only the raw text, we'll derive and explore the semantic properties of its words.  Imports Python code in one module gains access to the code in another module by the process of importing it. The import statement is the most common way of invoking the import machinery, but it is not the only way.<jupyter_code>from __future__ import absolute_import, division, print_function
import codecs
import glob
import logging
import multiprocessing
import os
import pprint
import re
import nltk
import gensim.models.word2vec as w2v
from gensim.models import KeyedVectors
from gensim.models import Word2Vec
from gensim.utils import simple_preprocess
import sklearn.manifold
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
%pylab inline<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>**Set up logging**<jupyter_code>logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)<jupyter_output><empty_output><jupyter_text>**Download NLTK tokenizer models (only the first time)**<jupyter_code>nltk.download("punkt")
nltk.download("stopwords")<jupyter_output>[nltk_data] Downloading package punkt to /home/keno/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to /home/keno/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Prepare Corpus **Load books from files**<jupyter_code>book_filenames = sorted(glob.glob("txtdata/*.txt"))
print("Found books:")
book_filenames<jupyter_output>Found books:<jupyter_text>**Combine the books into one string**<jupyter_code>corpus_raw = u""
for book_filename in book_filenames:
    print("Reading '{0}'...".format(book_filename))
    with codecs.open(book_filename, "r", "utf-8") as book_file:
        corpus_raw += book_file.read()
    print("Corpus is now {0} characters long".format(len(corpus_raw)))
    print()<jupyter_output>Reading 'txtdata/01_aufbauhaus.txt'...
Corpus is now 41834 characters long

Reading 'txtdata/02_bauhausbuch1.txt'...
Corpus is now 48291 characters long

Reading 'txtdata/03_staatlichesbauhaus.txt'...
Corpus is now 56550 characters long<jupyter_text>**Split the corpus into sentences**<jupyter_code>tokenizer = nltk.data.load('tokenizers/punkt/german.pickle')
raw_sentences = tokenizer.tokenize(corpus_raw)
# convert into a list of words
# remove unnnecessary, split into words, no hyphens
# list of words
def sentence_to_wordlist(raw):
    clean = re.sub("[^a-zA-ZäÄöÖüÜß]"," ", raw)
    clean = re.sub("[äÄ]", "ae", clean)
    clean = re.sub("[öÖ]", "oe", clean)
    clean = re.sub("[üÜ]", "ue", clean)
    clean = re.sub("[ß]", "ss", clean)
    words = clean.split()
    return words
#sentence where each word is tokenized
sentences = []
for raw_sentence in raw_sentences:
    if len(raw_sentence) > 0:
        sentences.append(sentence_to_wordlist(raw_sentence))
print(raw_sentences[5])
print(sentence_to_wordlist(raw_sentences[5]))
token_count = sum([len(sentence) for sentence in sentences])
print("The book corpus contains {0:,} tokens".format(token_count))<jupyter_output>The book corpus contains 7,699 tokens<jupyter_text>Train Word2Vec<jupyter_code>#ONCE we have vectors
#step 3 - build model
#3 main tasks that vectors help with
#DISTANCE, SIMILARITY, RANKING

# Dimensionality of the resulting word vectors.
#more dimensions, more computationally expensive to train
#but also more accurate
#more dimensions = more generalized
num_features = 300
# Minimum word count threshold.
min_word_count = 3

# Number of threads to run in parallel.
#more workers, faster we train
num_workers = multiprocessing.cpu_count()

# Context window length.
context_size = 7

# Downsample setting for frequent words.
#0 - 1e-5 is good for this
downsampling = 1e-3

# Seed for the RNG, to make the results reproducible.
#random number generator
#deterministic, good for debugging
seed = 1
thrones2vec = w2v.Word2Vec(
    sg=1,
    seed=seed,
    workers=num_workers,
    size=num_features,
    min_count=min_word_count,
    window=context_size,
    sample=downsampling
)
thrones2vec.build_vocab(sentences)
print("Word2Vec vocabulary length:", len(thrones2vec.wv.vocab))<jupyter_output>Word2Vec vocabulary length: 482<jupyter_text>**Start training, this might take a minute or two...**<jupyter_code>thrones2vec.train(sentences, total_words=token_count, epochs = 100 )<jupyter_output>2019-10-25 20:52:56,325 : INFO : training model with 4 workers on 482 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=7
2019-10-25 20:52:56,336 : INFO : worker thread finished; awaiting finish of 3 more threads
2019-10-25 20:52:56,338 : INFO : worker thread finished; awaiting finish of 2 more threads
2019-10-25 20:52:56,339 : INFO : worker thread finished; awaiting finish of 1 more threads
2019-10-25 20:52:56,368 : INFO : worker thread finished; awaiting finish of 0 more threads
2019-10-25 20:52:56,369 : INFO : EPOCH - 1 : training on 7699 raw words (3360 effective words) took 0.0s, 101420 effective words/s
2019-10-25 20:52:56,374 : INFO : worker thread finished; awaiting finish of 3 more threads
2019-10-25 20:52:56,374 : INFO : worker thread finished; awaiting finish of 2 more threads
2019-10-25 20:52:56,375 : INFO : worker thread finished; awaiting finish of 1 more threads
2019-10-25 20:52:56,404 : INFO : worker thread finished; awaiting finish of 0 more t[...]<jupyter_text>**Save to file, can be useful later**<jupyter_code>if not os.path.exists("trained"):
    os.makedirs("trained")
thrones2vec.save(os.path.join("trained", "thrones2vec.w2v"))<jupyter_output>2019-10-25 20:53:00,183 : INFO : saving Word2Vec object under trained/thrones2vec.w2v, separately None
2019-10-25 20:53:00,185 : INFO : not storing attribute vectors_norm
2019-10-25 20:53:00,186 : INFO : not storing attribute cum_table
2019-10-25 20:53:00,201 : INFO : saved trained/thrones2vec.w2v<jupyter_text>Explore the trained model.<jupyter_code>print(thrones2vec.most_similar('Wirtschaft', topn=5))<jupyter_output>/home/keno/.conda/envs/w2v/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).
  """Entry point for launching an IPython kernel.
2019-10-25 20:53:00,358 : INFO : precomputing L2-norms of word weight vectors<jupyter_text>Compress the word vectors into 2D space and plot them<jupyter_code>#my video - how to visualize a dataset easily
tsne = sklearn.manifold.TSNE(n_components=3, random_state=0)
all_word_vectors_matrix = thrones2vec.wv.syn0<jupyter_output>/home/keno/.conda/envs/w2v/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.vectors instead).
  """Entry point for launching an IPython kernel.<jupyter_text>**Train t-SNE, this could take a minute or two...**<jupyter_code>all_word_vectors_matrix_2d = tsne.fit_transform(all_word_vectors_matrix)<jupyter_output><empty_output><jupyter_text>**Plot the big picture**<jupyter_code>points = pd.DataFrame(
    [
        (word, coords[0], coords[1], coords[2])
        for word, coords in [
            (word, all_word_vectors_matrix_2d[thrones2vec.wv.vocab[word].index])
            for word in thrones2vec.wv.vocab
        ]
    ],
    columns=["word", "x", "y", "z"]
)
points.head(100)
sns.set_context("poster")
points.plot.scatter("x", "y", s=100, figsize=(100, 100))<jupyter_output><empty_output><jupyter_text>Explore semantic similarities between book characters **Words closest to the given word**<jupyter_code>thrones2vec.most_similar("Wirtschaft")
thrones2vec.most_similar("Bauhaus")
thrones2vec.most_similar("Studierenden")<jupyter_output>/home/keno/.conda/envs/w2v/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).
  """Entry point for launching an IPython kernel.<jupyter_text>**Linear relationships between word pairs**<jupyter_code>def nearest_similarity_cosmul(start1, end1, end2):
    similarities = thrones2vec.most_similar_cosmul(
        positive=[end2, start1],
        negative=[end1]
    )
    start2 = similarities[0][0]
    print("{start1} is related to {end1}, as {start2} is related to {end2}".format(**locals()))
    return start2
nearest_similarity_cosmul("Bauhaus", "Weimar", "Studierenden")
nearest_similarity_cosmul("Bauhaus", "Weimar", "Architekten")
nearest_similarity_cosmul("Form", "Tempo", "Meister")
thrones2vec.most_similar(positive=['Bauhaus'], negative=['Studierenden'])
thrones2vec.most_similar(positive=['Lehre'], negative=['Meister'])<jupyter_output>/home/keno/.conda/envs/w2v/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).
  """Entry point for launching an IPython kernel.
{'fraction_non_alphanumeric': 0.11330975016581915, 'fraction_numerical': 0.04919301348662392, 'mean_word_length': 5.7263940520446095, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '538281', 'license': 'MIT', 'n_tokens_mistral': 3408, 'n_tokens_neox': 3054, 'n_words': 805, 'path': 'BauhausMachine.ipynb', 'repo_name': 'kenokenobingo/BauhausMachine', 'size': '9046'}