<jupyter_start><jupyter_text>Word to Vec<jupyter_code>#! pip install plotly_express
import gensim
OHCO = ['book_id', 'chap_num', 'para_num', 'sent_num', 'token_num']
BAG = OHCO[:4] # Paragraphs
# BAG = OHCO[:5] # Sentences
window = 5
import pandas as pd
import numpy as np
from gensim.models import word2vec
from sklearn.manifold import TSNE
import plotly_express as px
%matplotlib inline
LIB = pd.read_csv('LIB.csv')
for i in LIB['book_title']:
    if 'Austen' in i:
        print(i)
aus_books = []
for i in LIB.iterrows():
    if 'austen' in i[1][3]:
        aus_books.append(i[1][0])

    
aus_books
TOKENS = pd.read_csv('TOKEN2.csv')
t1=TOKENS
corpus1 = t1[~t1.pos.str.match('NNPS?')]\
    .groupby(BAG)\
    .term_str.apply(lambda  x:  x.tolist())\
    .reset_index()['term_str'].tolist()

m1 = word2vec.Word2Vec(corpus1, window=window, min_count=200, workers=4)
# coords = pd.DataFrame(index=range(len(model.wv)))
# coords['label'] = [w for w in model.wv]
# coords['vector'] = coords['label'].apply(lambda x: model.wv.get_vector(x))
coords1 = pd.DataFrame(index=range(len(m1.wv.key_to_index)))
coords1['label'] = m1.wv.index_to_key
coords1['vector'] = coords1['label'].apply(lambda x: m1.wv.get_vector(x))
tsne_model1 = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)
tsne_values1 = tsne_model1.fit_transform(coords1['vector'].tolist())
coords1['x'] = tsne_values1[:,0]
coords1['y'] = tsne_values1[:,1]
coords1.head()
px.scatter(coords1, 'x', 'y', text='label', height=1000, title='Jane Austen Word Plot').update_traces(mode='text')
def complete_analogy_1(A, B, C, n=2):
    try:
        return m1.wv.most_similar(positive=[B, C], negative=[A])[0:n]
    except KeyError as e:
        print('Error:', e)
        return None
complete_analogy_1('mother', 'herself', 'father')
complete_analogy_1('sister', 'me', 'brother')
complete_analogy_1('you', 'thought', 'me')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17644006227296316, 'fraction_numerical': 0.031136481577581733, 'mean_word_length': 5.535593220338983, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '210574', 'license': 'MIT', 'n_tokens_mistral': 809, 'n_tokens_neox': 782, 'n_words': 150, 'path': 'nlp_proj/word_to_vec.ipynb', 'repo_name': 'DrakeWagner/projects', 'size': '1927'}