<jupyter_start><jupyter_text>Word2Vec Just some preliminary visualizations in Word2Vec...<jupyter_code># load packages
import os
import psycopg2 as pg
import pandas.io.sql as psql
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from textblob import Word
from gensim.models import Word2Vec
import multiprocessing 
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import matplotlib.cm as cm

# set cores, grab stop words
cores_available = multiprocessing.cpu_count() - 1
stop = stopwords.words('english')
os.chdir("/sfs/qumulo/qhome/kb7hp/git/dspg21oss/data/dspg21oss/")
repo_data = pd.read_csv("github_repos_157k.csv")
repo_data['description_clean'] = repo_data['description'].str.lower()
repo_data['description_clean'] = repo_data['description_clean'].str.replace(r'[^\w\s]+', '', regex=True)
repo_data['description_clean'] = repo_data['description_clean'].apply(lambda x:' '.join(x for x in x.split() if  not x.isdigit()))
repo_data['description_clean'] = repo_data['description_clean'].apply(lambda x:' '.join(x for x in x.split() if not x in stop))
repo_data['description_clean'] = repo_data['description_clean'].apply(lambda x:' '.join([Word(word).lemmatize() for word in x.split()]))
repo_data.head()
repo_list=[]
for i in repo_data['description_clean']:
    li = list(i.split(" "))
    repo_list.append(li)
repo_model = Word2Vec(repo_list, min_count=5, size=512, window=5, iter=5, workers=cores_available)

os.chdir("/sfs/qumulo/qhome/kb7hp/git/dspg21oss/data/dspg21oss/")
repo_model.save("repo_w2v_157k.bin")
os.chdir("/sfs/qumulo/qhome/kb7hp/git/dspg21oss/data/dspg21oss/")
repo_model = Word2Vec.load("repo_w2v_157k.bin")
print("Model loaded")
top_words = repo_model.wv.most_similar(positive=['blockchain'], topn=30)
top_words
%%capture
repo_vocab = list(repo_model.wv.vocab)
repo_x = repo_model[repo_vocab]
repo_tsne = TSNE(n_components=2)
repo_tsne_x = repo_tsne.fit_transform(repo_x)
repo_df = pd.DataFrame(repo_tsne_x, index=repo_vocab, columns=['x', 'y'])

keys = ['python', 'javascript', 'java', 'c', 'r', 'php', 'database', 'ai', 'ml', 'visualization', 'server', 'mac', 'linux']

embedding_clusters = []
word_clusters = []
for word in keys:
    embeddings = []
    words = []
    for similar_word, _ in repo_model.wv.most_similar(word, topn=15):
        words.append(similar_word)
        embeddings.append(repo_model[similar_word])
    embedding_clusters.append(embeddings)
    word_clusters.append(words)
    
embedding_clusters = np.array(embedding_clusters)
n, m, k = embedding_clusters.shape
tsne_model_en_2d = TSNE(perplexity=15, n_components=2, init='pca', n_iter=3500, random_state=32)
embeddings_en_2d = np.array(tsne_model_en_2d.fit_transform(embedding_clusters.reshape(n * m, k))).reshape(n, m, 2)
def tsne_plot_similar_words(title, labels, embedding_clusters, word_clusters, a, filename=None):
    plt.figure(figsize=(16, 9))
    colors = cm.rainbow(np.linspace(0, 1, len(labels)))
    for label, embeddings, words, color in zip(labels, embedding_clusters, word_clusters, colors):
        x = embeddings[:, 0]
        y = embeddings[:, 1]
        plt.scatter(x, y, c=color, alpha=a, label=label)
        for i, word in enumerate(words):
            plt.annotate(word, alpha=0.5, xy=(x[i], y[i]), xytext=(5, 2),
                         textcoords='offset points', ha='right', va='bottom', size=8)
    plt.legend(loc=4)
    plt.title(title)
    plt.grid(True)
    if filename:
        plt.savefig(filename, format='png', dpi=150, bbox_inches='tight')
    plt.show()
    
cats_plot = tsne_plot_similar_words('Top Word Vectors for Select Categories (Based on GitHub Descriptions)', 
                                    keys, embeddings_en_2d, word_clusters, 0.7, 'cats_plot.png')
cats_plot<jupyter_output>*c* argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with *x* & *y*.  Please use the *color* keyword-argument or provide a 2-D array with a single row if you intend to specify the same RGB or RGBA value for all points.
*c* argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with *x* & *y*.  Please use the *color* keyword-argument or provide a 2-D array with a single row if you intend to specify the same RGB or RGBA value for all points.
*c* argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with *x* & *y*.  Please use the *color* keyword-argument or provide a 2-D array with a single row if you intend to specify the same RGB or RGBA value for all points.
*c* argument looks like a single numeric RGB o[...]
{'fraction_non_alphanumeric': 0.12354651162790697, 'fraction_numerical': 0.016403654485049834, 'mean_word_length': 4.867235079171742, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '559893', 'license': 'MIT', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1625, 'n_words': 450, 'path': 'src/09_word2vec/01_word2vec.ipynb', 'repo_name': 'team-oss/dspg21oss', 'size': '4816'}