<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
import json
import os
from gensim.utils import simple_preprocess
from gensim.models.doc2vec import TaggedDocument, Doc2Vec
from gensim.models import Word2Vec
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import plotly.plotly as py
import plotly.graph_objs as go
import numpy as np
import plotly 
from collections import Counter
from tqdm import tqdm
with open('/home/msnow/site_configs/secrets.json', 'r') as fp:
    secrets = json.load(fp)
plotly.tools.set_credentials_file(username=secrets['plotly']['username'], api_key=secrets['plotly']['api_key'])<jupyter_output><empty_output><jupyter_text>Ancillary Functions<jupyter_code>class EmbedText:

    def __init__(self):
        # self.df = df
        self.model_d2v = None
        self.train_corpus = None
        # self.text_col = text_col
        # self.class_col = class_col

    def d2v_setup(self, df: pd.DataFrame, text_col: str, class_col: str = None, emb_size: int = 100):
        self.train_corpus = []
        for idx, row in df.iterrows():
            if class_col is None:
                self.train_corpus.append(TaggedDocument(row[text_col], [idx]))
            else:
                self.train_corpus.append(TaggedDocument(row[text_col], [row[class_col]]))
        self.model_d2v = Doc2Vec(vector_size=emb_size, dm=0, min_count=2, workers=6, epochs=1, min_alpha=0.025,
                                 alpha=0.025)
        self.model_d2v.build_vocab(self.train_corpus)

    def d2v_fit(self, epochs: int = 10, decay_rate: float = 0.002):
#         print('Fitting Doc2Vec')
        for _ in tqdm(range(epochs)):
            self.model_d2v.train(self.train_corpus, total_examples=self.model_d2v.corpus_count, epochs=epochs)
            self.model_d2v.alpha -= decay_rate
            self.model_d2v.min_alpha = self.model_d2v.alpha
        return self.model_d2v

    def d2v_embed_save(self, loc: str):
        self.model_d2v.save(loc)

    def d2v_embed_load(self, model_loc: str):
        self.model_d2v = Doc2Vec.load(model_loc)

    def d2v_vectors(self, df: pd.DataFrame, text_col: str):
        return df.loc[:, text_col].map(lambda x: self.model_d2v.infer_vector(x).tolist())

    def d2v_wordvecs(self, df: pd.DataFrame, text_col: str) -> pd.Series:
        return df.loc[:, text_col].map(
            lambda x: np.array([self.model_d2v.wv.word_vec(i) for i in x if i in self.model_d2v.wv.vocab]))<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>with open('../src/data/game_info_190509.json', 'r') as fp:
    df_game = pd.DataFrame(json.load(fp))
df_game.shape    
with open('../src/data/user_comments_190511.json', 'r') as fp:
    comment_list = json.load(fp)
df_comments = pd.DataFrame(comment_list)
df_comments.shape, df_comments.game_id.nunique()
with open('../src/data/user_ratings_190511.json', 'r') as fp:
    rating_list = json.load(fp)
df_ratings = pd.DataFrame(rating_list)
df_ratings.shape, df_ratings.game_id.nunique()<jupyter_output><empty_output><jupyter_text>Merge Data<jupyter_code>df_game.columns
df_ratings = df_ratings.merge(df_game[['name', 'game_id', 'BoardGameRank']], how='left', on='game_id')
df_comments = df_comments.merge(df_game[['name', 'game_id', 'BoardGameRank']], how='left', on='game_id')
df_ratings[['game_id', 'BoardGameRank', 'rating']].groupby(['game_id', 'BoardGameRank']).count().sort_values('rating', ascending=False).head()<jupyter_output><empty_output><jupyter_text>Create "sentences" from board game categories<jupyter_code># col_list = ['boardgameartist', 'boardgamedesigner', 'boardgamecategory', 'boardgamefamily', 'boardgamemechanic', 'boardgamepublisher', 'name', 'game_id']
col_list = ['boardgamedesigner', 'boardgamecategory', 'boardgamefamily', 'boardgamemechanic', 'name', 'game_id']
df_game_vecs = df_game[col_list].copy()
df_game_vecs['docs'] = df_game_vecs[col_list[0]]
for col in col_list[1:-2]:
    df_game_vecs['docs'] = df_game_vecs['docs'] + df_game_vecs[col]
df_game_vecs['id_names'] = df_game_vecs.name + df_game_vecs.game_id
    
d2v = EmbedText()
d2v.d2v_setup(df = df_game_vecs, text_col='docs', class_col='name')
d2v.d2v_fit()
d2v.model_d2v.docvecs.most_similar('7 Wonders', topn=20)
d2v_array = d2v.model_d2v.docvecs.vectors_docs[:2000,:]
tsne = TSNE(n_components=2)
X_tsne = tsne.fit_transform(d2v_array)
vect_size = 2000
trace1 = go.Scatter(
    x=X_tsne[:,0],
    y=X_tsne[:,1],
    mode='text',
    text = d2v.model_d2v.docvecs.offset2doctag[:vect_size],
    textposition='bottom center',
)
data = [trace1]
layout = go.Layout(
    margin=dict(
        l=0,
        r=0,
        b=0,
        t=0
    )
)
config = {'scrollZoom': True}
fig = go.Figure(data=data, layout=layout)
py.iplot(fig, filename='d2v_bgg', config=config)
d2v_array = d2v.model_d2v.docvecs.vectors_docs

type(d2v_array)
aa = d2v.model_d2v.wv.vectors
d2v_array.shape
d2v.model_d2v.docvecs.
df_game_vecs.loc[df_game_vecs.name==74]
df_game.head(2).T<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13712241653418125, 'fraction_numerical': 0.022853736089030206, 'mean_word_length': 4.567477876106195, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '549108', 'license': 'MIT', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1949, 'n_words': 316, 'path': 'notebooks/bgg_user_data.ipynb', 'repo_name': 'MichoelSnow/BGG', 'size': '5032'}