<jupyter_start><jupyter_text>Perform Doc2Vec  Read data This notebook trains a doc2vec model on the entire notebook dataset.   Import modules<jupyter_code>import pandas as pd
import numpy as np
import os
import gensim
from gensim.models.doc2vec import TaggedDocument<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>df = pd.read_json("../data/all-notebooks.json", orient="index")
df['filename'] = df['filename'].astype(str)
df.columns<jupyter_output><empty_output><jupyter_text>Ignore markdown, only consider code for now<jupyter_code>subdf = df
subdf = df[df.cell_type == "code"]<jupyter_output><empty_output><jupyter_text>Calculate the line number for each notebook<jupyter_code>subdf['cell_num']=subdf.groupby(['filename']).cumcount()+1
subdf['filename_with_cellnum'] = subdf['filename'] + "_" + subdf['cell_num'].astype(str)<jupyter_output><ipython-input-18-37fa39e16431>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  subdf['cell_num']=subdf.groupby(['filename']).cumcount()+1
<ipython-input-18-37fa39e16431>:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  subdf['filename_with_cellnum'] = subdf['filename'] + "_" + subdf['cell_num'].astype(str)<jupyter_text>Preprocessing Generate input data from raw source<jupyter_code>from gensim.utils import simple_preprocess

class MyDataframeCorpus(object):
    def __init__(self, source_df, text_col, tag_col):
        self.source_df = source_df
        self.text_col = text_col
        self.tag_col = tag_col

    def __iter__(self):
        for i, row in self.source_df.iterrows():
            yield TaggedDocument(words=simple_preprocess(row[self.text_col]), 
                                 tags=[row[self.tag_col]])<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>train_corpus = MyDataframeCorpus(subdf, 'source', 'filename_with_cellnum')
model = gensim.models.doc2vec.Doc2Vec(vector_size=300, min_count=2, epochs=40)
model.build_vocab(train_corpus)
model.train(train_corpus, total_examples=model.corpus_count, epochs=model.epochs)<jupyter_output><empty_output><jupyter_text>Test inference<jupyter_code>testinput = "import numpy as np"
vector = model.infer_vector(testinput.split(" "))
# print(vector)
sims = model.docvecs.most_similar([vector])
print(sims)

for result in sims[0:3]:
    display(df[df.filename == result[0]]['source'])<jupyter_output>[('1789576', 0.8671780824661255), ('90182', 0.8659797310829163), ('111694', 0.8597860336303711), ('646182', 0.8538728952407837), ('1222899', 0.8537805080413818), ('240013', 0.8515975475311279), ('8573758', 0.8492058515548706), ('111718', 0.8480663299560547), ('728277', 0.8475252389907837), ('1035483', 0.8474348783493042)]<jupyter_text>Save the model<jupyter_code>model.save("../model/notebook-doc2vec-model-mar24.model")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13290754257907542, 'fraction_numerical': 0.08394160583941605, 'mean_word_length': 6.631090487238979, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '453830', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1190, 'n_words': 203, 'path': 'backend/doc2vec-baselines/Generate-Doc2Vec-Model-From-Json/.ipynb_checkpoints/Doc2Vec_Generate_Doc2Vec_Model-checkpoint.ipynb', 'repo_name': 'WatVis/EDAssistant', 'size': '3288'}