<jupyter_start><jupyter_text>IMDB Movies Dataset Analysis using Doc2Vec  Preprocessing Step<jupyter_code>import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk import word_tokenize
import string

wordnet_lemmatizer = WordNetLemmatizer()
stopwords = nltk.corpus.stopwords.words('english')

def readSentenceAndPreprocess(sentence):
	# Remove stopwords
	record = unicode(sentence.lower(), "utf-8")
	record = word_tokenize(record)
	record = [wordnet_lemmatizer.lemmatize(x) for x in record if x not in string.punctuation]
	sentence = [w for w in record if w not in stopwords]
	return sentence<jupyter_output><empty_output><jupyter_text>Loading the model<jupyter_code>from gensim import models

model = models.Doc2Vec.load('imdb-doc2vec.model')<jupyter_output><empty_output><jupyter_text>Keyword-Test Paragraph Similarity Matrix<jupyter_code>from gensim import models
import pandas
import numpy as np

# get top 10 similar words to keyword from model
keywords = [["lion","king"], ["batman"],["pokemon"],["matrix"],["darth","vader"]]
print "Top 10 Similar words:"
for keyword in keywords:
    print "\n" + '_'.join(keyword)
    print pandas.DataFrame(model.wv.most_similar(positive=keyword, topn=10), columns=['Word','Score'])<jupyter_output>Top 10 Similar words:
lion_king
      Word     Score
0   rafiki  0.330048
1    simba  0.314177
2   matata  0.284346
3   pumbaa  0.282482
4  sabella  0.282138
5    timon  0.267771
6       1½  0.266465
7     1/2\  0.248916
8   hakuna  0.236737
9    marge  0.235552

batman
        Word     Score
0    batgirl  0.354142
1    penguin  0.295907
2       btas  0.272044
3   batwoman  0.271066
4         wb  0.268728
5    o'hearn  0.258377
6     gotham  0.246168
7      joker  0.243499
8  nightwing  0.242217
9    bartram  0.232965

pokemon
      Word     Score
0   celebi  0.475934
1  suicune  0.431710
2    4ever  0.345238
3  things\  0.265295
4  pikachu  0.231380
5      vol  0.227041
6  miramax  0.223460
7    misty  0.217513
8    brock  0.216896
9    lugia  0.216481

matrix
          Word     Score
0     reloaded  0.282041
1     morpheus  0.237963
2   simulation  0.213727
3       kaiser  0.192410
4  iconography  0.192259
5         j.j.  0.184706
6    \special\  0.182963
7     trickery  0.182767
8  [...]
{'fraction_non_alphanumeric': 0.08413672217353199, 'fraction_numerical': 0.1406660823838738, 'mean_word_length': 3.1966911764705883, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '430022', 'license': 'MIT', 'n_tokens_mistral': 1041, 'n_tokens_neox': 893, 'n_words': 237, 'path': 'nlp/doc2vec-sample.ipynb', 'repo_name': 'saisubramaniam147/ml-notebook', 'size': '2282'}