<jupyter_start><jupyter_text>Data Processing Functions for Word2Vec<jupyter_code>import pandas as pd
import numpy as np
# importing all necessary modules 
from nltk.tokenize import sent_tokenize, word_tokenize 
from nltk.corpus import stopwords
import warnings 
  
warnings.filterwarnings(action = 'ignore') 
  
import gensim 
from gensim.models import Word2Vec 
  
#  Reads ‘alice.txt’ file 
sample = open("../DATA_Store/frankenstein_fulltext.txt", "r", encoding="utf8") 
s = sample.read() 
  
# Replaces escape character with space 
f = s.replace("\n", " ")
  
data = [] 
  
# iterate through each sentence in the file 
for i in sent_tokenize(f): 
    temp = [] 
      
    # tokenize the sentence into words 
    for j in word_tokenize(i): 
        temp.append(j.lower())  # all words are lower case
  
    data.append(temp) 
# NEED TO IMPLEMENT PORTER STEMMER and REMOVE PUNCTUATION
# remove stop words
stop_words = set(stopwords.words('english')) 
filtered_data = [[w for w in data[i] if not w in stop_words] for i in range(len(data))] 
filtered_data
len(filtered_data)
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\
    level=logging.INFO)

# Set values for various parameters
num_features = 300    # Word vector dimensionality                      
min_word_count = 10   # Minimum word count                        
num_workers = -1      # Number of threads to run in parallel
context = 10          # Context window size                                                                                    
downsampling = 1e-3   # Downsample setting for frequent words

# Initialize and train the model (this will take some time)
from gensim.models import word2vec
print ("Training model...")
model = word2vec.Word2Vec(filtered_data, workers=num_workers, \
            size=num_features, min_count = min_word_count, \
            window = context, sample = downsampling)
model.most_similar("frankenstein")<jupyter_output><empty_output><jupyter_text>TSNE Analysis<jupyter_code>model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08653378706854643, 'fraction_numerical': 0.007778317938745746, 'mean_word_length': 3.0752475247524753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '85648', 'license': 'MIT', 'n_tokens_mistral': 649, 'n_tokens_neox': 610, 'n_words': 181, 'path': 'DATA_Processing/Data Processing Functions for Word2Vec.ipynb', 'repo_name': 'AniketPant02/frankenstein-word2vec', 'size': '2057'}