<jupyter_start><jupyter_text>Build Simple word2vec Model* Code Sampled from [<NAME>'s GitHub](https://github.com/lknelson/measuring_intersectionality/blob/main/scripts/00_measuringintersectionality_constructmodels.ipynb)<jupyter_code>import sys, os, string, glob, gensim
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize, sent_tokenize

# Import parser module.
module_path = os.path.abspath(os.path.join('..'))

if module_path not in sys.path:
    sys.path.append(module_path + '//Scripts')

from functions_xml_ET_parse import *

# Declare absolute path.
abs_dir = "/Users/quinn.wi/Documents/"

# Define tokenizer.
def fast_tokenize(text):
    
    # Get a list of punctuation marks
    punct = string.punctuation + '“' + '”' + '‘' + "’"
    
    lower_case = text.lower()
    lower_case = lower_case.replace('—', ' ').replace('\n', ' ')
    
    # Iterate through text removing punctuation characters
    no_punct = "".join([char for char in lower_case if char not in punct])
    
    # Split text over whitespace into list of words
    tokens = no_punct.split()
    
    return tokens
gensim.__version__<jupyter_output><empty_output><jupyter_text>Build Dataframe from XML<jupyter_code>%%time

"""
Declare variables.
"""

# Declare regex to simplify file paths below
regex = re.compile(r'.*/.*/(.*.xml)')

# Declare document level of file. Requires root starting point ('.').
doc_as_xpath = './/ns:div/[@type="entry"]'

# Declare date element of each document.
date_path = './ns:bibl/ns:date/[@when]'

# Declare person elements in each document.
person_path = './/ns:p/ns:persRef/[@ref]'

# Declare text level within each document.
text_path = './ns:div/[@type="docbody"]/ns:p'

"""
Build dataframe.
"""

dataframe = []

for file in glob.glob(abs_dir + 'Data/JQA/*/*.xml'):
    reFile = str(regex.search(file).group(1))
#         Call functions to create necessary variables and grab content.
    root = get_root(file)
    ns = get_namespace(root)

    for eachDoc in root.findall(doc_as_xpath, ns):
#             Call functions.
        entry = get_document_id(eachDoc, '{http://www.w3.org/XML/1998/namespace}id')
        date = get_date_from_attrValue(eachDoc, date_path, 'when', ns)
        people = get_peopleList_from_attrValue(eachDoc, person_path, 'ref', ns)
        text = get_textContent(eachDoc, text_path, ns)

        dataframe.append([reFile, entry, date, people, text])

dataframe = pd.DataFrame(dataframe, columns = ['file', 'entry', 'date', 'people', 'text'])

dataframe.head(4)<jupyter_output>CPU times: user 2.4 s, sys: 121 ms, total: 2.52 s
Wall time: 2.7 s<jupyter_text>Build w2v Model<jupyter_code>%%time

# Convert dataframe text field to list of sentences.
sentences = [sentence for text in dataframe['text'] for sentence in sent_tokenize(text)]
words_by_sentence = [fast_tokenize(sentence) for sentence in sentences]
words_by_sentence = [sentence for sentence in words_by_sentence if sentence != []]

# Get total number of words and unique words.
single_list_of_words = []
for l in words_by_sentence:
    for w in l:
        single_list_of_words.append(w)
print (f'Word total: {len(single_list_of_words)}\nUnique word total {len(set(single_list_of_words))}')

# Build model.
model = gensim.models.Word2Vec(words_by_sentence, window=5, vector_size=100,
                               min_count=10, sg=1, alpha=0.025, batch_words=10000, workers=4)

# Unused arguments:
# size=100, iter=5,

# Save model for later use
model.wv.save_word2vec_format(abs_dir + '/Data/Output/WordVectors/jqa_w2v.txt')<jupyter_output>Word total: 2041415
Unique word total 34687
CPU times: user 53.8 s, sys: 838 ms, total: 54.7 s
Wall time: 19.2 s
{'fraction_non_alphanumeric': 0.12812840043525572, 'fraction_numerical': 0.018770402611534277, 'mean_word_length': 4.193502824858757, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '314045', 'license': 'MIT', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1251, 'n_words': 320, 'path': 'Jupyter_Notebooks/WordVectors/w2v_Build-Model.ipynb', 'repo_name': 'NEU-Libraries/dsg-mhs', 'size': '3685'}