<jupyter_start><jupyter_text>ECIS 2021 - Needmining Training Notebook for word2vec embeddings***This is the Jupyter notbook holding code for training and saving word2vec embeddingsThe notebook has the following structure:1. Import2. Load Data3. Training4. Saving embeddings as file<jupyter_code>import logging  # Setting up the loggings to monitor gensim
logging.basicConfig(format="%(levelname)s - %(asctime)s: %(message)s", datefmt= '%H:%M:%S', level=logging.INFO)

from pathlib import Path
import pandas as pd  # For data handling
import csv
import re
import numpy as np

import gensim
from gensim.test.utils import get_tmpfile
from gensim.models import Word2Vec

import sys
import multiprocessing

# Count the number of cores available
cores = multiprocessing.cpu_count() 

# The csv file might contain very huge fields, therefore increase the field_size_limit:
csv.field_size_limit(sys.maxsize)
# path under which the model will be saved, only if save_model = True
save_path = '../models/w2v-amz.bin'

# path the the csv containing the trainingdata
DATA_PREPROCESSED = Path('../data/raw')

# boolean value which controls if we add a suffix of the publication to trump
add_suffix = False

'''
Training/Model parameter, only if load_model = False
see training section for an explaination of the parameters  
'''
min_count=5
window=10
vector_size=300
sample=6e-5 
negative=5
workers=cores-1
dm=0
dbow_words=1
alpha=0.03 
min_alpha=0.0007 

epochs = 5

np.random.seed(0) #set random seed
class Corpus(object):
    def __init__(self, file_path):
        self.file_path = file_path
    
    """An interator that yields sentences (lists of str)."""
    def __iter__(self):
        with open(self.file_path) as csv_file:
            csv_reader = csv.reader(csv_file)
            firstline = True
            i = 0
            for line in csv_reader:
                if firstline:
                    firstline = False
                    continue
                try:
                    text = line[2]
                except IndexError:
                    continue
                yield gensim.utils.simple_preprocess(text)
corpus = Corpus(DATA_PREPROCESSED / 'needs_sentences_filtered.csv')
model = Word2Vec(sentences=corpus, size=20, window = 5, negative=10, sg=1)
model.wv.save_word2vec_format(save_path, binary=True)<jupyter_output>INFO - 11:48:02: storing 34556x20 projection weights into ../models/w2v-amz.bin
{'fraction_non_alphanumeric': 0.08036454018227009, 'fraction_numerical': 0.024026512013256007, 'mean_word_length': 3.3592057761732854, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '421155', 'license': 'MIT', 'n_tokens_mistral': 785, 'n_tokens_neox': 726, 'n_words': 234, 'path': 'notebooks/01-sstahlmann-create-word-embeddings.ipynb', 'repo_name': 'SvenStahlmann/icis-2021', 'size': '2414'}